diff --git a/apis/kubedb/v1alpha1/mongodb_helpers.go b/apis/kubedb/v1alpha1/mongodb_helpers.go index dd822f58f2..a8b2abd96a 100644 --- a/apis/kubedb/v1alpha1/mongodb_helpers.go +++ b/apis/kubedb/v1alpha1/mongodb_helpers.go @@ -30,6 +30,8 @@ import ( apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + core_util "kmodules.xyz/client-go/core/v1" v1 "kmodules.xyz/client-go/core/v1" meta_util "kmodules.xyz/client-go/meta" appcat "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1" @@ -71,7 +73,7 @@ func (m MongoDB) ShardNodeTemplate() string { if m.Spec.ShardTopology == nil { return "" } - return fmt.Sprintf("%v${%v}", m.ShardCommonNodeName(), ShardAffinityTemplateVar) + return fmt.Sprintf("%s${%s}", m.ShardCommonNodeName(), ShardAffinityTemplateVar) } func (m MongoDB) ShardCommonNodeName() string { @@ -308,7 +310,7 @@ func (m *MongoDB) GetMonitoringVendor() string { return "" } -func (m *MongoDB) SetDefaults(mgVersion *v1alpha1.MongoDBVersion) { +func (m *MongoDB) SetDefaults(mgVersion *v1alpha1.MongoDBVersion, topology *core_util.Topology) { if m == nil { return } @@ -364,6 +366,19 @@ func (m *MongoDB) SetDefaults(mgVersion *v1alpha1.MongoDBVersion) { m.setDefaultProbes(&m.Spec.ShardTopology.Shard.PodTemplate, mgVersion) m.setDefaultProbes(&m.Spec.ShardTopology.ConfigServer.PodTemplate, mgVersion) m.setDefaultProbes(&m.Spec.ShardTopology.Mongos.PodTemplate, mgVersion) + + // set default affinity (PodAntiAffinity) + shardLabels := m.OffshootSelectors() + shardLabels[MongoDBShardLabelKey] = m.ShardNodeTemplate() + m.setDefaultAffinity(&m.Spec.ShardTopology.Shard.PodTemplate, shardLabels, topology, int(m.Spec.ShardTopology.Shard.Replicas)*int(m.Spec.ShardTopology.Shard.Shards)) + + configServerLabels := m.OffshootSelectors() + configServerLabels[MongoDBConfigLabelKey] = m.ConfigSvrNodeName() + m.setDefaultAffinity(&m.Spec.ShardTopology.ConfigServer.PodTemplate, configServerLabels, topology, int(m.Spec.ShardTopology.ConfigServer.Replicas)) + + mongosLabels := m.OffshootSelectors() + mongosLabels[MongoDBMongosLabelKey] = m.MongosNodeName() + m.setDefaultAffinity(&m.Spec.ShardTopology.Mongos.PodTemplate, mongosLabels, topology, int(m.Spec.ShardTopology.Mongos.Replicas)) } else { if m.Spec.Replicas == nil { m.Spec.Replicas = types.Int32P(1) @@ -378,6 +393,8 @@ func (m *MongoDB) SetDefaults(mgVersion *v1alpha1.MongoDBVersion) { // set default probes m.setDefaultProbes(m.Spec.PodTemplate, mgVersion) + // set default affinity (PodAntiAffinity) + m.setDefaultAffinity(m.Spec.PodTemplate, m.OffshootSelectors(), topology, int(*m.Spec.Replicas)) } } @@ -450,6 +467,46 @@ func (m *MongoDB) setDefaultProbes(podTemplate *ofst.PodTemplateSpec, mgVersion } } +// setDefaultAffinity +func (m *MongoDB) setDefaultAffinity(podTemplate *ofst.PodTemplateSpec, labels map[string]string, topology *core_util.Topology, totalPods int) { + if podTemplate == nil || podTemplate.Spec.Affinity != nil { + // Update topologyKey fields according to Kubernetes version + topology.ConvertAffinity(podTemplate.Spec.Affinity) + return + } + + podTemplate.Spec.Affinity = &core.Affinity{ + PodAntiAffinity: &core.PodAntiAffinity{ + // Prefer to not schedule multiple pods on the node with same zone + PreferredDuringSchedulingIgnoredDuringExecution: []core.WeightedPodAffinityTerm{ + { + Weight: 100, + PodAffinityTerm: core.PodAffinityTerm{ + Namespaces: []string{m.Namespace}, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: topology.LabelZone, + }, + }, + }, + }, + } + + // If there are more nodes than pods, don't schedule multiple pods on the same node + if topology.TotalNodes > totalPods { + podTemplate.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = []core.PodAffinityTerm{ + { + Namespaces: []string{m.Namespace}, + LabelSelector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + TopologyKey: core.LabelHostname, + }, + } + } +} + // setSecurityContext will set default PodSecurityContext. // These values will be applied only to newly created objects. // These defaultings should not be applied to DBs or dormantDBs, diff --git a/apis/kubedb/v1alpha1/mongodb_helpers_test.go b/apis/kubedb/v1alpha1/mongodb_helpers_test.go index f72bccfd7a..723d471fa5 100644 --- a/apis/kubedb/v1alpha1/mongodb_helpers_test.go +++ b/apis/kubedb/v1alpha1/mongodb_helpers_test.go @@ -25,8 +25,22 @@ import ( core "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + core_util "kmodules.xyz/client-go/core/v1" ) +var testTopology = &core_util.Topology{ + Regions: map[string][]string{ + "us-east-1": {"us-east-1a", "us-east-1b", "us-east-1c"}, + }, + TotalNodes: 100, + InstanceTypes: map[string]int{ + "n1-standard-4": 100, + }, + LabelZone: core.LabelZoneFailureDomain, + LabelRegion: core.LabelZoneRegion, + LabelInstanceType: core.LabelInstanceType, +} + func TestMongoDB_HostAddress(t *testing.T) { mongodb := &MongoDB{ ObjectMeta: metav1.ObjectMeta{ @@ -75,6 +89,8 @@ func TestMongoDB_HostAddress(t *testing.T) { }, } + mongodb.SetDefaults(&v1alpha1.MongoDBVersion{}, testTopology) + shardDSN := mongodb.HostAddress() t.Log(shardDSN) @@ -140,7 +156,7 @@ func TestMongoDB_ShardDSN(t *testing.T) { shardDSN := mongodb.ShardDSN(0) t.Log(shardDSN) - mongodb.SetDefaults(&v1alpha1.MongoDBVersion{}) + mongodb.SetDefaults(&v1alpha1.MongoDBVersion{}, testTopology) } func TestMongoDB_ConfigSvrDSN(t *testing.T) { @@ -214,5 +230,5 @@ func TestMongoDB_SetDefaults(t *testing.T) { }, } - mongodb.SetDefaults(&v1alpha1.MongoDBVersion{}) + mongodb.SetDefaults(&v1alpha1.MongoDBVersion{}, testTopology) } diff --git a/go.mod b/go.mod index 1da71fc947..6652824348 100644 --- a/go.mod +++ b/go.mod @@ -22,10 +22,10 @@ require ( k8s.io/apimachinery v0.16.5-beta.1 k8s.io/client-go v12.0.0+incompatible k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a - kmodules.xyz/client-go v0.0.0-20200125095840-f95182274a65 + kmodules.xyz/client-go v0.0.0-20200125212626-a094b2ba24c6 kmodules.xyz/crd-schema-fuzz v0.0.0-20191129174258-81f984340891 kmodules.xyz/custom-resources v0.0.0-20191130062942-f41b54f62419 - kmodules.xyz/monitoring-agent-api v0.0.0-20200125130554-3ed41c0ceff4 + kmodules.xyz/monitoring-agent-api v0.0.0-20200125202117-d3b3e33ce41f kmodules.xyz/objectstore-api v0.0.0-20191127144749-5881939b57f0 kmodules.xyz/offshoot-api v0.0.0-20200103145223-2c4f520520d2 kmodules.xyz/webhook-runtime v0.0.0-20191127075323-d4bfdee6974d diff --git a/go.sum b/go.sum index 2a7a0cbf8b..bf7ddb995c 100644 --- a/go.sum +++ b/go.sum @@ -778,15 +778,15 @@ kmodules.xyz/client-go v0.0.0-20191127054604-26981530831d/go.mod h1:OFxuKCiVR+MY kmodules.xyz/client-go v0.0.0-20191211192817-f1dcd02124ba/go.mod h1:OFxuKCiVR+MYlR2a08FkfaF+IbXkLe0xBetu2LTUuGI= kmodules.xyz/client-go v0.0.0-20200116162153-e083ae16abca h1:WErv3VRRePgp4oszIHgrFnxIaIHyfP9+930B99B8BBM= kmodules.xyz/client-go v0.0.0-20200116162153-e083ae16abca/go.mod h1:OFxuKCiVR+MYlR2a08FkfaF+IbXkLe0xBetu2LTUuGI= -kmodules.xyz/client-go v0.0.0-20200125095840-f95182274a65 h1:GOCyBO2SahVAgRkKl2FNZ95MuMhWpArraUjCFR8heR0= -kmodules.xyz/client-go v0.0.0-20200125095840-f95182274a65/go.mod h1:OFxuKCiVR+MYlR2a08FkfaF+IbXkLe0xBetu2LTUuGI= +kmodules.xyz/client-go v0.0.0-20200125212626-a094b2ba24c6 h1:pZvnFFQOJpSdYhm9h+83tfLgHZkfVmiuHHV8XtWDVSo= +kmodules.xyz/client-go v0.0.0-20200125212626-a094b2ba24c6/go.mod h1:OFxuKCiVR+MYlR2a08FkfaF+IbXkLe0xBetu2LTUuGI= kmodules.xyz/constants v0.0.0-20191024095500-cd4313df4aa6/go.mod h1:DbiFk1bJ1KEO94t1SlAn7tzc+Zz95rSXgyUKa2nzPmY= kmodules.xyz/crd-schema-fuzz v0.0.0-20191129174258-81f984340891 h1:2W/fqLbAurvupIZL3TEFrlb7DnOIhlOXpNgzgcFFSmA= kmodules.xyz/crd-schema-fuzz v0.0.0-20191129174258-81f984340891/go.mod h1:9NXNZ4xhqof0WngtIuo4vl+WoCQpLJEJcyuEo3mPpiM= kmodules.xyz/custom-resources v0.0.0-20191130062942-f41b54f62419 h1:o6KD8XMxdyRR3rqScTsWvcufFDT7vQBnYXpHUp6UtRg= kmodules.xyz/custom-resources v0.0.0-20191130062942-f41b54f62419/go.mod h1:EksiAQK3p3bVU5cNBjLT0z5V7er1tvWLBWILTuf6G/0= -kmodules.xyz/monitoring-agent-api v0.0.0-20200125130554-3ed41c0ceff4 h1:JoAMMESbYxoY+1GFIbIyRRda835uvnLc6vSeiGwqJck= -kmodules.xyz/monitoring-agent-api v0.0.0-20200125130554-3ed41c0ceff4/go.mod h1:q5Adit2fxEoY1jSkYiK1t856Ombl8cunYJBmgL43wwc= +kmodules.xyz/monitoring-agent-api v0.0.0-20200125202117-d3b3e33ce41f h1:BcVQqfRGPGZATwMLabMR6cFytqssE/wh+ZnfYSHIPwQ= +kmodules.xyz/monitoring-agent-api v0.0.0-20200125202117-d3b3e33ce41f/go.mod h1:q5Adit2fxEoY1jSkYiK1t856Ombl8cunYJBmgL43wwc= kmodules.xyz/objectstore-api v0.0.0-20191127144749-5881939b57f0 h1:ilgkGU/bteKRvH99piXxU8b2c1E9lwxb0sDpLLhy4iY= kmodules.xyz/objectstore-api v0.0.0-20191127144749-5881939b57f0/go.mod h1:AE1rz+T/3dlUwH5pLgCNghWeeFfeVBnBXKJVpsLHlmc= kmodules.xyz/offshoot-api v0.0.0-20200103145223-2c4f520520d2 h1:c/wxNy8KgaS8v/htruqtL0YgrKciUKroK1zXPVZHuqg= diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index ef6611f54b..f6d82e0f61 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -36,6 +36,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" + core_util "kmodules.xyz/client-go/core/v1" "kmodules.xyz/client-go/tools/queue" appcat_cs "kmodules.xyz/custom-resources/client/clientset/versioned" appcat_in "kmodules.xyz/custom-resources/client/informers/externalversions" @@ -61,6 +62,8 @@ type Controller struct { CertManagerClient cm.Interface // externalClient for crd ExternalClient ext_cs.Interface + // Cluster topology when the operator started + ClusterTopology *core_util.Topology } type Config struct { diff --git a/vendor/kmodules.xyz/client-go/core/v1/node.go b/vendor/kmodules.xyz/client-go/core/v1/node.go index c5106b9b33..c4f86bda29 100644 --- a/vendor/kmodules.xyz/client-go/core/v1/node.go +++ b/vendor/kmodules.xyz/client-go/core/v1/node.go @@ -23,6 +23,7 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" + "gomodules.xyz/version" core "k8s.io/api/core/v1" kerr "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -121,16 +122,114 @@ func IsMaster(node core.Node) bool { return ok17 || (ok16 && role16 == "master") } -func Topology(kc kubernetes.Interface) (regions map[string][]string, instances map[string]int, err error) { +type Topology struct { + Regions map[string][]string + TotalNodes int + InstanceTypes map[string]int + + LabelZone string + LabelRegion string + LabelInstanceType string + + // https://github.com/kubernetes/kubernetes/blob/v1.17.2/staging/src/k8s.io/api/core/v1/well_known_labels.go + + //LabelHostname = "kubernetes.io/hostname" + // + //LabelZoneFailureDomain = "failure-domain.beta.kubernetes.io/zone" + //LabelZoneRegion = "failure-domain.beta.kubernetes.io/region" + //LabelZoneFailureDomainStable = "topology.kubernetes.io/zone" + //LabelZoneRegionStable = "topology.kubernetes.io/region" + // + //LabelInstanceType = "beta.kubernetes.io/instance-type" + //LabelInstanceTypeStable = "node.kubernetes.io/instance-type" +} + +func (t Topology) ConvertAffinity(affinity *core.Affinity) { + if affinity == nil { + return + } + + t.convertPodAffinityTerm(affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + t.convertWeightedPodAffinityTerm(affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution) + + t.convertPodAffinityTerm(affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution) + t.convertWeightedPodAffinityTerm(affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution) +} + +func isZoneKey(key string) bool { + return key == core.LabelZoneFailureDomain || key == "topology.kubernetes.io/zone" +} + +func isRegionKey(key string) bool { + return key == core.LabelZoneRegion || key == "topology.kubernetes.io/region" +} + +func isInstanceTypeKey(key string) bool { + return key == core.LabelInstanceType || key == "node.kubernetes.io/instance-type" +} + +func (t Topology) convertPodAffinityTerm(terms []core.PodAffinityTerm) { + for i := range terms { + if isZoneKey(terms[i].TopologyKey) { + terms[i].TopologyKey = t.LabelZone + } + if isRegionKey(terms[i].TopologyKey) { + terms[i].TopologyKey = t.LabelRegion + } + if isInstanceTypeKey(terms[i].TopologyKey) { + terms[i].TopologyKey = t.LabelInstanceType + } + } +} + +func (t Topology) convertWeightedPodAffinityTerm(terms []core.WeightedPodAffinityTerm) { + for i := range terms { + if isZoneKey(terms[i].PodAffinityTerm.TopologyKey) { + terms[i].PodAffinityTerm.TopologyKey = t.LabelZone + } + if isRegionKey(terms[i].PodAffinityTerm.TopologyKey) { + terms[i].PodAffinityTerm.TopologyKey = t.LabelRegion + } + if isInstanceTypeKey(terms[i].PodAffinityTerm.TopologyKey) { + terms[i].PodAffinityTerm.TopologyKey = t.LabelInstanceType + } + } +} + +func DetectTopology(kc kubernetes.Interface) (*Topology, error) { // TODO: Use https://github.com/kubernetes/client-go/blob/kubernetes-1.17.0/metadata/interface.go once upgraded to 1.17 + var topology Topology + + info, err := kc.Discovery().ServerVersion() + if err != nil { + return nil, err + } + ver, err := version.NewVersion(info.GitVersion) + if err != nil { + return nil, err + } + ver = ver.ToMutator().ResetPrerelease().ResetMetadata().Done() + if ver.Major() >= 1 && ver.Minor() >= 17 { + topology.LabelZone = "topology.kubernetes.io/zone" + topology.LabelRegion = "topology.kubernetes.io/region" + topology.LabelInstanceType = "node.kubernetes.io/instance-type" + } else { + topology.LabelZone = core.LabelZoneFailureDomain + topology.LabelRegion = core.LabelZoneRegion + topology.LabelInstanceType = core.LabelInstanceType + } + topology.TotalNodes = 0 + mapRegion := make(map[string]sets.String) - instances = make(map[string]int) + instances := make(map[string]int) lister := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) { return kc.CoreV1().Nodes().List(opts) })) err = lister.EachListItem(context.Background(), metav1.ListOptions{Limit: 100}, func(obj runtime.Object) error { + topology.TotalNodes++ + m, err := meta.Accessor(obj) if err != nil { return err @@ -164,12 +263,15 @@ func Topology(kc kubernetes.Interface) (regions map[string][]string, instances m return nil }) if err != nil { - return nil, nil, err + return nil, err } - regions = make(map[string][]string) + regions := make(map[string][]string) for k, v := range mapRegion { regions[k] = v.List() } - return regions, instances, nil + topology.Regions = regions + topology.InstanceTypes = instances + + return &topology, nil } diff --git a/vendor/kmodules.xyz/monitoring-agent-api/api/v1/helpers.go b/vendor/kmodules.xyz/monitoring-agent-api/api/v1/helpers.go index 61eae5a281..51cd4ffe90 100644 --- a/vendor/kmodules.xyz/monitoring-agent-api/api/v1/helpers.go +++ b/vendor/kmodules.xyz/monitoring-agent-api/api/v1/helpers.go @@ -1,3 +1,19 @@ +/* +Copyright The Kmodules Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package v1 import ( diff --git a/vendor/modules.txt b/vendor/modules.txt index e457cc12be..b088f5dfc8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -784,7 +784,7 @@ k8s.io/utils/integer k8s.io/utils/path k8s.io/utils/pointer k8s.io/utils/trace -# kmodules.xyz/client-go v0.0.0-20200125095840-f95182274a65 +# kmodules.xyz/client-go v0.0.0-20200125212626-a094b2ba24c6 kmodules.xyz/client-go kmodules.xyz/client-go/core/v1 kmodules.xyz/client-go/discovery @@ -806,7 +806,7 @@ kmodules.xyz/custom-resources/client/informers/externalversions/appcatalog kmodules.xyz/custom-resources/client/informers/externalversions/appcatalog/v1alpha1 kmodules.xyz/custom-resources/client/informers/externalversions/internalinterfaces kmodules.xyz/custom-resources/client/listers/appcatalog/v1alpha1 -# kmodules.xyz/monitoring-agent-api v0.0.0-20200125130554-3ed41c0ceff4 +# kmodules.xyz/monitoring-agent-api v0.0.0-20200125202117-d3b3e33ce41f kmodules.xyz/monitoring-agent-api/api/v1 # kmodules.xyz/objectstore-api v0.0.0-20191127144749-5881939b57f0 kmodules.xyz/objectstore-api/api/v1