[k8s] Experimental Service Options (Azure#3179)

Added another `k8s-experimental` section called `serviceOptions`. I has 2 fields, `type` and `LoadBalancerIP`. `type` will be used as Service type if a Service is created. `loadBalancerIP` is put into the `loadBalancerIP` field in the Service, if a service is created. This allows individual modules to override default Service settings.
SlavyMihov · Jul 8, 2020 · cc192a0 · cc192a0
1 parent d7f043d
commit cc192a0
Show file tree

Hide file tree

Showing 9 changed files with 225 additions and 13 deletions.
diff --git a/...ent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CombinedKubernetesConfigProvider.cs b/...ent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CombinedKubernetesConfigProvider.cs
@@ -60,6 +60,7 @@ public CombinedKubernetesConfig GetCombinedConfig(IModule module, IRuntimeInfo r
                 experimentalOptions.ForEach(parameters => createOptions.NodeSelector = parameters.NodeSelector);
                 experimentalOptions.ForEach(parameters => createOptions.Resources = parameters.Resources);
                 experimentalOptions.ForEach(parameters => createOptions.SecurityContext = parameters.SecurityContext);
+                experimentalOptions.ForEach(parameters => createOptions.ServiceOptions = parameters.ServiceOptions);
                 experimentalOptions.ForEach(parameters => createOptions.DeploymentStrategy = parameters.DeploymentStrategy);
             }
 

diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CreatePodParameters.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Kubernetes/CreatePodParameters.cs
@@ -5,6 +5,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes
     using System.Linq;
     using k8s.Models;
     using Microsoft.Azure.Devices.Edge.Agent.Docker.Models;
+    using Microsoft.Azure.Devices.Edge.Agent.Kubernetes.EdgeDeployment.Service;
     using Microsoft.Azure.Devices.Edge.Util;
     using Microsoft.Azure.Devices.Edge.Util.Json;
     using Newtonsoft.Json;
@@ -21,7 +22,7 @@ public CreatePodParameters(
             IEnumerable<string> cmd,
             IEnumerable<string> entrypoint,
             string workingDir)
-            : this(env?.ToList(), exposedPorts, hostConfig, image, labels, cmd?.ToList(), entrypoint?.ToList(), workingDir, null, null, null, null, null)
+            : this(env?.ToList(), exposedPorts, hostConfig, image, labels, cmd?.ToList(), entrypoint?.ToList(), workingDir, null, null, null, null, null, null)
         {
         }
 
@@ -39,6 +40,7 @@ public CreatePodParameters(
             V1ResourceRequirements resources,
             IReadOnlyList<KubernetesModuleVolumeSpec> volumes,
             V1PodSecurityContext securityContext,
+            KubernetesServiceOptions serviceOptions,
             V1DeploymentStrategy strategy)
         {
             this.Env = Option.Maybe(env);
@@ -53,6 +55,7 @@ public CreatePodParameters(
             this.Resources = Option.Maybe(resources);
             this.Volumes = Option.Maybe(volumes);
             this.SecurityContext = Option.Maybe(securityContext);
+            this.ServiceOptions = Option.Maybe(serviceOptions);
             this.DeploymentStrategy = Option.Maybe(strategy);
         }
 
@@ -69,8 +72,9 @@ internal static CreatePodParameters Create(
             V1ResourceRequirements resources = null,
             IReadOnlyList<KubernetesModuleVolumeSpec> volumes = null,
             V1PodSecurityContext securityContext = null,
+            KubernetesServiceOptions serviceOptions = null,
             V1DeploymentStrategy deploymentStrategy = null)
-            => new CreatePodParameters(env, exposedPorts, hostConfig, image, labels, cmd, entrypoint, workingDir, nodeSelector, resources, volumes, securityContext, deploymentStrategy);
+            => new CreatePodParameters(env, exposedPorts, hostConfig, image, labels, cmd, entrypoint, workingDir, nodeSelector, resources, volumes, securityContext, serviceOptions, deploymentStrategy);
 
         [JsonProperty("env", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
         [JsonConverter(typeof(OptionConverter<IReadOnlyList<string>>))]
@@ -108,6 +112,10 @@ internal static CreatePodParameters Create(
         [JsonConverter(typeof(OptionConverter<V1PodSecurityContext>))]
         public Option<V1PodSecurityContext> SecurityContext { get; set; }
 
+        [JsonProperty("serviceOptions", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
+        [JsonConverter(typeof(OptionConverter<KubernetesServiceOptions>))]
+        public Option<KubernetesServiceOptions> ServiceOptions { get; set; }
+
         [JsonProperty("strategy", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
         [JsonConverter(typeof(OptionConverter<V1DeploymentStrategy>))]
         public Option<V1DeploymentStrategy> DeploymentStrategy { get; set; }

diff --git a/...icrosoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesExperimentalCreatePodParameters.cs b/...icrosoft.Azure.Devices.Edge.Agent.Kubernetes/KubernetesExperimentalCreatePodParameters.cs
@@ -5,6 +5,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes
     using System.Collections.Generic;
     using k8s.Models;
     using Microsoft.Azure.Devices.Edge.Agent.Kubernetes.EdgeDeployment;
+    using Microsoft.Azure.Devices.Edge.Agent.Kubernetes.EdgeDeployment.Service;
     using Microsoft.Azure.Devices.Edge.Util;
     using Microsoft.Extensions.Logging;
     using Newtonsoft.Json.Linq;
@@ -19,19 +20,23 @@ public class KubernetesExperimentalCreatePodParameters
 
         public Option<V1PodSecurityContext> SecurityContext { get; }
 
+        public Option<KubernetesServiceOptions> ServiceOptions { get; }
+
         public Option<V1DeploymentStrategy> DeploymentStrategy { get; }
 
         KubernetesExperimentalCreatePodParameters(
             Option<IDictionary<string, string>> nodeSelector,
             Option<V1ResourceRequirements> resources,
             Option<IReadOnlyList<KubernetesModuleVolumeSpec>> volumes,
             Option<V1PodSecurityContext> securityContext,
+            Option<KubernetesServiceOptions> serviceOptions,
             Option<V1DeploymentStrategy> deploymentStrategy)
         {
             this.NodeSelector = nodeSelector;
             this.Resources = resources;
             this.Volumes = volumes;
             this.SecurityContext = securityContext;
+            this.ServiceOptions = serviceOptions;
             this.DeploymentStrategy = deploymentStrategy;
         }
 
@@ -42,6 +47,7 @@ static class ExperimentalParameterNames
             public const string Resources = "Resources";
             public const string Volumes = "Volumes";
             public const string SecurityContext = "SecurityContext";
+            public const string ServiceOptions = "ServiceOptions";
             public const string DeploymentStrategy = "Strategy";
         }
 
@@ -74,10 +80,13 @@ static KubernetesExperimentalCreatePodParameters ParseParameters(JObject experim
             var securityContext = options.Get(ExperimentalParameterNames.SecurityContext)
                 .FlatMap(option => Option.Maybe(option.ToObject<V1PodSecurityContext>()));
 
+            var serviceOptions = options.Get(ExperimentalParameterNames.ServiceOptions)
+                .FlatMap(option => Option.Maybe(option.ToObject<KubernetesServiceOptions>()));
+
             var deploymentStrategy = options.Get(ExperimentalParameterNames.DeploymentStrategy)
                 .FlatMap(option => Option.Maybe(option.ToObject<V1DeploymentStrategy>()));
 
-            return new KubernetesExperimentalCreatePodParameters(nodeSelector, resources, volumes, securityContext, deploymentStrategy);
+            return new KubernetesExperimentalCreatePodParameters(nodeSelector, resources, volumes, securityContext, serviceOptions, deploymentStrategy);
         }
 
         static Dictionary<string, JToken> PrepareSupportedOptionsStore(JObject experimental)
@@ -103,6 +112,7 @@ static Dictionary<string, JToken> PrepareSupportedOptionsStore(JObject experimen
             ExperimentalParameterNames.Resources,
             ExperimentalParameterNames.Volumes,
             ExperimentalParameterNames.SecurityContext,
+            ExperimentalParameterNames.ServiceOptions,
             ExperimentalParameterNames.DeploymentStrategy
         };
 

diff --git a/...oft.Azure.Devices.Edge.Agent.Kubernetes/edgedeployment/service/KubernetesServiceMapper.cs b/...oft.Azure.Devices.Edge.Agent.Kubernetes/edgedeployment/service/KubernetesServiceMapper.cs
@@ -75,14 +75,21 @@ Option<V1Service> PrepareService(IModuleIdentity identity, KubernetesModule modu
                 //
                 // If the user wants to expose the ClusterIPs port externally, they should manually create a service to expose it.
                 // This gives the user more control as to how they want this to work.
-                var serviceType = onlyExposedPorts
-                    ? PortMapServiceType.ClusterIP
-                    : this.defaultMapServiceType;
+                var serviceType = this.GetServiceType(module, onlyExposedPorts);
+                var loadBalancerIP = module.Config.CreateOptions.ServiceOptions.FlatMap(so => so.LoadBalancerIP).OrDefault();
 
-                return Option.Some(new V1Service(metadata: serviceMeta, spec: new V1ServiceSpec(type: serviceType.ToString(), ports: servicePorts.Values.ToList(), selector: labels)));
+                return Option.Some(new V1Service(metadata: serviceMeta, spec: new V1ServiceSpec(type: serviceType.ToString(), loadBalancerIP: loadBalancerIP, ports: servicePorts.Values.ToList(), selector: labels)));
             }
 
             return Option.None<V1Service>();
         }
+
+        PortMapServiceType GetServiceType(KubernetesModule module, bool onlyExposedPorts)
+        {
+            var serviceType = module.Config.CreateOptions.ServiceOptions.FlatMap(so => so.Type);
+            return serviceType.GetOrElse(() => onlyExposedPorts
+                    ? PortMapServiceType.ClusterIP
+                    : this.defaultMapServiceType);
+        }
     }
 }
diff --git a/...ft.Azure.Devices.Edge.Agent.Kubernetes/edgedeployment/service/KubernetesServiceOptions.cs b/...ft.Azure.Devices.Edge.Agent.Kubernetes/edgedeployment/service/KubernetesServiceOptions.cs
@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft. All rights reserved.
+namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.EdgeDeployment.Service
+{
+    using System;
+    using Microsoft.Azure.Devices.Edge.Util;
+    using Microsoft.Azure.Devices.Edge.Util.Json;
+    using Newtonsoft.Json;
+
+    public class KubernetesServiceOptions
+    {
+        public KubernetesServiceOptions(string loadBalancerIP, string type)
+        {
+            PortMapServiceType serviceType;
+            this.LoadBalancerIP = Option.Maybe(loadBalancerIP);
+            this.Type = Enum.TryParse(type, true, out serviceType) ? Option.Some(serviceType) : Option.None<PortMapServiceType>();
+        }
+
+        [JsonProperty("LoadBalancerIP", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
+        [JsonConverter(typeof(OptionConverter<string>))]
+        public Option<string> LoadBalancerIP { get; }
+
+        [JsonProperty("Type", DefaultValueHandling = DefaultValueHandling.IgnoreAndPopulate)]
+        [JsonConverter(typeof(OptionConverter<PortMapServiceType>))]
+        public Option<PortMapServiceType> Type { get; }
+    }
+}
diff --git a/...osoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesExperimentalPodParametersTest.cs b/...osoft.Azure.Devices.Edge.Agent.Kubernetes.Test/KubernetesExperimentalPodParametersTest.cs
@@ -4,6 +4,7 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Kubernetes.Test
     using System.Collections.Generic;
     using System.Linq;
     using k8s.Models;
+    using Microsoft.Azure.Devices.Edge.Agent.Kubernetes.EdgeDeployment.Service;
     using Microsoft.Azure.Devices.Edge.Util;
     using Microsoft.Azure.Devices.Edge.Util.Test.Common;
     using Newtonsoft.Json.Linq;
@@ -68,6 +69,7 @@ public void IgnoresUnsupportedOptions()
             Assert.False(parameters.NodeSelector.HasValue);
             Assert.False(parameters.Resources.HasValue);
             Assert.False(parameters.SecurityContext.HasValue);
+            Assert.False(parameters.ServiceOptions.HasValue);
             Assert.False(parameters.DeploymentStrategy.HasValue);
         }
 
@@ -338,9 +340,84 @@ public void ParsesSomeDeploymentStrategyExperimentalOptions()
             var parameters = KubernetesExperimentalCreatePodParameters.Parse(experimental).OrDefault();
 
             Assert.True(parameters.DeploymentStrategy.HasValue);
-            var deploymentStrategy = parameters.DeploymentStrategy.OrDefault();
-            Assert.Equal("RollingUpdate", deploymentStrategy.Type);
-            Assert.Equal("1", deploymentStrategy.RollingUpdate.MaxUnavailable);
+            parameters.DeploymentStrategy.ForEach(deploymentStrategy => Assert.Equal("RollingUpdate", deploymentStrategy.Type));
+            parameters.DeploymentStrategy.ForEach(deploymentStrategy => Assert.Equal("1", deploymentStrategy.RollingUpdate.MaxUnavailable));
+        }
+
+        [Fact]
+        public void ParsesEmptyServiceOptions()
+        {
+            var experimental = new Dictionary<string, JToken>
+            {
+                ["k8s-experimental"] = JToken.Parse("{ serviceOptions: {  } }")
+            };
+
+            var parameters = KubernetesExperimentalCreatePodParameters.Parse(experimental).OrDefault();
+
+            Assert.True(parameters.ServiceOptions.HasValue);
+            parameters.ServiceOptions.ForEach(options => Assert.False(options.Type.HasValue));
+            parameters.ServiceOptions.ForEach(options => Assert.False(options.LoadBalancerIP.HasValue));
+        }
+
+        [Fact]
+        public void ParsesServiceOptions()
+        {
+            var experimental = new Dictionary<string, JToken>
+            {
+                ["k8s-experimental"] = JToken.Parse(@"{ serviceOptions: { type: ""nodeport"", loadbalancerip: ""100.1.2.3"" } }")
+            };
+
+            var parameters = KubernetesExperimentalCreatePodParameters.Parse(experimental).OrDefault();
+
+            Assert.True(parameters.ServiceOptions.HasValue);
+            parameters.ServiceOptions.ForEach(options =>
+            {
+                Assert.True(options.Type.HasValue);
+                options.Type.ForEach(t => Assert.Equal(PortMapServiceType.NodePort, t));
+            });
+            parameters.ServiceOptions.ForEach(options =>
+            {
+                Assert.True(options.LoadBalancerIP.HasValue);
+                options.LoadBalancerIP.ForEach(l => Assert.Equal("100.1.2.3", l));
+            });
+        }
+
+        [Fact]
+        public void ParsesServiceOptionsTypeOnly()
+        {
+            var experimental = new Dictionary<string, JToken>
+            {
+                ["k8s-experimental"] = JToken.Parse(@"{ serviceOptions: { type: ""LoadBalancer"" } }")
+            };
+
+            var parameters = KubernetesExperimentalCreatePodParameters.Parse(experimental).OrDefault();
+
+            Assert.True(parameters.ServiceOptions.HasValue);
+            parameters.ServiceOptions.ForEach(options =>
+            {
+                Assert.True(options.Type.HasValue);
+                options.Type.ForEach(t => Assert.Equal(PortMapServiceType.LoadBalancer, t));
+            });
+            parameters.ServiceOptions.ForEach(options => Assert.False(options.LoadBalancerIP.HasValue));
+        }
+
+        [Fact]
+        public void ParsesServiceOptionsLBIPOnly()
+        {
+            var experimental = new Dictionary<string, JToken>
+            {
+                ["k8s-experimental"] = JToken.Parse(@"{ serviceOptions: { loadbalancerip: ""any old string"" } }")
+            };
+
+            var parameters = KubernetesExperimentalCreatePodParameters.Parse(experimental).OrDefault();
+
+            Assert.True(parameters.ServiceOptions.HasValue);
+            parameters.ServiceOptions.ForEach(options => Assert.False(options.Type.HasValue));
+            parameters.ServiceOptions.ForEach(options =>
+            {
+                Assert.True(options.LoadBalancerIP.HasValue);
+                options.LoadBalancerIP.ForEach(l => Assert.Equal("any old string", l));
+            });
         }
     }
 }
diff --git a/....Devices.Edge.Agent.Kubernetes.Test/edgedeployment/service/KubernetesServiceMapperTest.cs b/....Devices.Edge.Agent.Kubernetes.Test/edgedeployment/service/KubernetesServiceMapperTest.cs
@@ -79,6 +79,66 @@ public void CreateServiceExposedPortsOnlyCreatesClusterIP()
             Assert.Equal(PortMapServiceType.ClusterIP.ToString(), service.Spec.Type);
         }
 
+        [Fact]
+        public void CreateServiceSetsServiceOptions()
+        {
+            var serviceOptions = new KubernetesServiceOptions("loadBalancerIP", "nodeport");
+            var module = CreateKubernetesModule(CreatePodParameters.Create(exposedPorts: ExposedPorts, serviceOptions: serviceOptions));
+            var mapper = new KubernetesServiceMapper(PortMapServiceType.ClusterIP);
+
+            Option<V1Service> result = mapper.CreateService(CreateIdentity, module, DefaultLabels);
+
+            Assert.True(result.HasValue);
+            var service = result.OrDefault();
+            Assert.Equal(PortMapServiceType.NodePort.ToString(), service.Spec.Type);
+            Assert.Equal("loadBalancerIP", service.Spec.LoadBalancerIP);
+        }
+
+        [Fact]
+        public void CreateServiceSetsServiceOptionsOverridesSetDefault()
+        {
+            var serviceOptions = new KubernetesServiceOptions("loadBalancerIP", "clusterIP");
+            var module = CreateKubernetesModule(CreatePodParameters.Create(hostConfig: HostPorts, serviceOptions: serviceOptions));
+            var mapper = new KubernetesServiceMapper(PortMapServiceType.LoadBalancer);
+
+            Option<V1Service> result = mapper.CreateService(CreateIdentity, module, DefaultLabels);
+
+            Assert.True(result.HasValue);
+            var service = result.OrDefault();
+            Assert.Equal(PortMapServiceType.ClusterIP.ToString(), service.Spec.Type);
+            Assert.Equal("loadBalancerIP", service.Spec.LoadBalancerIP);
+        }
+
+        [Fact]
+        public void CreateServiceSetsServiceOptionsNoIPOnNullLoadBalancerIP()
+        {
+            var serviceOptions = new KubernetesServiceOptions(null, "loadBalancer");
+            var module = CreateKubernetesModule(CreatePodParameters.Create(exposedPorts: ExposedPorts, serviceOptions: serviceOptions));
+            var mapper = new KubernetesServiceMapper(PortMapServiceType.ClusterIP);
+
+            Option<V1Service> result = mapper.CreateService(CreateIdentity, module, DefaultLabels);
+
+            Assert.True(result.HasValue);
+            var service = result.OrDefault();
+            Assert.Equal(PortMapServiceType.LoadBalancer.ToString(), service.Spec.Type);
+            Assert.Null(service.Spec.LoadBalancerIP);
+        }
+
+        [Fact]
+        public void CreateServiceSetsServiceOptionsSetDefaultOnNullType()
+        {
+            var serviceOptions = new KubernetesServiceOptions("loadBalancerIP", null);
+            var module = CreateKubernetesModule(CreatePodParameters.Create(hostConfig: HostPorts, serviceOptions: serviceOptions));
+            var mapper = new KubernetesServiceMapper(PortMapServiceType.LoadBalancer);
+
+            Option<V1Service> result = mapper.CreateService(CreateIdentity, module, DefaultLabels);
+
+            Assert.True(result.HasValue);
+            var service = result.OrDefault();
+            Assert.Equal(PortMapServiceType.LoadBalancer.ToString(), service.Spec.Type);
+            Assert.Equal("loadBalancerIP", service.Spec.LoadBalancerIP);
+        }
+
         [Fact]
         public void CreateServiceExposedPortsOnlyCreatesExposedPortService()
         {

diff --git a/kubernetes/doc/create-options.md b/kubernetes/doc/create-options.md
@@ -22,6 +22,7 @@ We added CreateOptions for experimental features on Kubernetes. These options "o
     "resources": [{...}],
     "nodeSelector": {...},
     "securityContext": {...},
+    "service-options": {...},
     "strategy": {...}
   }
 }
@@ -143,6 +144,25 @@ A `securityContext` section of config used to apply a pod security context to a
 }
 ```
 
+## Apply Service options
+
+EdgeAgent creates a service for each module that exposes one or more ports. Default service types (typically ClusterIp) are assigned to the service.  This does not allow the use to mix service types on an edge deployment. If provided, the `serviceOptions.type` field will override the `type` option for the module's ServiceSpec. Also, if provided, the `serviceOptions.loadBalancerIP` field will be assigned to the `loadBalancerIP` field.
+
+`EdgeAgent` doesn't do any translations or interpretations of values but simply assigns value from module deployment to `type` and `loadBalancerIP` parameter of a service spec. Valid `type` options are "ClusterIP", "NodePort", and "LoadBalancer."
+
+### CreateOptions
+
+```json
+{
+  "k8s-experimental": {
+    "serviceOptions" : {
+      "loadBalancerIP" : "100.23.201.78",
+      "type" : "LoadBalancer"
+    }
+  }
+}
+```
+
 ## Apply Deployment strategy
 
 EdgeAgent uses the default deployment strategy for handling pod replicas. This doesn't always have the expected effects, especially when dealing with persistent volumes. The user may assign this section to get more desireable behavior. This section has the same structure as the Kubernetes [Deployment Strategy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#deploymentstrategy-v1-apps) object.