Implementing audit dynamic configuration (kubernetes#7392)

Signed-off-by: mmerrill3 <[email protected]>

docs/cluster_spec.md

@@ -246,6 +246,24 @@ You could use the [fileAssets](https://github.com/kubernetes/kops/blob/master/do
 
 Example policy file can be found [here](https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/audit/audit-policy.yaml)
 
+#### dynamic audit configuration
+
+Read more about this here: https://kubernetes.io/docs/tasks/debug-application-cluster/audit/#dynamic-backend
+
+```yaml
+spec:
+  kubeAPIServer:
+    auditDynamicConfiguration: true
+```
+
+By enabling this feature you are allowing for auditsinks to be registered with the API server.  For information on audit sinks please read [Audit Sink](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.13/#auditsink-v1alpha1-auditregistration).  This feature is only supported in kubernetes versions greater than 1.13.  Currently, this feature is alpha and requires enabling the feature gate and a runtime config.
+
+**Note** For kubernetes versions greater than 1.13, this is an alpha feature that requires the API auditregistration.k8s.io/v1alpha1 to be enabled as a runtime-config option, and the feature gate DynamicAuditing to be also enabled.  The options --feature-gates=DynamicAuditing=true and --runtime-config=auditregistration.k8s.io/v1alpha1=true must be enabled on the API server in addition to this flag.  See the sections for how to enable feature gates [here](https://github.com/kubernetes/kops/blob/master/docs/cluster_spec.md#feature-gates).  See the section on how to enable alphas APIs in the runtime config [here](https://github.com/kubernetes/kops/blob/master/docs/cluster_spec.md#runtimeconfig).  
+Also, an audit policy should be provided in the file assets section.  If the flag is omitted, no events are logged.
+You could use the [fileAssets](https://github.com/kubernetes/kops/blob/master/docs/cluster_spec.md#fileassets)  feature to push an advanced audit policy file on the master nodes.
+
+Example policy file can be found [here](https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/audit/audit-policy.yaml)
+
 #### bootstrap tokens
 
 Read more about this here: https://kubernetes.io/docs/reference/access-authn-authz/bootstrap-tokens/

k8s/crds/kops.k8s.io_clusters.yaml

@@ -776,6 +776,10 @@ spec:
                   items:
                     type: string
                   type: array
+                auditDynamicConfiguration:
+                  description: AuditDynamicConfiguration enables dynamic audit configuration
+                    via AuditSinks
+                  type: boolean
                 auditLogFormat:
                   description: AuditLogFormat flag specifies the format type for audit
                     log files.

nodeup/pkg/model/BUILD.bazel

@@ -95,6 +95,7 @@ go_test(
         "//pkg/flagbuilder:go_default_library",
         "//pkg/testutils:go_default_library",
         "//upup/pkg/fi:go_default_library",
+        "//upup/pkg/fi/nodeup/nodetasks:go_default_library",
         "//util/pkg/exec:go_default_library",
         "//vendor/github.com/blang/semver:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",

nodeup/pkg/model/kube_apiserver.go

@@ -347,6 +347,11 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 		}
 	}
 
+	//remove elements from the spec that are not enabled yet
+	if b.Cluster.Spec.KubeAPIServer.AuditDynamicConfiguration != nil && !b.IsKubernetesGTE("1.13") {
+		b.Cluster.Spec.KubeAPIServer.AuditDynamicConfiguration = nil
+	}
+
 	// build the kube-apiserver flags for the service
 	flags, err := flagbuilder.BuildFlagsList(b.Cluster.Spec.KubeAPIServer)
 	if err != nil {

nodeup/pkg/model/kube_apiserver_test.go

@@ -17,13 +17,49 @@ limitations under the License.
 package model
 
 import (
+	"bytes"
+	"strings"
 	"testing"
 
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/upup/pkg/fi"
+	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
 )
 
+func Test_KubeAPIServer_Builder(t *testing.T) {
+	basedir := "tests/apiServer/auditDynamicConfiguration"
+
+	context := &fi.ModelBuilderContext{
+		Tasks: make(map[string]fi.Task),
+	}
+
+	nodeUpModelContext, err := BuildNodeupModelContext(basedir)
+	if err != nil {
+		t.Fatalf("error loading model %q: %v", basedir, err)
+		return
+	}
+
+	builder := KubeAPIServerBuilder{NodeupModelContext: nodeUpModelContext}
+
+	err = builder.Build(context)
+	if err != nil {
+		t.Fatalf("error from KubeAPIServerBuilder buildKubeletConfig: %v", err)
+		return
+	}
+	if task, ok := context.Tasks["File//etc/kubernetes/manifests/kube-apiserver.manifest"]; !ok {
+		t.Error("did not find the kubernetes API manifest after the build")
+	} else {
+		nodeTask, _ := task.(*nodetasks.File)
+		reader, _ := nodeTask.Contents.Open()
+		buf := new(bytes.Buffer)
+		buf.ReadFrom(reader)
+		s := buf.String()
+		if strings.Contains(s, "--audit-dynamic-configuration") {
+			t.Error("Older versions of k8s should not have --audit-dynamic-configuration flag")
+		}
+	}
+}
 func Test_KubeAPIServer_BuildFlags(t *testing.T) {
 	grid := []struct {
 		config   kops.KubeAPIServerConfig
@@ -100,9 +136,23 @@ func Test_KubeAPIServer_BuildFlags(t *testing.T) {
 		},
 		{
 			kops.KubeAPIServerConfig{
-				ServiceAccountKeyFile: []string{"/srv/kubernetes/server.key", "/srv/kubernetes/service-account.key"},
+				AuditDynamicConfiguration: &[]bool{true}[0],
+				ServiceAccountKeyFile:     []string{"/srv/kubernetes/server.key", "/srv/kubernetes/service-account.key"},
+			},
+			"--audit-dynamic-configuration=true --insecure-port=0 --secure-port=0 --service-account-key-file=/srv/kubernetes/server.key --service-account-key-file=/srv/kubernetes/service-account.key",
+		},
+		{
+			kops.KubeAPIServerConfig{
+				AuditDynamicConfiguration: &[]bool{false}[0],
+			},
+			"--audit-dynamic-configuration=false --insecure-port=0 --secure-port=0",
+		},
+		{
+			kops.KubeAPIServerConfig{
+
+				AuditDynamicConfiguration: &[]bool{true}[0],
 			},
-			"--insecure-port=0 --secure-port=0 --service-account-key-file=/srv/kubernetes/server.key --service-account-key-file=/srv/kubernetes/service-account.key",
+			"--audit-dynamic-configuration=true --insecure-port=0 --secure-port=0",
 		},
 	}
 

nodeup/pkg/model/tests/apiServer/auditDynamicConfiguration/cluster.yaml

@@ -0,0 +1,63 @@
+apiVersion: kops.k8s.io/v1alpha2
+kind: Cluster
+metadata:
+  creationTimestamp: "2016-12-10T22:42:27Z"
+  name: minimal.example.com
+spec:
+  kubeAPIServer:
+    auditDynamicConfiguration: true
+  kubernetesApiAccess:
+  - 0.0.0.0/0
+  channel: stable
+  cloudProvider: aws
+  configBase: memfs://clusters.example.com/minimal.example.com
+  etcdClusters:
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: master-us-test-1a
+    name: main
+  - etcdMembers:
+    - instanceGroup: master-us-test-1a
+      name: master-us-test-1a
+    name: events
+  kubelet:
+    featureGates:
+      ExperimentalCriticalPodAnnotation: "true"
+      AllowExtTrafficLocalEndpoints: "false"
+    podManifestPath: "/etc/kubernetes/manifests"
+  kubernetesVersion: v1.6.0
+  masterInternalName: api.internal.minimal.example.com
+  masterPublicName: api.minimal.example.com
+  networkCIDR: 172.20.0.0/16
+  networking:
+    kubenet: {}
+  nonMasqueradeCIDR: 100.64.0.0/10
+  sshAccess:
+    - 0.0.0.0/0
+  topology:
+    masters: public
+    nodes: public
+  subnets:
+  - cidr: 172.20.32.0/19
+    name: us-test-1a
+    type: Public
+    zone: us-test-1a
+
+---
+
+apiVersion: kops.k8s.io/v1alpha2
+kind: InstanceGroup
+metadata:
+  creationTimestamp: "2016-12-10T22:42:28Z"
+  name: masters
+  labels:
+    kops.k8s.io/cluster: minimal.example.com
+spec:
+  associatePublicIp: true
+  image: kope.io/k8s-1.4-debian-jessie-amd64-hvm-ebs-2016-10-21
+  machineType: t2.medium
+  maxSize: 1
+  minSize: 1
+  role: Master
+  subnets:
+  - us-test-1a

pkg/apis/kops/componentconfig.go

@@ -453,6 +453,9 @@ type KubeAPIServerConfig struct {
 
 	// Amount of time to retain Kubernetes events
 	EventTTL *metav1.Duration `json:"eventTTL,omitempty" flag:"event-ttl"`
+
+	// AuditDynamicConfiguration enables dynamic audit configuration via AuditSinks
+	AuditDynamicConfiguration *bool `json:"auditDynamicConfiguration,omitempty" flag:"audit-dynamic-configuration"`
 }
 
 // KubeControllerManagerConfig is the configuration for the controller

pkg/apis/kops/v1alpha1/componentconfig.go

@@ -453,6 +453,9 @@ type KubeAPIServerConfig struct {
 
 	// Amount of time to retain Kubernetes events
 	EventTTL *metav1.Duration `json:"eventTTL,omitempty" flag:"event-ttl"`
+
+	// AuditDynamicConfiguration enables dynamic audit configuration via AuditSinks
+	AuditDynamicConfiguration *bool `json:"auditDynamicConfiguration,omitempty" flag:"audit-dynamic-configuration"`
 }
 
 // KubeControllerManagerConfig is the configuration for the controller

pkg/apis/kops/v1alpha2/componentconfig.go

@@ -453,6 +453,9 @@ type KubeAPIServerConfig struct {
 
 	// Amount of time to retain Kubernetes events
 	EventTTL *metav1.Duration `json:"eventTTL,omitempty" flag:"event-ttl"`
+
+	// AuditDynamicConfiguration enables dynamic audit configuration via AuditSinks
+	AuditDynamicConfiguration *bool `json:"auditDynamicConfiguration,omitempty" flag:"audit-dynamic-configuration"`
 }
 
 // KubeControllerManagerConfig is the configuration for the controller