Generate TLS certs for calico typha (kubernetes-sigs#5258)

* Generate TLS certs for calico typha

Change-Id: I3883f49c124c52d0fc5b900ca2b44e4e2ed0d707

* Add group vars note

Change-Id: I63550dfef616e884efdbd42010a90b2c04c5eb69

inventory/sample/group_vars/k8s-cluster/k8s-net-calico.yml

@@ -34,5 +34,8 @@
 # Use typha (only with kdd)
 # typha_enabled: false
 
+# Generate TLS certs for secure typha<->calico-node communication
+# typha_secure: false
+
 # Number of typha replicas
 # typha_replicas: 1

roles/download/defaults/main.yml

@@ -71,6 +71,7 @@ calico_cni_version: "v3.7.3"
 calico_policy_version: "v3.7.3"
 calico_rr_version: "v0.6.1"
 calico_typha_version: "v3.7.3"
+typha_enabled: false
 
 flannel_version: "v0.11.0"
 flannel_cni_version: "v0.3.0"
@@ -525,7 +526,7 @@ downloads:
       - calico-rr
 
   calico_typha:
-    enabled: "{{ typha_enabled == 'calico' }}"
+    enabled: "{{ typha_enabled }}"
     container: true
     repo: "{{ calico_typha_image_repo }}"
     tag: "{{ calico_typha_image_tag }}"

roles/network_plugin/calico/defaults/main.yml

@@ -74,4 +74,8 @@ typha_enabled: false
 # Number of typha replicas
 typha_replicas: 1
 
+# Generate certifcates for typha<->calico-node communication
+typha_secure: false
+
+
 calico_feature_control: {}

roles/network_plugin/calico/files/make-ssl-typha.sh

@@ -0,0 +1,89 @@
+#!/bin/bash
+
+# Author: Smana [email protected]
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o pipefail
+usage()
+{
+    cat << EOF
+Create self signed certificates
+
+Usage : $(basename $0) -f <config> [-d <ssldir>]
+      -h | --help         : Show this message
+      -f | --config       : Openssl configuration file
+      -d | --ssldir       : Directory where the certificates will be installed
+      -c | --cadir        : Directory where the existing CA is located
+
+               ex :
+               $(basename $0) -f openssl.conf -d /srv/ssl
+EOF
+}
+
+# Options parsing
+while (($#)); do
+    case "$1" in
+        -h | --help)   usage;   exit 0;;
+        -f | --config) CONFIG=${2}; shift 2;;
+        -d | --ssldir) SSLDIR="${2}"; shift 2;;
+        -c | --cadir) CADIR="${2}"; shift 2;;
+        *)
+            usage
+            echo "ERROR : Unknown option"
+            exit 3
+        ;;
+    esac
+done
+
+if [ -z ${CONFIG} ]; then
+    echo "ERROR: the openssl configuration file is missing. option -f"
+    exit 1
+fi
+if [ -z ${SSLDIR} ]; then
+    SSLDIR="/etc/calico/certs"
+fi
+
+tmpdir=$(mktemp -d /tmp/calico_typha_certs.XXXXXX)
+trap 'rm -rf "${tmpdir}"' EXIT
+cd "${tmpdir}"
+
+mkdir -p "${SSLDIR} ${CADIR}"
+
+# Root CA
+if [ -e "$CADIR/ca.key" ]; then
+  # Reuse existing CA
+  cp $CADIR/{ca.crt,ca.key} .
+else
+  openssl genrsa -out ca.key 2048 > /dev/null 2>&1
+  openssl req -x509 -new -nodes -key ca.key -days 3650 -out ca.crt -subj "/CN=calico-typha-ca" > /dev/null 2>&1
+fi
+
+# Typha server
+openssl genrsa -out typha-server.key 2048 > /dev/null 2>&1
+openssl req -new -key typha-server.key -out typha-server.csr -subj "/CN=typha-server" -config ${CONFIG} > /dev/null 2>&1
+openssl x509 -req -in typha-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-server.crt -days 3650 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+
+# Typha client
+openssl genrsa -out typha-client.key 2048 > /dev/null 2>&1
+openssl req -new -key typha-client.key -out typha-client.csr -subj "/CN=typha-client" -config ${CONFIG} > /dev/null 2>&1
+openssl x509 -req -in typha-client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out typha-client.crt -days 3650 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1
+
+# Install certs
+if [ -e "$CADIR/ca.key" ]; then
+    # No pass existing CA
+    rm -f ca.crt ca.key
+fi
+
+mv {*.crt,*.key} ${SSLDIR}/

roles/network_plugin/calico/files/openssl.conf

@@ -0,0 +1,20 @@
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+
+[req_distinguished_name]
+
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = digitalSignature, keyEncipherment
+
+[ ssl_client ]
+extendedKeyUsage = clientAuth, serverAuth
+basicConstraints = CA:FALSE
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid,issuer
+
+[ v3_ca ]
+basicConstraints = CA:TRUE
+keyUsage = cRLSign, digitalSignature, keyCertSign
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid:always,issuer

roles/network_plugin/calico/tasks/install.yml

@@ -57,6 +57,12 @@
     - {s: "{{ kube_etcd_key_file }}", d: "key.pem"}
   when: calico_datastore == "etcd"
 
+- name: Calico | Generate typha certs
+  include_tasks: typha_certs.yml
+  when:
+    - typha_secure
+    - inventory_hostname == groups['kube-master'][0]
+
 - name: Calico | Install calicoctl wrapper script
   template:
     src: "calicoctl.{{ calico_datastore }}.sh.j2"

roles/network_plugin/calico/tasks/typha_certs.yml

@@ -0,0 +1,49 @@
+---
+- name: Calico | Check if typha-server exists
+  command: "{{ bin_dir }}/kubectl -n kube-system get secret typha-server"
+  register: typha_server_secret
+  changed_when: false
+  failed_when: false
+
+- name: Calico | Ensure calico certs dir
+  file:
+    path: /etc/calico/certs
+    state: directory
+  when: typha_server_secret.rc != 0
+
+- name: Calico | Copy ssl script for typha certs
+  copy:
+    src: make-ssl-typha.sh
+    dest: "{{ bin_dir }}/make-ssl-typha.sh"
+    mode: 0755
+  when: typha_server_secret.rc != 0
+
+- name: Calico | Copy ssl config for typha certs
+  copy:
+    src: openssl.conf
+    dest: /etc/calico/certs/openssl.conf
+    mode: 0644
+  when: typha_server_secret.rc != 0
+
+- name: Calico | Generate typha certs
+  command: >-
+    {{ bin_dir }}/make-ssl-typha.sh
+    -f /etc/calico/certs/openssl.conf
+    -c {{ kube_cert_dir }}
+    -d /etc/calico/certs
+  when: typha_server_secret.rc != 0
+
+- name: Calico | Create typha tls secrets
+  command: >-
+    {{ bin_dir }}/kubectl -n kube-system
+    create secret tls {{ item.name }}
+    --cert {{ item.cert }}
+    --key {{ item.key }}
+  with_items:
+    - name: typha-server
+      cert: /etc/calico/certs/typha-server.crt
+      key: /etc/calico/certs/typha-server.key
+    - name: typha-client
+      cert: /etc/calico/certs/typha-client.crt
+      key: /etc/calico/certs/typha-client.key
+  when: typha_server_secret.rc != 0

roles/network_plugin/calico/templates/calico-config.yml.j2

@@ -9,7 +9,7 @@ data:
   etcd_ca: "/calico-secrets/ca_cert.crt"
   etcd_cert: "/calico-secrets/cert.crt"
   etcd_key: "/calico-secrets/key.pem"
-{% elif calico_datastore == "kdd" and typha_enabled == "true" %}
+{% elif calico_datastore == "kdd" and typha_enabled %}
   # To enable Typha, set this to "calico-typha" *and* set a non-zero value for Typha replicas
   # below.  We recommend using Typha if you have more than 50 nodes. Above 100 nodes it is
   # essential.

roles/network_plugin/calico/templates/calico-node.yml.j2

@@ -155,14 +155,24 @@ spec:
             # Use Kubernetes API as the backing datastore.
             - name: DATASTORE_TYPE
               value: "kubernetes"
-  {% if typha_enabled == "true" %}
+{% if typha_enabled %}
             # Typha support: controlled by the ConfigMap.
             - name: FELIX_TYPHAK8SSERVICENAME
               valueFrom:
                 configMapKeyRef:
                   name: calico-config
                   key: typha_service_name
-  {% endif %}
+{% if typha_secure %}
+            - name: FELIX_TYPHACN
+              value: typha-server
+            - name: FELIX_TYPHACAFILE
+              value: /etc/typha-ca/ca.crt
+            - name: FELIX_TYPHACERTFILE
+              value: /etc/typha-client/typha-client.crt
+            - name: FELIX_TYPHAKEYFILE
+              value: /etc/typha-client/typha-client.key
+{% endif %}
+{% endif %}
             # Wait for the datastore.
             - name: WAIT_FOR_DATASTORE
               value: "true"
@@ -296,6 +306,16 @@ spec:
             - name: xtables-lock
               mountPath: /run/xtables.lock
               readOnly: false
+{% if typha_secure %}
+            - name: typha-client
+              mountPath: /etc/typha-client
+              readOnly: true
+            - name: typha-cacert
+              subPath: ca.crt
+              mountPath: /etc/typha-ca/ca.crt
+              readOnly: true
+{% endif %}
+
       volumes:
         # Used by calico/node.
         - name: lib-modules
@@ -332,6 +352,19 @@ spec:
         - name: host-local-net-dir
           hostPath:
             path: /var/lib/cni/networks
+{% endif %}
+{% if typha_enabled and typha_secure %}
+        - name: typha-client
+          secret:
+            secretName: typha-client
+            items:
+               - key: tls.crt
+                 path: typha-client.crt
+               - key: tls.key
+                 path: typha-client.key
+        - name: typha-cacert
+          hostPath:
+            path: "/etc/kubernetes/ssl/"
 {% endif %}
   updateStrategy:
     rollingUpdate:

roles/network_plugin/calico/templates/calico-typha.yml.j2

@@ -62,6 +62,7 @@ spec:
       # Since Calico can't network a pod until Typha is up, we need to run Typha itself
       # as a host-networked pod.
       serviceAccountName: calico-node
+      priorityClassName: system-cluster-critical
       containers:
       - image: {{ calico_typha_image_repo }}:{{ calico_typha_image_tag }}
         name: calico-typha
@@ -86,6 +87,24 @@ spec:
             value: "kubernetes"
           - name: TYPHA_HEALTHENABLED
             value: "true"
+{% if typha_secure %}
+          - name: TYPHA_CAFILE
+            value: /etc/ca/ca.crt
+          - name: TYPHA_CLIENTCN
+            value: typha-client
+          - name: TYPHA_SERVERCERTFILE
+            value: /etc/typha/server_certificate.pem
+          - name: TYPHA_SERVERKEYFILE
+            value: /etc/typha/server_key.pem
+        volumeMounts:
+          - mountPath: /etc/typha
+            name: typha-server
+            readOnly: true
+          - mountPath: /etc/ca/ca.crt
+            subPath: ca.crt
+            name: cacert
+            readOnly: true
+{% endif %}
           # Uncomment these lines to enable prometheus metrics.  Since Typha is host-networked,
           # this opens a port on the host, which may need to be secured.
           #- name: TYPHA_PROMETHEUSMETRICSENABLED
@@ -127,6 +146,20 @@ spec:
             host: localhost
 {% endif %}
           periodSeconds: 10
+{% if typha_secure %}
+      volumes:
+         - name: typha-server
+           secret:
+             secretName: typha-server
+             items:
+                - key: tls.crt
+                  path: server_certificate.pem
+                - key: tls.key
+                  path: server_key.pem
+         - name: cacert
+           hostPath:
+             path: "{{ kube_cert_dir }}"
+{% endif %}
 
 ---
 

tests/files/gce_centos7-calico-ha.yml

@@ -10,5 +10,8 @@ kube_network_plugin: calico
 download_localhost: true
 download_run_once: true
 deploy_netchecker: true
-dns_min_replicas: 1
 cloud_provider: gce
+dns_min_replicas: 1
+typha_enabled: true
+calico_backend: kdd
+typha_secure: true

tests/files/packet_centos7-calico-ha.yml

@@ -9,3 +9,6 @@ download_localhost: true
 download_run_once: true
 deploy_netchecker: true
 dns_min_replicas: 1
+typha_enabled: true
+calico_backend: kdd
+typha_secure: true