Merge pull request openshift#1518 from squat/federation_experiment

contrib/federation-experiment: add federate tool

contrib/federation-experiment/README.md

@@ -0,0 +1,65 @@
+# Federation Experiment
+
+This README documents the usage of an experimental tool for creating federated Tectonic clusters. 
+
+## Dependencies
+The `federate` tool depends on the following utilities:
+* jq
+* kubectl
+
+## Getting Started
+To begin, create a number of Tectonic clusters using the installer and copy their respective kubeconfig files to a known directory.
+These kubeconfigs must use client certificate authentication.
+*Note* the `federate` tool will not work with kubeconfigs generated by the Tectonic Console.
+*Note*: the name given to the kubeconfig will be the cluster's name in the federation.
+```sh
+cd contrib/federation-experiment
+mkdir kubeconfigs
+cp /path/to/first/kubeconfig kubeconfigs/cluster1
+cp /path/to/second/kubeconfig kubeconfigs/cluster2
+```
+
+Identify the cluster you would like to use as the host cluster for the federation control plane by setting `FEDERATE_HOST` to be the filename of the cluster's corresponding kubeconfig.
+*Note*: if you leave this unset, the `federate` tool will use the first cluster alphabetically.
+```sh
+export FEDERATE_HOST=cluster1
+```
+
+Identify the name and ID of the AWS hosted zone you would like to use for service federation.
+*Note*: the zone name must end with a trailing `.`.
+```sh
+export DNS_ZONE_NAME=tectonic.dev.coreos.systems.
+export DNS_ZONE_ID=Z38L6YZDN7JBF7
+```
+
+Run the `federate` command with the path to the directory of kubeconfigs as the first argument:
+```sh
+./federate kubeconfigs
+```
+
+If the `federate` tool exited cleanly, the hosted control plane will now be deployed to `$FEDERATE_HOST`; congratulations!
+
+## Verify
+Confirm that `federate` created a `clusters` directory in the working directory and that it contains a manifest for each of the Tectonic clusters in the federation.
+
+The `federate` tool will have merged all of the kubeconfigs into one kubeconfig located in the current directory. It will also have created two additional contexts for managing the federation, `federation` and `host`. Switch to the `federation` context and verify that the clusters are ready:
+```sh
+kubectl config use-context federation --kubeconfig=kubeconfig
+kubectl get clusters -w --kubeconfig=kubeconfig
+```
+
+Once `kubectl` reports all clusters as `Ready` you can begin to deploy workloads to the federation.
+
+## Integration With Console
+This federation cluster management is enabled by default in the Tectonic Console in Tectonic versions >= 1.6.4-tectonic.1. Alternatively you can use this Console image: `quay.io/coreos/tectonic-console:multi-cluster-dev`.
+To configure the Tectonic Console and take advantage of this feature, follow these steps:
+
+1. Create a Kubernetes federation using the `federate` tool.
+2. Ensure that the Tectonic clusters in your federation are running a recent tag of the Tectonic Console image, e.g. `multi-cluster-dev`. If they are not, update necessary deployments and specify this image tag in the `PodSpec` section.
+3. In your browser, navigate to the Tectonic Console for each of the clusters in the federation and open the developer tools.
+4. In the developer tools console, type the following to create two local storage entries:
+ ```js
+ localStorage.setItem('federation-apiserver-token', '<the token printed by the federate tool>');
+ localStorage.setItem('federation-apiserver-url', '<the url printed by the federate tool>');
+ ```
+5. Refresh the Tectonic Console and verify that a cluster selection UI is now available under the Tectonic logo. *Note*: local storage settings will persist across refreshes and browser restarts but are tied to a specific Chrome user. This will not work across multiple incognito mode sessions.

contrib/federation-experiment/federate

@@ -0,0 +1,201 @@
+#!/bin/sh
+
+set -e -u -o pipefail
+
+if [ $# -eq 0 ]; then
+ echo "The first argument must be a path to a directory of kubeconfigs"
+ exit 1
+fi
+
+KUBECONFIGS_DIR=$1
+NEW_CLUSTERS=
+CLUSTERS=
+HOST=
+KUBECTL="kubectl --kubeconfig kubeconfig"
+
+# Find new clusters
+for f in $KUBECONFIGS_DIR/*; do
+ [ ! -f "$f" ] && continue
+ NEW_CLUSTERS="$NEW_CLUSTERS $(basename $f)"
+ mv $f $f.yaml
+ mkdir -p $f
+ kubectl --kubeconfig $f.yaml config view --flatten -o json > $f/kubeconfig
+ rm $f.yaml
+done
+
+# Find clusters
+for d in $KUBECONFIGS_DIR/*; do
+ [ ! -d "$d" ] && continue
+ [ "$(basename $d)" = "federation" ] && continue
+ CLUSTERS="$CLUSTERS $(basename $d)"
+done
+
+# Set the host cluster
+HOST=${FEDERATE_HOST:-$(echo $CLUSTERS | cut -d " " -f 1)}
+
+# Merge kubeconfigs
+for cluster in $CLUSTERS; do
+ $KUBECTL config set-cluster $cluster
+
+ $KUBECTL config set clusters.${cluster}.server \
+ "$(jq -r '.clusters[0].cluster.server' $KUBECONFIGS_DIR/${cluster}/kubeconfig)"
+
+ $KUBECTL config set clusters.${cluster}.certificate-authority-data \
+ "$(jq -r '.clusters[0].cluster."certificate-authority-data"' $KUBECONFIGS_DIR/${cluster}/kubeconfig)"
+
+ $KUBECTL config set-credentials $cluster
+
+ $KUBECTL config set users.${cluster}.client-certificate-data \
+ "$(jq -r '.users[0].user."client-certificate-data"' $KUBECONFIGS_DIR/${cluster}/kubeconfig)"
+
+ $KUBECTL config set users.${cluster}.client-key-data \
+ "$(jq -r '.users[0].user."client-key-data"' $KUBECONFIGS_DIR/${cluster}/kubeconfig)"
+
+ $KUBECTL config set-context $cluster \
+ --cluster=${cluster} \
+ --user=${cluster}
+done
+
+# Create host context for convenience
+$KUBECTL config set-context host \
+ --cluster=${HOST} \
+ --user=${HOST}
+
+# Create federation objects
+$KUBECTL config use-context host
+
+# Create federation namespace
+$KUBECTL apply -f manifests/federation-namespace.yaml
+
+# Create cluster role binding
+$KUBECTL apply -f manifests/federation-cluster-role-binding.yaml
+
+# Create federation apiserver service
+$KUBECTL apply -f manifests/federation-apiserver-service.yaml
+
+# Create federation apiserver secret
+if ! $KUBECTL describe secrets federation-apiserver-secrets -n federation ; then
+ $KUBECTL create secret generic federation-apiserver-secrets \
+ --from-literal="known-tokens.csv=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),admin,admin"\
+ -n federation
+fi
+
+# Create persistent volume claim
+$KUBECTL apply -f manifests/federation-etcd-persistent-volume-claim.yaml
+
+# Create federation apiserver configmap
+if ! $KUBECTL describe configmap federation-apiserver -n federation ; then
+ $KUBECTL create configmap federation-apiserver \
+ -n federation \
+ --from-literal=advertise-address="$( \
+ $KUBECTL get service federation-apiserver \
+ -n federation -o jsonpath='{.spec.clusterIP}')"
+fi
+
+# Create federation apiserver deployment
+$KUBECTL apply -f manifests/federation-apiserver-deployment.yaml
+
+# Create federation apiserver ingress
+HOST_CONSOLE_URL=$(jq -r '.clusters[0].cluster.server' $KUBECONFIGS_DIR/$HOST/kubeconfig | sed 's/-api\././g')
+HOST_CONSOLE_DOMAIN=$(echo $HOST_CONSOLE_URL | sed 's|https\{0,1\}://||g' | cut -d ":" -f 1)
+cat > manifests/federation-apiserver-ingress.yaml <<EOF
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+ annotations:
+ ingress.kubernetes.io/rewrite-target: /
+ ingress.kubernetes.io/secure-backends: "true"
+ ingress.kubernetes.io/ssl-passthrough: "true"
+ kubernetes.io/ingress.class: tectonic
+ name: federation-apiserver
+ namespace: federation
+spec:
+ rules:
+ - host: $HOST_CONSOLE_DOMAIN
+ http:
+ paths:
+ - backend:
+ serviceName: federation-apiserver
+ servicePort: 443
+ path: /federation
+ - host: federation.$HOST_CONSOLE_DOMAIN
+ http:
+ paths:
+ - backend:
+ serviceName: federation-apiserver
+ servicePort: 443
+ path: /
+EOF
+$KUBECTL apply -f manifests/federation-apiserver-ingress.yaml
+
+# Get the federation token
+FEDERATION_TOKEN=$($KUBECTL get secret federation-apiserver-secrets -n federation -o go-template='{{index .data "known-tokens.csv"}}' | base64 --decode | cut -d "," -f 1)
+
+# Create federation kubeconfig
+$KUBECTL config set-cluster federation \
+ --server="$HOST_CONSOLE_URL"/federation \
+ --insecure-skip-tls-verify=true
+$KUBECTL config set-credentials federation \
+ --token="$FEDERATION_TOKEN"
+$KUBECTL config set-context federation \
+ --cluster=federation \
+ --user=federation
+$KUBECTL config use-context federation
+mkdir -p "$KUBECONFIGS_DIR"/federation
+$KUBECTL config view --flatten --minify > "$KUBECONFIGS_DIR"/federation/kubeconfig
+
+# Wait for federation apiserver to become available
+until $KUBECTL get clusters 2>/dev/null; do
+ printf "waiting for federation apiserver to become available at %s/federation\n" "$HOST_CONSOLE_URL"
+ sleep 5
+done
+
+# Create federation kubeconfig secret
+$KUBECTL config use-context host
+if ! $KUBECTL describe secret federation-apiserver-kubeconfig -n federation ; then
+ $KUBECTL create secret generic federation-apiserver-kubeconfig \
+ --from-file="$KUBECONFIGS_DIR"/federation/kubeconfig \
+ -n federation
+fi
+
+# Create federation controller manager configmap
+if ! $KUBECTL describe configmap federation-controller-manager -n federation ; then
+ $KUBECTL create configmap federation-controller-manager \
+ --from-literal=zone-id=${DNS_ZONE_ID} \
+ --from-literal=zone-name=${DNS_ZONE_NAME} \
+ -n federation
+fi
+
+# Create federation controller manager deployment
+$KUBECTL apply -f manifests/federation-controller-manager-deployment.yaml
+
+# Create cluster kubeconfig secrets and resources
+mkdir -p clusters
+for cluster in ${CLUSTERS}; do
+ if ! $KUBECTL describe secret ${cluster} -n federation ; then
+ $KUBECTL create secret generic ${cluster} \
+ --from-file=$KUBECONFIGS_DIR/${cluster}/kubeconfig \
+ -n federation
+ fi
+
+ cat > clusters/${cluster}.yaml <<EOF
+apiVersion: federation/v1beta1
+kind: Cluster
+metadata:
+ name: ${cluster}
+ annotations:
+ federation.alpha.coreos.com/console: "$(jq -r '.clusters[0].cluster.server' $KUBECONFIGS_DIR/${cluster}/kubeconfig | sed 's/-api\././g')"
+spec:
+ serverAddressByClientCIDRs:
+ - clientCIDR: "0.0.0.0/0"
+ serverAddress: "$(jq -r '.clusters[0].cluster.server' $KUBECONFIGS_DIR/${cluster}/kubeconfig)"
+ secretRef:
+ name: ${cluster}
+EOF
+done
+
+$KUBECTL config use-context federation
+$KUBECTL apply -f clusters --validate=false
+
+printf "The federation apiserver can be reached at: %s/federation\n" "$HOST_CONSOLE_URL"
+printf "The token for the federation apiserver is: %s\n" "$FEDERATION_TOKEN"

contrib/federation-experiment/manifests/federation-apiserver-deployment.yaml

@@ -0,0 +1,61 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: federation-apiserver
+ namespace: federation
+ labels:
+ app: federated-cluster
+spec:
+ template:
+ metadata:
+ name: federation-apiserver
+ labels:
+ app: federated-cluster
+ module: federation-apiserver
+ spec:
+ containers:
+ - name: apiserver
+ image: gcr.io/google_containers/hyperkube-amd64:v1.6.0
+ env:
+ - name: ADVERTISE_ADDRESS
+ valueFrom:
+ configMapKeyRef:
+ name: federation-apiserver
+ key: advertise-address
+ command:
+ - /hyperkube
+ - federation-apiserver
+ - --bind-address=0.0.0.0
+ - --etcd-servers=http://localhost:2379
+ - --secure-port=443
+ - --advertise-address=$(ADVERTISE_ADDRESS)
+ - --token-auth-file=/srv/kubernetes/known-tokens.csv
+ ports:
+ - containerPort: 443
+ name: https
+ - containerPort: 8080
+ name: local
+ volumeMounts:
+ - name: federation-apiserver-secrets
+ mountPath: /srv/kubernetes/
+ readOnly: true
+ - name: etcd
+ image: quay.io/coreos/etcd:v3.0.7
+ command:
+ - "/usr/local/bin/etcd"
+ args:
+ - "--data-dir=/var/lib/etcd"
+ volumeMounts:
+ - name: etcd-data
+ mountPath: /var/lib/etcd
+ tolerations:
+ - key: node-role.kubernetes.io/master
+ operator: Exists
+ effect: NoSchedule
+ volumes:
+ - name: federation-apiserver-secrets
+ secret:
+ secretName: federation-apiserver-secrets
+ - name: etcd-data 
+ persistentVolumeClaim:
+ claimName: federation-apiserver-etcd

contrib/federation-experiment/manifests/federation-apiserver-service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: federation-apiserver
+ namespace: federation
+ labels:
+ app: federated-cluster
+spec:
+ selector:
+ app: federated-cluster
+ module: federation-apiserver
+ ports:
+ - name: https
+ protocol: TCP
+ port: 443
+ targetPort: 443

contrib/federation-experiment/manifests/federation-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+ name: federation-user
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: cluster-admin
+subjects:
+- kind: ServiceAccount
+ name: default
+ namespace: federation