cleanup overhall

.bootkube_env

@@ -58,5 +58,4 @@ export OS_SVC_USERNAME=demo                   ### OPENSTACK SVC USERNAME      ##
 export OS_SVC_PASSWORD=password               ### OPENSTACK SVC PASSWORD      ###
 
 # CONTAINER REGISTRY PREPARATION
-
 export REGISTRY_ADMIN_PASSWORD=password

01-bootkube-gen-up

@@ -50,6 +50,8 @@ done
 
 
 ### PREPARE THE ENVIRONMENT:
+sudo service docker restart
+sudo service docker enable
 source /opt/demo/.bootkube_env
 echo "export KUBE_MASTER=${KUBE_IP}" >> .bootkube_env
 source /opt/demo/.bootkube_env

02-bootkube-addons

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Copyright 2017 The Bootkube-CI Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+### Declare colors to use during the running of this script:
+declare -r GREEN="\033[0;32m"
+declare -r RED="\033[0;31m"
+declare -r YELLOW="\033[0;33m"
+
+function echo_green {
+  echo -e "${GREEN}$1"; tput sgr0
+}
+function echo_red {
+  echo -e "${RED}$1"; tput sgr0
+}
+function echo_yellow {
+  echo -e "${YELLOW}$1"; tput sgr0
+}
+
+source /opt/demo/.bootkube_env
+
+echo_yellow "\nDeploying Heapster add-on for Kubernetes:"
+kubectl apply -f https://raw.githubusercontent.com/charter-ctec/charter-nfv-demo/master/scripts/kubernetes/manifests/heapster.yaml
+
+echo -e -n "Waiting for Heapster deployment to complete..."
+while true; do
+  running_count=$(kubectl get pods -n kube-system --no-headers 2>/dev/null | grep "1/1" | grep "heapster" | wc -l)
+  ### Expect all components to be out of a "ContainerCreating" state before collecting log data (this includes CrashLoopBackOff states):
+  if [ "$running_count" -ge 1 ]; then
+    break
+  fi
+  echo -n "."
+  sleep 1
+done
+echo_green "HEAPSTER ADD-ON READY!"
+echo ""
+
+echo_yellow "\nDeploying InfluxDB add-on for Kubernetes:"
+kubectl apply -f https://raw.githubusercontent.com/charter-ctec/charter-nfv-demo/master/scripts/kubernetes/manifests/influxdb.yaml
+
+echo -e -n "Waiting for InfluxDB deployment to complete..."
+while true; do
+  running_count=$(kubectl get pods -n kube-system --no-headers 2>/dev/null | grep "1/1" | grep "influxdb" | wc -l)
+  ### Expect all components to be out of a "ContainerCreating" state before collecting log data (this includes CrashLoopBackOff states):
+  if [ "$running_count" -ge 1 ]; then
+    break
+  fi
+  echo -n "."
+  sleep 1
+done
+echo_green "INFLUX ADD-ON READY!"
+echo ""
+
+echo_green "\nList of deployed Kubernetes add-ons:"
+kubectl cluster-info

02-bootkube-node-add → 03-bootkube-node-add

@@ -28,23 +28,23 @@ function echo_yellow {
   echo -e "${YELLOW}$1"; tput sgr0
 }
 
-source .bootkube_env
+source /opt/demo/.bootkube_env
 
 ## NEW INSTALLATIONS:
-for node in $(cat ./deploy-node/nodes); do
-  echo -e "Creating demo directory on ${node} in...";
+for node in $(cat /opt/demo/deploy-node/nodes); do
+  echo_yellow "\nCreating demo directory on ${node} in...";
   ssh -o "StrictHostKeyChecking no" -tq ${node} "mkdir ${HOME}/demo";
   ssh -tq ${node} "sudo mv ${HOME}/demo ${BOOTKUBE_DIR}";
-  echo -e "Copying Kubernetes node files to ${node}...";
+  echo_yellow "\nCopying Kubernetes node files to ${node}...";
   scp ${BOOTKUBE_DIR}/deploy-node/node-add ${node}:${BOOTKUBE_DIR};
-  scp ${BOOTKUBE_DIR}/clean-kube-gen ${node}:${BOOTKUBE_DIR};
+  scp ${BOOTKUBE_DIR}/clean-kube-wipe ${node}:${BOOTKUBE_DIR};
   scp ${BOOTKUBE_DIR}/.bootkube_env ${node}:${BOOTKUBE_DIR};
   scp ${BOOTKUBE_DIR}/.bootkube/tls/ca.crt ${node}:${BOOTKUBE_DIR};
   scp ${BOOTKUBE_DIR}/.bootkube/auth/kubeconfig ${node}:${BOOTKUBE_DIR};
-  scp ${BOOTKUBE_DIR}/video.cfg ${node}:${BOOTKUBE_DIR};
-  echo -e "Attempting to deploy ${node} as a worker node...";
+  echo_yellow "\nAttempting to deploy ${node} as a worker node...";
   ssh -tq ${node} "cd ${BOOTKUBE_DIR}/ && \
     sudo chown root:root ${BOOTKUBE_DIR}/kubeconfig && \
     sudo mkdir -p ${BOOTKUBE_DIR}/backups/ && \
     ./node-add";
+  echo_green "\n${node} has been added to the Kubernetes Master at ${KUBE_MASTER}!"
 done

06-infrastructure-up → 07-infrastructure-up

@@ -34,6 +34,7 @@ source /opt/demo/.bootkube_env
 source /opt/demo/.bootkube_env
 echo -e -n "Bringing up MariaDB..."
 helm install --name=mariadb ${HELM_REPO}/mariadb --namespace=openstack \
+  --set volume.size=${MARIADB_SIZE} \
   --set pod.replicas.server=${MARIADB_REPLICAS}
 
 ### RUN VARIOUS CEPH CHECKS BEFORE CONTINUING:

08-armada_launch_monitoring → 09-armada-launch-monitoring

@@ -30,6 +30,14 @@ function echo_yellow {
 
 source /opt/demo/.bootkube_env
 
+echo_yellow "\nNow deploying Weave Scope to the cluster:"
+kubectl apply --namespace kube-system -f "https://cloud.weave.works/k8s/scope.yaml?k8s-version=$(kubectl version | base64 | tr -d '\n')"
+kubectl patch svc/weave-scope-app -n kube-system -p '{"spec": {"ports": [{"name": "app","nodePort": 30905,"port": 80,"protocol": "TCP","targetPort": 4040}],"type": "NodePort"}}'
+
+echo_yellow "\nDeploying the Kubernetes-Dashboard:"
+kubectl create -f https://raw.githubusercontent.com/kubernetes/dashboard/v1.6.3/src/deploy/kubernetes-dashboard.yaml
+kubectl patch svc/kubernetes-dashboard -n kube-system -p '{"spec": {"ports": [{"nodePort": 30906,"port": 80,"protocol": "TCP","targetPort": 9090}],"type": "NodePort"}}'
+
 echo_yellow "\nCleaning any potential stale jobs before launching Armada deployment:"
 export job_array=`(kubectl get jobs -n monitoring | (awk -F"," '{print $1}') | (cut -d' ' -f1) | (sed '1d') | (awk '/[^[:upper:] ]/') | grep -v ceph)`
 for job in $job_array;do
@@ -39,3 +47,13 @@ done
 echo_yellow "\nUsing Armada to deploy Prometheus and Grafana:"
 sudo docker exec armada armada tiller --status
 sudo docker exec armada armada apply /examples/monitoring.yaml
+
+echo_green "\nCLUSTER MONITORING READY!"
+echo_green "\nUse the following ports to reach Monitoring Services:"
+kubectl get svc -n monitoring
+
+echo_green "\nUse the following ports to reach Kubernetes Services:"
+kubectl get svc -n kube-system
+
+echo_green "\nList of deployed Kubernetes add-ons:"
+kubectl cluster-info

10-boot-vm-newton → 11-boot-vm-newton

@@ -35,6 +35,8 @@ OS_CMD="kubectl exec -n openstack ${OS_TOOLKIT} -- openstack ${KEYSTONE_CREDS} -
 OS_PROJECT=$($OS_CMD project show ctec -f value -c id)
 OS_SEC_GROUP=$($OS_CMD security group list -f csv | grep $OS_PROJECT | grep "default" | awk -F "," '{ print $1 }'  | tr -d '"')
 
+kubectl cp /opt/demo/video.cfg openstack/$OS_TOOLKIT:/tmp/video.cfg
+
 $OS_CMD security group rule create $OS_SEC_GROUP \
   --protocol icmp \
   --src-ip 0.0.0.0/0
@@ -61,7 +63,7 @@ $OS_CMD server create --nic net-id=$NETWORK_ID \
   --image=$VM_IMAGE_ID \
   --key-name="ctec-test" \
   --security-group=$OS_SEC_GROUP \
-  --user-data=/tmp/config/video.cfg \
+  --user-data=/tmp/video.cfg \
   "ctec-vm"
 
 VM_ID=$($OS_CMD server list -f value -c ID)

clean-kube-nodes

@@ -28,15 +28,17 @@ function echo_yellow {
   echo -e "${YELLOW}$1"; tput sgr0
 }
 
-source .bootkube_env
+source /opt/demo/.bootkube_env
 
 ## REMOVING NODE INSTALLATIONS:
 for node in $(cat /opt/demo/deploy-node/nodes); do
-  echo -e "Disabling kubelet.service, and stopping/removing containers for ${node}";
+  echo_yellow "\nDisabling kubelet.service on and stopping/removing containers for ${node}...";
     ssh -tq ${node} "cd ${BOOTKUBE_DIR} && \
-    ./clean-kube-gen";
-  echo -e "Telling the Kubernetes Master to remove ${node}";
-  kubectl drain ${node} --force --ignore-daemonsets;
+    ./clean-kube-wipe";
+  echo_yellow "\nTelling the Kubernetes Master at ${KUBE_MASTER} to remove ${node}...";
+# DRAIN NODES: NOT TYPICALLY USED, MAY HANG NODE
+# kubectl drain ${node} --force --ignore-daemonsets;
   kubectl delete node ${node};
   ssh -tq ${node} "sudo rm -rf ${BOOTKUBE_DIR}";
+  echo_green "\n${node} has been removed from the cluster!";
 done

clean-kube-gen → clean-kube-wipe

@@ -30,12 +30,14 @@ function echo_yellow {
 }
 
 source /opt/demo/.bootkube_env
-echo_red "\nWARNING!!! Removing entire Kubernetes environment from system..."
-{ sudo systemctl stop kubelet.service
+echo_red "\nWARNING!!! Removing entire Kubernetes environment from ${HOSTNAME}..."
+  sudo systemctl stop kubelet.service
   sudo systemctl disable kubelet.service
   sudo systemctl restart docker
   sudo docker stop $(sudo docker ps -a | grep k8s| cut -c1-20 | xargs sudo docker stop)
   sudo docker rm -f $(sudo docker ps -a | grep k8s| cut -c1-20 | xargs sudo docker stop)
+  sudo systemctl stop docker
+  sudo systemctl disable docker
   sudo rm -rf /etc/kubernetes/
   sudo rm -rf /var/etcd
   sudo rm -rf /var/run/calico
@@ -46,27 +48,27 @@ echo_red "\nWARNING!!! Removing entire Kubernetes environment from system..."
   sudo rm -rf /var/run/lock/api-server.lock
   sudo rm -rf /var/run/lock/etcd.lock
   sudo rm -rf /var/run/lock/pod-checkpointer.lock
-  sudo rm -rf $BOOTKUBE_DIR/log/cluster-info*
-  sudo rm -rf $BOOTKUBE_DIR/.bootkube
+  sudo rm -rf /opt/demo/log/cluster-info*
+  sudo rm -rf /opt/demo/.bootkube
   sudo ip link set flannel.1 down
   sudo hostess del ${KUBE_DNS_API} ${KUBE_MASTER}
   sudo hostess del kubernetes ${KUBE_MASTER}
-  sudo cp $BOOTKUBE_DIR/backups/resolv.conf /etc/resolv.conf
+  sudo cp /opt/demo/backups/resolv.conf /etc/resolv.conf
   sudo rm -rf /var/lib/ceph/* /var/lib/openstack-helm/*
   ### These can optionally be enabled, but a menu is soon to come to clean up the following binaries:
-  sudo rm -rf /usr/local/bin/bootkube
-  sudo rm -rf /usr/local/bin/kubectl
-  sudo rm -rf /usr/local/bin/kubelet
-  sudo rm -rf /usr/local/bin/helm
-  sudo rm -rf /usr/local/bin/hostess
-  sudo rm -rf /opt/cni
+  # sudo rm -rf /usr/local/bin/bootkube
+  # sudo rm -rf /usr/local/bin/kubectl
+  # sudo rm -rf /usr/local/bin/kubelet
+  # sudo rm -rf /usr/local/bin/helm
+  # sudo rm -rf /usr/local/bin/hostess
+  # sudo rm -rf /opt/cni
   sudo rm -rf /etc/systemd/system/kubelet.service
   sudo systemctl daemon-reload
   ### These can optionally be enabled, but Docker will need to be reinstalled, and volumes will be perm removed:
-  sudo apt-get remove docker.io -y
-  sudo rm -rf /var/lib/docker
-  sudo rm -rf /var/lib/kubelet
-  sudo rm -rf /var/lib/dockershim
-} &> /dev/null
-
+  # sudo apt-get remove docker.io -y
+  # sudo rm -rf /var/lib/docker
+  # sudo rm -rf /var/lib/kubelet
+  # sudo rm -rf /var/lib/dockershim
+  sudo rm -rf /var/lib/openstack-helm/*
+  sudo rm -rf /var/lib/nova
 echo_green "\nCOMPLETE!\n"

clean-osh

@@ -37,6 +37,13 @@ helm delete --purge osh-etcd
 helm delete --purge osh-libvirt
 helm delete --purge osh-openvswitch
 helm delete --purge osh-memcached
+helm delete --purge ctec-grafana
+helm delete --purge ctec-prometheus
+helm delete --purge ceph-monitoring-config
+
+echo_yellow "\nDELETING MONITORING PVC's:"
+kubectl delete pvc/alertmanager-pvc
+kubectl delete pvc/prometheus-pvc
 
 echo_yellow "\nDELETING OPENSTACK COMPONENTS:"
 helm delete --purge osh-horizon &
@@ -68,6 +75,11 @@ sleep 5
 kubectl delete jobs/`(kubectl get jobs -n openstack | grep nova-clean | cut -d " " -f1)` -n openstack
 kubectl delete jobs/`(kubectl get jobs -n openstack | grep nova-db-drop | cut -d " " -f1)` -n openstack
 
+echo_yellow "\nDELETING ANY REMAINING CRONJOBS:"
+kubectl delete cronjob/`(kubectl get cronjob -n openstack | grep keystone-credential | cut -d " " -f1)` -n openstack
+kubectl delete cronjob/`(kubectl get cronjob -n openstack | grep keystone-fernet | cut -d " " -f1)` -n openstack
+kubectl delete cronjob/`(kubectl get cronjob -n openstack | grep cinder | cut -d " " -f1)` -n openstack
+
 echo_yellow "\nDELETING DATABASE COMPONENTS:"
 kubectl delete pvc mysql-data-mariadb-0 -n openstack
 kubectl delete pvc mysql-data-mariadb-1 -n openstack
@@ -86,15 +98,6 @@ helm delete --purge ceph-openstack-config
 helm delete --purge ceph
 sudo rm -rf /var/lib/openstack-helm/ceph
 
-#echo_yellow "\nDELETING DATABASE COMPONENTS:"
-#kubectl delete pvc mysql-data-mariadb-0 -n openstack
-#kubectl delete pvc mysql-data-mariadb-1 -n openstack
-#kubectl delete pvc mysql-data-mariadb-2 -n openstack
-#kubectl delete po/mariadb-0 --grace-period=0 --force -n openstack
-#kubectl delete po/mariadb-1 --grace-period=0 --force -n openstack
-#kubectl delete po/mariadb-2 --grace-period=0 --force -n openstack
-#helm delete --purge mariadb
-
 echo_yellow "\nDELETING DEMO NAMESPACES:"
 kubectl delete namespace openstack
 kubectl delete namespace ceph
@@ -105,3 +108,5 @@ helm delete --purge default-nfs
 echo_yellow "\nDELETING OPENSTACK-HELM LOCAL STORAGE DIRS:"
 sudo rm -rf /var/lib/openstack-helm/nfs
 sudo rm -rf /var/lib/openstack-helm/*
+
+echo_green "\nDONE!"

deploy-node/node-add

@@ -136,7 +136,9 @@ sudo systemctl daemon-reload
 echo_yellow "\nEnabling and restarting the kubelet service..."
 sudo systemctl enable kubelet.service
 sudo systemctl restart kubelet.service
+sudo service docker restart
 echo_yellow "\nCopying the kubelet config..."
 sudo cp ${BOOTKUBE_DIR}/kubeconfig /etc/kubernetes/
 echo_yellow "\nCopying the kubelet certificate..."
 sudo cp ${BOOTKUBE_DIR}/ca.crt /etc/kubernetes/
+echo_green "\nCOMPLETE! ${HOSTNAME} has been added to the cluster."

deploy-node/node-clean

@@ -34,5 +34,5 @@ source /opt/demo/.bootkube_env
 for node in $(cat /opt/demo/deploy-node/nodes); do
   echo -e "Cleaning $node:"
   ssh -tq $node "cd ${BOOTKUBE_DIR} && \
-    ./clean-kube-gen";
+    ./clean-kube-wipe";
 done