add VMAS scale up/down support for openshift (Azure#2896)

* add tgzreader to filesystem

* validate that openshift can only be started with one master

* remove ExternalMasterHostname and RouterLBHostname from OpenShiftConfig

* add VMAS scale up/down support for openshift

* minor test fixes

cmd/scale.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -18,6 +19,7 @@ import (
 	"github.com/Azure/acs-engine/pkg/armhelpers/utils"
 	"github.com/Azure/acs-engine/pkg/helpers"
 	"github.com/Azure/acs-engine/pkg/i18n"
+	"github.com/Azure/acs-engine/pkg/openshift/filesystem"
 	"github.com/Azure/acs-engine/pkg/operations"
 	"github.com/leonelquinteros/gotext"
 
@@ -51,8 +53,8 @@ type scaleCmd struct {
 
 const (
 	scaleName             = "scale"
-	scaleShortDescription = "Scale an existing Kubernetes cluster"
-	scaleLongDescription  = "Scale an existing Kubernetes cluster by specifying increasing or decreasing the node count of an agentpool"
+	scaleShortDescription = "Scale an existing Kubernetes or OpenShift cluster"
+	scaleLongDescription  = "Scale an existing Kubernetes or OpenShift cluster by specifying increasing or decreasing the node count of an agentpool"
 )
 
 // NewScaleCmd run a command to upgrade a Kubernetes cluster
@@ -237,12 +239,32 @@ func (sc *scaleCmd) run(cmd *cobra.Command, args []string) error {
 				vmsToDelete = append(vmsToDelete, indexToVM[i])
 			}
 
-			if orchestratorInfo.OrchestratorType == api.Kubernetes {
-				err = sc.drainNodes(vmsToDelete)
+			switch orchestratorInfo.OrchestratorType {
+			case api.Kubernetes:
+				kubeConfig, err := acsengine.GenerateKubeConfig(sc.containerService.Properties, sc.location)
+				if err != nil {
+					log.Errorf("failed to generate kube config: %v", err)
+					return err
+				}
+				err = sc.drainNodes(kubeConfig, vmsToDelete)
 				if err != nil {
 					log.Errorf("Got error %+v, while draining the nodes to be deleted", err)
 					return err
 				}
+			case api.OpenShift:
+				bundle := bytes.NewReader(sc.containerService.Properties.OrchestratorProfile.OpenShiftConfig.ConfigBundles["master"])
+				fs, err := filesystem.NewTGZReader(bundle)
+				if err != nil {
+					return fmt.Errorf("failed to read master bundle: %v", err)
+				}
+				kubeConfig, err := fs.ReadFile("etc/origin/master/admin.kubeconfig")
+				if err != nil {
+					return fmt.Errorf("failed to read kube config: %v", err)
+				}
+				err = sc.drainNodes(string(kubeConfig), vmsToDelete)
+				if err != nil {
+					return fmt.Errorf("Got error %v, while draining the nodes to be deleted", err)
+				}
 			}
 
 			errList := operations.ScaleDownVMs(sc.client, sc.logger, sc.resourceGroupName, vmsToDelete...)
@@ -320,6 +342,14 @@ func (sc *scaleCmd) run(cmd *cobra.Command, args []string) error {
 	addValue(parametersJSON, sc.agentPool.Name+"Count", countForTemplate)
 
 	switch orchestratorInfo.OrchestratorType {
+	case api.OpenShift:
+		err = transformer.NormalizeForOpenShiftVMASScalingUp(sc.logger, sc.agentPool.Name, templateJSON)
+		if err != nil {
+			return fmt.Errorf("error tranforming the template for scaling template %s: %v", sc.apiModelPath, err)
+		}
+		if sc.agentPool.IsAvailabilitySets() {
+			addValue(parametersJSON, fmt.Sprintf("%sOffset", sc.agentPool.Name), highestUsedIndex+1)
+		}
 	case api.Kubernetes:
 		err = transformer.NormalizeForK8sVMASScalingUp(sc.logger, templateJSON)
 		if err != nil {
@@ -387,11 +417,7 @@ func addValue(m paramsMap, k string, v interface{}) {
 	}
 }
 
-func (sc *scaleCmd) drainNodes(vmsToDelete []string) error {
-	kubeConfig, err := acsengine.GenerateKubeConfig(sc.containerService.Properties, sc.location)
-	if err != nil {
-		log.Fatalf("failed to generate kube config: %v", err) // TODO: cleanup
-	}
+func (sc *scaleCmd) drainNodes(kubeConfig string, vmsToDelete []string) error {
 	masterURL := sc.masterFQDN
 	if !strings.HasPrefix(masterURL, "https://") {
 		masterURL = fmt.Sprintf("https://%s", masterURL)
@@ -401,7 +427,7 @@ func (sc *scaleCmd) drainNodes(vmsToDelete []string) error {
 	defer close(errChan)
 	for _, vmName := range vmsToDelete {
 		go func(vmName string) {
-			err = operations.SafelyDrainNode(sc.client, sc.logger,
+			err := operations.SafelyDrainNode(sc.client, sc.logger,
 				masterURL, kubeConfig, vmName, time.Duration(60)*time.Minute)
 			if err != nil {
 				log.Errorf("Failed to drain node %s, got error %v", vmName, err)

parts/openshift/openshiftnodescript.sh

@@ -23,3 +23,8 @@ update-ca-trust
 # note: ${SERVICE_TYPE}-node crash loops until master is up
 systemctl enable ${SERVICE_TYPE}-node.service
 systemctl start ${SERVICE_TYPE}-node.service &
+
+while [[ $(KUBECONFIG=/etc/origin/node/node.kubeconfig oc get node $(hostname) -o template \
+    --template '{{`{{range .status.conditions}}{{if eq .type "Ready"}}{{.status}}{{end}}{{end}}`}}') != True ]]; do
+    sleep 1
+done

pkg/acsengine/defaults.go

@@ -769,8 +769,11 @@ func setStorageDefaults(a *api.Properties) {
 }
 
 func openShiftSetDefaultCerts(a *api.Properties) (bool, error) {
-	externalMasterHostname := fmt.Sprintf("%s.%s.cloudapp.azure.com", a.MasterProfile.DNSPrefix, a.AzProfile.Location)
-	routerLBHostname := fmt.Sprintf("%s-router.%s.cloudapp.azure.com", a.MasterProfile.DNSPrefix, a.AzProfile.Location)
+	if len(a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["master"]) > 0 &&
+		len(a.OrchestratorProfile.OpenShiftConfig.ConfigBundles["bootstrap"]) > 0 {
+		return true, nil
+	}
+
 	c := certgen.Config{
 		Master: &certgen.Master{
 			Hostname: fmt.Sprintf("%s-master-%s-0", DefaultOpenshiftOrchestratorName, GenerateClusterID(a)),
@@ -779,7 +782,7 @@ func openShiftSetDefaultCerts(a *api.Properties) (bool, error) {
 			},
 			Port: 8443,
 		},
-		ExternalMasterHostname:  externalMasterHostname,
+		ExternalMasterHostname:  fmt.Sprintf("%s.%s.cloudapp.azure.com", a.MasterProfile.DNSPrefix, a.AzProfile.Location),
 		ClusterUsername:         a.OrchestratorProfile.OpenShiftConfig.ClusterUsername,
 		ClusterPassword:         a.OrchestratorProfile.OpenShiftConfig.ClusterPassword,
 		EnableAADAuthentication: a.OrchestratorProfile.OpenShiftConfig.EnableAADAuthentication,
@@ -792,8 +795,6 @@ func openShiftSetDefaultCerts(a *api.Properties) (bool, error) {
 			Location:        a.AzProfile.Location,
 		},
 	}
-	a.OrchestratorProfile.OpenShiftConfig.ExternalMasterHostname = externalMasterHostname
-	a.OrchestratorProfile.OpenShiftConfig.RouterLBHostname = routerLBHostname
 
 	err := c.PrepareMasterCerts()
 	if err != nil {
@@ -832,12 +833,12 @@ func openShiftSetDefaultCerts(a *api.Properties) (bool, error) {
 	return true, nil
 }
 
-type writeFn func(filesystem.Filesystem) error
+type writeFn func(filesystem.Writer) error
 
 func getConfigBundle(write writeFn) ([]byte, error) {
 	b := &bytes.Buffer{}
 
-	fs, err := filesystem.NewTGZFile(b)
+	fs, err := filesystem.NewTGZWriter(b)
 	if err != nil {
 		return nil, err
 	}

pkg/acsengine/engine.go

@@ -1949,8 +1949,8 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 				Location               string
 			}{
 				ConfigBundle:           base64.StdEncoding.EncodeToString(cs.Properties.OrchestratorProfile.OpenShiftConfig.ConfigBundles["master"]),
-				ExternalMasterHostname: cs.Properties.OrchestratorProfile.OpenShiftConfig.ExternalMasterHostname,
-				RouterLBHostname:       cs.Properties.OrchestratorProfile.OpenShiftConfig.RouterLBHostname,
+				ExternalMasterHostname: fmt.Sprintf("%s.%s.cloudapp.azure.com", cs.Properties.MasterProfile.DNSPrefix, cs.Properties.AzProfile.Location),
+				RouterLBHostname:       fmt.Sprintf("%s-router.%s.cloudapp.azure.com", cs.Properties.MasterProfile.DNSPrefix, cs.Properties.AzProfile.Location),
 				Location:               cs.Properties.AzProfile.Location,
 			})
 			return b.String(), err

pkg/acsengine/transform/transform.go

@@ -3,6 +3,7 @@ package transform
 import (
 	"fmt"
 	"log"
+	"runtime"
 	"sort"
 	"strings"
 
@@ -28,6 +29,7 @@ const (
 	createOptionFieldName          = "createOption"
 	tagsFieldName                  = "tags"
 	managedDiskFieldName           = "managedDisk"
+	outputsFieldName               = "outputs"
 
 	// ARM resource Types
 	nsgResourceType  = "Microsoft.Network/networkSecurityGroups"
@@ -97,6 +99,55 @@ func (t *Transformer) NormalizeForVMSSScaling(logger *logrus.Entry, templateMap
 	return nil
 }
 
+// NormalizeForOpenShiftVMASScalingUp takes a template and removes elements that
+// are unwanted in a OpenShift VMAS scale up/down case
+func (t *Transformer) NormalizeForOpenShiftVMASScalingUp(logger *logrus.Entry, agentPoolName string, templateMap map[string]interface{}) (err error) {
+	defer func() {
+		// catch a failed type assertion and return it as an error.  This saves
+		// needing to write `if foo, ok := bar.(*Baz); ok` everywhere below.
+		if r := recover(); r != nil {
+			e, ok := r.(*runtime.TypeAssertionError)
+			if ok {
+				err = e
+			} else {
+				panic(r)
+			}
+		}
+	}()
+
+	isNeeded := func(name interface{}) bool {
+		return strings.Contains(name.(string), agentPoolName+"VMNamePrefix")
+	}
+
+	// only include resources including <agentPoolName>VMNamePrefix in their
+	// name (VM, VM extension, NIC).
+	resources := []interface{}{}
+	for _, res := range templateMap[resourcesFieldName].([]interface{}) {
+		res := res.(map[string]interface{})
+
+		if !isNeeded(res[nameFieldName]) {
+			continue
+		}
+
+		// remove dependencies to removed resources
+		depends := []interface{}{}
+		for _, depend := range res[dependsOnFieldName].([]interface{}) {
+			if isNeeded(depend) {
+				depends = append(depends, depend)
+			}
+		}
+		res[dependsOnFieldName] = depends
+
+		resources = append(resources, res)
+	}
+	templateMap[resourcesFieldName] = resources
+
+	// remove all outputs: they may depend on deleted resources
+	templateMap[outputsFieldName] = []interface{}{}
+
+	return
+}
+
 // NormalizeForK8sVMASScalingUp takes a template and removes elements that are unwanted in a K8s VMAS scale up/down case
 func (t *Transformer) NormalizeForK8sVMASScalingUp(logger *logrus.Entry, templateMap map[string]interface{}) error {
 	if err := t.NormalizeMasterResourcesForScaling(logger, templateMap); err != nil {

pkg/acsengine/transform/transform_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/Azure/acs-engine/pkg/helpers"
 	"github.com/Azure/acs-engine/pkg/i18n"
+	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/sirupsen/logrus"
 )
@@ -63,6 +64,69 @@ func TestNormalizeForK8sVMASScalingUpWithVnet(t *testing.T) {
 	ValidateTemplate(templateMap, expectedFileContents, "TestNormalizeForK8sVMASScalingUpWithVnet")
 }
 
+func TestNormalizeForOpenShiftVMASScalingUp(t *testing.T) {
+	RegisterTestingT(t)
+
+	tests := []struct {
+		agentPoolName       string
+		templateMap         map[string]interface{}
+		expectedTemplateMap map[string]interface{}
+		expectedErr         bool
+	}{
+		{
+			// a badly constructed input should result in an error, not a panic
+			expectedErr: true,
+		},
+		{
+			agentPoolName: "compute",
+			templateMap: map[string]interface{}{
+				"resources": []interface{}{
+					map[string]interface{}{
+						"name": "foo",
+					},
+					map[string]interface{}{
+						"name": "barVMNamePrefix",
+						"dependsOn": []interface{}{
+							"foo",
+						},
+					},
+					map[string]interface{}{
+						"name": "computeVMNamePrefix",
+						"dependsOn": []interface{}{
+							"foo",
+							"barVMNamePrefix",
+							"computeVMNamePrefix",
+						},
+					},
+				},
+				"outputs": []interface{}{
+					"junk",
+				},
+			},
+			expectedTemplateMap: map[string]interface{}{
+				"resources": []interface{}{
+					map[string]interface{}{
+						"name": "computeVMNamePrefix",
+						"dependsOn": []interface{}{
+							"computeVMNamePrefix",
+						},
+					},
+				},
+				"outputs": []interface{}{},
+			},
+		},
+	}
+
+	transformer := &Transformer{}
+
+	for i, test := range tests {
+		fmt.Fprintf(GinkgoWriter, "test %d\n", i)
+		err := transformer.NormalizeForOpenShiftVMASScalingUp(nil, test.agentPoolName, test.templateMap)
+		Expect(err != nil).To(Equal(test.expectedErr))
+		Expect(test.templateMap).To(BeEquivalentTo(test.expectedTemplateMap))
+	}
+}
+
 func TestNormalizeResourcesForK8sMasterUpgrade(t *testing.T) {
 	RegisterTestingT(t)
 	logger := logrus.New().WithField("testName", "TestNormalizeResourcesForK8sMasterUpgrade")

pkg/api/converterfromapi.go

@@ -661,6 +661,7 @@ func convertOpenShiftConfigToVLabs(api *OpenShiftConfig, vl *vlabs.OpenShiftConf
 	vl.ClusterUsername = api.ClusterUsername
 	vl.ClusterPassword = api.ClusterPassword
 	vl.EnableAADAuthentication = api.EnableAADAuthentication
+	vl.ConfigBundles = api.ConfigBundles
 }
 
 func convertDcosConfigToVLabs(api *DcosConfig, vl *vlabs.DcosConfig) {
@@ -987,6 +988,7 @@ func convertAgentPoolProfileToVLabs(api *AgentPoolProfile, p *vlabs.AgentPoolPro
 		p.ImageRef.Name = api.ImageRef.Name
 		p.ImageRef.ResourceGroup = api.ImageRef.ResourceGroup
 	}
+	p.Role = vlabs.AgentPoolProfileRole(api.Role)
 }
 
 func convertDiagnosticsProfileToV20160930(api *DiagnosticsProfile, dp *v20160930.DiagnosticsProfile) {

pkg/api/convertertoapi.go

@@ -649,6 +649,7 @@ func convertVLabsOpenShiftConfig(vlabs *vlabs.OpenShiftConfig, api *OpenShiftCon
 	api.ClusterUsername = vlabs.ClusterUsername
 	api.ClusterPassword = vlabs.ClusterPassword
 	api.EnableAADAuthentication = vlabs.EnableAADAuthentication
+	api.ConfigBundles = vlabs.ConfigBundles
 }
 
 func convertVLabsKubernetesConfig(vlabs *vlabs.KubernetesConfig, api *KubernetesConfig) {

pkg/api/types.go

@@ -345,9 +345,7 @@ type OpenShiftConfig struct {
 	// EnableAADAuthentication is temporary, do not rely on it.
 	EnableAADAuthentication bool `json:"enableAADAuthentication,omitempty"`
 
-	ConfigBundles          map[string][]byte `json:"-"`
-	ExternalMasterHostname string            `json:"-"`
-	RouterLBHostname       string            `json:"-"`
+	ConfigBundles map[string][]byte `json:"configBundles,omitempty"`
 }
 
 // MasterProfile represents the definition of the master cluster

pkg/api/vlabs/types.go

@@ -338,6 +338,8 @@ type OpenShiftConfig struct {
 
 	// EnableAADAuthentication is temporary, do not rely on it.
 	EnableAADAuthentication bool `json:"enableAADAuthentication,omitempty"`
+
+	ConfigBundles map[string][]byte `json:"configBundles,omitempty"`
 }
 
 // MasterProfile represents the definition of the master cluster

pkg/api/vlabs/validate.go

@@ -280,7 +280,10 @@ func validateImageNameAndGroup(name, resourceGroup string) error {
 }
 
 // Validate implements APIObject
-func (m *MasterProfile) Validate() error {
+func (m *MasterProfile) Validate(o *OrchestratorProfile) error {
+	if o.OrchestratorType == OpenShift && m.Count != 1 {
+		return errors.New("openshift can only deployed with one master")
+	}
 	if m.ImageRef != nil {
 		if err := validateImageNameAndGroup(m.ImageRef.Name, m.ImageRef.ResourceGroup); err != nil {
 			return err
@@ -454,7 +457,7 @@ func (a *Properties) Validate(isUpdate bool) error {
 	if e := a.validateAddons(); e != nil {
 		return e
 	}
-	if e := a.MasterProfile.Validate(); e != nil {
+	if e := a.MasterProfile.Validate(a.OrchestratorProfile); e != nil {
 		return e
 	}
 	if e := validateUniqueProfileNames(a.AgentPoolProfiles); e != nil {