Skip to content

Commit

Permalink
Verify machine-controller CRDs before deleting workers (kubermatic#683)
Browse files Browse the repository at this point in the history
* Verify machine-controller CRDs before deleting workers

* Ignore unparam lint error for defaultRetryBackoff
  • Loading branch information
xmudrii authored and kubermatic-bot committed Sep 24, 2019
1 parent 5b07ced commit cbea136
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 3 deletions.
1 change: 1 addition & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,4 @@ issues:
- "don't use underscores in Go names; func SetDefaults_SystemPackages should be SetDefaultsSystemPackages"
- "G101: Potential hardcoded credentials"
- "G106: Use of ssh InsecureIgnoreHostKey should be audited"
- "`defaultRetryBackoff` - `retries` always receives `3`"
12 changes: 12 additions & 0 deletions pkg/installer/installation/reset.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,18 @@ func destroyWorkers(s *state.State) error {
return errors.Wrap(lastErr, "unable to build kubernetes clientset")
}

_ = wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) {
lastErr = machinecontroller.VerifyCRDs(s)
if lastErr != nil {
return false, nil
}
return true, nil
})
if lastErr != nil {
s.Logger.Info("Skipping deleting worker nodes because machine-controller CRDs are not deployed")
return nil
}

_ = wait.ExponentialBackoff(defaultRetryBackoff(3), func() (bool, error) {
lastErr = machinecontroller.DestroyWorkers(s)
if lastErr != nil {
Expand Down
31 changes: 28 additions & 3 deletions pkg/templates/machinecontroller/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/kubermatic/kubeone/pkg/state"

corev1 "k8s.io/api/core/v1"
apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
errorsutil "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
Expand Down Expand Up @@ -73,6 +74,32 @@ func WaitReady(s *state.State) error {
return nil
}

// VerifyCRDs verifies are Cluster-API CRDs deployed
func VerifyCRDs(s *state.State) error {
bgCtx := context.Background()
crd := &apiextensions.CustomResourceDefinition{}

// Verify MachineDeployment CRD
key := dynclient.ObjectKey{Name: "machinedeployments.cluster.k8s.io"}
if err := s.DynamicClient.Get(bgCtx, key, crd); err != nil {
return errors.Wrap(err, "MachineDeployments CRD is not deployed")
}

// Verify MachineSet CRD
key = dynclient.ObjectKey{Name: "machinesets.cluster.k8s.io"}
if err := s.DynamicClient.Get(bgCtx, key, crd); err != nil {
return errors.Wrap(err, "MachineSet CRD is not deployed")
}

// Verify Machine CRD
key = dynclient.ObjectKey{Name: "machines.cluster.k8s.io"}
if err := s.DynamicClient.Get(bgCtx, key, crd); err != nil {
return errors.Wrap(err, "MachineSet CRD is not deployed")
}

return nil
}

// DestroyWorkers destroys all MachineDeployment, MachineSet and Machine objects
func DestroyWorkers(s *state.State) error {
if !s.Cluster.MachineController.Deploy {
Expand Down Expand Up @@ -116,11 +143,9 @@ func DestroyWorkers(s *state.State) error {
s.Logger.Info("Deleting MachineDeployment objects…")
mdList := &clusterv1alpha1.MachineDeploymentList{}
if err := s.DynamicClient.List(bgCtx, dynclient.InNamespace(MachineControllerNamespace), mdList); err != nil {
if errorsutil.IsTimeout(err) || errorsutil.IsServerTimeout(err) {
if !errorsutil.IsNotFound(err) {
return errors.Wrap(err, "unable to list machinedeployment objects")
}
s.Logger.Info("Skipping deleting worker nodes because MachineDeployments CRD is not deployed")
return nil
}
for i := range mdList.Items {
if err := s.DynamicClient.Delete(bgCtx, &mdList.Items[i]); err != nil {
Expand Down

0 comments on commit cbea136

Please sign in to comment.