diff --git a/cmd/argocd-util/main.go b/cmd/argocd-util/main.go index f9e2a0f90f23b..856ac886ef64a 100644 --- a/cmd/argocd-util/main.go +++ b/cmd/argocd-util/main.go @@ -1,26 +1,31 @@ package main import ( + "bufio" "context" "fmt" + "io" "io/ioutil" "os" "os/exec" - "strings" "syscall" + "github.com/argoproj/argo-cd/common" + "github.com/argoproj/argo-cd/util" "github.com/ghodss/yaml" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "github.com/argoproj/argo-cd/common" "github.com/argoproj/argo-cd/errors" - "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1" - appclientset "github.com/argoproj/argo-cd/pkg/client/clientset/versioned" "github.com/argoproj/argo-cd/util/cli" "github.com/argoproj/argo-cd/util/db" "github.com/argoproj/argo-cd/util/dex" @@ -36,9 +41,15 @@ import ( const ( // CLIName is the name of the CLI cliName = "argocd-util" - // YamlSeparator separates sections of a YAML file - yamlSeparator = "\n---\n" + yamlSeparator = "---\n" +) + +var ( + configMapResource = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"} + secretResource = schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"} + applicationsResource = schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "applications"} + appprojectsResource = schema.GroupVersionResource{Group: "argoproj.io", Version: "v1alpha1", Resource: "appprojects"} ) // NewCommand returns a new instance of an argocd command @@ -195,94 +206,153 @@ func NewGenDexConfigCommand() *cobra.Command { func NewImportCommand() *cobra.Command { var ( clientConfig clientcmd.ClientConfig + prune bool + dryRun bool ) var command = cobra.Command{ Use: "import SOURCE", Short: "Import Argo CD data from stdin (specify `-') or a file", - RunE: func(c *cobra.Command, args []string) error { + Run: func(c *cobra.Command, args []string) { if len(args) != 1 { c.HelpFunc()(c, args) os.Exit(1) } + config, err := clientConfig.ClientConfig() + config.QPS = 100 + config.Burst = 50 + errors.CheckError(err) + namespace, _, err := clientConfig.Namespace() + errors.CheckError(err) + acdClients := newArgoCDClientsets(config, namespace) - var ( - input []byte - err error - newSettings *settings.ArgoCDSettings - newRepos []*v1alpha1.Repository - newClusters []*v1alpha1.Cluster - newApps []*v1alpha1.Application - newRBACCM *apiv1.ConfigMap - ) - + var input []byte if in := args[0]; in == "-" { input, err = ioutil.ReadAll(os.Stdin) - errors.CheckError(err) } else { input, err = ioutil.ReadFile(in) - errors.CheckError(err) } - inputStrings := strings.Split(string(input), yamlSeparator) - - err = yaml.Unmarshal([]byte(inputStrings[0]), &newSettings) - errors.CheckError(err) - - err = yaml.Unmarshal([]byte(inputStrings[1]), &newRepos) - errors.CheckError(err) - - err = yaml.Unmarshal([]byte(inputStrings[2]), &newClusters) - errors.CheckError(err) - - err = yaml.Unmarshal([]byte(inputStrings[3]), &newApps) errors.CheckError(err) + var dryRunMsg string + if dryRun { + dryRunMsg = " (dry run)" + } - err = yaml.Unmarshal([]byte(inputStrings[4]), &newRBACCM) + // pruneObjects tracks live objects and it's current resource version. any remaining + // items in this map indicates the resource should be pruned since it no longer appears + // in the backup + pruneObjects := make(map[kube.ResourceKey]string) + configMaps, err := acdClients.configMaps.List(metav1.ListOptions{}) errors.CheckError(err) - - config, err := clientConfig.ClientConfig() + for _, cm := range configMaps.Items { + cmName := cm.GetName() + if cmName == common.ArgoCDConfigMapName || cmName == common.ArgoCDRBACConfigMapName { + pruneObjects[kube.ResourceKey{Group: "", Kind: "ConfigMap", Name: cm.GetName()}] = cm.GetResourceVersion() + } + } + secrets, err := acdClients.secrets.List(metav1.ListOptions{}) errors.CheckError(err) - namespace, _, err := clientConfig.Namespace() + for _, secret := range secrets.Items { + if isArgoCDSecret(nil, secret) { + pruneObjects[kube.ResourceKey{Group: "", Kind: "Secret", Name: secret.GetName()}] = secret.GetResourceVersion() + } + } + applications, err := acdClients.applications.List(metav1.ListOptions{}) errors.CheckError(err) - kubeClientset := kubernetes.NewForConfigOrDie(config) - - settingsMgr := settings.NewSettingsManager(context.Background(), kubeClientset, namespace) - err = settingsMgr.SaveSettings(newSettings) + for _, app := range applications.Items { + pruneObjects[kube.ResourceKey{Group: "argoproj.io", Kind: "Application", Name: app.GetName()}] = app.GetResourceVersion() + } + projects, err := acdClients.projects.List(metav1.ListOptions{}) errors.CheckError(err) - db := db.NewDB(namespace, settingsMgr, kubeClientset) + for _, proj := range projects.Items { + pruneObjects[kube.ResourceKey{Group: "argoproj.io", Kind: "AppProject", Name: proj.GetName()}] = proj.GetResourceVersion() + } - _, err = kubeClientset.CoreV1().ConfigMaps(namespace).Create(newRBACCM) + // Create or replace existing object + objs, err := kube.SplitYAML(string(input)) errors.CheckError(err) - - for _, repo := range newRepos { - _, err := db.CreateRepository(context.Background(), repo) - if err != nil { - log.Warn(err) + for _, obj := range objs { + gvk := obj.GroupVersionKind() + key := kube.ResourceKey{Group: gvk.Group, Kind: gvk.Kind, Name: obj.GetName()} + resourceVersion, exists := pruneObjects[key] + delete(pruneObjects, key) + var dynClient dynamic.ResourceInterface + switch obj.GetKind() { + case "Secret": + dynClient = acdClients.secrets + case "ConfigMap": + dynClient = acdClients.configMaps + case "AppProject": + dynClient = acdClients.projects + case "Application": + dynClient = acdClients.applications } - } - - for _, cluster := range newClusters { - _, err := db.CreateCluster(context.Background(), cluster) - if err != nil { - log.Warn(err) + if !exists { + if !dryRun { + _, err = dynClient.Create(obj, metav1.CreateOptions{}) + errors.CheckError(err) + } + fmt.Printf("%s/%s %s created%s\n", gvk.Group, gvk.Kind, obj.GetName(), dryRunMsg) + } else { + if !dryRun { + obj.SetResourceVersion(resourceVersion) + _, err = dynClient.Update(obj, metav1.UpdateOptions{}) + errors.CheckError(err) + } + fmt.Printf("%s/%s %s replaced%s\n", gvk.Group, gvk.Kind, obj.GetName(), dryRunMsg) } } - appClientset := appclientset.NewForConfigOrDie(config) - for _, app := range newApps { - out, err := appClientset.ArgoprojV1alpha1().Applications(namespace).Create(app) - errors.CheckError(err) - log.Println(out) + // Delete objects not in backup + for key := range pruneObjects { + if prune { + var dynClient dynamic.ResourceInterface + switch key.Kind { + case "Secret": + dynClient = acdClients.secrets + case "AppProject": + dynClient = acdClients.projects + case "Application": + dynClient = acdClients.applications + default: + log.Fatalf("Unexpected kind '%s' in prune list", key.Kind) + } + if !dryRun { + err = dynClient.Delete(key.Name, &metav1.DeleteOptions{}) + errors.CheckError(err) + } + fmt.Printf("%s/%s %s pruned%s\n", key.Group, key.Kind, key.Name, dryRunMsg) + } else { + fmt.Printf("%s/%s %s needs pruning\n", key.Group, key.Kind, key.Name) + } } - - return nil }, } clientConfig = cli.AddKubectlFlagsToCmd(&command) + command.Flags().BoolVar(&dryRun, "dry-run", false, "Print what will be performed") + command.Flags().BoolVar(&prune, "prune", false, "Prune secrets, applications and projects which do not appear in the backup") return &command } +type argoCDClientsets struct { + configMaps dynamic.ResourceInterface + secrets dynamic.ResourceInterface + applications dynamic.ResourceInterface + projects dynamic.ResourceInterface +} + +func newArgoCDClientsets(config *rest.Config, namespace string) *argoCDClientsets { + dynamicIf, err := dynamic.NewForConfig(config) + errors.CheckError(err) + return &argoCDClientsets{ + configMaps: dynamicIf.Resource(configMapResource).Namespace(namespace), + secrets: dynamicIf.Resource(secretResource).Namespace(namespace), + applications: dynamicIf.Resource(applicationsResource).Namespace(namespace), + projects: dynamicIf.Resource(appprojectsResource).Namespace(namespace), + } +} + // NewExportCommand defines a new command for exporting Kubernetes and Argo CD resources. func NewExportCommand() *cobra.Command { var ( @@ -292,75 +362,48 @@ func NewExportCommand() *cobra.Command { var command = cobra.Command{ Use: "export", Short: "Export all Argo CD data to stdout (default) or a file", - RunE: func(c *cobra.Command, args []string) error { + Run: func(c *cobra.Command, args []string) { config, err := clientConfig.ClientConfig() errors.CheckError(err) namespace, _, err := clientConfig.Namespace() errors.CheckError(err) - kubeClientset := kubernetes.NewForConfigOrDie(config) - - settingsMgr := settings.NewSettingsManager(context.Background(), kubeClientset, namespace) - settings, err := settingsMgr.GetSettings() - errors.CheckError(err) - // certificate data is included in secrets that are exported alongside - settings.Certificate = nil - db := db.NewDB(namespace, settingsMgr, kubeClientset) - clusters, err := db.ListClusters(context.Background()) - errors.CheckError(err) - - repoURLs, err := db.ListRepoURLs(context.Background()) - errors.CheckError(err) - repos := make([]*v1alpha1.Repository, len(repoURLs)) - for i := range repoURLs { - repo, err := db.GetRepository(context.Background(), repoURLs[i]) + var writer io.Writer + if out == "-" { + writer = os.Stdout + } else { + f, err := os.Create(out) errors.CheckError(err) - repos = append(repos, repo) + defer util.Close(f) + writer = bufio.NewWriter(f) } - appClientset := appclientset.NewForConfigOrDie(config) - apps, err := appClientset.ArgoprojV1alpha1().Applications(namespace).List(metav1.ListOptions{}) + acdClients := newArgoCDClientsets(config, namespace) + acdConfigMap, err := acdClients.configMaps.Get(common.ArgoCDConfigMapName, metav1.GetOptions{}) errors.CheckError(err) - - rbacCM, err := kubeClientset.CoreV1().ConfigMaps(namespace).Get(common.ArgoCDRBACConfigMapName, metav1.GetOptions{}) + export(writer, *acdConfigMap) + acdRBACConfigMap, err := acdClients.configMaps.Get(common.ArgoCDRBACConfigMapName, metav1.GetOptions{}) errors.CheckError(err) + export(writer, *acdRBACConfigMap) - // remove extraneous cruft from output - rbacCM.ObjectMeta = metav1.ObjectMeta{ - Name: rbacCM.ObjectMeta.Name, - } - - // remove extraneous cruft from output - for idx, app := range apps.Items { - apps.Items[idx].ObjectMeta = metav1.ObjectMeta{ - Name: app.ObjectMeta.Name, - Finalizers: app.ObjectMeta.Finalizers, - } - apps.Items[idx].Status = v1alpha1.ApplicationStatus{ - History: app.Status.History, + referencedSecrets := getReferencedSecrets(*acdConfigMap) + secrets, err := acdClients.secrets.List(metav1.ListOptions{}) + errors.CheckError(err) + for _, secret := range secrets.Items { + if isArgoCDSecret(referencedSecrets, secret) { + export(writer, secret) } - apps.Items[idx].Operation = nil } - - // take a list of exportable objects, marshal them to YAML, - // and return a string joined by a delimiter - output := func(delimiter string, oo ...interface{}) string { - out := make([]string, 0) - for _, o := range oo { - data, err := yaml.Marshal(o) - errors.CheckError(err) - out = append(out, string(data)) - } - return strings.Join(out, delimiter) - }(yamlSeparator, settings, clusters.Items, repos, apps.Items, rbacCM) - - if out == "-" { - fmt.Println(output) - } else { - err = ioutil.WriteFile(out, []byte(output), 0644) - errors.CheckError(err) + projects, err := acdClients.projects.List(metav1.ListOptions{}) + errors.CheckError(err) + for _, proj := range projects.Items { + export(writer, proj) + } + applications, err := acdClients.applications.List(metav1.ListOptions{}) + errors.CheckError(err) + for _, app := range applications.Items { + export(writer, app) } - return nil }, } @@ -370,13 +413,109 @@ func NewExportCommand() *cobra.Command { return &command } -// NewClusterConfig returns a new instance of `argocd-util cluster-kubeconfig` command +// getReferencedSecrets examines the argocd-cm config for any referenced repo secrets and returns a +// map of all referenced secrets. +func getReferencedSecrets(un unstructured.Unstructured) map[string]bool { + var cm apiv1.ConfigMap + err := runtime.DefaultUnstructuredConverter.FromUnstructured(un.Object, &cm) + errors.CheckError(err) + referencedSecrets := make(map[string]bool) + if reposRAW, ok := cm.Data["repositories"]; ok { + repoCreds := make([]settings.RepoCredentials, 0) + err := yaml.Unmarshal([]byte(reposRAW), &repoCreds) + errors.CheckError(err) + for _, cred := range repoCreds { + if cred.PasswordSecret != nil { + referencedSecrets[cred.PasswordSecret.Name] = true + } + if cred.SSHPrivateKeySecret != nil { + referencedSecrets[cred.SSHPrivateKeySecret.Name] = true + } + if cred.UsernameSecret != nil { + referencedSecrets[cred.UsernameSecret.Name] = true + } + } + } + if helmReposRAW, ok := cm.Data["helm.repositories"]; ok { + helmRepoCreds := make([]settings.HelmRepoCredentials, 0) + err := yaml.Unmarshal([]byte(helmReposRAW), &helmRepoCreds) + errors.CheckError(err) + for _, cred := range helmRepoCreds { + if cred.CASecret != nil { + referencedSecrets[cred.CASecret.Name] = true + } + if cred.CertSecret != nil { + referencedSecrets[cred.CertSecret.Name] = true + } + if cred.KeySecret != nil { + referencedSecrets[cred.KeySecret.Name] = true + } + if cred.UsernameSecret != nil { + referencedSecrets[cred.UsernameSecret.Name] = true + } + if cred.PasswordSecret != nil { + referencedSecrets[cred.PasswordSecret.Name] = true + } + } + } + return referencedSecrets +} + +// isArgoCDSecret returns whether or not the given secret is a part of Argo CD configuration +// (e.g. argocd-secret, repo credentials, or cluster credentials) +func isArgoCDSecret(repoSecretRefs map[string]bool, un unstructured.Unstructured) bool { + secretName := un.GetName() + if secretName == common.ArgoCDSecretName { + return true + } + if repoSecretRefs != nil { + if _, ok := repoSecretRefs[secretName]; ok { + return true + } + } + if labels := un.GetLabels(); labels != nil { + if _, ok := labels[common.LabelKeySecretType]; ok { + return true + } + } + if annotations := un.GetAnnotations(); annotations != nil { + if annotations[common.AnnotationKeyManagedBy] == common.AnnotationValueManagedByArgoCD { + return true + } + } + return false +} + +// export writes the unstructured object and removes extraneous cruft from output before writing +func export(w io.Writer, un unstructured.Unstructured) { + name := un.GetName() + finalizers := un.GetFinalizers() + apiVersion := un.GetAPIVersion() + kind := un.GetKind() + labels := un.GetLabels() + annotations := un.GetAnnotations() + unstructured.RemoveNestedField(un.Object, "metadata") + un.SetName(name) + un.SetFinalizers(finalizers) + un.SetAPIVersion(apiVersion) + un.SetKind(kind) + un.SetLabels(labels) + un.SetAnnotations(annotations) + data, err := yaml.Marshal(un.Object) + errors.CheckError(err) + _, err = w.Write(data) + errors.CheckError(err) + _, err = w.Write([]byte(yamlSeparator)) + errors.CheckError(err) +} + +// NewClusterConfig returns a new instance of `argocd-util kubeconfig` command func NewClusterConfig() *cobra.Command { var ( clientConfig clientcmd.ClientConfig ) var command = &cobra.Command{ - Use: "cluster-kubeconfig CLUSTER_URL OUTPUT_PATH", + Use: "kubeconfig CLUSTER_URL OUTPUT_PATH", Short: "Generates kubeconfig for the specified cluster", Run: func(c *cobra.Command, args []string) { if len(args) != 2 { @@ -387,12 +526,8 @@ func NewClusterConfig() *cobra.Command { output := args[1] conf, err := clientConfig.ClientConfig() errors.CheckError(err) - namespace, wasSpecified, err := clientConfig.Namespace() + namespace, _, err := clientConfig.Namespace() errors.CheckError(err) - if !(wasSpecified) { - namespace = "argocd" - } - kubeclientset, err := kubernetes.NewForConfig(conf) errors.CheckError(err) diff --git a/util/kube/kube.go b/util/kube/kube.go index 6941030b066e5..d05a500497551 100644 --- a/util/kube/kube.go +++ b/util/kube/kube.go @@ -384,7 +384,7 @@ func SplitYAML(out string) ([]*unstructured.Unstructured, error) { } remObj, err := Remarshal(&obj) if err != nil { - log.Warnf("Failed to remarshal oject: %v", err) + log.Debugf("Failed to remarshal oject: %v", err) } else { obj = *remObj }