From f1ebfc7766d0c0423e1f1820febdf9cd3e1ddedb Mon Sep 17 00:00:00 2001 From: Tiexin Guo Date: Fri, 4 Mar 2022 21:25:05 +0800 Subject: [PATCH] feat: add "dependsOn" in config, add config dependency check (#270) * chore: merge main branch * docs: update docs for dependencies * feat: config dependency check with unit test * feat: config dependency check with unit test * chore: rename and add comments * fix: readme link error * fix: update dependsOn to array according to code review * feat: refactor dependsOn from str to array according to code review --- README.md | 2 +- docs/plugins/argocdapp_plugin.md | 4 +- docs/plugins/githubactions-golang_plugin.md | 2 + docs/plugins/githubactions-nodejs_plugin.md | 2 + docs/plugins/githubactions-python_plugin.md | 2 + docs/plugins/trello-github-integ_plugin.md | 4 +- examples/quickstart.yaml | 1 + internal/pkg/configloader/config.go | 16 ++--- .../pkg/configloader/configloader_test.go | 45 ++++++++++++++ internal/pkg/configloader/validation.go | 61 +++++++++++++++---- 10 files changed, 117 insertions(+), 22 deletions(-) create mode 100644 internal/pkg/configloader/configloader_test.go diff --git a/README.md b/README.md index 8299d7049..2e96881ea 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ If you want to get a quick start, follow our [quick start](./docs/quickstart_en. ## Configuration -This is an example of DevStream config: [examples/quick_start.yaml](./examples/quick_start.yaml). +This is an example of DevStream config: [examples/quickstart.yaml](./examples/quickstart.yaml). Remember to open this configuration file, modify all FULL_UPPER_CASE_STRINGS (like YOUR_GITHUB_USERNAME, for example) in it to your own. diff --git a/docs/plugins/argocdapp_plugin.md b/docs/plugins/argocdapp_plugin.md index 177d6f1df..75821e3cf 100644 --- a/docs/plugins/argocdapp_plugin.md +++ b/docs/plugins/argocdapp_plugin.md @@ -18,6 +18,8 @@ tools: # version of the plugin # checkout the version from the GitHub releases version: 0.2.0 + # optional; if specified, dtm will make sure the dependency is applied first before handling this tool. + dependsOn: [ "TOOL1_NAME.TOOL1_KIND", "TOOL2_NAME.TOOL2_KIND" ] # options for the plugin options: # information on the ArgoCD application @@ -41,5 +43,3 @@ tools: # Helm chart repo URL, this is only an example, do not use this repoURL: https://github.com/ironcore864/openstream-gitops-test.git ``` - -Currently, all the parameters in the example above are mandatory. diff --git a/docs/plugins/githubactions-golang_plugin.md b/docs/plugins/githubactions-golang_plugin.md index fe5564460..cdaa61be3 100644 --- a/docs/plugins/githubactions-golang_plugin.md +++ b/docs/plugins/githubactions-golang_plugin.md @@ -23,6 +23,8 @@ tools: # version of the plugin # checkout the version from the GitHub releases version: 0.2.0 + # optional; if specified, dtm will make sure the dependency is applied first before handling this tool. + dependsOn: [ "TOOL1_NAME.TOOL1_KIND", "TOOL2_NAME.TOOL2_KIND" ] # options for the plugin options: # the repo's owner. It should be case-sensitive here; strictly use your GitHub user name; please change the value below. diff --git a/docs/plugins/githubactions-nodejs_plugin.md b/docs/plugins/githubactions-nodejs_plugin.md index 031bc6bb4..0f90cd4e4 100644 --- a/docs/plugins/githubactions-nodejs_plugin.md +++ b/docs/plugins/githubactions-nodejs_plugin.md @@ -15,6 +15,8 @@ tools: # version of the plugin # checkout the version from the GitHub releases version: 0.2.0 + # optional; if specified, dtm will make sure the dependency is applied first before handling this tool. + dependsOn: [ "TOOL1_NAME.TOOL1_KIND", "TOOL2_NAME.TOOL2_KIND" ] # options for the plugin options: # the repo's owner. It should be case-sensitive here; strictly use your GitHub user name; please change the value below. diff --git a/docs/plugins/githubactions-python_plugin.md b/docs/plugins/githubactions-python_plugin.md index ab29bb9e1..a9ab50e02 100644 --- a/docs/plugins/githubactions-python_plugin.md +++ b/docs/plugins/githubactions-python_plugin.md @@ -15,6 +15,8 @@ tools: # version of the plugin # checkout the version from the GitHub releases version: 0.2.0 + # optional; if specified, dtm will make sure the dependency is applied first before handling this tool. + dependsOn: [ "TOOL1_NAME.TOOL1_KIND", "TOOL2_NAME.TOOL2_KIND" ] # options for the plugin options: # the repo's owner. It should be case-sensitive here; strictly use your GitHub user name; please change the value below. diff --git a/docs/plugins/trello-github-integ_plugin.md b/docs/plugins/trello-github-integ_plugin.md index 3345bab1f..4a9652eeb 100644 --- a/docs/plugins/trello-github-integ_plugin.md +++ b/docs/plugins/trello-github-integ_plugin.md @@ -24,9 +24,11 @@ tools: # kind of this plugin kind: trello-github-integ # version of the plugin + # checkout the version from the GitHub releases version: 0.2.0 + # optional; if specified, dtm will make sure the dependency is applied first before handling this tool. + dependsOn: [ "TOOL1_NAME.TOOL1_KIND", "TOOL2_NAME.TOOL2_KIND" ] # options for the plugin - # checkout the version from the GitHub releases options: # the repo's owner. It should be case-sensitive here; strictly use your GitHub user name; please change the value below. owner: YOUR_GITHUB_USERNAME diff --git a/examples/quickstart.yaml b/examples/quickstart.yaml index 2b02ae633..75b7f82c1 100644 --- a/examples/quickstart.yaml +++ b/examples/quickstart.yaml @@ -13,6 +13,7 @@ tools: plugin: kind: githubactions-golang version: 0.2.0 + dependsOn: ["go-webapp-repo.github-repo-scaffolding-golang"] options: owner: YOUR_GITHUB_USERNAME_CASE_SENSITIVE repo: go-webapp-devstream-demo diff --git a/internal/pkg/configloader/config.go b/internal/pkg/configloader/config.go index 3b8324907..2f542b4d0 100644 --- a/internal/pkg/configloader/config.go +++ b/internal/pkg/configloader/config.go @@ -27,16 +27,18 @@ type Tool struct { // contain only lowercase alphanumeric characters, '-' or '.' // start with an alphanumeric character // end with an alphanumeric character - Name string `yaml:"name"` - Plugin Plugin `yaml:"plugin"` - Options map[string]interface{} `yaml:"options"` + Name string `yaml:"name"` + Plugin Plugin `yaml:"plugin"` + DependsOn []string `yaml:"dependsOn"` + Options map[string]interface{} `yaml:"options"` } func (t *Tool) DeepCopy() *Tool { var retTool = Tool{ - Name: t.Name, - Plugin: t.Plugin, - Options: map[string]interface{}{}, + Name: t.Name, + Plugin: t.Plugin, + DependsOn: t.DependsOn, + Options: map[string]interface{}{}, } for k, v := range t.Options { retTool.Options[k] = v @@ -69,7 +71,7 @@ func LoadConf(fname string) *Config { return nil } - errs := config.Validate() + errs := validateConfig(&config) if len(errs) != 0 { for _, e := range errs { diff --git a/internal/pkg/configloader/configloader_test.go b/internal/pkg/configloader/configloader_test.go new file mode 100644 index 000000000..50f9548c7 --- /dev/null +++ b/internal/pkg/configloader/configloader_test.go @@ -0,0 +1,45 @@ +package configloader + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDependencyPass(t *testing.T) { + tools := []Tool{ + {Name: "argocd", Plugin: Plugin{Kind: "argocd"}}, + {Name: "argocdapp", Plugin: Plugin{Kind: "argocdapp"}, DependsOn: []string{"argocd.argocd"}}, + } + errors := validateDependency(tools) + assert.Equal(t, len(errors), 0, "Dependency check passed.") + +} + +func TestDependencyNotExist(t *testing.T) { + tools := []Tool{ + {Name: "argocdapp", Plugin: Plugin{Kind: "argocdapp"}, DependsOn: []string{"argocd.argocd"}}, + } + errors := validateDependency(tools) + assert.Equal(t, len(errors), 1) + +} + +func TestMultipleDependencies(t *testing.T) { + tools := []Tool{ + {Name: "argocd", Plugin: Plugin{Kind: "argocd"}}, + {Name: "repo", Plugin: Plugin{Kind: "github"}}, + {Name: "argocdapp", Plugin: Plugin{Kind: "argocdapp"}, DependsOn: []string{"argocd.argocd", "repo.github"}}, + } + errors := validateDependency(tools) + assert.Equal(t, len(errors), 0) +} + +func TestEmptyDependency(t *testing.T) { + tools := []Tool{ + {Name: "argocd", Plugin: Plugin{Kind: "argocd"}}, + {Name: "argocdapp", Plugin: Plugin{Kind: "argocdapp"}, DependsOn: []string{}}, + } + errors := validateDependency(tools) + assert.Equal(t, len(errors), 0) +} diff --git a/internal/pkg/configloader/validation.go b/internal/pkg/configloader/validation.go index 5cb8ced51..ed21630bb 100644 --- a/internal/pkg/configloader/validation.go +++ b/internal/pkg/configloader/validation.go @@ -2,36 +2,75 @@ package configloader import ( "fmt" + "strings" "k8s.io/apimachinery/pkg/util/validation" ) -func (c *Config) Validate() []error { - retErrors := make([]error, 0) +func validateConfig(config *Config) []error { + errors := make([]error, 0) - for _, t := range c.Tools { - retErrors = append(retErrors, t.validate()...) + for _, t := range config.Tools { + errors = append(errors, validateTool(&t)...) } - return retErrors + + errors = append(errors, validateDependency(config.Tools)...) + + return errors } -func (t *Tool) validate() []error { - retErrors := make([]error, 0) +func validateTool(t *Tool) []error { + errors := make([]error, 0) // Name if t.Name == "" { - retErrors = append(retErrors, fmt.Errorf("name is empty")) + errors = append(errors, fmt.Errorf("name is empty")) } errs := validation.IsDNS1123Subdomain(t.Name) for _, e := range errs { - retErrors = append(retErrors, fmt.Errorf("name %s is invalid: %s", t.Name, e)) + errors = append(errors, fmt.Errorf("name %s is invalid: %s", t.Name, e)) } // Plugin if t.Plugin.Kind == "" { - retErrors = append(retErrors, fmt.Errorf("plugin.kind is empty")) + errors = append(errors, fmt.Errorf("plugin.kind is empty")) + } + + return errors +} + +func validateDependency(tools []Tool) []error { + errors := make([]error, 0) + + // config "set" (map) + toolMap := make(map[string]bool) + // creating the set + for _, tool := range tools { + key := fmt.Sprintf("%s.%s", tool.Name, tool.Plugin.Kind) + toolMap[key] = true + } + + for _, tool := range tools { + // no dependency, pass + if len(tool.DependsOn) == 0 { + continue + } + + // for each dependency + for _, dependency := range tool.DependsOn { + // skip empty string + dependency = strings.TrimSpace(dependency) + if dependency == "" { + continue + } + + // generate an error if the dependency isn't in the config set, + if _, ok := toolMap[dependency]; !ok { + errors = append(errors, fmt.Errorf("tool %s's dependency %s doesn't exist in the config", tool.Name, dependency)) + } + } } - return retErrors + return errors }