feat: add "dependsOn" in config, add config dependency check (devstre…

…am-io#270)

* chore: merge main branch

* docs: update docs for dependencies

* feat: config dependency check with unit test

* feat: config dependency check with unit test

* chore: rename and add comments

* fix: readme link error

* fix: update dependsOn to array according to code review

* feat: refactor dependsOn from str to array according to code review

README.md

@@ -84,7 +84,7 @@ If you want to get a quick start, follow our [quick start](./docs/quickstart_en.
 
 ## Configuration
 
-This is an example of DevStream config: [examples/quick_start.yaml](./examples/quick_start.yaml).
+This is an example of DevStream config: [examples/quickstart.yaml](./examples/quickstart.yaml).
 
 Remember to open this configuration file, modify all FULL_UPPER_CASE_STRINGS (like YOUR_GITHUB_USERNAME, for example) in it to your own.
 

docs/plugins/argocdapp_plugin.md

@@ -18,6 +18,8 @@ tools:
  # version of the plugin
  # checkout the version from the GitHub releases
  version: 0.2.0
+ # optional; if specified, dtm will make sure the dependency is applied first before handling this tool.
+ dependsOn: [ "TOOL1_NAME.TOOL1_KIND", "TOOL2_NAME.TOOL2_KIND" ]
  # options for the plugin
  options:
  # information on the ArgoCD application
@@ -41,5 +43,3 @@ tools:
  # Helm chart repo URL, this is only an example, do not use this
  repoURL: https://github.com/ironcore864/openstream-gitops-test.git
 ```
-
-Currently, all the parameters in the example above are mandatory.

docs/plugins/githubactions-golang_plugin.md

@@ -23,6 +23,8 @@ tools:
  # version of the plugin
  # checkout the version from the GitHub releases
  version: 0.2.0
+ # optional; if specified, dtm will make sure the dependency is applied first before handling this tool.
+ dependsOn: [ "TOOL1_NAME.TOOL1_KIND", "TOOL2_NAME.TOOL2_KIND" ]
  # options for the plugin
  options:
  # the repo's owner. It should be case-sensitive here; strictly use your GitHub user name; please change the value below.

docs/plugins/githubactions-nodejs_plugin.md

@@ -15,6 +15,8 @@ tools:
  # version of the plugin
  # checkout the version from the GitHub releases
  version: 0.2.0
+ # optional; if specified, dtm will make sure the dependency is applied first before handling this tool.
+ dependsOn: [ "TOOL1_NAME.TOOL1_KIND", "TOOL2_NAME.TOOL2_KIND" ]
  # options for the plugin
  options:
  # the repo's owner. It should be case-sensitive here; strictly use your GitHub user name; please change the value below.

docs/plugins/githubactions-python_plugin.md

@@ -15,6 +15,8 @@ tools:
  # version of the plugin
  # checkout the version from the GitHub releases
  version: 0.2.0
+ # optional; if specified, dtm will make sure the dependency is applied first before handling this tool.
+ dependsOn: [ "TOOL1_NAME.TOOL1_KIND", "TOOL2_NAME.TOOL2_KIND" ]
  # options for the plugin
  options:
  # the repo's owner. It should be case-sensitive here; strictly use your GitHub user name; please change the value below.

docs/plugins/trello-github-integ_plugin.md

@@ -24,9 +24,11 @@ tools:
  # kind of this plugin
  kind: trello-github-integ
  # version of the plugin
+ # checkout the version from the GitHub releases
  version: 0.2.0
+ # optional; if specified, dtm will make sure the dependency is applied first before handling this tool.
+ dependsOn: [ "TOOL1_NAME.TOOL1_KIND", "TOOL2_NAME.TOOL2_KIND" ]
  # options for the plugin
- # checkout the version from the GitHub releases
  options:
  # the repo's owner. It should be case-sensitive here; strictly use your GitHub user name; please change the value below.
  owner: YOUR_GITHUB_USERNAME

examples/quickstart.yaml

@@ -13,6 +13,7 @@ tools:
  plugin:
  kind: githubactions-golang
  version: 0.2.0
+ dependsOn: ["go-webapp-repo.github-repo-scaffolding-golang"]
  options:
  owner: YOUR_GITHUB_USERNAME_CASE_SENSITIVE
  repo: go-webapp-devstream-demo

internal/pkg/configloader/config.go

@@ -27,16 +27,18 @@ type Tool struct {
  // contain only lowercase alphanumeric characters, '-' or '.'
  // start with an alphanumeric character
  // end with an alphanumeric character
- Name string `yaml:"name"`
- Plugin Plugin `yaml:"plugin"`
- Options map[string]interface{} `yaml:"options"`
+ Name string `yaml:"name"`
+ Plugin Plugin `yaml:"plugin"`
+ DependsOn []string `yaml:"dependsOn"`
+ Options map[string]interface{} `yaml:"options"`
 }
 
 func (t *Tool) DeepCopy() *Tool {
  var retTool = Tool{
- Name: t.Name,
- Plugin: t.Plugin,
- Options: map[string]interface{}{},
+ Name: t.Name,
+ Plugin: t.Plugin,
+ DependsOn: t.DependsOn,
+ Options: map[string]interface{}{},
  }
  for k, v := range t.Options {
  retTool.Options[k] = v
@@ -69,7 +71,7 @@ func LoadConf(fname string) *Config {
  return nil
  }
 
- errs := config.Validate()
+ errs := validateConfig(&config)
 
  if len(errs) != 0 {
  for _, e := range errs {

internal/pkg/configloader/configloader_test.go

@@ -0,0 +1,45 @@
+package configloader
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestDependencyPass(t *testing.T) {
+ tools := []Tool{
+ {Name: "argocd", Plugin: Plugin{Kind: "argocd"}},
+ {Name: "argocdapp", Plugin: Plugin{Kind: "argocdapp"}, DependsOn: []string{"argocd.argocd"}},
+ }
+ errors := validateDependency(tools)
+ assert.Equal(t, len(errors), 0, "Dependency check passed.")
+
+}
+
+func TestDependencyNotExist(t *testing.T) {
+ tools := []Tool{
+ {Name: "argocdapp", Plugin: Plugin{Kind: "argocdapp"}, DependsOn: []string{"argocd.argocd"}},
+ }
+ errors := validateDependency(tools)
+ assert.Equal(t, len(errors), 1)
+
+}
+
+func TestMultipleDependencies(t *testing.T) {
+ tools := []Tool{
+ {Name: "argocd", Plugin: Plugin{Kind: "argocd"}},
+ {Name: "repo", Plugin: Plugin{Kind: "github"}},
+ {Name: "argocdapp", Plugin: Plugin{Kind: "argocdapp"}, DependsOn: []string{"argocd.argocd", "repo.github"}},
+ }
+ errors := validateDependency(tools)
+ assert.Equal(t, len(errors), 0)
+}
+
+func TestEmptyDependency(t *testing.T) {
+ tools := []Tool{
+ {Name: "argocd", Plugin: Plugin{Kind: "argocd"}},
+ {Name: "argocdapp", Plugin: Plugin{Kind: "argocdapp"}, DependsOn: []string{}},
+ }
+ errors := validateDependency(tools)
+ assert.Equal(t, len(errors), 0)
+}

internal/pkg/configloader/validation.go

@@ -2,36 +2,75 @@ package configloader
 
 import (
  "fmt"
+ "strings"
 
  "k8s.io/apimachinery/pkg/util/validation"
 )
 
-func (c *Config) Validate() []error {
- retErrors := make([]error, 0)
+func validateConfig(config *Config) []error {
+ errors := make([]error, 0)
 
- for _, t := range c.Tools {
- retErrors = append(retErrors, t.validate()...)
+ for _, t := range config.Tools {
+ errors = append(errors, validateTool(&t)...)
  }
- return retErrors
+
+ errors = append(errors, validateDependency(config.Tools)...)
+
+ return errors
 }
 
-func (t *Tool) validate() []error {
- retErrors := make([]error, 0)
+func validateTool(t *Tool) []error {
+ errors := make([]error, 0)
 
  // Name
  if t.Name == "" {
- retErrors = append(retErrors, fmt.Errorf("name is empty"))
+ errors = append(errors, fmt.Errorf("name is empty"))
  }
 
  errs := validation.IsDNS1123Subdomain(t.Name)
  for _, e := range errs {
- retErrors = append(retErrors, fmt.Errorf("name %s is invalid: %s", t.Name, e))
+ errors = append(errors, fmt.Errorf("name %s is invalid: %s", t.Name, e))
  }
 
  // Plugin
  if t.Plugin.Kind == "" {
- retErrors = append(retErrors, fmt.Errorf("plugin.kind is empty"))
+ errors = append(errors, fmt.Errorf("plugin.kind is empty"))
+ }
+
+ return errors
+}
+
+func validateDependency(tools []Tool) []error {
+ errors := make([]error, 0)
+
+ // config "set" (map)
+ toolMap := make(map[string]bool)
+ // creating the set
+ for _, tool := range tools {
+ key := fmt.Sprintf("%s.%s", tool.Name, tool.Plugin.Kind)
+ toolMap[key] = true
+ }
+
+ for _, tool := range tools {
+ // no dependency, pass
+ if len(tool.DependsOn) == 0 {
+ continue
+ }
+
+ // for each dependency
+ for _, dependency := range tool.DependsOn {
+ // skip empty string
+ dependency = strings.TrimSpace(dependency)
+ if dependency == "" {
+ continue
+ }
+
+ // generate an error if the dependency isn't in the config set,
+ if _, ok := toolMap[dependency]; !ok {
+ errors = append(errors, fmt.Errorf("tool %s's dependency %s doesn't exist in the config", tool.Name, dependency))
+ }
+ }
  }
 
- return retErrors
+ return errors
 }