feat: Application specific parameter override files (argoproj#5038)

* feat: Support application specific parameter override files

Signed-off-by: jannfis <[email protected]>

cmd/argocd/commands/app.go

@@ -1055,7 +1055,7 @@ func getLocalObjectsString(app *argoappv1.Application, local, localRepoRoot, app
 	res, err := repository.GenerateManifests(local, localRepoRoot, app.Spec.Source.TargetRevision, &repoapiclient.ManifestRequest{
 		Repo:              &argoappv1.Repository{Repo: app.Spec.Source.RepoURL},
 		AppLabelKey:       appLabelKey,
-		AppLabelValue:     app.Name,
+		AppName:           app.Name,
 		Namespace:         app.Spec.Destination.Namespace,
 		ApplicationSource: &app.Spec.Source,
 		KustomizeOptions:  kustomizeOptions,

controller/state.go

@@ -160,7 +160,7 @@ func (m *appStateManager) getRepoObjs(app *v1alpha1.Application, source v1alpha1
 		Revision:          revision,
 		NoCache:           noCache,
 		AppLabelKey:       appLabelKey,
-		AppLabelValue:     app.Name,
+		AppName:           app.Name,
 		Namespace:         app.Spec.Destination.Namespace,
 		ApplicationSource: &source,
 		Plugins:           tools,

docs/user-guide/parameters.md

@@ -54,6 +54,8 @@ argocd app create redis --repo https://github.com/helm/charts.git --path stable/
 
 ## Store Overrides In Git
 
+> The following is available from v1.8 or later
+
 The config management tool specific overrides can be specified in `.argocd-source.yaml` file stored in the source application
 directory in the Git repository.
 
@@ -77,3 +79,16 @@ The `.argocd-source` is trying to solve two following main use cases:
 for projects like [argocd-image-updater](https://github.com/argoproj-labs/argocd-image-updater).
 - Support "discovering" applications in the Git repository by projects like [applicationset](https://github.com/argoproj-labs/applicationset)
 (see [git files generator](https://github.com/argoproj-labs/applicationset/blob/master/examples/git-files-discovery.yaml))
+
+> The following is available from v1.9 or later
+
+You can also store parameter overrides in an application specific file, if you
+are sourcing multiple applications from a single path in your repository.
+
+The application specific file must be named `.argocd-source-<appname>.yaml`,
+where `<appname>` is the name of the application the overrides are valid for.
+
+If there exists an non-application specific `.argocd-source.yaml`, parameters
+included in that file will be merged first, and then the application specific
+parameters are merged, which can also contain overrides to the parameters
+stored in the non-application specific file.

reposerver/cache/cache.go

@@ -69,12 +69,12 @@ func (c *Cache) SetApps(repoUrl, revision string, apps map[string]string) error
 	return c.cache.SetItem(listApps(repoUrl, revision), apps, c.repoCacheExpiration, apps == nil)
 }
 
-func manifestCacheKey(revision string, appSrc *appv1.ApplicationSource, namespace string, appLabelKey string, appLabelValue string) string {
-	return fmt.Sprintf("mfst|%s|%s|%s|%s|%d", appLabelKey, appLabelValue, revision, namespace, appSourceKey(appSrc))
+func manifestCacheKey(revision string, appSrc *appv1.ApplicationSource, namespace string, appLabelKey string, appName string) string {
+	return fmt.Sprintf("mfst|%s|%s|%s|%s|%d", appLabelKey, appName, revision, namespace, appSourceKey(appSrc))
 }
 
-func (c *Cache) GetManifests(revision string, appSrc *appv1.ApplicationSource, namespace string, appLabelKey string, appLabelValue string, res *CachedManifestResponse) error {
-	err := c.cache.GetItem(manifestCacheKey(revision, appSrc, namespace, appLabelKey, appLabelValue), res)
+func (c *Cache) GetManifests(revision string, appSrc *appv1.ApplicationSource, namespace string, appLabelKey string, appName string, res *CachedManifestResponse) error {
+	err := c.cache.GetItem(manifestCacheKey(revision, appSrc, namespace, appLabelKey, appName), res)
 
 	if err != nil {
 		return err
@@ -87,9 +87,9 @@ func (c *Cache) GetManifests(revision string, appSrc *appv1.ApplicationSource, n
 
 	// If the expected hash of the cache entry does not match the actual hash value...
 	if hash != res.CacheEntryHash {
-		log.Warnf("Manifest hash did not match expected value, treating as a cache miss: %s", appLabelValue)
+		log.Warnf("Manifest hash did not match expected value, treating as a cache miss: %s", appName)
 
-		err = c.DeleteManifests(revision, appSrc, namespace, appLabelKey, appLabelValue)
+		err = c.DeleteManifests(revision, appSrc, namespace, appLabelKey, appName)
 		if err != nil {
 			return fmt.Errorf("Unable to delete manifest after hash mismatch, %v", err)
 		}
@@ -104,7 +104,7 @@ func (c *Cache) GetManifests(revision string, appSrc *appv1.ApplicationSource, n
 	return nil
 }
 
-func (c *Cache) SetManifests(revision string, appSrc *appv1.ApplicationSource, namespace string, appLabelKey string, appLabelValue string, res *CachedManifestResponse) error {
+func (c *Cache) SetManifests(revision string, appSrc *appv1.ApplicationSource, namespace string, appLabelKey string, appName string, res *CachedManifestResponse) error {
 
 	// Generate and apply the cache entry hash, before writing
 	if res != nil {
@@ -116,11 +116,11 @@ func (c *Cache) SetManifests(revision string, appSrc *appv1.ApplicationSource, n
 		res.CacheEntryHash = hash
 	}
 
-	return c.cache.SetItem(manifestCacheKey(revision, appSrc, namespace, appLabelKey, appLabelValue), res, c.repoCacheExpiration, res == nil)
+	return c.cache.SetItem(manifestCacheKey(revision, appSrc, namespace, appLabelKey, appName), res, c.repoCacheExpiration, res == nil)
 }
 
-func (c *Cache) DeleteManifests(revision string, appSrc *appv1.ApplicationSource, namespace string, appLabelKey string, appLabelValue string) error {
-	return c.cache.SetItem(manifestCacheKey(revision, appSrc, namespace, appLabelKey, appLabelValue), "", c.repoCacheExpiration, true)
+func (c *Cache) DeleteManifests(revision string, appSrc *appv1.ApplicationSource, namespace string, appLabelKey string, appName string) error {
+	return c.cache.SetItem(manifestCacheKey(revision, appSrc, namespace, appLabelKey, appName), "", c.repoCacheExpiration, true)
 }
 
 func appDetailsCacheKey(revision string, appSrc *appv1.ApplicationSource) string {

reposerver/repository/repository.go

@@ -57,6 +57,8 @@ const (
 	cachedManifestGenerationPrefix = "Manifest generation error (cached)"
 	helmDepUpMarkerFile            = ".argocd-helm-dep-up"
 	allowConcurrencyFile           = ".argocd-allow-concurrency"
+	repoSourceFile                 = ".argocd-source.yaml"
+	appSourceFile                  = ".argocd-source-%s.yaml"
 )
 
 // Service implements ManifestService interface
@@ -324,7 +326,7 @@ func (s *Service) runManifestGen(repoRoot, commitSHA, cacheKey string, ctxSrc op
 			// Retrieve a new copy (if available) of the cached response: this ensures we are updating the latest copy of the cache,
 			// rather than a copy of the cache that occurred before (a potentially lengthy) manifest generation.
 			innerRes := &cache.CachedManifestResponse{}
-			cacheErr := s.cache.GetManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppLabelValue, innerRes)
+			cacheErr := s.cache.GetManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppName, innerRes)
 			if cacheErr != nil && cacheErr != reposervercache.ErrCacheMiss {
 				log.Warnf("manifest cache set error %s: %v", q.ApplicationSource.String(), cacheErr)
 				return nil, cacheErr
@@ -339,7 +341,7 @@ func (s *Service) runManifestGen(repoRoot, commitSHA, cacheKey string, ctxSrc op
 			// Update the cache to include failure information
 			innerRes.NumberOfConsecutiveFailures++
 			innerRes.MostRecentError = err.Error()
-			cacheErr = s.cache.SetManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppLabelValue, innerRes)
+			cacheErr = s.cache.SetManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppName, innerRes)
 			if cacheErr != nil {
 				log.Warnf("manifest cache set error %s: %v", q.ApplicationSource.String(), cacheErr)
 				return nil, cacheErr
@@ -358,7 +360,7 @@ func (s *Service) runManifestGen(repoRoot, commitSHA, cacheKey string, ctxSrc op
 	}
 	manifestGenResult.Revision = commitSHA
 	manifestGenResult.VerifyResult = ctx.verificationResult
-	err = s.cache.SetManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppLabelValue, &manifestGenCacheEntry)
+	err = s.cache.SetManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppName, &manifestGenCacheEntry)
 	if err != nil {
 		log.Warnf("manifest cache set error %s/%s: %v", q.ApplicationSource.String(), cacheKey, err)
 	}
@@ -373,7 +375,7 @@ func (s *Service) runManifestGen(repoRoot, commitSHA, cacheKey string, ctxSrc op
 // If true is returned, either the second or third parameter (but not both) will contain a value from the cache (a ManifestResponse, or error, respectively)
 func (s *Service) getManifestCacheEntry(cacheKey string, q *apiclient.ManifestRequest, firstInvocation bool) (bool, interface{}, error) {
 	res := cache.CachedManifestResponse{}
-	err := s.cache.GetManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppLabelValue, &res)
+	err := s.cache.GetManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppName, &res)
 	if err == nil {
 
 		// The cache contains an existing value
@@ -392,7 +394,7 @@ func (s *Service) getManifestCacheEntry(cacheKey string, q *apiclient.ManifestRe
 					// After X minutes, reset the cache and retry the operation (eg perhaps the error is ephemeral and has passed)
 					if elapsedTimeInMinutes >= s.initConstants.PauseGenerationOnFailureForMinutes {
 						// We can now try again, so reset the cache state and run the operation below
-						err = s.cache.DeleteManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppLabelValue)
+						err = s.cache.DeleteManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppName)
 						if err != nil {
 							log.Warnf("manifest cache set error %s/%s: %v", q.ApplicationSource.String(), cacheKey, err)
 						}
@@ -406,7 +408,7 @@ func (s *Service) getManifestCacheEntry(cacheKey string, q *apiclient.ManifestRe
 
 					if res.NumberOfCachedResponsesReturned >= s.initConstants.PauseGenerationOnFailureForRequests {
 						// We can now try again, so reset the error cache state and run the operation below
-						err = s.cache.DeleteManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppLabelValue)
+						err = s.cache.DeleteManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppName)
 						if err != nil {
 							log.Warnf("manifest cache set error %s/%s: %v", q.ApplicationSource.String(), cacheKey, err)
 						}
@@ -424,7 +426,7 @@ func (s *Service) getManifestCacheEntry(cacheKey string, q *apiclient.ManifestRe
 					// Increment the number of returned cached responses and push that new value to the cache
 					// (if we have not already done so previously in this function)
 					res.NumberOfCachedResponsesReturned++
-					err = s.cache.SetManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppLabelValue, &res)
+					err = s.cache.SetManifests(cacheKey, q.ApplicationSource, q.Namespace, q.AppLabelKey, q.AppName, &res)
 					if err != nil {
 						log.Warnf("manifest cache set error %s/%s: %v", q.ApplicationSource.String(), cacheKey, err)
 					}
@@ -504,7 +506,7 @@ func helmTemplate(appPath string, repoRoot string, env *v1alpha1.Env, q *apiclie
 	}
 
 	templateOpts := &helm.TemplateOpts{
-		Name:        q.AppLabelValue,
+		Name:        q.AppName,
 		Namespace:   q.Namespace,
 		KubeVersion: text.SemVer(q.KubeVersion),
 		APIVersions: q.ApiVersions,
@@ -577,7 +579,7 @@ func helmTemplate(appPath string, repoRoot string, env *v1alpha1.Env, q *apiclie
 		}
 	}
 	if templateOpts.Name == "" {
-		templateOpts.Name = q.AppLabelValue
+		templateOpts.Name = q.AppName
 	}
 	for i, j := range templateOpts.Set {
 		templateOpts.Set[i] = env.Envsubst(j)
@@ -628,7 +630,7 @@ func GenerateManifests(appPath, repoRoot, revision string, q *apiclient.Manifest
 	var targetObjs []*unstructured.Unstructured
 	var dest *v1alpha1.ApplicationDestination
 
-	appSourceType, err := GetAppSourceType(q.ApplicationSource, appPath)
+	appSourceType, err := GetAppSourceType(q.ApplicationSource, appPath, q.AppName)
 	if err != nil {
 		return nil, err
 	}
@@ -685,8 +687,8 @@ func GenerateManifests(appPath, repoRoot, revision string, q *apiclient.Manifest
 		}
 
 		for _, target := range targets {
-			if q.AppLabelKey != "" && q.AppLabelValue != "" && !kube.IsCRD(target) {
-				err = argokube.SetAppInstanceLabel(target, q.AppLabelKey, q.AppLabelValue)
+			if q.AppLabelKey != "" && q.AppName != "" && !kube.IsCRD(target) {
+				err = argokube.SetAppInstanceLabel(target, q.AppLabelKey, q.AppName)
 				if err != nil {
 					return nil, err
 				}
@@ -712,7 +714,7 @@ func GenerateManifests(appPath, repoRoot, revision string, q *apiclient.Manifest
 
 func newEnv(q *apiclient.ManifestRequest, revision string) *v1alpha1.Env {
 	return &v1alpha1.Env{
-		&v1alpha1.EnvEntry{Name: "ARGOCD_APP_NAME", Value: q.AppLabelValue},
+		&v1alpha1.EnvEntry{Name: "ARGOCD_APP_NAME", Value: q.AppName},
 		&v1alpha1.EnvEntry{Name: "ARGOCD_APP_NAMESPACE", Value: q.Namespace},
 		&v1alpha1.EnvEntry{Name: "ARGOCD_APP_REVISION", Value: revision},
 		&v1alpha1.EnvEntry{Name: "ARGOCD_APP_SOURCE_REPO_URL", Value: q.Repo.Repo},
@@ -721,38 +723,54 @@ func newEnv(q *apiclient.ManifestRequest, revision string) *v1alpha1.Env {
 	}
 }
 
-func mergeSourceParameters(source *v1alpha1.ApplicationSource, path string) error {
-	appFilePath := filepath.Join(path, ".argocd-source.yaml")
-	info, err := os.Stat(appFilePath)
-	if os.IsNotExist(err) {
-		return nil
-	} else if info != nil && info.IsDir() {
-		return nil
-	} else if err != nil {
-		return err
-	}
-	patch, err := json.Marshal(source)
-	if err != nil {
-		return err
-	}
+// mergeSourceParameters merges parameter overrides from one or more files in
+// the Git repo into the given ApplicationSource objects.
+//
+// If .argocd-source.yaml exists at application's path in repository, it will
+// be read and merged. If appName is not the empty string, and a file named
+// .argocd-source-<appName>.yaml exists, it will also be read and merged.
+func mergeSourceParameters(source *v1alpha1.ApplicationSource, path, appName string) error {
+	repoFilePath := filepath.Join(path, repoSourceFile)
+	overrides := []string{repoFilePath}
+	if appName != "" {
+		overrides = append(overrides, filepath.Join(path, fmt.Sprintf(appSourceFile, appName)))
+	}
+
+	var merged v1alpha1.ApplicationSource = *source.DeepCopy()
+
+	for _, filename := range overrides {
+		info, err := os.Stat(filename)
+		if os.IsNotExist(err) {
+			continue
+		} else if info != nil && info.IsDir() {
+			continue
+		} else if err != nil {
+			// filename should be part of error message here
+			return err
+		}
 
-	data, err := ioutil.ReadFile(appFilePath)
-	if err != nil {
-		return err
-	}
-	data, err = yaml.YAMLToJSON(data)
-	if err != nil {
-		return err
-	}
-	data, err = jsonpatch.MergePatch(data, patch)
-	if err != nil {
-		return err
-	}
-	var merged v1alpha1.ApplicationSource
-	err = json.Unmarshal(data, &merged)
-	if err != nil {
-		return err
+		data, err := json.Marshal(merged)
+		if err != nil {
+			return fmt.Errorf("%s: %v", filename, err)
+		}
+		patch, err := ioutil.ReadFile(filename)
+		if err != nil {
+			return fmt.Errorf("%s: %v", filename, err)
+		}
+		patch, err = yaml.YAMLToJSON(patch)
+		if err != nil {
+			return fmt.Errorf("%s: %v", filename, err)
+		}
+		data, err = jsonpatch.MergePatch(data, patch)
+		if err != nil {
+			return fmt.Errorf("%s: %v", filename, err)
+		}
+		err = json.Unmarshal(data, &merged)
+		if err != nil {
+			return fmt.Errorf("%s: %v", filename, err)
+		}
 	}
+
 	// make sure only config management tools related properties are used and ignore everything else
 	merged.Chart = source.Chart
 	merged.Path = source.Path
@@ -764,10 +782,10 @@ func mergeSourceParameters(source *v1alpha1.ApplicationSource, path string) erro
 }
 
 // GetAppSourceType returns explicit application source type or examines a directory and determines its application source type
-func GetAppSourceType(source *v1alpha1.ApplicationSource, path string) (v1alpha1.ApplicationSourceType, error) {
-	err := mergeSourceParameters(source, path)
+func GetAppSourceType(source *v1alpha1.ApplicationSource, path, appName string) (v1alpha1.ApplicationSourceType, error) {
+	err := mergeSourceParameters(source, path, appName)
 	if err != nil {
-		return "", fmt.Errorf("error while parsing .argocd-source.yaml: %v", err)
+		return "", fmt.Errorf("error while parsing source parameters: %v", err)
 	}
 
 	appSourceType, err := source.ExplicitType()
@@ -1049,7 +1067,7 @@ func (s *Service) GetAppDetails(ctx context.Context, q *apiclient.RepoServerAppD
 		appPath := ctx.appPath
 
 		res := &apiclient.RepoAppDetailsResponse{}
-		appSourceType, err := GetAppSourceType(q.Source, appPath)
+		appSourceType, err := GetAppSourceType(q.Source, appPath, q.AppName)
 		if err != nil {
 			return nil, err
 		}

reposerver/repository/repository.proto

@@ -15,7 +15,8 @@ message ManifestRequest {
     string revision = 2;
     bool noCache = 3;
     string appLabelKey = 4;
-    string appLabelValue = 5;
+    // Name of the application for which the request is triggered
+    string appName = 5;
     string namespace = 8;
     github.com.argoproj.argo_cd.pkg.apis.application.v1alpha1.ApplicationSource applicationSource = 10;
     repeated github.com.argoproj.argo_cd.pkg.apis.application.v1alpha1.Repository repos = 11;
@@ -65,6 +66,7 @@ message RepoServerAppDetailsQuery {
     github.com.argoproj.argo_cd.pkg.apis.application.v1alpha1.ApplicationSource source = 2;
     repeated github.com.argoproj.argo_cd.pkg.apis.application.v1alpha1.Repository repos = 3;
     github.com.argoproj.argo_cd.pkg.apis.application.v1alpha1.KustomizeOptions kustomizeOptions = 4;
+    string appName = 5;
 }
 
 // RepoAppDetailsResponse application details