Merge branch 'v0.19.x'

# Conflicts:
#	pkg/steampipeconfig/modconfig/mod_resources.go

design/mod_deps.md

@@ -0,0 +1,29 @@
+
+ModParseContext has LoadedDependencyMods modconfig.ModMap
+
+currently keyed by mod name - change to key by full name of locked version
+
+GetLockedModVersionConstraint()
+FullName()
+
+Usage
+
+1) loadModDependencies
+```go 
+func loadModDependencies(mod *modconfig.Mod, parseCtx *parse.ModParseContext) error {
+    ...
+    for _, requiredModVersion := range mod.Require.Mods {
+        // if we have a locked version, update the required version to reflect this
+        lockedVersion, err := parseCtx.WorkspaceLock.GetLockedModVersionConstraint(requiredModVersion, mod)
+        if err != nil {
+            errors = append(errors, err)
+            continue
+        }
+        if lockedVersion != nil {
+            requiredModVersion = lockedVersion
+        }
+
+        // have we already loaded a mod which satisfied this
+        if loadedMod, ok := parseCtx.LoadedDependencyMods[requiredModVersion.Name]; ok {
+
+```

pkg/modinstaller/install_data.go

@@ -64,17 +64,18 @@ func (d *InstallData) GetAvailableUpdates() (versionmap.DependencyVersionMap, er
 
 // onModInstalled is called when a dependency is satisfied by installing a mod version
 func (d *InstallData) onModInstalled(dependency *ResolvedModRef, modDef *modconfig.Mod, parent *modconfig.Mod) {
-	parentPath := parent.GetModDependencyPath()
+	parentPath := parent.GetInstallCacheKey()
 	// get the constraint from the parent (it must be there)
-	modVersion := parent.Require.GetModDependency(dependency.Name)
+	modVersionConstraint := parent.Require.GetModDependency(dependency.Name).Constraint.Original
+
 	// update lock
-	d.NewLock.InstallCache.Add(dependency.Name, modDef.ShortName, modDef.Version, modVersion.Constraint.Original, parentPath)
+	d.NewLock.InstallCache.Add(dependency.Name, modDef.ShortName, modDef.Version, modVersionConstraint, parentPath)
 }
 
 // addExisting is called when a dependency is satisfied by a mod which is already installed
 func (d *InstallData) addExisting(dependencyName string, existingDep *modconfig.Mod, constraint *versionhelpers.Constraints, parent *modconfig.Mod) {
 	// update lock
-	parentPath := parent.GetModDependencyPath()
+	parentPath := parent.GetInstallCacheKey()
 	d.NewLock.InstallCache.Add(dependencyName, existingDep.ShortName, existingDep.Version, constraint.Original, parentPath)
 }
 
@@ -107,13 +108,13 @@ func (d *InstallData) onInstallComplete() {
 }
 
 func (d *InstallData) GetUpdatedTree() treeprint.Tree {
-	return d.Upgraded.GetDependencyTree(d.WorkspaceMod.GetModDependencyPath())
+	return d.Upgraded.GetDependencyTree(d.WorkspaceMod.GetInstallCacheKey())
 }
 
 func (d *InstallData) GetInstalledTree() treeprint.Tree {
-	return d.Installed.GetDependencyTree(d.WorkspaceMod.GetModDependencyPath())
+	return d.Installed.GetDependencyTree(d.WorkspaceMod.GetInstallCacheKey())
 }
 
 func (d *InstallData) GetUninstalledTree() treeprint.Tree {
-	return d.Uninstalled.GetDependencyTree(d.WorkspaceMod.GetModDependencyPath())
+	return d.Uninstalled.GetDependencyTree(d.WorkspaceMod.GetInstallCacheKey())
 }

pkg/modinstaller/mod_installer.go

@@ -184,7 +184,7 @@ func (i *ModInstaller) InstallWorkspaceDependencies() (err error) {
 }
 
 func (i *ModInstaller) GetModList() string {
-	return i.installData.Lock.GetModList(i.workspaceMod.GetModDependencyPath())
+	return i.installData.Lock.GetModList(i.workspaceMod.GetInstallCacheKey())
 }
 
 func (i *ModInstaller) installMods(mods []*modconfig.ModVersionConstraint, parent *modconfig.Mod) error {
@@ -383,7 +383,7 @@ func (i *ModInstaller) install(dependency *ResolvedModRef, parent *modconfig.Mod
 			return nil, err
 		}
 		// now the mod is installed in it's final location, set mod dependency path
-		if err := i.setModDependencyPath(modDef, destPath); err != nil {
+		if err := i.setModDependencyConfig(modDef, destPath); err != nil {
 			return nil, err
 		}
 	}
@@ -428,24 +428,31 @@ func (i *ModInstaller) getDependencyDestPath(dependencyFullName string) string {
 }
 
 func (i *ModInstaller) loadDependencyMod(modVersion *versionmap.ResolvedVersionConstraint) (*modconfig.Mod, error) {
-	modPath := i.getDependencyDestPath(modconfig.ModVersionFullName(modVersion.Name, modVersion.Version))
+	modPath := i.getDependencyDestPath(modconfig.BuildModDependencyPath(modVersion.Name, modVersion.Version))
 	modDef, err := i.loadModfile(modPath, false)
 	if err != nil {
 		return nil, err
 	}
 	if modDef == nil {
 		return nil, fmt.Errorf("failed to load mod from %s", modPath)
 	}
-	if err := i.setModDependencyPath(modDef, modPath); err != nil {
+	if err := i.setModDependencyConfig(modDef, modPath); err != nil {
 		return nil, err
 	}
 	return modDef, nil
 
 }
 
 // set the mod dependency path
-func (i *ModInstaller) setModDependencyPath(mod *modconfig.Mod, modPath string) (err error) {
-	mod.ModDependencyPath, err = filepath.Rel(i.modsPath, modPath)
+func (i *ModInstaller) setModDependencyConfig(mod *modconfig.Mod, modPath string) (err error) {
+	mod.DependencyPath, err = filepath.Rel(i.modsPath, modPath)
+	// parse the dependency path to get the depdency name and version
+	dependencyName, version, err := modconfig.ParseModDependencyPath(mod.DependencyPath)
+	if err != nil {
+		return err
+	}
+	mod.DependencyName = dependencyName
+	mod.Version = version
 	return
 }
 
@@ -457,6 +464,7 @@ func (i *ModInstaller) loadModfile(modPath string, createDefault bool) (*modconf
 		}
 		return nil, nil
 	}
+
 	mod, err := parse.ParseModDefinition(modPath)
 	if err != nil {
 		return nil, err

pkg/modinstaller/mod_installer_prune.go

@@ -13,7 +13,7 @@ func (i *ModInstaller) Prune() (versionmap.VersionListMap, error) {
 	// now delete any mod folders which are not in the lock file
 	for name, versions := range unusedMods {
 		for _, version := range versions {
-			depPath := i.getDependencyDestPath(modconfig.ModVersionFullName(name, version))
+			depPath := i.getDependencyDestPath(modconfig.BuildModDependencyPath(name, version))
 			if err := i.deleteDependencyItem(depPath); err != nil {
 				return nil, err
 			}

pkg/modinstaller/resolved_mod_ref.go

@@ -45,5 +45,5 @@ func (r *ResolvedModRef) setGitReference() {
 
 // FullName returns name in the format <dependency name>@v<dependencyVersion>
 func (r *ResolvedModRef) FullName() string {
-	return modconfig.ModVersionFullName(r.Name, r.Version)
+	return modconfig.BuildModDependencyPath(r.Name, r.Version)
 }

pkg/modinstaller/summary_builder.go

@@ -34,7 +34,7 @@ func getVerb(verb string) string {
 
 func BuildInstallSummary(installData *InstallData) string {
 	// for now treat an install as update - we only install deps which are in the mod.sp but missing in the mod folder
-	modDependencyPath := installData.WorkspaceMod.GetModDependencyPath()
+	modDependencyPath := installData.WorkspaceMod.GetInstallCacheKey()
 	installCount, installedTreeString := getInstallationResultString(installData.Installed, modDependencyPath)
 	uninstallCount, uninstalledTreeString := getInstallationResultString(installData.Uninstalled, modDependencyPath)
 	upgradeCount, upgradeTreeString := getInstallationResultString(installData.Upgraded, modDependencyPath)

pkg/steampipeconfig/inputvars/collect_variables.go

@@ -144,9 +144,12 @@ func CollectVariableValuesFromModRequire(mod *modconfig.Mod, parseCtx *parse.Mod
 	res := make(InputValues)
 	if mod.Require != nil {
 		for _, depModConstraint := range mod.Require.Mods {
-			// find the short name for this mod
-			depMod, ok := parseCtx.LoadedDependencyMods[depModConstraint.Name]
-			if !ok {
+			// find the loaded dep mod which satisfies this constraint
+			depMod, err := parseCtx.GetLoadedDependencyMod(depModConstraint, mod)
+			if err != nil {
+				return nil, err
+			}
+			if depMod == nil {
 				return nil, fmt.Errorf("depency mod %s is not loaded", depMod.Name())
 			}
 

pkg/steampipeconfig/load_mod.go

@@ -21,7 +21,7 @@ import (
 // if CreatePseudoResources flag is set, construct hcl resources for files with specific extensions
 // NOTE: it is an error if there is more than 1 mod defined, however zero mods is acceptable
 // - a default mod will be created assuming there are any resource files
-func LoadMod(modPath string, parseCtx *parse.ModParseContext) (mod *modconfig.Mod, errAndWarnings *modconfig.ErrorAndWarnings) {
+func LoadMod(modPath string, parseCtx *parse.ModParseContext, opts ...LoadModOption) (mod *modconfig.Mod, errAndWarnings *modconfig.ErrorAndWarnings) {
 	defer func() {
 		if r := recover(); r != nil {
 			errAndWarnings = modconfig.NewErrorsAndWarning(helpers.ToError(r))
@@ -32,6 +32,12 @@ func LoadMod(modPath string, parseCtx *parse.ModParseContext) (mod *modconfig.Mo
 	if err != nil {
 		return nil, modconfig.NewErrorsAndWarning(err)
 	}
+
+	// apply opts to mod
+	for _, o := range opts {
+		o(mod)
+	}
+
 	// load the mod dependencies
 	if err := loadModDependencies(mod, parseCtx); err != nil {
 		return nil, modconfig.NewErrorsAndWarning(err)
@@ -88,22 +94,16 @@ func loadModDependencies(mod *modconfig.Mod, parseCtx *parse.ModParseContext) er
 		}
 
 		for _, requiredModVersion := range mod.Require.Mods {
-			// if we have a locked version, update the required version to reflect this
-			lockedVersion, err := parseCtx.WorkspaceLock.GetLockedModVersionConstraint(requiredModVersion, mod)
+
+			// have we already loaded a mod which satisfied this
+			loadedMod, err := parseCtx.GetLoadedDependencyMod(requiredModVersion, mod)
 			if err != nil {
-				errors = append(errors, err)
-				continue
+				return err
 			}
-			if lockedVersion != nil {
-				requiredModVersion = lockedVersion
+			if loadedMod != nil {
+				continue
 			}
 
-			// have we already loaded a mod which satisfied this
-			if loadedMod, ok := parseCtx.LoadedDependencyMods[requiredModVersion.Name]; ok {
-				if requiredModVersion.Constraint.Check(loadedMod.Version) {
-					continue
-				}
-			}
 			if err := loadModDependency(requiredModVersion, parseCtx); err != nil {
 				errors = append(errors, err)
 			}
@@ -148,19 +148,16 @@ func loadModDependency(modDependency *modconfig.ModVersionConstraint, parseCtx *
 	childRunCtx.BlockTypes = parseCtx.BlockTypes
 	childRunCtx.ParentParseCtx = parseCtx
 
-	mod, errAndWarnings := LoadMod(dependencyPath, childRunCtx)
+	// NOTE: pass in the version and dependency path of the mod - these must be set before it loads its depdencies
+	mod, errAndWarnings := LoadMod(dependencyPath, childRunCtx, WithDependencyConfig(modDependency.Name, version))
 	if errAndWarnings.GetError() != nil {
 		return errAndWarnings.GetError()
 	}
 
-	// set the version and dependency path of the mod
-	mod.Version = version
-	mod.ModDependencyPath = modDependency.Name
-
 	// update loaded dependency mods
-	parseCtx.LoadedDependencyMods[modDependency.Name] = mod
+	parseCtx.AddLoadedDependencyMod(mod)
 	if parseCtx.ParentParseCtx != nil {
-		parseCtx.ParentParseCtx.LoadedDependencyMods[modDependency.Name] = mod
+		parseCtx.ParentParseCtx.AddLoadedDependencyMod(mod)
 	}
 
 	return nil
@@ -192,7 +189,7 @@ func loadModResources(modPath string, parseCtx *parse.ModParseContext) (*modconf
 		return nil, modconfig.NewErrorsAndWarning(plugin.DiagsToError("Failed to load all mod files", diags))
 	}
 
-	// parse all hcl files.
+	// parse all hcl files (NOTE - this reads the CurrentMod out of ParseContext and adds to it)
 	mod, errAndWarnings := parse.ParseMod(fileData, pseudoResources, parseCtx)
 	if errAndWarnings.GetError() == nil {
 		// now add fully populated mod to the parent run context
@@ -205,7 +202,7 @@ func loadModResources(modPath string, parseCtx *parse.ModParseContext) (*modconf
 	return mod, errAndWarnings
 }
 
-// search the parent folder for a mod installatio which satisfied the given mod dependency
+// search the parent folder for a mod installation which satisfied the given mod dependency
 func findInstalledDependency(modDependency *modconfig.ModVersionConstraint, parentFolder string) (string, *semver.Version, error) {
 	shortDepName := filepath.Base(modDependency.Name)
 	entries, err := os.ReadDir(parentFolder)

pkg/steampipeconfig/load_mod_option.go

@@ -0,0 +1,18 @@
+package steampipeconfig
+
+import (
+	"fmt"
+	"github.com/Masterminds/semver"
+	"github.com/turbot/steampipe/pkg/steampipeconfig/modconfig"
+)
+
+type LoadModOption = func(mod *modconfig.Mod)
+
+func WithDependencyConfig(modDependencyName string, version *semver.Version) LoadModOption {
+	return func(mod *modconfig.Mod) {
+		mod.Version = version
+		// build the ModDependencyPath from the modDependencyName and the version
+		mod.DependencyPath = fmt.Sprintf("%s@v%s", modDependencyName, version.String())
+		mod.DependencyName = modDependencyName
+	}
+}

pkg/steampipeconfig/load_mod_variables.go

@@ -23,7 +23,7 @@ func LoadVariableDefinitions(variablePath string, parseCtx *parse.ModParseContex
 		return nil, errAndWarnings.GetError()
 	}
 
-	variableMap := modconfig.NewModVariableMap(mod, parseCtx.LoadedDependencyMods)
+	variableMap := modconfig.NewModVariableMap(mod, parseCtx.GetTopLevelDependencyMods())
 
 	return variableMap, nil
 }
@@ -75,7 +75,7 @@ func getInputVariables(variableMap map[string]*modconfig.Variable, validate bool
 		return nil, err
 	}
 
-	// build map of depedency mod variable values declared in the mod 'Require' section
+	// build map of dependency mod variable values declared in the mod 'Require' section
 	depModVarValues, err := inputvars.CollectVariableValuesFromModRequire(mod, parseCtx)
 	if err != nil {
 		return nil, err