Skip to content

Commit

Permalink
Replace "Package" with "Workspace" (vercel#2746)
Browse files Browse the repository at this point in the history 
* Change packageInfos to WorkspaceInfos and consolidate types
* Rename TopologicalGraph field to WorkspaceGraph
* Add field comments for fields in Context struct
* Rename PackageNames to WorkspaceNames
* Remove type assertion now that type is more strict

Co-authored-by: Chris Olszewski <[email protected]>
  • Loading branch information
@mehulkar @chris-olszewski
mehulkar and chris-olszewski authored Nov 17, 2022
1 parent 1e3b1b1 commit 1b84680
Show file tree
Hide file tree
Showing 13 changed files with 157 additions and 122 deletions.
60 changes: 37 additions & 23 deletions cli/internal/context/context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/hashicorp/go-multierror"
"github.com/vercel/turbo/cli/internal/core"
"github.com/vercel/turbo/cli/internal/fs"
"github.com/vercel/turbo/cli/internal/graph"
"github.com/vercel/turbo/cli/internal/lockfile"
"github.com/vercel/turbo/cli/internal/packagemanager"
"github.com/vercel/turbo/cli/internal/turbopath"
Expand Down Expand Up @@ -48,13 +49,27 @@ func (w *Warnings) append(err error) {

// Context of the CLI
type Context struct {
// TODO(gsoltis): should the RootPackageJSON be included in PackageInfos?
PackageInfos map[interface{}]*fs.PackageJSON
PackageNames []string
TopologicalGraph dag.AcyclicGraph
RootNode string
Lockfile lockfile.Lockfile
PackageManager *packagemanager.PackageManager
// WorkspaceInfos contains the contents of package.json for every workspace
// TODO(gsoltis): should the RootPackageJSON be included in WorkspaceInfos?
WorkspaceInfos graph.WorkspaceInfos

// WorkspaceNames is all the names of the workspaces
WorkspaceNames []string

// WorkspaceGraph is a graph of workspace dependencies
// (based on package.json dependencies and devDependencies)
WorkspaceGraph dag.AcyclicGraph

// RootNode is a sigil identifying the root workspace
RootNode string

// Lockfile is a struct to read the lockfile based on the package manager
Lockfile lockfile.Lockfile

// PackageManager is an abstraction for all the info a package manager
// can give us about the repo.
PackageManager *packagemanager.PackageManager

// Used to arbitrate access to the graph. We parallelise most build operations
// and Go maps aren't natively threadsafe so this is needed.
mutex sync.Mutex
Expand Down Expand Up @@ -129,13 +144,12 @@ func isWorkspaceReference(packageVersion string, dependencyVersion string, cwd s

// SinglePackageGraph constructs a Context instance from a single package.
func SinglePackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *fs.PackageJSON) (*Context, error) {
packageInfos := make(map[interface{}]*fs.PackageJSON)
packageInfos[util.RootPkgName] = rootPackageJSON
workspaceInfos := map[string]*fs.PackageJSON{util.RootPkgName: rootPackageJSON}
c := &Context{
PackageInfos: packageInfos,
RootNode: core.ROOT_NODE_NAME,
WorkspaceInfos: workspaceInfos,
RootNode: core.ROOT_NODE_NAME,
}
c.TopologicalGraph.Connect(dag.BasicEdge(util.RootPkgName, core.ROOT_NODE_NAME))
c.WorkspaceGraph.Connect(dag.BasicEdge(util.RootPkgName, core.ROOT_NODE_NAME))
packageManager, err := packagemanager.GetPackageManager(repoRoot, rootPackageJSON)
if err != nil {
return nil, err
Expand All @@ -148,7 +162,7 @@ func SinglePackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *
func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *fs.PackageJSON) (*Context, error) {
c := &Context{}
rootpath := repoRoot.ToStringDuringMigration()
c.PackageInfos = make(map[interface{}]*fs.PackageJSON)
c.WorkspaceInfos = make(graph.WorkspaceInfos)
c.RootNode = core.ROOT_NODE_NAME

var warnings Warnings
Expand Down Expand Up @@ -193,7 +207,7 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f
return nil, err
}
populateGraphWaitGroup := &errgroup.Group{}
for _, pkg := range c.PackageInfos {
for _, pkg := range c.WorkspaceInfos {
pkg := pkg
populateGraphWaitGroup.Go(func() error {
return c.populateTopologicGraphForPackageJSON(pkg, rootpath, pkg.Name, &warnings)
Expand All @@ -210,7 +224,7 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f
if err != nil {
return nil, fmt.Errorf("failed to resolve dependencies for root package: %v", err)
}
c.PackageInfos[util.RootPkgName] = rootPackageJSON
c.WorkspaceInfos[util.RootPkgName] = rootPackageJSON

return c, warnings.errorOrNil()
}
Expand Down Expand Up @@ -283,9 +297,9 @@ func (c *Context) populateTopologicGraphForPackageJSON(pkg *fs.PackageJSON, root

// split out internal vs. external deps
for depName, depVersion := range depMap {
if item, ok := c.PackageInfos[depName]; ok && isWorkspaceReference(item.Version, depVersion, pkg.Dir.ToStringDuringMigration(), rootpath) {
if item, ok := c.WorkspaceInfos[depName]; ok && isWorkspaceReference(item.Version, depVersion, pkg.Dir.ToStringDuringMigration(), rootpath) {
internalDepsSet.Add(depName)
c.TopologicalGraph.Connect(dag.BasicEdge(vertexName, depName))
c.WorkspaceGraph.Connect(dag.BasicEdge(vertexName, depName))
} else {
externalUnresolvedDepsSet.Add(depName)
}
Expand Down Expand Up @@ -318,7 +332,7 @@ func (c *Context) populateTopologicGraphForPackageJSON(pkg *fs.PackageJSON, root

// when there are no internal dependencies, we need to still add these leafs to the graph
if internalDepsSet.Len() == 0 {
c.TopologicalGraph.Connect(dag.BasicEdge(pkg.Name, core.ROOT_NODE_NAME))
c.WorkspaceGraph.Connect(dag.BasicEdge(pkg.Name, core.ROOT_NODE_NAME))
}
pkg.ExternalDeps = make([]string, 0, externalDepSet.Cardinality())
for _, v := range externalDepSet.ToSlice() {
Expand Down Expand Up @@ -352,15 +366,15 @@ func (c *Context) parsePackageJSON(repoRoot turbopath.AbsoluteSystemPath, pkgJSO
if err != nil {
return err
}
c.TopologicalGraph.Add(pkg.Name)
c.WorkspaceGraph.Add(pkg.Name)
pkg.PackageJSONPath = turbopath.AnchoredSystemPathFromUpstream(relativePkgJSONPath)
pkg.Dir = turbopath.AnchoredSystemPathFromUpstream(filepath.Dir(relativePkgJSONPath))
if c.PackageInfos[pkg.Name] != nil {
existing := c.PackageInfos[pkg.Name]
if c.WorkspaceInfos[pkg.Name] != nil {
existing := c.WorkspaceInfos[pkg.Name]
return fmt.Errorf("Failed to add workspace \"%s\" from %s, it already exists at %s", pkg.Name, pkg.Dir, existing.Dir)
}
c.PackageInfos[pkg.Name] = pkg
c.PackageNames = append(c.PackageNames, pkg.Name)
c.WorkspaceInfos[pkg.Name] = pkg
c.WorkspaceNames = append(c.WorkspaceNames, pkg.Name)
}
return nil
}
Expand Down
2 changes: 1 addition & 1 deletion cli/internal/core/engine.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -314,7 +314,7 @@ func (e *Engine) ValidatePersistentDependencies(graph *graph.CompleteGraph) erro
}

// Get information about the package
pkg, pkgExists := graph.PackageInfos[packageName]
pkg, pkgExists := graph.WorkspaceInfos[packageName]
if !pkgExists {
return fmt.Errorf("Cannot find package %v", packageName)
}
Expand Down
28 changes: 14 additions & 14 deletions cli/internal/core/engine_persistent_deps_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ var _workspaceGraphDefinition = map[string][]string{
func TestPrepare_PersistentDependencies_Topological(t *testing.T) {
completeGraph, workspaces := _buildCompleteGraph(_workspaceGraphDefinition)

engine := NewEngine(&completeGraph.TopologicalGraph)
engine := NewEngine(&completeGraph.WorkspaceGraph)

// Make this Task Graph:
// dev
Expand Down Expand Up @@ -63,7 +63,7 @@ func TestPrepare_PersistentDependencies_Topological(t *testing.T) {

func TestPrepare_PersistentDependencies_SameWorkspace(t *testing.T) {
completeGraph, workspaces := _buildCompleteGraph(_workspaceGraphDefinition)
engine := NewEngine(&completeGraph.TopologicalGraph)
engine := NewEngine(&completeGraph.WorkspaceGraph)

// Make this Task Graph:
// build
Expand Down Expand Up @@ -113,7 +113,7 @@ func TestPrepare_PersistentDependencies_SameWorkspace(t *testing.T) {

func TestPrepare_PersistentDependencies_WorkspaceSpecific(t *testing.T) {
completeGraph, workspaces := _buildCompleteGraph(_workspaceGraphDefinition)
engine := NewEngine(&completeGraph.TopologicalGraph)
engine := NewEngine(&completeGraph.WorkspaceGraph)

// Make this Task Graph:
// build
Expand Down Expand Up @@ -163,7 +163,7 @@ func TestPrepare_PersistentDependencies_WorkspaceSpecific(t *testing.T) {

func TestPrepare_PersistentDependencies_CrossWorkspace(t *testing.T) {
completeGraph, workspaces := _buildCompleteGraph(_workspaceGraphDefinition)
engine := NewEngine(&completeGraph.TopologicalGraph)
engine := NewEngine(&completeGraph.WorkspaceGraph)

// Make this Task Graph:
// workspace-a#dev
Expand Down Expand Up @@ -205,8 +205,8 @@ func TestPrepare_PersistentDependencies_CrossWorkspace(t *testing.T) {
func TestPrepare_PersistentDependencies_RootWorkspace(t *testing.T) {
completeGraph, workspaces := _buildCompleteGraph(_workspaceGraphDefinition)
// Add in a "dev" task into the root workspace, so it exists
completeGraph.PackageInfos["//"].Scripts["dev"] = "echo \"root dev task\""
engine := NewEngine(&completeGraph.TopologicalGraph)
completeGraph.WorkspaceInfos["//"].Scripts["dev"] = "echo \"root dev task\""
engine := NewEngine(&completeGraph.WorkspaceGraph)

// Make this Task Graph:
// build
Expand Down Expand Up @@ -255,7 +255,7 @@ func TestPrepare_PersistentDependencies_RootWorkspace(t *testing.T) {

func TestPrepare_PersistentDependencies_Unimplemented(t *testing.T) {
completeGraph, workspaces := _buildCompleteGraph(_workspaceGraphDefinition)
engine := NewEngine(&completeGraph.TopologicalGraph)
engine := NewEngine(&completeGraph.WorkspaceGraph)

// Make this Task Graph:
// dev
Expand All @@ -270,7 +270,7 @@ func TestPrepare_PersistentDependencies_Unimplemented(t *testing.T) {

// Remove "dev" script from workspace-c. workspace-a|b will still implement,
// but since no topological dependencies implement, this test can ensure there is no error
delete(completeGraph.PackageInfos["workspace-c"].Scripts, "dev")
delete(completeGraph.WorkspaceInfos["workspace-c"].Scripts, "dev")

// "dev": dependsOn: ["^dev"] (dev is persistent, but workspace-c does not implement dev)
engine.AddTask(&Task{
Expand Down Expand Up @@ -313,9 +313,9 @@ func TestPrepare_PersistentDependencies_Topological_SkipDepImplementedTask(t *te
// └── workspace-c#dev

// remove b's dev script, so there's a skip in the middle
delete(completeGraph.PackageInfos["workspace-b"].Scripts, "dev")
delete(completeGraph.WorkspaceInfos["workspace-b"].Scripts, "dev")

engine := NewEngine(&completeGraph.TopologicalGraph)
engine := NewEngine(&completeGraph.WorkspaceGraph)

// "dev": dependsOn: ["^dev"] (where dev is persistent)
engine.AddTask(&Task{
Expand Down Expand Up @@ -351,7 +351,7 @@ func TestPrepare_PersistentDependencies_Topological_WithALittleExtra(t *testing.
}

completeGraph, workspaces := _buildCompleteGraph(workspaceGraphDefinition)
engine := NewEngine(&completeGraph.TopologicalGraph)
engine := NewEngine(&completeGraph.WorkspaceGraph)

// Make this Task Graph:
// build
Expand Down Expand Up @@ -412,7 +412,7 @@ func TestPrepare_PersistentDependencies_CrossWorkspace_DownstreamPersistent(t *t
"workspace-z": {}, // no dependencies
}
completeGraph, workspaces := _buildCompleteGraph(workspaceGraphDefinition)
engine := NewEngine(&completeGraph.TopologicalGraph)
engine := NewEngine(&completeGraph.WorkspaceGraph)

// Make this Task Graph:
//
Expand Down Expand Up @@ -511,8 +511,8 @@ func _buildCompleteGraph(workspaceEasyDefinition map[string][]string) (*graph.Co

// build completeGraph struct
completeGraph := &graph.CompleteGraph{
TopologicalGraph: workspaceGraph,
PackageInfos: workspaceInfos,
WorkspaceGraph: workspaceGraph,
WorkspaceInfos: workspaceInfos,
}

return completeGraph, workspaces
Expand Down
23 changes: 15 additions & 8 deletions cli/internal/graph/graph.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,17 +12,24 @@ import (
)

// WorkspaceInfos holds information about each workspace in the monorepo.
type WorkspaceInfos map[interface{}]*fs.PackageJSON
type WorkspaceInfos map[string]*fs.PackageJSON

// CompleteGraph represents the common state inferred from the filesystem and pipeline.
// It is not intended to include information specific to a particular run.
type CompleteGraph struct {
// Topo
TopologicalGraph dag.AcyclicGraph
Pipeline fs.Pipeline
PackageInfos WorkspaceInfos
GlobalHash string
RootNode string
// WorkspaceGraph expresses the dependencies between packages
WorkspaceGraph dag.AcyclicGraph

// Pipeline is config from turbo.json
Pipeline fs.Pipeline

// WorkspaceInfos stores the package.json contents by package name
WorkspaceInfos WorkspaceInfos

// GlobalHash is the hash of all global dependencies
GlobalHash string

RootNode string
}

// GetPackageTaskVisitor wraps a `visitor` function that is used for walking the TaskGraph
Expand All @@ -32,7 +39,7 @@ func (g *CompleteGraph) GetPackageTaskVisitor(ctx gocontext.Context, visitor fun
return func(taskID string) error {
packageName, taskName := util.GetPackageTaskFromId(taskID)

pkg, ok := g.PackageInfos[packageName]
pkg, ok := g.WorkspaceInfos[packageName]
if !ok {
return fmt.Errorf("cannot find package %v for task %v", packageName, taskID)
}
Expand Down
32 changes: 20 additions & 12 deletions cli/internal/prune/prune.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package prune
import (
"bufio"
"fmt"

"github.com/vercel/turbo/cli/internal/config"

"github.com/spf13/cobra"
Expand Down Expand Up @@ -93,7 +94,7 @@ func (p *prune) prune(opts *opts) error {
return errors.Wrap(err, "could not construct graph")
}
p.base.Logger.Trace("scope", "value", opts.scope)
target, scopeIsValid := ctx.PackageInfos[opts.scope]
target, scopeIsValid := ctx.WorkspaceInfos[opts.scope]
if !scopeIsValid {
return errors.Errorf("invalid scope: package %v not found", opts.scope)
}
Expand Down Expand Up @@ -142,12 +143,18 @@ func (p *prune) prune(opts *opts) error {
}
}
workspaces := []turbopath.AnchoredSystemPath{}
targets := []interface{}{opts.scope}
internalDeps, err := ctx.TopologicalGraph.Ancestors(opts.scope)
targets := []string{opts.scope}
internalDeps, err := ctx.WorkspaceGraph.Ancestors(opts.scope)
if err != nil {
return errors.Wrap(err, "could find traverse the dependency graph to find topological dependencies")
}
targets = append(targets, internalDeps.List()...)

// Use for loop so we can coerce to string
// .List() returns a list of interface{} types, but
// we know they are strings.
for _, dep := range internalDeps.List() {
targets = append(targets, dep.(string))
}

lockfileKeys := make([]string, 0, len(rootPackageJSON.TransitiveDeps))
lockfileKeys = append(lockfileKeys, rootPackageJSON.TransitiveDeps...)
Expand All @@ -156,33 +163,34 @@ func (p *prune) prune(opts *opts) error {
if internalDep == ctx.RootNode {
continue
}
workspaces = append(workspaces, ctx.PackageInfos[internalDep].Dir)
originalDir := ctx.PackageInfos[internalDep].Dir.RestoreAnchor(p.base.RepoRoot)

workspaces = append(workspaces, ctx.WorkspaceInfos[internalDep].Dir)
originalDir := ctx.WorkspaceInfos[internalDep].Dir.RestoreAnchor(p.base.RepoRoot)
info, err := originalDir.Lstat()
if err != nil {
return errors.Wrapf(err, "failed to lstat %s", originalDir)
}
targetDir := ctx.PackageInfos[internalDep].Dir.RestoreAnchor(fullDir)
targetDir := ctx.WorkspaceInfos[internalDep].Dir.RestoreAnchor(fullDir)
if err := targetDir.MkdirAllMode(info.Mode()); err != nil {
return errors.Wrapf(err, "failed to create folder %s for %v", targetDir, internalDep)
}

if err := fs.RecursiveCopy(ctx.PackageInfos[internalDep].Dir.ToStringDuringMigration(), targetDir.ToStringDuringMigration()); err != nil {
if err := fs.RecursiveCopy(ctx.WorkspaceInfos[internalDep].Dir.ToStringDuringMigration(), targetDir.ToStringDuringMigration()); err != nil {
return errors.Wrapf(err, "failed to copy %v into %v", internalDep, targetDir)
}
if opts.docker {
jsonDir := outDir.UntypedJoin("json", ctx.PackageInfos[internalDep].PackageJSONPath.ToStringDuringMigration())
jsonDir := outDir.UntypedJoin("json", ctx.WorkspaceInfos[internalDep].PackageJSONPath.ToStringDuringMigration())
if err := jsonDir.EnsureDir(); err != nil {
return errors.Wrapf(err, "failed to create folder %v for %v", jsonDir, internalDep)
}
if err := fs.RecursiveCopy(ctx.PackageInfos[internalDep].PackageJSONPath.ToStringDuringMigration(), jsonDir.ToStringDuringMigration()); err != nil {
if err := fs.RecursiveCopy(ctx.WorkspaceInfos[internalDep].PackageJSONPath.ToStringDuringMigration(), jsonDir.ToStringDuringMigration()); err != nil {
return errors.Wrapf(err, "failed to copy %v into %v", internalDep, jsonDir)
}
}

lockfileKeys = append(lockfileKeys, ctx.PackageInfos[internalDep].TransitiveDeps...)
lockfileKeys = append(lockfileKeys, ctx.WorkspaceInfos[internalDep].TransitiveDeps...)

p.base.UI.Output(fmt.Sprintf(" - Added %v", ctx.PackageInfos[internalDep].Name))
p.base.UI.Output(fmt.Sprintf(" - Added %v", ctx.WorkspaceInfos[internalDep].Name))
}
p.base.Logger.Trace("new workspaces", "value", workspaces)

Expand Down
Loading

0 comments on commit 1b84680

Please sign in to comment.