Improve FS overlays

cli/daemon/dash/ai/codegen.go

@@ -312,7 +312,7 @@ func updateCode(ctx context.Context, services []Service, app *apps.Instance, ove
 		}
 		rtn.Errors = overlays.validationErrors(perrs)
 	}()
-	for p, olay := range overlays.list {
+	for p, olay := range overlays.items {
 		astFile, err := parser.ParseFile(fset, p.ToIO(), olay.content, parser.ParseComments|parser.AllErrors)
 		if err != nil {
 			perrs.AddStd(err)

cli/daemon/dash/ai/overlay.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"go/token"
-	"io/fs"
+	"io"
 	"os"
 	"strings"
 	"time"
@@ -17,6 +17,7 @@ import (
 	"encr.dev/pkg/idents"
 	"encr.dev/pkg/paths"
 	meta "encr.dev/proto/encore/parser/meta/v1"
+	"encr.dev/v2/internals/parsectx"
 	"encr.dev/v2/internals/perr"
 )
 
@@ -102,7 +103,7 @@ func newServicePaths(app *apps.Instance) (*servicePaths, error) {
 
 // An overlay is a virtual file that is used to store the source code of an endpoint or types
 // It automatically generates a header with pkg name and imports.
-// It implements fs.FileInfo and fs.DirEntry interfaces
+// It implements os.FileInfo and os.DirEntry interfaces
 type overlay struct {
 	path         paths.FS
 	endpoint     *Endpoint
@@ -112,11 +113,11 @@ type overlay struct {
 	headerOffset token.Position
 }
 
-func (o *overlay) Type() fs.FileMode {
+func (o *overlay) Type() os.FileMode {
 	return o.Mode()
 }
 
-func (o *overlay) Info() (fs.FileInfo, error) {
+func (o *overlay) Info() (os.FileInfo, error) {
 	return o, nil
 }
 
@@ -128,8 +129,8 @@ func (o *overlay) Size() int64 {
 	return int64(len(o.content))
 }
 
-func (o *overlay) Mode() fs.FileMode {
-	return fs.ModePerm
+func (o *overlay) Mode() os.FileMode {
+	return os.ModePerm
 }
 
 func (o *overlay) ModTime() time.Time {
@@ -145,25 +146,25 @@ func (o *overlay) Sys() any {
 	panic("implement me")
 }
 
-func (o *overlay) Stat() (fs.FileInfo, error) {
+func (o *overlay) Stat() (os.FileInfo, error) {
 	return o, nil
 }
 
-func (o *overlay) File() fs.File {
-	return &overlayFile{o, bytes.NewReader(o.content)}
+func (o *overlay) Reader() io.ReadCloser {
+	return &overlayReader{o, bytes.NewReader(o.content)}
 }
 
-// overlayFile is a wrapper which implements the fs.File interface
-type overlayFile struct {
+// overlayReader is a wrapper around the overlay to implement io.ReadCloser
+type overlayReader struct {
 	*overlay
 	*bytes.Reader
 }
 
-func (o *overlayFile) Close() error { return nil }
+func (o *overlayReader) Close() error { return nil }
 
 var (
-	_ fs.FileInfo = (*overlay)(nil)
-	_ fs.DirEntry = (*overlay)(nil)
+	_ os.FileInfo = (*overlay)(nil)
+	_ os.DirEntry = (*overlay)(nil)
 )
 
 func newOverlays(app *apps.Instance, overwrite bool, services ...Service) (*overlays, error) {
@@ -172,7 +173,7 @@ func newOverlays(app *apps.Instance, overwrite bool, services ...Service) (*over
 		return nil, err
 	}
 	o := &overlays{
-		list:  map[paths.FS]*overlay{},
+		items: map[paths.FS]*overlay{},
 		paths: svcPaths,
 	}
 	for _, s := range services {
@@ -190,30 +191,29 @@ func newOverlays(app *apps.Instance, overwrite bool, services ...Service) (*over
 }
 
 // overlays is a collection of virtual files that are used to store the source code of endpoints and types
-// in memory. It's modelled as a filesystem and implements the fs.ReadFileFS, fs.ReadDirFS and fs.StatFS interfaces.
-// It's used as an overlay for the Go and Encore parsers to include our in-progress code in the parsing process.
+// in memory. It's modelled as a replacement for the os package.
 type overlays struct {
-	list  map[paths.FS]*overlay
+	items map[paths.FS]*overlay
 	paths *servicePaths
 }
 
-func (o *overlays) Stat(name string) (fs.FileInfo, error) {
-	f, ok := o.list[paths.FS(name)]
+func (o *overlays) Stat(name string) (os.FileInfo, error) {
+	f, ok := o.items[paths.FS(name)]
 	if !ok {
 		// else return the filesystem file
 		return os.Stat(name)
 	}
 	return f, nil
 }
 
-func (o *overlays) ReadDir(name string) ([]fs.DirEntry, error) {
-	entries := map[string]fs.DirEntry{}
+func (o *overlays) ReadDir(name string) ([]os.DirEntry, error) {
+	entries := map[string]os.DirEntry{}
 	osFiles, err := os.ReadDir(name)
 	for _, f := range osFiles {
 		entries[f.Name()] = f
 	}
 	dir := paths.FS(name)
-	for _, info := range o.list {
+	for _, info := range o.items {
 		if dir == info.path.Dir() {
 			entries[info.path.Base()] = info
 		}
@@ -226,40 +226,40 @@ func (o *overlays) ReadDir(name string) ([]fs.DirEntry, error) {
 
 func (o *overlays) PkgOverlay() map[string][]byte {
 	files := map[string][]byte{}
-	for f, info := range o.list {
+	for f, info := range o.items {
 		files[f.ToIO()] = info.content
 	}
 	return files
 }
 
 func (o *overlays) ReadFile(name string) ([]byte, error) {
-	f, ok := o.list[paths.FS(name)]
+	f, ok := o.items[paths.FS(name)]
 	if !ok {
 		// else return the filesystem file
 		return os.ReadFile(name)
 	}
 	return f.content, nil
 }
 
-func (o *overlays) Open(name string) (fs.File, error) {
-	f, ok := o.list[paths.FS(name)]
+func (o *overlays) Open(name string) (io.ReadCloser, error) {
+	f, ok := o.items[paths.FS(name)]
 	if !ok {
 		// else return the filesystem file
 		return os.Open(name)
 	}
-	return f.File(), nil
+	return f.Reader(), nil
 }
 
 func (o *overlays) pkgPaths() []paths.Pkg {
 	pkgs := map[paths.Pkg]struct{}{}
-	for _, info := range o.list {
+	for _, info := range o.items {
 		pkgs[o.paths.PkgPath(info.service.Name)] = struct{}{}
 	}
 	return maps.Keys(pkgs)
 }
 
 func (o *overlays) get(p paths.FS) (*overlay, bool) {
-	rtn, ok := o.list[p]
+	rtn, ok := o.items[p]
 	return rtn, ok
 }
 
@@ -317,7 +317,7 @@ func (o *overlays) add(s Service, e *Endpoint) error {
 	}
 	offset, content := toSrcFile(p, s.Name, e.EndpointSource)
 	e.EndpointSource = string(content[offset.Offset:])
-	o.list[p] = &overlay{
+	o.items[p] = &overlay{
 		path:         p,
 		endpoint:     e,
 		service:      &s,
@@ -331,7 +331,7 @@ func (o *overlays) add(s Service, e *Endpoint) error {
 	}
 	offset, content = toSrcFile(p, s.Name, e.TypeSource)
 	e.TypeSource = string(content[offset.Offset:])
-	o.list[p] = &overlay{
+	o.items[p] = &overlay{
 		path:         p,
 		endpoint:     e,
 		service:      &s,
@@ -343,7 +343,5 @@ func (o *overlays) add(s Service, e *Endpoint) error {
 }
 
 var (
-	_ fs.ReadFileFS = (*overlays)(nil)
-	_ fs.ReadDirFS  = (*overlays)(nil)
-	_ fs.StatFS     = (*overlays)(nil)
+	_ parsectx.OverlaidOSFS = (*overlays)(nil)
 )

pkg/errinsrc/internal/golocation.go

@@ -112,14 +112,17 @@ func FromGoTokenPositions(start token.Position, end token.Position, fileReaders
 func convertSingleGoPositionToRange(filename string, fileBody []byte, start token.Position) (end token.Position) {
 	fs := token.NewFileSet()
 	file, err := parser.ParseFile(fs, filename, fileBody, parser.ParseComments)
-
 	if err != nil || file == nil {
-		fileBodyStr := string(fileBody)
-		f := fs.File(1)
-		lineStart := int(f.LineStart(start.Line)) - 1
-		offset := lineStart + start.Column - 1
-		pos := token.Pos(GuessEndColumn(fileBodyStr, offset) - 1)
-		return fs.Position(pos)
+		end = start
+		// If the file is not parsable for some reason (e.g. syntax error), we can't determine the end position
+		// based on ast.Nodes. If so, we can fall back on guessing the end position by looking for common delimiters
+		offset, ok := findPositionOffset(start, fileBody)
+		if !ok {
+			return end
+		}
+		endOffset := GuessEndColumn(fileBody, offset)
+		end.Column += endOffset - offset
+		return end
 	}
 
 	var match ast.Node
@@ -143,12 +146,30 @@ func convertSingleGoPositionToRange(filename string, fileBody []byte, start toke
 	return start
 }
 
-func GuessEndColumn(line string, startColumn int) int {
+func findPositionOffset(pos token.Position, data []byte) (int, bool) {
+	line, col := 1, 1
+	for i, c := range data {
+		if line == pos.Line && col == pos.Column {
+			return i, true
+		} else if line > pos.Line {
+			return -1, false
+		}
+		if c == '\n' {
+			line++
+			col = 1
+		} else {
+			col++
+		}
+	}
+	return -1, false
+}
+
+func GuessEndColumn(data []byte, offset int) int {
 	var params, brackets, braces int
 	inBackticks := false
 
-	for i := startColumn; i < len(line); i++ {
-		switch line[i] {
+	for i := offset; i < len(data); i++ {
+		switch data[i] {
 		case '(':
 			params++
 		case '[':
@@ -183,5 +204,5 @@ func GuessEndColumn(line string, startColumn int) int {
 		}
 	}
 
-	return len(line) + 1
+	return len(data) + 1
 }

pkg/errinsrc/srcrender.go

@@ -198,7 +198,7 @@ linePrintLoop:
 				// Try and guess the atom where the error is
 				// if the currentCause.Start()/currentCause.End() point is the same position
 				if endCol <= startCol {
-					endCol = GuessEndColumn(sc.Text(), startCol)
+					endCol = GuessEndColumn(sc.Bytes(), startCol)
 				}
 
 				// Work out how long the indicator is

v2/codegen/rewrite/rewrite.go

@@ -40,7 +40,7 @@ func (r *Rewriter) Append(data []byte) {
 	}
 	r.segs = append(r.segs, segment{
 		start: int(start),
-		end:   int(start) + len(data),
+		end:   int(start),
 		data:  data,
 	})
 }

v2/internals/parsectx/pctx.go

@@ -48,8 +48,9 @@ type Context struct {
 	// Errs contains encountered errors.
 	Errs *perr.List
 
-	// Overlay is a map of file paths to their contents.
-	Overlay OverlayFS
+	// Overlay is an optional replacement for reading files using the os pkg.
+	// If unset, os is used instead.
+	Overlay OverlaidOSFS
 }
 
 func (c *Context) ReadFile(dir string) ([]byte, error) {
@@ -99,10 +100,14 @@ func (c *Context) PkgOverlay() map[string][]byte {
 	return c.Overlay.PkgOverlay()
 }
 
-type OverlayFS interface {
-	fs.ReadDirFS
-	fs.ReadFileFS
-	fs.StatFS
+// OverlaidOSFS is an interface that allows overlaying the os package with custom implementations.
+// This is used to allow for e.g. in-memory files. The name parameters are os paths, like the
+// corresponding methods in the os package.
+type OverlaidOSFS interface {
+	ReadDir(name string) ([]os.DirEntry, error)
+	ReadFile(name string) ([]byte, error)
+	Stat(name string) (os.FileInfo, error)
+	Open(name string) (io.ReadCloser, error)
 	PkgOverlay() map[string][]byte
 }