Skip to content

Commit

Permalink
[ENC-307] Generic Response Types (encoredev#3)
Browse files Browse the repository at this point in the history
This commit gets generic response types working under Go 1.18 and
allows the system to return a response struct with a generic field
within it.

This commit also adds the ability for the encore parser to understand
and obey `//go:build` tags, which helps with things like ARCH, OS,
language level or tooling tags filters on files - preventing us trying
to parse a file we can't or shouldn't.
  • Loading branch information
DomBlack authored and eandre committed Feb 8, 2022
1 parent 375d0e9 commit b2a71f3
Show file tree
Hide file tree
Showing 17 changed files with 759 additions and 282 deletions.
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ require (
github.com/golang/protobuf v1.5.2
github.com/google/go-cmp v0.5.6
github.com/gorilla/websocket v1.4.2
github.com/hashicorp/go-version v1.4.0 // indirect
github.com/hashicorp/go-version v1.4.0
github.com/hashicorp/yamux v0.0.0-20200609203250-aecfd211c9ce
github.com/jackc/pgproto3/v2 v2.0.7
github.com/jackc/pgx/v4 v4.10.1
Expand All @@ -29,11 +29,11 @@ require (
go.uber.org/goleak v1.1.10
go4.org v0.0.0-20201209231011-d4a079459e60
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
golang.org/x/mod v0.5.0
golang.org/x/mod v0.5.1
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d
golang.org/x/tools v0.1.5
golang.org/x/tools v0.1.9
golang.zx2c4.com/wireguard v0.0.0-20210728232740-bad6caeb82ed
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20210803171230-4253848d036c
golang.zx2c4.com/wireguard/windows v0.4.1
Expand Down
9 changes: 9 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -605,6 +605,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1 h1:/vn0k+RBvwlxEmP5E7SZMqNxPhfMVFEJiykr15/0XKM=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
Expand Down Expand Up @@ -693,6 +695,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q=
golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -750,6 +754,8 @@ golang.org/x/net v0.0.0-20210504132125-bbd867fde50d/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180227000427-d7d64896b5ff/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down Expand Up @@ -868,6 +874,7 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E=
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down Expand Up @@ -955,6 +962,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/tools v0.1.9 h1:j9KsMiaP1c3B0OTQGth0/k+miLGTgLsAFUCrF2vLcF8=
golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
13 changes: 12 additions & 1 deletion parser/dirs.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package parser
import (
"fmt"
"go/ast"
"go/build"
goparser "go/parser"
"go/scanner"
"go/token"
Expand Down Expand Up @@ -62,7 +63,7 @@ func walkDir(dir, rel string, walkFn walkFunc) error {
}

// parseDir is like go/parser.ParseDir but it constructs *est.File objects instead.
func parseDir(fset *token.FileSet, dir, relPath string, filter func(os.FileInfo) bool, mode goparser.Mode) (pkgs map[string]*ast.Package, files []*est.File, err error) {
func parseDir(buildContext build.Context, fset *token.FileSet, dir, relPath string, filter func(os.FileInfo) bool, mode goparser.Mode) (pkgs map[string]*ast.Package, files []*est.File, err error) {
fd, err := os.Open(dir)
if err != nil {
return nil, nil, err
Expand All @@ -88,6 +89,16 @@ func parseDir(fset *token.FileSet, dir, relPath string, filter func(os.FileInfo)
return nil, nil, err
}

// Check if this file should be part of the build
matched, err := buildContext.MatchFile(dir, d.Name())
if err != nil {
errors.Add(token.Position{Filename: filename}, err.Error())
continue
}
if !matched {
continue
}

src, err := goparser.ParseFile(fset, filename, contents, mode)
if err != nil || !src.Pos().IsValid() {
// Parse error or invalid file
Expand Down
4 changes: 3 additions & 1 deletion parser/dirs_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package parser

import (
"go/build"
goparser "go/parser"
"go/token"
"os"
Expand Down Expand Up @@ -127,7 +128,8 @@ package fo/;
c.Assert(err, qt.IsNil, qt.Commentf("test #%d", i))

fs := token.NewFileSet()
pkgs, files, err := parseDir(fs, base, ".", nil, goparser.ParseComments)
context := build.Default
pkgs, files, err := parseDir(context, fs, base, ".", nil, goparser.ParseComments)
if test.Err != "" {
c.Assert(err, qt.ErrorMatches, test.Err)
continue
Expand Down
17 changes: 16 additions & 1 deletion parser/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package parser
import (
"fmt"
"go/ast"
"go/build"
goparser "go/parser"
"go/scanner"
"go/token"
Expand Down Expand Up @@ -147,6 +148,17 @@ func (p *parser) Parse() (res *Result, err error) {
}, nil
}

// encoreBuildContext creates a build context that mirrors what we pass onto the go compiler once the we trigger a build
// of the application. This allows us to ignore `go` files which would be exlcuded during the build.
//
// For instance if a file has the directive `//go:build !encore` in it
func encoreBuildContext() build.Context {
buildContext := build.Default
buildContext.ToolTags = append(buildContext.ToolTags, "encore")

return buildContext
}

// collectPackages collects and parses the regular Go AST
// for all subdirectories in the root.
func collectPackages(fs *token.FileSet, rootDir, rootImportPath string, mode goparser.Mode, parseTests bool) ([]*est.Package, error) {
Expand All @@ -155,8 +167,11 @@ func collectPackages(fs *token.FileSet, rootDir, rootImportPath string, mode gop
filter := func(f os.FileInfo) bool {
return parseTests || !strings.HasSuffix(f.Name(), "_test.go")
}

buildContext := encoreBuildContext()

err := walkDirs(rootDir, func(dir, relPath string, files []os.FileInfo) error {
ps, pkgFiles, err := parseDir(fs, dir, relPath, filter, mode)
ps, pkgFiles, err := parseDir(buildContext, fs, dir, relPath, filter, mode)
if err != nil {
// If the error is an error list, it means we have a parsing error.
// Keep going with other directories in that case.
Expand Down
60 changes: 60 additions & 0 deletions parser/parser_go1.18.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
//go:build go1.18
// +build go1.18

package parser

import (
"go/ast"
"reflect"

"encr.dev/parser/est"
schema "encr.dev/proto/encore/parser/schema/v1"
)

func init() {
additionalTypeResolver = go118ResolveType
}

func go118ResolveType(p *parser, pkg *est.Package, file *est.File, expr ast.Expr) *schema.Type {
switch expr := expr.(type) {
// Needed for generic types with single generic parameters: `X[Index]` (i.e. `Vector[string]`)
case *ast.IndexExpr:
return resolveWithTypeArguments(p, pkg, file, expr.X, expr.Index)

// Needed for generic types with multiple generic parameters: `X[A, B]` (i.e. `Skiplist[string, string]`)
case *ast.IndexListExpr:
return resolveWithTypeArguments(p, pkg, file, expr.X, expr.Indices...)
}

return nil
}

// resolveWithTypeArguments first resolves the parameterized declaration of `ident`, before resolving each of
// the `typeArguments` to concrete types. It then returns a `*schema.Name` representing that instantiated type.
func resolveWithTypeArguments(p *parser, pkg *est.Package, file *est.File, ident ast.Expr, typeArguments ...ast.Expr) *schema.Type {
parameterizedType := p.resolveType(pkg, file, ident, nil)

named := parameterizedType.GetNamed()
if named == nil {
p.errf(ident.Pos(), "expected to get a named type, got %+v", reflect.TypeOf(parameterizedType.Typ))
return parameterizedType
}

decl := p.decls[named.Id]
if decl == nil {
p.errf(ident.Pos(), "unable to find decl referenced")
panic(bailout{})
}

if len(decl.TypeParams) != len(typeArguments) {
p.errf(ident.Pos(), "expected %d type parameters, got %d for reference to %s", len(decl.TypeParams), len(typeArguments), decl.Name)
panic(bailout{})
}

named.TypeArguments = make([]*schema.Type, len(decl.TypeParams))
for idx, expr := range typeArguments {
named.TypeArguments[idx] = p.resolveType(pkg, file, expr, nil)
}

return parameterizedType
}
67 changes: 27 additions & 40 deletions parser/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,14 @@ import (
"github.com/fatih/structtag"

"encr.dev/parser/est"
"encr.dev/parser/internal/names"
schema "encr.dev/proto/encore/parser/schema/v1"
)

var additionalTypeResolver = func(p *parser, pkg *est.Package, file *est.File, expr ast.Expr) *schema.Type { return nil }

// resolveType parses the schema from a type expression.
func (p *parser) resolveDecl(pkg *est.Package, file *est.File, expr ast.Expr) *schema.Decl {
typ := p.resolveType(pkg, file, expr)
typ := p.resolveType(pkg, file, expr, nil)
n := typ.GetNamed()
if n == nil {
p.errf(expr.Pos(), "%s is not a named type", types.ExprString(expr))
Expand All @@ -27,29 +28,38 @@ func (p *parser) resolveDecl(pkg *est.Package, file *est.File, expr ast.Expr) *s
}

// resolveType parses the schema from a type expression.
func (p *parser) resolveType(pkg *est.Package, file *est.File, expr ast.Expr) *schema.Type {
func (p *parser) resolveType(pkg *est.Package, file *est.File, expr ast.Expr, typeParameters typeParameterLookup) *schema.Type {
expr = deref(expr)
names := p.names[pkg]
pkgNames := p.names[pkg]

switch expr := expr.(type) {
case *ast.Ident:
// Check if we have a type parameter defined for this
if ref, ok := typeParameters[expr.Name]; ok {
return &schema.Type{Typ: &schema.Type_TypeParameter{TypeParameter: ref}}
}

// Local type name or universe scope
if d, ok := names.Decls[expr.Name]; ok && d.Type == token.TYPE {
return p.parseDecl(pkg, d)
} else if b, ok := builtinTypes[expr.Name]; ok {
if d, ok := pkgNames.Decls[expr.Name]; ok && d.Type == token.TYPE {
return p.parseDecl(pkg, d, typeParameters)
}

// Finally check if it's a built-in type
if b, ok := builtinTypes[expr.Name]; ok {
return &schema.Type{
Typ: &schema.Type_Builtin{Builtin: b},
}
}

p.errf(expr.Pos(), "undefined type: %s", expr.Name)

case *ast.SelectorExpr:
// pkg.T
if pkgName, ok := expr.X.(*ast.Ident); ok {
pkgPath := names.Files[file].NameToPath[pkgName.Name]
pkgPath := pkgNames.Files[file].NameToPath[pkgName.Name]
if otherPkg, ok := p.pkgMap[pkgPath]; ok {
if d, ok := p.names[otherPkg].Decls[expr.Sel.Name]; ok && d.Type == token.TYPE {
return p.parseDecl(otherPkg, d)
return p.parseDecl(otherPkg, d, typeParameters)
}
} else {
return p.parseEncoreBuiltin(expr.Pos(), pkgPath, expr.Sel.Name)
Expand All @@ -73,7 +83,7 @@ func (p *parser) resolveType(pkg *est.Package, file *est.File, expr ast.Expr) *s
}

for _, field := range expr.Fields.List {
typ := p.resolveType(pkg, file, field.Type)
typ := p.resolveType(pkg, file, field.Type, typeParameters)
if len(field.Names) == 0 {
p.err(field.Pos(), "cannot use anonymous fields in Encore struct types")
}
Expand Down Expand Up @@ -111,12 +121,12 @@ func (p *parser) resolveType(pkg *est.Package, file *est.File, expr ast.Expr) *s
return &schema.Type{Typ: &schema.Type_Struct{Struct: st}}

case *ast.MapType:
key := p.resolveType(pkg, file, expr.Key)
value := p.resolveType(pkg, file, expr.Value)
key := p.resolveType(pkg, file, expr.Key, typeParameters)
value := p.resolveType(pkg, file, expr.Value, typeParameters)
return &schema.Type{Typ: &schema.Type_Map{Map: &schema.Map{Key: key, Value: value}}}

case *ast.ArrayType:
elem := p.resolveType(pkg, file, expr.Elt)
elem := p.resolveType(pkg, file, expr.Elt, typeParameters)
// Translate []byte to BYTES
if b, ok := elem.Typ.(*schema.Type_Builtin); ok && b.Builtin == schema.Builtin_UINT8 {
return &schema.Type{Typ: &schema.Type_Builtin{Builtin: schema.Builtin_BYTES}}
Expand All @@ -133,39 +143,16 @@ func (p *parser) resolveType(pkg *est.Package, file *est.File, expr ast.Expr) *s
p.err(expr.Pos(), "cannot use function types in Encore schema definitions")

default:
if resolvedType := additionalTypeResolver(p, pkg, file, expr); resolvedType != nil {
return resolvedType
}

p.errf(expr.Pos(), "%s is not a supported type; got %+v", types.ExprString(expr), reflect.TypeOf(expr))
}

panic(bailout{})
}

// parseDecl parses the type from a package declaration.
func (p *parser) parseDecl(pkg *est.Package, d *names.PkgDecl) *schema.Type {
key := pkg.ImportPath + "." + d.Name
decl, ok := p.declMap[key]
if !ok {
// We haven't parsed this yet; do so now.
// Allocate a decl immediately so that we can properly handle
// recursive types by short-circuiting above the second time we get here.
id := uint32(len(p.decls))
typ := d.Spec.(*ast.TypeSpec).Type
decl = &schema.Decl{
Id: id,
Name: d.Name,
Doc: d.Doc,
Loc: parseLoc(d.File, typ),
// Type is set below
}
p.declMap[key] = decl
p.decls = append(p.decls, decl)
decl.Type = p.resolveType(pkg, d.File, d.Spec.(*ast.TypeSpec).Type)
}

return &schema.Type{Typ: &schema.Type_Named{
Named: &schema.Named{Id: decl.Id},
}}
}

func (p *parser) parseEncoreBuiltin(pos token.Pos, pkgPath, name string) *schema.Type {
switch {
case pkgPath == uuidImportPath && name == "UUID":
Expand Down
41 changes: 41 additions & 0 deletions parser/schema_go1.16.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
//go:build go1.16 && !go1.18
// +build go1.16,!go1.18

package parser

import (
"go/ast"

"encr.dev/parser/est"
"encr.dev/parser/internal/names"
schema "encr.dev/proto/encore/parser/schema/v1"
)

// parseDecl parses the type from a package declaration.
func (p *parser) parseDecl(pkg *est.Package, d *names.PkgDecl, _ typeParameterLookup) *schema.Type {
key := pkg.ImportPath + "." + d.Name
decl, ok := p.declMap[key]
if !ok {
// We haven't parsed this yet; do so now.
// Allocate a decl immediately so that we can properly handle
// recursive types by short-circuiting above the second time we get here.
id := uint32(len(p.decls))
typ := d.Spec.(*ast.TypeSpec).Type
decl = &schema.Decl{
Id: id,
Name: d.Name,
Doc: d.Doc,
TypeParams: nil,
Loc: parseLoc(d.File, typ),
// Type is set below
}
p.declMap[key] = decl
p.decls = append(p.decls, decl)

decl.Type = p.resolveType(pkg, d.File, d.Spec.(*ast.TypeSpec).Type, nil)
}

return &schema.Type{Typ: &schema.Type_Named{
Named: &schema.Named{Id: decl.Id},
}}
}
Loading

0 comments on commit b2a71f3

Please sign in to comment.