Skip to content

Commit

Permalink
syntax: handle UnaryExpr{Op:STAR, X:nil} (google#158)
Browse files Browse the repository at this point in the history
In "def f(*, x)", the unary expression has no operand.
Document this and be appropriately defensive.
Add another test (and example) of walk.

Also:
- visit Comprehension.Body before Clauses and fix existing TestWalk.
- move existing TestWalk

Fixes google#157
  • Loading branch information
adonovan authored Feb 22, 2019
1 parent 4eb7695 commit 34a3319
Show file tree
Hide file tree
Showing 4 changed files with 120 additions and 61 deletions.
56 changes: 0 additions & 56 deletions syntax/parse_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -434,62 +434,6 @@ func TestParseErrors(t *testing.T) {
}
}

func TestWalk(t *testing.T) {
const src = `
for x in y:
if x:
pass
else:
f([2*x for x in "abc"])
`
// TODO(adonovan): test that it finds all syntax.Nodes
// (compare against a reflect-based implementation).
// TODO(adonovan): test that the result of f is used to prune
// the descent.
f, err := syntax.Parse("hello.go", src, 0)
if err != nil {
t.Fatal(err)
}

var buf bytes.Buffer
var depth int
syntax.Walk(f, func(n syntax.Node) bool {
if n == nil {
depth--
return true
}
fmt.Fprintf(&buf, "%s%s\n",
strings.Repeat(" ", depth),
strings.TrimPrefix(reflect.TypeOf(n).String(), "*syntax."))
depth++
return true
})
got := buf.String()
want := `
File
ForStmt
Ident
Ident
IfStmt
Ident
BranchStmt
ExprStmt
CallExpr
Ident
Comprehension
ForClause
Ident
Literal
BinaryExpr
Literal
Ident`
got = strings.TrimSpace(got)
want = strings.TrimSpace(want)
if got != want {
t.Errorf("got %s, want %s", got, want)
}
}

// dataFile is the same as starlarktest.DataFile.
// We make a copy to avoid a dependency cycle.
var dataFile = func(pkgdir, filename string) string {
Expand Down
11 changes: 9 additions & 2 deletions syntax/syntax.go
Original file line number Diff line number Diff line change
Expand Up @@ -480,15 +480,22 @@ func (x *TupleExpr) Span() (start, end Position) {
}

// A UnaryExpr represents a unary expression: Op X.
//
// As a special case it may also represent the star
// parameter in def f(*args) or def f(*, x).
type UnaryExpr struct {
commentsRef
OpPos Position
Op Token
X Expr
X Expr // may be nil if Op==STAR
}

func (x *UnaryExpr) Span() (start, end Position) {
_, end = x.X.Span()
if x.X != nil {
_, end = x.X.Span()
} else {
end = x.OpPos.add("*")
}
return x.OpPos, end
}

Expand Down
11 changes: 8 additions & 3 deletions syntax/walk.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ package syntax
// recursively for each non-nil child of n.
// Walk then calls f(nil).
func Walk(n Node, f func(Node) bool) {
if n == nil {
panic("nil")
}
if !f(n) {
return
}
Expand All @@ -31,8 +34,8 @@ func Walk(n Node, f func(Node) bool) {
walkStmts(n.False, f)

case *AssignStmt:
Walk(n.RHS, f)
Walk(n.LHS, f)
Walk(n.RHS, f)

case *DefStmt:
Walk(n.Name, f)
Expand Down Expand Up @@ -97,10 +100,10 @@ func Walk(n Node, f func(Node) bool) {
}

case *Comprehension:
Walk(n.Body, f)
for _, clause := range n.Clauses {
Walk(clause, f)
}
Walk(n.Body, f)

case *IfClause:
Walk(n.Cond, f)
Expand All @@ -122,7 +125,9 @@ func Walk(n Node, f func(Node) bool) {
}

case *UnaryExpr:
Walk(n.X, f)
if n.X != nil {
Walk(n.X, f)
}

case *BinaryExpr:
Walk(n.X, f)
Expand Down
103 changes: 103 additions & 0 deletions syntax/walk_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package syntax_test

import (
"bytes"
"fmt"
"log"
"reflect"
"strings"
"testing"

"go.starlark.net/syntax"
)

func TestWalk(t *testing.T) {
const src = `
for x in y:
if x:
pass
else:
f([2*x for x in "abc"])
`
// TODO(adonovan): test that it finds all syntax.Nodes
// (compare against a reflect-based implementation).
// TODO(adonovan): test that the result of f is used to prune
// the descent.
f, err := syntax.Parse("hello.go", src, 0)
if err != nil {
t.Fatal(err)
}

var buf bytes.Buffer
var depth int
syntax.Walk(f, func(n syntax.Node) bool {
if n == nil {
depth--
return true
}
fmt.Fprintf(&buf, "%s%s\n",
strings.Repeat(" ", depth),
strings.TrimPrefix(reflect.TypeOf(n).String(), "*syntax."))
depth++
return true
})
got := buf.String()
want := `
File
ForStmt
Ident
Ident
IfStmt
Ident
BranchStmt
ExprStmt
CallExpr
Ident
Comprehension
BinaryExpr
Literal
Ident
ForClause
Ident
Literal`
got = strings.TrimSpace(got)
want = strings.TrimSpace(want)
if got != want {
t.Errorf("got %s, want %s", got, want)
}
}

// ExampleWalk demonstrates the use of Walk to
// enumerate the identifiers in a Starlark source file
// containing a nonsense program with varied grammar.
func ExampleWalk() {
const src = `
load("library", "a")
def b(c, *, d=e):
f += {g: h}
i = -(j)
return k.l[m + n]
for o in [p for q, r in s if t]:
u(lambda: v, w[x:y:z])
`
f, err := syntax.Parse("hello.star", src, 0)
if err != nil {
log.Fatal(err)
}

var idents []string
syntax.Walk(f, func(n syntax.Node) bool {
if id, ok := n.(*syntax.Ident); ok {
idents = append(idents, id.Name)
}
return true
})
fmt.Println(strings.Join(idents, " "))

// The identifer 'a' appears in both LoadStmt.From[0] and LoadStmt.To[0].

// Output:
// a a b c d e f g h i j k l m n o p q r s t u v w x y z
}

0 comments on commit 34a3319

Please sign in to comment.