Skip to content

Commit

Permalink
syntax: improve REPL parsing (google#98)
Browse files Browse the repository at this point in the history 
Previously, the REPL used a heuristic: it would consume a single line
and attempt to parse it; if that failed, it would consume lines up to
a blank line then parse the whole as a file. This was suboptimal for
various reasons: it failed to parse lines ending with an unfinished
multi-line string literal, for example, and it would prematurely
stop reading even while parentheses were open.

This change integrates the REPL with the scanner and parser (as Python
does). The REPL invokes a new parser entry point, ParseCompoundStmt,
that consumes only enough input to parse a compound statement, defined
as (a) blank line, (b) a semicolon-separated list of simple statements
all on one line, or (c) a complex statement such as def, if or for.

If the 'src' value provided to the scanner is a function of type
func() ([]byte, error), then the scanner will call it each time
it runs out of input.

Fixes google#81
  • Loading branch information
@adonovan
adonovan authored Jan 4, 2019
1 parent 9d97771 commit 30e71c6
Show file tree
Hide file tree
Showing 7 changed files with 347 additions and 121 deletions.
123 changes: 50 additions & 73 deletions repl/repl.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,11 @@ package repl // import "go.starlark.net/repl"
// This is not necessarily a bug.

import (
"bytes"
"context"
"fmt"
"io"
"os"
"os/signal"
"strings"

"github.com/chzyer/readline"
"go.starlark.net/starlark"
Expand Down Expand Up @@ -88,98 +87,76 @@ func rep(rl *readline.Instance, thread *starlark.Thread, globals starlark.String

thread.SetLocal("context", ctx)

rl.SetPrompt(">>> ")
line, err := rl.Readline()
if err != nil {
return err // may be ErrInterrupt
}
eof := false

if l := strings.TrimSpace(line); l == "" || l[0] == '#' {
return nil // blank or comment
// readline returns EOF, ErrInterrupted, or a line including "\n".
rl.SetPrompt(">>> ")
readline := func() ([]byte, error) {
line, err := rl.Readline()
rl.SetPrompt("... ")
if err != nil {
if err == io.EOF {
eof = true
}
return nil, err
}
return []byte(line + "\n"), nil
}

// If the line contains a well-formed expression, evaluate it.
if _, err := syntax.ParseExpr("<stdin>", line, 0); err == nil {
if v, err := starlark.Eval(thread, "<stdin>", line, globals); err != nil {
PrintError(err)
} else if v != starlark.None {
fmt.Println(v)
// parse
f, err := syntax.ParseCompoundStmt("<stdin>", readline)
if err != nil {
if eof {
return io.EOF
}
PrintError(err)
return nil
}

// If the input so far is a single load or assignment statement,
// execute it without waiting for a blank line.
if f, err := syntax.Parse("<stdin>", line, 0); err == nil && len(f.Stmts) == 1 {
switch f.Stmts[0].(type) {
case *syntax.AssignStmt, *syntax.LoadStmt:
// Execute it as a file.
if err := execFileNoFreeze(thread, line, globals); err != nil {
PrintError(err)
}
if expr := soleExpr(f); expr != nil {
// eval
v, err := starlark.EvalExpr(thread, expr, globals)
if err != nil {
PrintError(err)
return nil
}
}

// Otherwise assume it is the first of several
// comprising a file, followed by a blank line.
var buf bytes.Buffer
fmt.Fprintln(&buf, line)
for {
rl.SetPrompt("... ")
line, err := rl.Readline()
if err != nil {
return err // may be ErrInterrupt
// print
if v != starlark.None {
fmt.Println(v)
}
if l := strings.TrimSpace(line); l == "" {
break // blank
} else {
// compile
prog, err := starlark.FileProgram(f, globals.Has)
if err != nil {
PrintError(err)
return nil
}
fmt.Fprintln(&buf, line)
}
text := buf.Bytes()

// Try parsing it once more as an expression,
// such as a call spread over several lines:
// f(
// 1,
// 2
// )
if _, err := syntax.ParseExpr("<stdin>", text, 0); err == nil {
if v, err := starlark.Eval(thread, "<stdin>", text, globals); err != nil {

// execute (but do not freeze)
res, err := prog.Init(thread, globals)
if err != nil {
PrintError(err)
} else if v != starlark.None {
fmt.Println(v)
}
return nil
}

// Execute it as a file.
if err := execFileNoFreeze(thread, text, globals); err != nil {
PrintError(err)
// The global names from the previous call become
// the predeclared names of this call.
// If execution failed, some globals may be undefined.
for k, v := range res {
globals[k] = v
}
}

return nil
}

// execFileNoFreeze is starlark.ExecFile without globals.Freeze().
func execFileNoFreeze(thread *starlark.Thread, src interface{}, globals starlark.StringDict) error {
_, prog, err := starlark.SourceProgram("<stdin>", src, globals.Has)
if err != nil {
return err
}

res, err := prog.Init(thread, globals)

// The global names from the previous call become
// the predeclared names of this call.

// Copy globals back to the caller's map.
// If execution failed, some globals may be undefined.
for k, v := range res {
globals[k] = v
func soleExpr(f *syntax.File) syntax.Expr {
if len(f.Stmts) == 1 {
if stmt, ok := f.Stmts[0].(*syntax.ExprStmt); ok {
return stmt.X
}
}

return err
return nil
}

// PrintError prints the error to stderr,
Expand Down
52 changes: 47 additions & 5 deletions starlark/eval.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,21 +285,36 @@ func SourceProgram(filename string, src interface{}, isPredeclared func(string)
if err != nil {
return nil, nil, err
}
prog, err := FileProgram(f, isPredeclared)
return f, prog, err
}

// FileProgram produces a new program by resolving,
// and compiling the Starlark source file syntax tree.
// On success, it returns the compiled program.
//
// Resolving a syntax tree mutates it.
// Do not call FileProgram more than once on the same file.
//
// The isPredeclared predicate reports whether a name is
// a pre-declared identifier of the current module.
// Its typical value is predeclared.Has,
// where predeclared is a StringDict of pre-declared values.
func FileProgram(f *syntax.File, isPredeclared func(string) bool) (*Program, error) {
if err := resolve.File(f, isPredeclared, Universe.Has); err != nil {
return f, nil, err
return nil, err
}

var pos syntax.Position
if len(f.Stmts) > 0 {
pos = syntax.Start(f.Stmts[0])
} else {
pos = syntax.MakePosition(&filename, 1, 1)
pos = syntax.MakePosition(&f.Path, 1, 1)
}

compiled := compile.File(f.Stmts, pos, "<toplevel>", f.Locals, f.Globals)

return f, &Program{compiled}, nil
return &Program{compiled}, nil
}

// CompiledProgram produces a new program from the representation
Expand All @@ -324,7 +339,7 @@ func (prog *Program) Init(thread *Thread, predeclared StringDict) (StringDict, e

_, err := Call(thread, toplevel, nil, nil)

// Convert the global environment to a map and freeze it.
// Convert the global environment to a map.
// We return a (partial) map even in case of error.
return toplevel.Globals(), err
}
Expand Down Expand Up @@ -368,21 +383,48 @@ func makeToplevelFunction(funcode *compile.Funcode, predeclared StringDict) *Fun
// If Eval fails during evaluation, it returns an *EvalError
// containing a backtrace.
func Eval(thread *Thread, filename string, src interface{}, env StringDict) (Value, error) {
f, err := ExprFunc(filename, src, env)
expr, err := syntax.ParseExpr(filename, src, 0)
if err != nil {
return nil, err
}
f, err := makeExprFunc(expr, env)
if err != nil {
return nil, err
}
return Call(thread, f, nil, nil)
}

// EvalExpr resolves and evaluates an expression within the
// specified (predeclared) environment.
//
// Resolving an expression mutates it.
// Do not call EvalExpr more than once for the same expression.
//
// Evaluation cannot mutate the environment dictionary itself,
// though it may modify variables reachable from the dictionary.
//
// If Eval fails during evaluation, it returns an *EvalError
// containing a backtrace.
func EvalExpr(thread *Thread, expr syntax.Expr, env StringDict) (Value, error) {
fn, err := makeExprFunc(expr, env)
if err != nil {
return nil, err
}
return Call(thread, fn, nil, nil)
}

// ExprFunc returns a no-argument function
// that evaluates the expression whose source is src.
func ExprFunc(filename string, src interface{}, env StringDict) (*Function, error) {
expr, err := syntax.ParseExpr(filename, src, 0)
if err != nil {
return nil, err
}
return makeExprFunc(expr, env)
}

// makeExprFunc returns a no-argument function whose body is expr.
func makeExprFunc(expr syntax.Expr, env StringDict) (*Function, error) {
locals, err := resolve.Expr(expr, env.Has, Universe.Has)
if err != nil {
return nil, err
Expand Down
46 changes: 42 additions & 4 deletions syntax/parse.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,38 @@ func Parse(filename string, src interface{}, mode Mode) (f *File, err error) {
return f, nil
}

// ParseCompoundStmt parses a single compound statement:
// a blank line, a def, for, while, or if statement, or a
// semicolon-separated list of simple statements followed
// by a newline. These are the units on which the REPL operates.
// ParseCompoundStmt does not consume any following input.
// The parser calls the readline function each
// time it needs a new line of input.
func ParseCompoundStmt(filename string, readline func() ([]byte, error)) (f *File, err error) {
in, err := newScanner(filename, readline, false)
if err != nil {
return nil, err
}

p := parser{in: in}
defer p.in.recover(&err)

p.nextToken() // read first lookahead token

var stmts []Stmt
switch p.tok {
case DEF, IF, FOR, WHILE:
stmts = p.parseStmt(stmts)
case NEWLINE:
// blank line
default:
// Don't consume newline, to avoid blocking again.
stmts = p.parseSimpleStmt(stmts, false)
}

return &File{Path: filename, Stmts: stmts}, nil
}

// ParseExpr parses a Starlark expression.
// See Parse for explanation of parameters.
func ParseExpr(filename string, src interface{}, mode Mode) (expr Expr, err error) {
Expand All @@ -58,6 +90,10 @@ func ParseExpr(filename string, src interface{}, mode Mode) (expr Expr, err erro
defer p.in.recover(&err)

p.nextToken() // read first lookahead token

// TODO(adonovan): Python's eval would use the equivalent of
// parseExpr here, which permits an unparenthesized tuple.
// We should too.
expr = p.parseTest()

// A following newline (e.g. "f()\n") appears outside any brackets,
Expand Down Expand Up @@ -114,7 +150,7 @@ func (p *parser) parseStmt(stmts []Stmt) []Stmt {
} else if p.tok == WHILE {
return append(stmts, p.parseWhileStmt())
}
return p.parseSimpleStmt(stmts)
return p.parseSimpleStmt(stmts, true)
}

func (p *parser) parseDefStmt() Stmt {
Expand Down Expand Up @@ -219,7 +255,8 @@ func (p *parser) parseForLoopVariables() Expr {
}

// simple_stmt = small_stmt (SEMI small_stmt)* SEMI? NEWLINE
func (p *parser) parseSimpleStmt(stmts []Stmt) []Stmt {
// In REPL mode, it does not consume the NEWLINE.
func (p *parser) parseSimpleStmt(stmts []Stmt, consumeNL bool) []Stmt {
for {
stmts = append(stmts, p.parseSmallStmt())
if p.tok != SEMI {
Expand All @@ -231,9 +268,10 @@ func (p *parser) parseSimpleStmt(stmts []Stmt) []Stmt {
}
}
// EOF without NEWLINE occurs in `if x: pass`, for example.
if p.tok != EOF {
if p.tok != EOF && consumeNL {
p.consume(NEWLINE)
}

return stmts
}

Expand Down Expand Up @@ -355,7 +393,7 @@ func (p *parser) parseSuite() []Stmt {
return stmts
}

return p.parseSimpleStmt(nil)
return p.parseSimpleStmt(nil, true)
}

func (p *parser) parseIdent() *Ident {
Expand Down
Loading

0 comments on commit 30e71c6

Please sign in to comment.