starlark: API additions for improved debugging (google#76)

This change adds a number of small features to improve debugging.
The actual debugger API will come in a later change.

- Thread.Name : an optional string field that describes the purpose of
  the thread, for use in debugging.
- (*Program).Filename: a method that reports the file of the program.
  Also, a String method that returns the same thing.
  Also, a test that it reports the correct location even for
  an empty file (a special case of the previous implementation).
- (*Frame).Local(i int): a method to return the value of a local
  variable of an active frame, such as one might use in debugger's
  stack trace.
- ExprFunc: creates a starlark.Function from a given expression,
  such as one might use in a debugger REPL.

cmd/starlark/starlark.go

@@ -71,13 +71,15 @@ func main() {
 			// Execute specified file.
 			filename = flag.Arg(0)
 		}
+		thread.Name = "exec " + filename
 		globals, err = starlark.ExecFile(thread, filename, src, nil)
 		if err != nil {
 			repl.PrintError(err)
 			os.Exit(1)
 		}
 	case flag.NArg() == 0:
 		fmt.Println("Welcome to Starlark (go.starlark.net)")
+		thread.Name = "REPL"
 		repl.REPL(thread, globals)
 	default:
 		log.Fatal("want at most one Starlark file name")

internal/compile/codegen_test.go

@@ -64,7 +64,7 @@ func TestPlusFolding(t *testing.T) {
 			t.Errorf("#%d: %v", i, err)
 			continue
 		}
-		got := disassemble(Expr(expr, locals))
+		got := disassemble(Expr(expr, "<expr>", locals))
 		if test.want != got {
 			t.Errorf("expression <<%s>> generated <<%s>>, want <<%s>>",
 				test.src, got, test.want)

internal/compile/compile.go

@@ -406,13 +406,14 @@ func idents(ids []*syntax.Ident) []Ident {
 }
 
 // Expr compiles an expression to a program consisting of a single toplevel function.
-func Expr(expr syntax.Expr, locals []*syntax.Ident) *Funcode {
+func Expr(expr syntax.Expr, name string, locals []*syntax.Ident) *Funcode {
+	pos := syntax.Start(expr)
 	stmts := []syntax.Stmt{&syntax.ReturnStmt{Result: expr}}
-	return File(stmts, locals, nil).Toplevel
+	return File(stmts, pos, name, locals, nil).Toplevel
 }
 
 // File compiles the statements of a file into a program.
-func File(stmts []syntax.Stmt, locals, globals []*syntax.Ident) *Program {
+func File(stmts []syntax.Stmt, pos syntax.Position, name string, locals, globals []*syntax.Ident) *Program {
 	pcomp := &pcomp{
 		prog: &Program{
 			Globals: idents(globals),
@@ -421,13 +422,7 @@ func File(stmts []syntax.Stmt, locals, globals []*syntax.Ident) *Program {
 		constants: make(map[interface{}]uint32),
 		functions: make(map[*Funcode]uint32),
 	}
-
-	var pos syntax.Position
-	if len(stmts) > 0 {
-		pos = syntax.Start(stmts[0])
-	}
-
-	pcomp.prog.Toplevel = pcomp.function("<toplevel>", pos, stmts, locals, nil)
+	pcomp.prog.Toplevel = pcomp.function(name, pos, stmts, locals, nil)
 
 	return pcomp.prog
 }

repl/repl.go

@@ -215,7 +215,7 @@ func MakeLoad() func(thread *starlark.Thread, module string) (starlark.StringDic
 			cache[module] = nil
 
 			// Load it.
-			thread := &starlark.Thread{Load: thread.Load}
+			thread := &starlark.Thread{Name: "exec " + module, Load: thread.Load}
 			globals, err := starlark.ExecFile(thread, module, nil, nil)
 			e = &entry{globals, err}
 

starlark/debug.go

@@ -0,0 +1,18 @@
+package starlark
+
+// This file defines an experimental API for the debugging tools.
+// Some of these declarations expose details of internal packages.
+// (The debugger makes liberal use of exported fields of unexported types.)
+// Breaking changes may occur without notice.
+
+// Local returns the value of the i'th local variable.
+// It may be nil if not yet assigned.
+//
+// Local may be called only for frames whose Callable is a *Function (a
+// function defined by Starlark source code), and only while the frame
+// is active; it will panic otherwise.
+//
+// This function is provided only for debugging tools.
+//
+// THIS API IS EXPERIMENTAL AND MAY CHANGE WITHOUT NOTICE.
+func (fr *Frame) Local(i int) Value { return fr.locals[i] }

starlark/eval.go

@@ -28,6 +28,9 @@ const debug = false
 // such as its call stack and thread-local storage.
 // The Thread is threaded throughout the evaluator.
 type Thread struct {
+	// Name is an optional name that describes the thread, for debugging.
+	Name string
+
 	// frame is the current Starlark execution frame.
 	frame *Frame
 
@@ -113,6 +116,7 @@ type Frame struct {
 	callable Callable        // current function (or toplevel) or built-in
 	posn     syntax.Position // source position of PC, set during error
 	callpc   uint32          // PC of position of active call, set during call
+	locals   []Value         // local variables, for debugger
 }
 
 // The Frames of a thread are structured as a spaghetti stack, not a
@@ -204,6 +208,11 @@ type Program struct {
 // the compiler version into the cache key when reusing compiled code.
 const CompilerVersion = compile.Version
 
+// Filename returns the name of the file from which this program was loaded.
+func (prog *Program) Filename() string { return prog.compiled.Toplevel.Pos.Filename() }
+
+func (prog *Program) String() string { return prog.Filename() }
+
 // NumLoads returns the number of load statements in the compiled program.
 func (prog *Program) NumLoads() int { return len(prog.compiled.Loads) }
 
@@ -270,7 +279,14 @@ func SourceProgram(filename string, src interface{}, isPredeclared func(string)
 		return f, nil, err
 	}
 
-	compiled := compile.File(f.Stmts, f.Locals, f.Globals)
+	var pos syntax.Position
+	if len(f.Stmts) > 0 {
+		pos = syntax.Start(f.Stmts[0])
+	} else {
+		pos = syntax.MakePosition(&filename, 1, 1)
+	}
+
+	compiled := compile.File(f.Stmts, pos, "<toplevel>", f.Locals, f.Globals)
 
 	return f, &Program{compiled}, nil
 }
@@ -282,11 +298,11 @@ func CompiledProgram(in io.Reader) (*Program, error) {
 	if err != nil {
 		return nil, err
 	}
-	prog, err := compile.DecodeProgram(data)
+	compiled, err := compile.DecodeProgram(data)
 	if err != nil {
 		return nil, err
 	}
-	return &Program{prog}, nil
+	return &Program{compiled}, nil
 }
 
 // Init creates a set of global variables for the program,
@@ -341,6 +357,16 @@ func makeToplevelFunction(funcode *compile.Funcode, predeclared StringDict) *Fun
 // If Eval fails during evaluation, it returns an *EvalError
 // containing a backtrace.
 func Eval(thread *Thread, filename string, src interface{}, env StringDict) (Value, error) {
+	f, err := ExprFunc(filename, src, env)
+	if err != nil {
+		return nil, err
+	}
+	return Call(thread, f, nil, nil)
+}
+
+// ExprFunc returns a no-argument function
+// that evaluates the expression whose source is src.
+func ExprFunc(filename string, src interface{}, env StringDict) (*Function, error) {
 	expr, err := syntax.ParseExpr(filename, src, 0)
 	if err != nil {
 		return nil, err
@@ -351,9 +377,7 @@ func Eval(thread *Thread, filename string, src interface{}, env StringDict) (Val
 		return nil, err
 	}
 
-	fn := makeToplevelFunction(compile.Expr(expr, locals), env)
-
-	return Call(thread, fn, nil, nil)
+	return makeToplevelFunction(compile.Expr(expr, "<expr>", locals), env), nil
 }
 
 // The following functions are primitive operations of the byte code interpreter.

starlark/eval_test.go

@@ -337,11 +337,11 @@ f()
 			caller.Position(), caller.Callable().Name(), msg)
 	}
 	thread := &starlark.Thread{Print: print}
-	if _, err := starlark.ExecFile(thread, "foo.go", src, nil); err != nil {
+	if _, err := starlark.ExecFile(thread, "foo.star", src, nil); err != nil {
 		t.Fatal(err)
 	}
-	want := "foo.go:2: <toplevel>: hello\n" +
-		"foo.go:3: f: hello, world\n"
+	want := "foo.star:2: <toplevel>: hello\n" +
+		"foo.star:3: f: hello, world\n"
 	if got := buf.String(); got != want {
 		t.Errorf("output was %s, want %s", got, want)
 	}
@@ -452,6 +452,21 @@ func TestRepeatedExec(t *testing.T) {
 	}
 }
 
+// TestEmptyFilePosition ensures that even Programs
+// from empty files have a valid position.
+func TestEmptyPosition(t *testing.T) {
+	var predeclared starlark.StringDict
+	for _, content := range []string{"", "empty = False"} {
+		_, prog, err := starlark.SourceProgram("hello.star", content, predeclared.Has)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if got, want := prog.Filename(), "hello.star"; got != want {
+			t.Errorf("Program.Filename() = %q, want %q", got, want)
+		}
+	}
+}
+
 // TestUnpackUserDefined tests that user-defined
 // implementations of starlark.Value may be unpacked.
 func TestUnpackUserDefined(t *testing.T) {
@@ -483,3 +498,54 @@ def somefunc():
 		t.Fatal("docstring not found")
 	}
 }
+
+func TestFrameLocals(t *testing.T) {
+	// trace prints a nice stack trace including argument
+	// values of calls to Starlark functions.
+	trace := func(thread *starlark.Thread) string {
+		buf := new(bytes.Buffer)
+		for fr := thread.TopFrame(); fr != nil; fr = fr.Parent() {
+			fmt.Fprintf(buf, "%s(", fr.Callable().Name())
+			if fn, ok := fr.Callable().(*starlark.Function); ok {
+				for i := 0; i < fn.NumParams(); i++ {
+					if i > 0 {
+						buf.WriteString(", ")
+					}
+					name, _ := fn.Param(i)
+					fmt.Fprintf(buf, "%s=%s", name, fr.Local(i))
+				}
+			} else {
+				buf.WriteString("...") // a built-in function
+			}
+			buf.WriteString(")\n")
+		}
+		return buf.String()
+	}
+
+	var got string
+	builtin := func(thread *starlark.Thread, _ *starlark.Builtin, _ starlark.Tuple, _ []starlark.Tuple) (starlark.Value, error) {
+		got = trace(thread)
+		return starlark.None, nil
+	}
+	predeclared := starlark.StringDict{
+		"builtin": starlark.NewBuiltin("builtin", builtin),
+	}
+	_, err := starlark.ExecFile(&starlark.Thread{}, "foo.star", `
+def f(x, y): builtin()
+def g(z): f(z, z*z)
+g(7)
+`, predeclared)
+	if err != nil {
+		t.Errorf("ExecFile failed: %v", err)
+	}
+
+	var want = `
+builtin(...)
+f(x=7, y=49)
+g(z=7)
+<toplevel>()
+`[1:]
+	if got != want {
+		t.Errorf("got <<%s>>, want <<%s>>", got, want)
+	}
+}

starlark/example_test.go

@@ -28,6 +28,7 @@ squares = [x*x for x in range(10)]
 `
 
 	thread := &starlark.Thread{
+		Name:  "example",
 		Print: func(_ *starlark.Thread, msg string) { fmt.Println(msg) },
 	}
 	predeclared := starlark.StringDict{
@@ -90,7 +91,7 @@ func ExampleThread_Load_sequential() {
 
 			// Load and initialize the module in a new thread.
 			data := fakeFilesystem[module]
-			thread := &starlark.Thread{Load: load}
+			thread := &starlark.Thread{Name: "exec " + module, Load: load}
 			globals, err := starlark.ExecFile(thread, module, data, nil)
 			e = &entry{globals, err}
 
@@ -100,7 +101,7 @@ func ExampleThread_Load_sequential() {
 		return e.globals, e.err
 	}
 
-	thread := &starlark.Thread{Load: load}
+	thread := &starlark.Thread{Name: "exec c.star", Load: load}
 	globals, err := load(thread, "c.star")
 	if err != nil {
 		log.Fatal(err)
@@ -250,6 +251,7 @@ func (c *cache) get(cc *cycleChecker, module string) (starlark.StringDict, error
 
 func (c *cache) doLoad(cc *cycleChecker, module string) (starlark.StringDict, error) {
 	thread := &starlark.Thread{
+		Name:  "exec " + module,
 		Print: func(_ *starlark.Thread, msg string) { fmt.Println(msg) },
 		Load: func(_ *starlark.Thread, module string) (starlark.StringDict, error) {
 			// Tunnel the cycle-checker state for this "thread of loading".

starlark/interp.go

@@ -53,6 +53,8 @@ func call(thread *Thread, args Tuple, kwargs []Tuple) (Value, error) {
 		return nil, fr.errorf(fr.Position(), "%v", err)
 	}
 
+	fr.locals = locals // for debugger
+
 	if vmdebug {
 		fmt.Printf("Entering %s @ %s\n", f.Name, f.Position(0))
 		fmt.Printf("%d stack, %d locals\n", len(stack), len(locals))
@@ -561,5 +563,8 @@ loop:
 			err = fr.errorf(f.Position(savedpc), "%s", err.Error())
 		}
 	}
+
+	fr.locals = nil
+
 	return result, err
 }