diff options
-rw-r--r-- | internal/compile/compile.go | 14 | ||||
-rw-r--r-- | resolve/binding.go | 74 | ||||
-rw-r--r-- | resolve/resolve.go | 49 | ||||
-rw-r--r-- | resolve/resolve_test.go | 4 | ||||
-rw-r--r-- | starlark/eval.go | 3 | ||||
-rw-r--r-- | starlarkstruct/struct_test.go | 4 | ||||
-rw-r--r-- | syntax/binding.go | 48 | ||||
-rw-r--r-- | syntax/parse.go | 18 | ||||
-rw-r--r-- | syntax/parse_test.go | 20 | ||||
-rw-r--r-- | syntax/syntax.go | 45 | ||||
-rw-r--r-- | syntax/walk.go | 8 |
11 files changed, 147 insertions, 140 deletions
diff --git a/internal/compile/compile.go b/internal/compile/compile.go index 423ae0b..3a8d385 100644 --- a/internal/compile/compile.go +++ b/internal/compile/compile.go @@ -985,7 +985,7 @@ func (fcomp *fcomp) setPos(pos syntax.Position) { // set emits code to store the top-of-stack value // to the specified local, cell, or global variable. func (fcomp *fcomp) set(id *syntax.Ident) { - bind := id.Binding + bind := id.Binding.(*resolve.Binding) switch bind.Scope { case resolve.Local: fcomp.emit1(SETLOCAL, uint32(bind.Index)) @@ -1002,7 +1002,7 @@ func (fcomp *fcomp) set(id *syntax.Ident) { // lookup emits code to push the value of the specified variable. func (fcomp *fcomp) lookup(id *syntax.Ident) { - bind := id.Binding + bind := id.Binding.(*resolve.Binding) if bind.Scope != resolve.Universal { // (universal lookup can't fail) fcomp.setPos(id.NamePos) } @@ -1149,7 +1149,7 @@ func (fcomp *fcomp) stmt(stmt syntax.Stmt) { } case *syntax.DefStmt: - fcomp.function(stmt.Def, stmt.Name.Name, &stmt.Function) + fcomp.function(stmt.Function.(*resolve.Function)) fcomp.set(stmt.Name) case *syntax.ForStmt: @@ -1428,7 +1428,7 @@ func (fcomp *fcomp) expr(e syntax.Expr) { fcomp.call(e) case *syntax.LambdaExpr: - fcomp.function(e.Lambda, "lambda", &e.Function) + fcomp.function(e.Function.(*resolve.Function)) default: start, _ := e.Span() @@ -1777,9 +1777,9 @@ func (fcomp *fcomp) comprehension(comp *syntax.Comprehension, clauseIndex int) { log.Fatalf("%s: unexpected comprehension clause %T", start, clause) } -func (fcomp *fcomp) function(pos syntax.Position, name string, f *syntax.Function) { +func (fcomp *fcomp) function(f *resolve.Function) { // Evaluation of the defaults may fail, so record the position. - fcomp.setPos(pos) + fcomp.setPos(f.Pos) // To reduce allocation, we emit a combined tuple // for the defaults and the freevars. @@ -1821,7 +1821,7 @@ func (fcomp *fcomp) function(pos syntax.Position, name string, f *syntax.Functio fcomp.emit1(MAKETUPLE, uint32(ndefaults+len(f.FreeVars))) - funcode := fcomp.pcomp.function(name, pos, f.Body, f.Locals, f.FreeVars) + funcode := fcomp.pcomp.function(f.Name, f.Pos, f.Body, f.Locals, f.FreeVars) if debug { // TODO(adonovan): do compilations sequentially not as a tree, diff --git a/resolve/binding.go b/resolve/binding.go new file mode 100644 index 0000000..6b99f4b --- /dev/null +++ b/resolve/binding.go @@ -0,0 +1,74 @@ +// Copyright 2019 The Bazel Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package resolve + +import "go.starlark.net/syntax" + +// This file defines resolver data types saved in the syntax tree. +// We cannot guarantee API stability for these types +// as they are closely tied to the implementation. + +// A Binding contains resolver information about an identifer. +// The resolver populates the Binding field of each syntax.Identifier. +// The Binding ties together all identifiers that denote the same variable. +type Binding struct { + Scope Scope + + // Index records the index into the enclosing + // - {DefStmt,File}.Locals, if Scope==Local + // - DefStmt.FreeVars, if Scope==Free + // - File.Globals, if Scope==Global. + // It is zero if Scope is Predeclared, Universal, or Undefined. + Index int + + First *syntax.Ident // first binding use (iff Scope==Local/Free/Global) +} + +// The Scope of Binding indicates what kind of scope it has. +type Scope uint8 + +const ( + Undefined Scope = iota // name is not defined + Local // name is local to its function or file + Cell // name is function-local but shared with a nested function + Free // name is cell of some enclosing function + Global // name is global to module + Predeclared // name is predeclared for this module (e.g. glob) + Universal // name is universal (e.g. len) +) + +var scopeNames = [...]string{ + Undefined: "undefined", + Local: "local", + Cell: "cell", + Free: "free", + Global: "global", + Predeclared: "predeclared", + Universal: "universal", +} + +func (scope Scope) String() string { return scopeNames[scope] } + +// A Module contains resolver information about a file. +// The resolver populates the Module field of each syntax.File. +type Module struct { + Locals []*Binding // the file's (comprehension-)local variables + Globals []*Binding // the file's global variables +} + +// A Function contains resolver information about a named or anonymous function. +// The resolver populates the Function field of each syntax.DefStmt and syntax.LambdaExpr. +type Function struct { + Pos syntax.Position // of DEF or LAMBDA + Name string // name of def, or "lambda" + Params []syntax.Expr // param = ident | ident=expr | * | *ident | **ident + Body []syntax.Stmt // contains synthetic 'return expr' for lambda + + HasVarargs bool // whether params includes *args (convenience) + HasKwargs bool // whether params includes **kwargs (convenience) + NumKwonlyParams int // number of keyword-only optional parameters + Locals []*Binding // this function's local/cell variables, parameters first + FreeVars []*Binding // enclosing cells to capture in closure +} diff --git a/resolve/resolve.go b/resolve/resolve.go index d4945ce..2811b65 100644 --- a/resolve/resolve.go +++ b/resolve/resolve.go @@ -108,7 +108,8 @@ var ( LoadBindsGlobally = false // load creates global not file-local bindings (deprecated) ) -// File resolves the specified file. +// File resolves the specified file and records information about the +// module in file.Module. // // The isPredeclared and isUniversal predicates report whether a name is // a pre-declared identifier (visible in the current module) or a @@ -131,8 +132,10 @@ func File(file *syntax.File, isPredeclared, isUniversal func(name string) bool) // Function bodies may contain forward references to later global declarations. r.resolveNonLocalUses(r.env) - file.Locals = r.moduleLocals - file.Globals = r.moduleGlobals + file.Module = &Module{ + Locals: r.moduleLocals, + Globals: r.moduleGlobals, + } if len(r.errors) > 0 { return r.errors @@ -180,20 +183,6 @@ func newResolver(isPredeclared, isUniversal func(name string) bool) *resolver { } } -// Declare aliases for types and constants that logically belong here. - -type Binding = syntax.Binding - -const ( - Undefined = syntax.UndefinedScope - Local = syntax.LocalScope - Cell = syntax.CellScope - Free = syntax.FreeScope - Global = syntax.GlobalScope - Predeclared = syntax.PredeclaredScope - Universal = syntax.UniversalScope -) - type resolver struct { // env is the current local environment: // a linked list of blocks, innermost first. @@ -244,7 +233,7 @@ type block struct { parent *block // nil for file block // In the file (root) block, both these fields are nil. - function *syntax.Function // only for function blocks + function *Function // only for function blocks comp *syntax.Comprehension // only for comprehension blocks // bindings maps a name to its binding. @@ -272,7 +261,7 @@ func (b *block) bind(name string, bind *Binding) { func (b *block) String() string { if b.function != nil { - return "function block at " + fmt.Sprint(b.function.Span()) + return "function block at " + fmt.Sprint(b.function.Pos) } if b.comp != nil { return "comprehension block at " + fmt.Sprint(b.comp.Span()) @@ -502,7 +491,14 @@ func (r *resolver) stmt(stmt syntax.Stmt) { r.errorf(stmt.Def, doesnt+"support nested def") } r.bind(stmt.Name) - r.function(stmt.Def, stmt.Name.Name, &stmt.Function) + fn := &Function{ + Name: stmt.Name.Name, + Pos: stmt.Def, + Params: stmt.Params, + Body: stmt.Body, + } + stmt.Function = fn + r.function(fn, stmt.Def) case *syntax.ForStmt: if !AllowGlobalReassign && r.container().function == nil { @@ -773,7 +769,14 @@ func (r *resolver) expr(e syntax.Expr) { if !AllowLambda { r.errorf(e.Lambda, doesnt+"support lambda") } - r.function(e.Lambda, "lambda", &e.Function) + fn := &Function{ + Name: "lambda", + Pos: e.Lambda, + Params: e.Params, + Body: []syntax.Stmt{&syntax.ReturnStmt{Result: e.Body}}, + } + e.Function = fn + r.function(fn, e.Lambda) case *syntax.ParenExpr: r.expr(e.X) @@ -783,7 +786,7 @@ func (r *resolver) expr(e syntax.Expr) { } } -func (r *resolver) function(pos syntax.Position, name string, function *syntax.Function) { +func (r *resolver) function(function *Function, pos syntax.Position) { // Resolve defaults in enclosing environment. for _, param := range function.Params { if binary, ok := param.(*syntax.BinaryExpr); ok { @@ -944,7 +947,7 @@ func (r *resolver) lookupLexical(use use, env *block) (bind *Binding) { } if debug { fmt.Printf("creating freevar %v in function at %s: %s\n", - len(env.function.FreeVars), fmt.Sprint(env.function.Span()), use.id.Name) + len(env.function.FreeVars), env.function.Pos, use.id.Name) } } diff --git a/resolve/resolve_test.go b/resolve/resolve_test.go index d87a591..090dbc5 100644 --- a/resolve/resolve_test.go +++ b/resolve/resolve_test.go @@ -59,7 +59,7 @@ func TestDefVarargsAndKwargsSet(t *testing.T) { if err := resolve.File(file, isPredeclared, isUniversal); err != nil { t.Fatal(err) } - fn := file.Stmts[0].(*syntax.DefStmt) + fn := file.Stmts[0].(*syntax.DefStmt).Function.(*resolve.Function) if !fn.HasVarargs { t.Error("HasVarargs not set") } @@ -78,7 +78,7 @@ func TestLambdaVarargsAndKwargsSet(t *testing.T) { if err := resolve.File(file, isPredeclared, isUniversal); err != nil { t.Fatal(err) } - lam := file.Stmts[0].(*syntax.AssignStmt).RHS.(*syntax.LambdaExpr) + lam := file.Stmts[0].(*syntax.AssignStmt).RHS.(*syntax.LambdaExpr).Function.(*resolve.Function) if !lam.HasVarargs { t.Error("HasVarargs not set") } diff --git a/starlark/eval.go b/starlark/eval.go index 3fcc45d..b35eb76 100644 --- a/starlark/eval.go +++ b/starlark/eval.go @@ -329,7 +329,8 @@ func FileProgram(f *syntax.File, isPredeclared func(string) bool) (*Program, err pos = syntax.MakePosition(&f.Path, 1, 1) } - compiled := compile.File(f.Stmts, pos, "<toplevel>", f.Locals, f.Globals) + module := f.Module.(*resolve.Module) + compiled := compile.File(f.Stmts, pos, "<toplevel>", module.Locals, module.Globals) return &Program{compiled}, nil } diff --git a/starlarkstruct/struct_test.go b/starlarkstruct/struct_test.go index 81936a9..8e6a93d 100644 --- a/starlarkstruct/struct_test.go +++ b/starlarkstruct/struct_test.go @@ -9,8 +9,8 @@ import ( "path/filepath" "testing" - "go.starlark.net/starlark" "go.starlark.net/resolve" + "go.starlark.net/starlark" "go.starlark.net/starlarkstruct" "go.starlark.net/starlarktest" ) @@ -67,7 +67,7 @@ func (sym *symbol) Name() string { return sym.name } func (sym *symbol) String() string { return sym.name } func (sym *symbol) Type() string { return "symbol" } func (sym *symbol) Freeze() {} // immutable -func (sym *symbol) Truth() starlark.Bool { return starlark.True } +func (sym *symbol) Truth() starlark.Bool { return starlark.True } func (sym *symbol) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: %s", sym.Type()) } func (sym *symbol) CallInternal(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { diff --git a/syntax/binding.go b/syntax/binding.go deleted file mode 100644 index 062e1b9..0000000 --- a/syntax/binding.go +++ /dev/null @@ -1,48 +0,0 @@ -package syntax - -// This file defines resolver data types referenced by the syntax tree. -// We cannot guarantee API stability for these types -// as they are closely tied to the implementation. - -// A Binding ties together all identifiers that denote the same variable. -// The resolver computes a binding for every Ident. -// -// Where possible, refer to this type using the alias resolve.Binding. -type Binding struct { - Scope Scope - - // Index records the index into the enclosing - // - {DefStmt,File}.Locals, if Scope==Local - // - DefStmt.FreeVars, if Scope==Free - // - File.Globals, if Scope==Global. - // It is zero if Scope is Predeclared, Universal, or Undefined. - Index int - - First *Ident // first binding use (iff Scope==Local/Free/Global) -} - -// The Scope of Binding indicates what kind of scope it has. -// Where possible, refer to these constants using the aliases resolve.Local, etc. -type Scope uint8 - -const ( - UndefinedScope Scope = iota // name is not defined - LocalScope // name is local to its function or file - CellScope // name is function-local but shared with a nested function - FreeScope // name is cell of some enclosing function - GlobalScope // name is global to module - PredeclaredScope // name is predeclared for this module (e.g. glob) - UniversalScope // name is universal (e.g. len) -) - -var scopeNames = [...]string{ - UndefinedScope: "undefined", - LocalScope: "local", - CellScope: "cell", - FreeScope: "free", - GlobalScope: "global", - PredeclaredScope: "predeclared", - UniversalScope: "universal", -} - -func (scope Scope) String() string { return scopeNames[scope] } diff --git a/syntax/parse.go b/syntax/parse.go index a5f12e1..0e4d284 100644 --- a/syntax/parse.go +++ b/syntax/parse.go @@ -164,13 +164,10 @@ func (p *parser) parseDefStmt() Stmt { p.consume(COLON) body := p.parseSuite() return &DefStmt{ - Def: defpos, - Name: id, - Function: Function{ - StartPos: defpos, - Params: params, - Body: body, - }, + Def: defpos, + Name: id, + Params: params, + Body: body, } } @@ -569,11 +566,8 @@ func (p *parser) parseLambda(allowCond bool) Expr { return &LambdaExpr{ Lambda: lambda, - Function: Function{ - StartPos: lambda, - Params: params, - Body: []Stmt{&ReturnStmt{Result: body}}, - }, + Params: params, + Body: body, } } diff --git a/syntax/parse_test.go b/syntax/parse_test.go index 87a0395..76f9eb3 100644 --- a/syntax/parse_test.go +++ b/syntax/parse_test.go @@ -45,7 +45,7 @@ func TestExprParseTrees(t *testing.T) { {`a + b not in c`, `(BinaryExpr X=(BinaryExpr X=a Op=+ Y=b) Op=not in Y=c)`}, {`lambda x, *args, **kwargs: None`, - `(LambdaExpr Function=(Function Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=((ReturnStmt Result=None))))`}, + `(LambdaExpr Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=None)`}, {`{"one": 1}`, `(DictExpr List=((DictEntry Key="one" Value=1)))`}, {`a[i]`, @@ -109,9 +109,9 @@ func TestExprParseTrees(t *testing.T) { {`f(*args, **kwargs)`, `(CallExpr Fn=f Args=((UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)))`}, {`lambda *args, *, x=1, **kwargs: 0`, - `(LambdaExpr Function=(Function Params=((UnaryExpr Op=* X=args) (UnaryExpr Op=*) (BinaryExpr X=x Op== Y=1) (UnaryExpr Op=** X=kwargs)) Body=((ReturnStmt Result=0))))`}, + `(LambdaExpr Params=((UnaryExpr Op=* X=args) (UnaryExpr Op=*) (BinaryExpr X=x Op== Y=1) (UnaryExpr Op=** X=kwargs)) Body=0)`}, {`lambda *, a, *b: 0`, - `(LambdaExpr Function=(Function Params=((UnaryExpr Op=*) a (UnaryExpr Op=* X=b)) Body=((ReturnStmt Result=0))))`}, + `(LambdaExpr Params=((UnaryExpr Op=*) a (UnaryExpr Op=* X=b)) Body=0)`}, {`a if b else c`, `(CondExpr Cond=b True=a False=c)`}, {`a and not b`, @@ -174,20 +174,20 @@ else: `(IfStmt Cond=True True=((LoadStmt Module="" From=(a c) To=(a b))))`}, {`def f(x, *args, **kwargs): pass`, - `(DefStmt Name=f Function=(Function Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=((BranchStmt Token=pass))))`}, + `(DefStmt Name=f Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=((BranchStmt Token=pass)))`}, {`def f(**kwargs, *args): pass`, - `(DefStmt Name=f Function=(Function Params=((UnaryExpr Op=** X=kwargs) (UnaryExpr Op=* X=args)) Body=((BranchStmt Token=pass))))`}, + `(DefStmt Name=f Params=((UnaryExpr Op=** X=kwargs) (UnaryExpr Op=* X=args)) Body=((BranchStmt Token=pass)))`}, {`def f(a, b, c=d): pass`, - `(DefStmt Name=f Function=(Function Params=(a b (BinaryExpr X=c Op== Y=d)) Body=((BranchStmt Token=pass))))`}, + `(DefStmt Name=f Params=(a b (BinaryExpr X=c Op== Y=d)) Body=((BranchStmt Token=pass)))`}, {`def f(a, b=c, d): pass`, - `(DefStmt Name=f Function=(Function Params=(a (BinaryExpr X=b Op== Y=c) d) Body=((BranchStmt Token=pass))))`}, // TODO(adonovan): fix this + `(DefStmt Name=f Params=(a (BinaryExpr X=b Op== Y=c) d) Body=((BranchStmt Token=pass)))`}, // TODO(adonovan): fix this {`def f(): def g(): pass pass def h(): pass`, - `(DefStmt Name=f Function=(Function Body=((DefStmt Name=g Function=(Function Body=((BranchStmt Token=pass)))) (BranchStmt Token=pass))))`}, + `(DefStmt Name=f Body=((DefStmt Name=g Body=((BranchStmt Token=pass))) (BranchStmt Token=pass)))`}, {"f();g()", `(ExprStmt X=(CallExpr Fn=f))`}, {"f();", @@ -227,7 +227,7 @@ print(x)`, pass pass`, - `(DefStmt Name=f Function=(Function Body=((BranchStmt Token=pass)))) + `(DefStmt Name=f Body=((BranchStmt Token=pass))) (BranchStmt Token=pass) (BranchStmt Token=pass)`}, {`pass; pass`, @@ -291,7 +291,7 @@ func TestCompoundStmt(t *testing.T) { `(ExprStmt X=(CallExpr Fn=f))`}, // complex statements {"def f():\n pass\n\n", - `(DefStmt Name=f Function=(Function Body=((BranchStmt Token=pass))))`}, + `(DefStmt Name=f Body=((BranchStmt Token=pass)))`}, {"if cond:\n pass\n\n", `(IfStmt Cond=cond True=((BranchStmt Token=pass)))`}, // Even as a 1-liner, the following blank line is required. diff --git a/syntax/syntax.go b/syntax/syntax.go index 759f946..b4817c1 100644 --- a/syntax/syntax.go +++ b/syntax/syntax.go @@ -70,9 +70,7 @@ type File struct { Path string Stmts []Stmt - // set by resolver: - Locals []*Binding // this file's (comprehension-)local variables - Globals []*Binding // this file's global variables + Module interface{} // a *resolve.Module, set by resolver } func (x *File) Span() (start, end Position) { @@ -118,36 +116,19 @@ func (x *AssignStmt) Span() (start, end Position) { return } -// A Function represents the common parts of LambdaExpr and DefStmt. -type Function struct { - commentsRef - StartPos Position // position of DEF or LAMBDA token - Params []Expr // param = ident | ident=expr | * | *ident | **ident - Body []Stmt - - // set by resolver: - HasVarargs bool // whether params includes *args (convenience) - HasKwargs bool // whether params includes **kwargs (convenience) - NumKwonlyParams int // number of keyword-only optional parameters - Locals []*Binding // this function's local/cell variables, parameters first - FreeVars []*Binding // enclosing cells to capture in closure -} - -func (x *Function) Span() (start, end Position) { - _, end = x.Body[len(x.Body)-1].Span() - return x.StartPos, end -} - // A DefStmt represents a function definition. type DefStmt struct { commentsRef - Def Position - Name *Ident - Function + Def Position + Name *Ident + Params []Expr // param = ident | ident=expr | * | *ident | **ident + Body []Stmt + + Function interface{} // a *resolve.Function, set by resolver } func (x *DefStmt) Span() (start, end Position) { - _, end = x.Function.Body[len(x.Body)-1].Span() + _, end = x.Body[len(x.Body)-1].Span() return x.Def, end } @@ -260,8 +241,7 @@ type Ident struct { NamePos Position Name string - // set by resolver - Binding *Binding + Binding interface{} // a *resolver.Binding, set by resolver } func (x *Ident) Span() (start, end Position) { @@ -425,11 +405,14 @@ func (x *DictEntry) Span() (start, end Position) { type LambdaExpr struct { commentsRef Lambda Position - Function + Params []Expr // param = ident | ident=expr | * | *ident | **ident + Body Expr + + Function interface{} // a *resolve.Function, set by resolver } func (x *LambdaExpr) Span() (start, end Position) { - _, end = x.Function.Body[len(x.Body)-1].Span() + _, end = x.Body.Span() return x.Lambda, end } diff --git a/syntax/walk.go b/syntax/walk.go index ad2aa88..1491149 100644 --- a/syntax/walk.go +++ b/syntax/walk.go @@ -39,10 +39,10 @@ func Walk(n Node, f func(Node) bool) { case *DefStmt: Walk(n.Name, f) - for _, param := range n.Function.Params { + for _, param := range n.Params { Walk(param, f) } - walkStmts(n.Function.Body, f) + walkStmts(n.Body, f) case *ForStmt: Walk(n.Vars, f) @@ -144,10 +144,10 @@ func Walk(n Node, f func(Node) bool) { } case *LambdaExpr: - for _, param := range n.Function.Params { + for _, param := range n.Params { Walk(param, f) } - walkStmts(n.Function.Body, f) + Walk(n.Body, f) default: panic(n) |