aboutsummaryrefslogtreecommitdiff
path: root/syntax/parse_test.go
diff options
context:
space:
mode:
Diffstat (limited to 'syntax/parse_test.go')
-rw-r--r--syntax/parse_test.go487
1 files changed, 487 insertions, 0 deletions
diff --git a/syntax/parse_test.go b/syntax/parse_test.go
new file mode 100644
index 0000000..fedbb3e
--- /dev/null
+++ b/syntax/parse_test.go
@@ -0,0 +1,487 @@
+// Copyright 2017 The Bazel Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package syntax_test
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ "go/build"
+ "io/ioutil"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+
+ "go.starlark.net/internal/chunkedfile"
+ "go.starlark.net/starlarktest"
+ "go.starlark.net/syntax"
+)
+
+func TestExprParseTrees(t *testing.T) {
+ for _, test := range []struct {
+ input, want string
+ }{
+ {`print(1)`,
+ `(CallExpr Fn=print Args=(1))`},
+ {"print(1)\n",
+ `(CallExpr Fn=print Args=(1))`},
+ {`x + 1`,
+ `(BinaryExpr X=x Op=+ Y=1)`},
+ {`[x for x in y]`,
+ `(Comprehension Body=x Clauses=((ForClause Vars=x X=y)))`},
+ {`[x for x in (a if b else c)]`,
+ `(Comprehension Body=x Clauses=((ForClause Vars=x X=(ParenExpr X=(CondExpr Cond=b True=a False=c)))))`},
+ {`x[i].f(42)`,
+ `(CallExpr Fn=(DotExpr X=(IndexExpr X=x Y=i) Name=f) Args=(42))`},
+ {`x.f()`,
+ `(CallExpr Fn=(DotExpr X=x Name=f))`},
+ {`x+y*z`,
+ `(BinaryExpr X=x Op=+ Y=(BinaryExpr X=y Op=* Y=z))`},
+ {`x%y-z`,
+ `(BinaryExpr X=(BinaryExpr X=x Op=% Y=y) Op=- Y=z)`},
+ {`a + b not in c`,
+ `(BinaryExpr X=(BinaryExpr X=a Op=+ Y=b) Op=not in Y=c)`},
+ {`lambda x, *args, **kwargs: None`,
+ `(LambdaExpr Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=None)`},
+ {`{"one": 1}`,
+ `(DictExpr List=((DictEntry Key="one" Value=1)))`},
+ {`a[i]`,
+ `(IndexExpr X=a Y=i)`},
+ {`a[i:]`,
+ `(SliceExpr X=a Lo=i)`},
+ {`a[:j]`,
+ `(SliceExpr X=a Hi=j)`},
+ {`a[::]`,
+ `(SliceExpr X=a)`},
+ {`a[::k]`,
+ `(SliceExpr X=a Step=k)`},
+ {`[]`,
+ `(ListExpr)`},
+ {`[1]`,
+ `(ListExpr List=(1))`},
+ {`[1,]`,
+ `(ListExpr List=(1))`},
+ {`[1, 2]`,
+ `(ListExpr List=(1 2))`},
+ {`()`,
+ `(TupleExpr)`},
+ {`(4,)`,
+ `(ParenExpr X=(TupleExpr List=(4)))`},
+ {`(4)`,
+ `(ParenExpr X=4)`},
+ {`(4, 5)`,
+ `(ParenExpr X=(TupleExpr List=(4 5)))`},
+ {`1, 2, 3`,
+ `(TupleExpr List=(1 2 3))`},
+ {`1, 2,`,
+ `unparenthesized tuple with trailing comma`},
+ {`{}`,
+ `(DictExpr)`},
+ {`{"a": 1}`,
+ `(DictExpr List=((DictEntry Key="a" Value=1)))`},
+ {`{"a": 1,}`,
+ `(DictExpr List=((DictEntry Key="a" Value=1)))`},
+ {`{"a": 1, "b": 2}`,
+ `(DictExpr List=((DictEntry Key="a" Value=1) (DictEntry Key="b" Value=2)))`},
+ {`{x: y for (x, y) in z}`,
+ `(Comprehension Curly Body=(DictEntry Key=x Value=y) Clauses=((ForClause Vars=(ParenExpr X=(TupleExpr List=(x y))) X=z)))`},
+ {`{x: y for a in b if c}`,
+ `(Comprehension Curly Body=(DictEntry Key=x Value=y) Clauses=((ForClause Vars=a X=b) (IfClause Cond=c)))`},
+ {`-1 + +2`,
+ `(BinaryExpr X=(UnaryExpr Op=- X=1) Op=+ Y=(UnaryExpr Op=+ X=2))`},
+ {`"foo" + "bar"`,
+ `(BinaryExpr X="foo" Op=+ Y="bar")`},
+ {`-1 * 2`, // prec(unary -) > prec(binary *)
+ `(BinaryExpr X=(UnaryExpr Op=- X=1) Op=* Y=2)`},
+ {`-x[i]`, // prec(unary -) < prec(x[i])
+ `(UnaryExpr Op=- X=(IndexExpr X=x Y=i))`},
+ {`a | b & c | d`, // prec(|) < prec(&)
+ `(BinaryExpr X=(BinaryExpr X=a Op=| Y=(BinaryExpr X=b Op=& Y=c)) Op=| Y=d)`},
+ {`a or b and c or d`,
+ `(BinaryExpr X=(BinaryExpr X=a Op=or Y=(BinaryExpr X=b Op=and Y=c)) Op=or Y=d)`},
+ {`a and b or c and d`,
+ `(BinaryExpr X=(BinaryExpr X=a Op=and Y=b) Op=or Y=(BinaryExpr X=c Op=and Y=d))`},
+ {`f(1, x=y)`,
+ `(CallExpr Fn=f Args=(1 (BinaryExpr X=x Op== Y=y)))`},
+ {`f(*args, **kwargs)`,
+ `(CallExpr Fn=f Args=((UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)))`},
+ {`lambda *args, *, x=1, **kwargs: 0`,
+ `(LambdaExpr Params=((UnaryExpr Op=* X=args) (UnaryExpr Op=*) (BinaryExpr X=x Op== Y=1) (UnaryExpr Op=** X=kwargs)) Body=0)`},
+ {`lambda *, a, *b: 0`,
+ `(LambdaExpr Params=((UnaryExpr Op=*) a (UnaryExpr Op=* X=b)) Body=0)`},
+ {`a if b else c`,
+ `(CondExpr Cond=b True=a False=c)`},
+ {`a and not b`,
+ `(BinaryExpr X=a Op=and Y=(UnaryExpr Op=not X=b))`},
+ {`[e for x in y if cond1 if cond2]`,
+ `(Comprehension Body=e Clauses=((ForClause Vars=x X=y) (IfClause Cond=cond1) (IfClause Cond=cond2)))`}, // github.com/google/skylark/issues/53
+ } {
+ e, err := syntax.ParseExpr("foo.star", test.input, 0)
+ var got string
+ if err != nil {
+ got = stripPos(err)
+ } else {
+ got = treeString(e)
+ }
+ if test.want != got {
+ t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want)
+ }
+ }
+}
+
+func TestStmtParseTrees(t *testing.T) {
+ for _, test := range []struct {
+ input, want string
+ }{
+ {`print(1)`,
+ `(ExprStmt X=(CallExpr Fn=print Args=(1)))`},
+ {`return 1, 2`,
+ `(ReturnStmt Result=(TupleExpr List=(1 2)))`},
+ {`return`,
+ `(ReturnStmt)`},
+ {`for i in "abc": break`,
+ `(ForStmt Vars=i X="abc" Body=((BranchStmt Token=break)))`},
+ {`for i in "abc": continue`,
+ `(ForStmt Vars=i X="abc" Body=((BranchStmt Token=continue)))`},
+ {`for x, y in z: pass`,
+ `(ForStmt Vars=(TupleExpr List=(x y)) X=z Body=((BranchStmt Token=pass)))`},
+ {`if True: pass`,
+ `(IfStmt Cond=True True=((BranchStmt Token=pass)))`},
+ {`if True: break`,
+ `(IfStmt Cond=True True=((BranchStmt Token=break)))`},
+ {`if True: continue`,
+ `(IfStmt Cond=True True=((BranchStmt Token=continue)))`},
+ {`if True: pass
+else:
+ pass`,
+ `(IfStmt Cond=True True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))`},
+ {"if a: pass\nelif b: pass\nelse: pass",
+ `(IfStmt Cond=a True=((BranchStmt Token=pass)) False=((IfStmt Cond=b True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))))`},
+ {`x, y = 1, 2`,
+ `(AssignStmt Op== LHS=(TupleExpr List=(x y)) RHS=(TupleExpr List=(1 2)))`},
+ {`x[i] = 1`,
+ `(AssignStmt Op== LHS=(IndexExpr X=x Y=i) RHS=1)`},
+ {`x.f = 1`,
+ `(AssignStmt Op== LHS=(DotExpr X=x Name=f) RHS=1)`},
+ {`(x, y) = 1`,
+ `(AssignStmt Op== LHS=(ParenExpr X=(TupleExpr List=(x y))) RHS=1)`},
+ {`load("", "a", b="c")`,
+ `(LoadStmt Module="" From=(a c) To=(a b))`},
+ {`if True: load("", "a", b="c")`, // load needn't be at toplevel
+ `(IfStmt Cond=True True=((LoadStmt Module="" From=(a c) To=(a b))))`},
+ {`def f(x, *args, **kwargs):
+ pass`,
+ `(DefStmt Name=f Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=((BranchStmt Token=pass)))`},
+ {`def f(**kwargs, *args): pass`,
+ `(DefStmt Name=f Params=((UnaryExpr Op=** X=kwargs) (UnaryExpr Op=* X=args)) Body=((BranchStmt Token=pass)))`},
+ {`def f(a, b, c=d): pass`,
+ `(DefStmt Name=f Params=(a b (BinaryExpr X=c Op== Y=d)) Body=((BranchStmt Token=pass)))`},
+ {`def f(a, b=c, d): pass`,
+ `(DefStmt Name=f Params=(a (BinaryExpr X=b Op== Y=c) d) Body=((BranchStmt Token=pass)))`}, // TODO(adonovan): fix this
+ {`def f():
+ def g():
+ pass
+ pass
+def h():
+ pass`,
+ `(DefStmt Name=f Body=((DefStmt Name=g Body=((BranchStmt Token=pass))) (BranchStmt Token=pass)))`},
+ {"f();g()",
+ `(ExprStmt X=(CallExpr Fn=f))`},
+ {"f();",
+ `(ExprStmt X=(CallExpr Fn=f))`},
+ {"f();g()\n",
+ `(ExprStmt X=(CallExpr Fn=f))`},
+ {"f();\n",
+ `(ExprStmt X=(CallExpr Fn=f))`},
+ } {
+ f, err := syntax.Parse("foo.star", test.input, 0)
+ if err != nil {
+ t.Errorf("parse `%s` failed: %v", test.input, stripPos(err))
+ continue
+ }
+ if got := treeString(f.Stmts[0]); test.want != got {
+ t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want)
+ }
+ }
+}
+
+// TestFileParseTrees tests sequences of statements, and particularly
+// handling of indentation, newlines, line continuations, and blank lines.
+func TestFileParseTrees(t *testing.T) {
+ for _, test := range []struct {
+ input, want string
+ }{
+ {`x = 1
+print(x)`,
+ `(AssignStmt Op== LHS=x RHS=1)
+(ExprStmt X=(CallExpr Fn=print Args=(x)))`},
+ {"if cond:\n\tpass",
+ `(IfStmt Cond=cond True=((BranchStmt Token=pass)))`},
+ {"if cond:\n\tpass\nelse:\n\tpass",
+ `(IfStmt Cond=cond True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))`},
+ {`def f():
+ pass
+pass
+
+pass`,
+ `(DefStmt Name=f Body=((BranchStmt Token=pass)))
+(BranchStmt Token=pass)
+(BranchStmt Token=pass)`},
+ {`pass; pass`,
+ `(BranchStmt Token=pass)
+(BranchStmt Token=pass)`},
+ {"pass\npass",
+ `(BranchStmt Token=pass)
+(BranchStmt Token=pass)`},
+ {"pass\n\npass",
+ `(BranchStmt Token=pass)
+(BranchStmt Token=pass)`},
+ {`x = (1 +
+2)`,
+ `(AssignStmt Op== LHS=x RHS=(ParenExpr X=(BinaryExpr X=1 Op=+ Y=2)))`},
+ {`x = 1 \
++ 2`,
+ `(AssignStmt Op== LHS=x RHS=(BinaryExpr X=1 Op=+ Y=2))`},
+ } {
+ f, err := syntax.Parse("foo.star", test.input, 0)
+ if err != nil {
+ t.Errorf("parse `%s` failed: %v", test.input, stripPos(err))
+ continue
+ }
+ var buf bytes.Buffer
+ for i, stmt := range f.Stmts {
+ if i > 0 {
+ buf.WriteByte('\n')
+ }
+ writeTree(&buf, reflect.ValueOf(stmt))
+ }
+ if got := buf.String(); test.want != got {
+ t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want)
+ }
+ }
+}
+
+// TestCompoundStmt tests handling of REPL-style compound statements.
+func TestCompoundStmt(t *testing.T) {
+ for _, test := range []struct {
+ input, want string
+ }{
+ // blank lines
+ {"\n",
+ ``},
+ {" \n",
+ ``},
+ {"# comment\n",
+ ``},
+ // simple statement
+ {"1\n",
+ `(ExprStmt X=1)`},
+ {"print(1)\n",
+ `(ExprStmt X=(CallExpr Fn=print Args=(1)))`},
+ {"1;2;3;\n",
+ `(ExprStmt X=1)(ExprStmt X=2)(ExprStmt X=3)`},
+ {"f();g()\n",
+ `(ExprStmt X=(CallExpr Fn=f))(ExprStmt X=(CallExpr Fn=g))`},
+ {"f();\n",
+ `(ExprStmt X=(CallExpr Fn=f))`},
+ {"f(\n\n\n\n\n\n\n)\n",
+ `(ExprStmt X=(CallExpr Fn=f))`},
+ // complex statements
+ {"def f():\n pass\n\n",
+ `(DefStmt Name=f Body=((BranchStmt Token=pass)))`},
+ {"if cond:\n pass\n\n",
+ `(IfStmt Cond=cond True=((BranchStmt Token=pass)))`},
+ // Even as a 1-liner, the following blank line is required.
+ {"if cond: pass\n\n",
+ `(IfStmt Cond=cond True=((BranchStmt Token=pass)))`},
+ // github.com/google/starlark-go/issues/121
+ {"a; b; c\n",
+ `(ExprStmt X=a)(ExprStmt X=b)(ExprStmt X=c)`},
+ {"a; b c\n",
+ `invalid syntax`},
+ } {
+
+ // Fake readline input from string.
+ // The ! suffix, which would cause a parse error,
+ // tests that the parser doesn't read more than necessary.
+ sc := bufio.NewScanner(strings.NewReader(test.input + "!"))
+ readline := func() ([]byte, error) {
+ if sc.Scan() {
+ return []byte(sc.Text() + "\n"), nil
+ }
+ return nil, sc.Err()
+ }
+
+ var got string
+ f, err := syntax.ParseCompoundStmt("foo.star", readline)
+ if err != nil {
+ got = stripPos(err)
+ } else {
+ for _, stmt := range f.Stmts {
+ got += treeString(stmt)
+ }
+ }
+ if test.want != got {
+ t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want)
+ }
+ }
+}
+
+func stripPos(err error) string {
+ s := err.Error()
+ if i := strings.Index(s, ": "); i >= 0 {
+ s = s[i+len(": "):] // strip file:line:col
+ }
+ return s
+}
+
+// treeString prints a syntax node as a parenthesized tree.
+// Idents are printed as foo and Literals as "foo" or 42.
+// Structs are printed as (type name=value ...).
+// Only non-empty fields are shown.
+func treeString(n syntax.Node) string {
+ var buf bytes.Buffer
+ writeTree(&buf, reflect.ValueOf(n))
+ return buf.String()
+}
+
+func writeTree(out *bytes.Buffer, x reflect.Value) {
+ switch x.Kind() {
+ case reflect.String, reflect.Int, reflect.Bool:
+ fmt.Fprintf(out, "%v", x.Interface())
+ case reflect.Ptr, reflect.Interface:
+ if elem := x.Elem(); elem.Kind() == 0 {
+ out.WriteString("nil")
+ } else {
+ writeTree(out, elem)
+ }
+ case reflect.Struct:
+ switch v := x.Interface().(type) {
+ case syntax.Literal:
+ switch v.Token {
+ case syntax.STRING:
+ fmt.Fprintf(out, "%q", v.Value)
+ case syntax.BYTES:
+ fmt.Fprintf(out, "b%q", v.Value)
+ case syntax.INT:
+ fmt.Fprintf(out, "%d", v.Value)
+ }
+ return
+ case syntax.Ident:
+ out.WriteString(v.Name)
+ return
+ }
+ fmt.Fprintf(out, "(%s", strings.TrimPrefix(x.Type().String(), "syntax."))
+ for i, n := 0, x.NumField(); i < n; i++ {
+ f := x.Field(i)
+ if f.Type() == reflect.TypeOf(syntax.Position{}) {
+ continue // skip positions
+ }
+ name := x.Type().Field(i).Name
+ if name == "commentsRef" {
+ continue // skip comments fields
+ }
+ if f.Type() == reflect.TypeOf(syntax.Token(0)) {
+ fmt.Fprintf(out, " %s=%s", name, f.Interface())
+ continue
+ }
+
+ switch f.Kind() {
+ case reflect.Slice:
+ if n := f.Len(); n > 0 {
+ fmt.Fprintf(out, " %s=(", name)
+ for i := 0; i < n; i++ {
+ if i > 0 {
+ out.WriteByte(' ')
+ }
+ writeTree(out, f.Index(i))
+ }
+ out.WriteByte(')')
+ }
+ continue
+ case reflect.Ptr, reflect.Interface:
+ if f.IsNil() {
+ continue
+ }
+ case reflect.Int:
+ if f.Int() != 0 {
+ fmt.Fprintf(out, " %s=%d", name, f.Int())
+ }
+ continue
+ case reflect.Bool:
+ if f.Bool() {
+ fmt.Fprintf(out, " %s", name)
+ }
+ continue
+ }
+ fmt.Fprintf(out, " %s=", name)
+ writeTree(out, f)
+ }
+ fmt.Fprintf(out, ")")
+ default:
+ fmt.Fprintf(out, "%T", x.Interface())
+ }
+}
+
+func TestParseErrors(t *testing.T) {
+ filename := starlarktest.DataFile("syntax", "testdata/errors.star")
+ for _, chunk := range chunkedfile.Read(filename, t) {
+ _, err := syntax.Parse(filename, chunk.Source, 0)
+ switch err := err.(type) {
+ case nil:
+ // ok
+ case syntax.Error:
+ chunk.GotError(int(err.Pos.Line), err.Msg)
+ default:
+ t.Error(err)
+ }
+ chunk.Done()
+ }
+}
+
+func TestFilePortion(t *testing.T) {
+ // Imagine that the Starlark file or expression print(x.f) is extracted
+ // from the middle of a file in some hypothetical template language;
+ // see https://github.com/google/starlark-go/issues/346. For example:
+ // --
+ // {{loop x seq}}
+ // {{print(x.f)}}
+ // {{end}}
+ // --
+ fp := syntax.FilePortion{Content: []byte("print(x.f)"), FirstLine: 2, FirstCol: 4}
+ file, err := syntax.Parse("foo.template", fp, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ span := fmt.Sprint(file.Stmts[0].Span())
+ want := "foo.template:2:4 foo.template:2:14"
+ if span != want {
+ t.Errorf("wrong span: got %q, want %q", span, want)
+ }
+}
+
+// dataFile is the same as starlarktest.DataFile.
+// We make a copy to avoid a dependency cycle.
+var dataFile = func(pkgdir, filename string) string {
+ return filepath.Join(build.Default.GOPATH, "src/go.starlark.net", pkgdir, filename)
+}
+
+func BenchmarkParse(b *testing.B) {
+ filename := dataFile("syntax", "testdata/scan.star")
+ b.StopTimer()
+ data, err := ioutil.ReadFile(filename)
+ if err != nil {
+ b.Fatal(err)
+ }
+ b.StartTimer()
+
+ for i := 0; i < b.N; i++ {
+ _, err := syntax.Parse(filename, data, 0)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-20 12:53:52 +0000