diff options
author | alandonovan <adonovan@google.com> | 2021-02-08 12:20:22 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-02-08 12:20:22 -0500 |
commit | 0a10e4fe7402e37a43d9b62c15bfeac1cd4ef272 (patch) | |
tree | 33fd5d4434befd04b6dcc7650f57896cbb992951 | |
parent | bc864be25151710e5d1c992993e31b5543e779e1 (diff) | |
download | starlark-go-0a10e4fe7402e37a43d9b62c15bfeac1cd4ef272.tar.gz |
syntax: support setting initial line/col for scanner (#349)
The new FilePortion type, which may be provided to the scanner,
parser, or ExecFile functions, combines a piece of text along
with its start line/column numbers, for applications that
extract a Starlark expression from the middle of a larger file.
Fixes https://github.com/google/starlark-go/issues/346
-rw-r--r-- | syntax/parse.go | 2 | ||||
-rw-r--r-- | syntax/parse_test.go | 21 | ||||
-rw-r--r-- | syntax/scan.go | 17 |
3 files changed, 38 insertions, 2 deletions
diff --git a/syntax/parse.go b/syntax/parse.go index 0281e4b..50b8087 100644 --- a/syntax/parse.go +++ b/syntax/parse.go @@ -28,7 +28,7 @@ const ( // If src != nil, ParseFile parses the source from src and the filename // is only used when recording position information. // The type of the argument for the src parameter must be string, -// []byte, or io.Reader. +// []byte, io.Reader, or FilePortion. // If src == nil, ParseFile parses the file specified by filename. func Parse(filename string, src interface{}, mode Mode) (f *File, err error) { in, err := newScanner(filename, src, mode&RetainComments != 0) diff --git a/syntax/parse_test.go b/syntax/parse_test.go index 76f9eb3..6052e79 100644 --- a/syntax/parse_test.go +++ b/syntax/parse_test.go @@ -439,6 +439,27 @@ func TestParseErrors(t *testing.T) { } } +func TestFilePortion(t *testing.T) { + // Imagine that the Starlark file or expression print(x.f) is extracted + // from the middle of a file in some hypothetical template language; + // see https://github.com/google/starlark-go/issues/346. For example: + // -- + // {{loop x seq}} + // {{print(x.f)}} + // {{end}} + // -- + fp := syntax.FilePortion{Content: []byte("print(x.f)"), FirstLine: 2, FirstCol: 4} + file, err := syntax.Parse("foo.template", fp, 0) + if err != nil { + t.Fatal(err) + } + span := fmt.Sprint(file.Stmts[0].Span()) + want := "foo.template:2:4 foo.template:2:14" + if span != want { + t.Errorf("wrong span: got %q, want %q", span, want) + } +} + // dataFile is the same as starlarktest.DataFile. // We make a copy to avoid a dependency cycle. var dataFile = func(pkgdir, filename string) string { diff --git a/syntax/scan.go b/syntax/scan.go index 53d9f5c..a162264 100644 --- a/syntax/scan.go +++ b/syntax/scan.go @@ -182,6 +182,15 @@ var tokenNames = [...]string{ WHILE: "while", } +// A FilePortion describes the content of a portion of a file. +// Callers may provide a FilePortion for the src argument of Parse +// when the desired initial line and column numbers are not (1, 1), +// such as when an expression is parsed from within larger file. +type FilePortion struct { + Content []byte + FirstLine, FirstCol int32 +} + // A Position describes the location of a rune of input. type Position struct { file *string // filename (indirect for compactness) @@ -249,8 +258,12 @@ type scanner struct { } func newScanner(filename string, src interface{}, keepComments bool) (*scanner, error) { + var firstLine, firstCol int32 = 1, 1 + if portion, ok := src.(FilePortion); ok { + firstLine, firstCol = portion.FirstLine, portion.FirstCol + } sc := &scanner{ - pos: Position{file: &filename, Line: 1, Col: 1}, + pos: MakePosition(&filename, firstLine, firstCol), indentstk: make([]int, 1, 10), // []int{0} + spare capacity lineStart: true, keepComments: keepComments, @@ -279,6 +292,8 @@ func readSource(filename string, src interface{}) ([]byte, error) { return nil, err } return data, nil + case FilePortion: + return src.Content, nil case nil: return ioutil.ReadFile(filename) default: |