From e187687ca024f24af4c24ee28fd7b4e55ffb6ac1 Mon Sep 17 00:00:00 2001
From: David Srbecky <dsrbecky@google.com>
Date: Wed, 5 Aug 2015 17:15:13 +0100
Subject: Keep track of api filename for the purpose of error reporting.

Change-Id: Icccfa73b4f01d4d0f418a62d1bbb5f8ed12fe0b8
---
 api/api.go                                        |  2 +-
 api/apic/commands/command.go                      |  3 ++-
 api/apic/format/format.go                         |  2 +-
 api/apic/validate/no_unreachables_test.go         |  2 +-
 api/parser/parser.go                              |  4 ++--
 api/parser/parser_test.go                         | 13 ++++++++-----
 api/resolver/resolve_test.go                      |  2 +-
 gfxapi/gles/glsl/preprocessor/constants.go        |  6 ------
 gfxapi/gles/glsl/preprocessor/lexer.go            |  4 ++--
 gfxapi/gles/glsl/preprocessor/preprocessorImpl.go |  5 +++--
 parse/cst_test.go                                 |  4 ++--
 parse/parser.go                                   |  4 ++--
 parse/parser_test.go                              | 10 +++++-----
 parse/reader.go                                   | 16 +++++++++-------
 parse/reader_test.go                              |  2 +-
 parse/source.go                                   | 20 ++++++++++++++++++++
 parse/token.go                                    | 12 ++++++------
 tools/generate_gles_api/generate_api.go           |  4 ++--
 tools/generate_gles_api/main.go                   |  2 +-
 19 files changed, 69 insertions(+), 48 deletions(-)
 create mode 100644 parse/source.go

diff --git a/api/api.go b/api/api.go
index 2d1f9d6b5..1492dedcf 100644
--- a/api/api.go
+++ b/api/api.go
@@ -61,7 +61,7 @@ func (p *Processor) Parse(path string) (*ast.API, parse.ErrorList) {
 	if err != nil {
 		return nil, parse.ErrorList{parse.Error{Message: err.Error()}}
 	}
-	return parser.Parse(string(info))
+	return parser.Parse(path, string(info))
 }
 
 // Resolve resolves the api file with the DefaultProcessor.
diff --git a/api/apic/commands/command.go b/api/apic/commands/command.go
index 59ed20f09..627acc48a 100644
--- a/api/apic/commands/command.go
+++ b/api/apic/commands/command.go
@@ -95,8 +95,9 @@ func CheckErrors(apiName string, errs parse.ErrorList) error {
 	}
 	for _, e := range errs {
 		if e.At != nil {
+			filename := e.At.Token().Source.Filename
 			line, column := e.At.Token().Cursor()
-			fmt.Fprintf(os.Stderr, "%s:%v:%v: %s\n", apiName, line, column, e.Message)
+			fmt.Fprintf(os.Stderr, "%s:%v:%v: %s\n", filename, line, column, e.Message)
 		} else {
 			fmt.Fprintf(os.Stderr, "%s: %s\n", apiName, e.Message)
 		}
diff --git a/api/apic/format/format.go b/api/apic/format/format.go
index aa3b35cd4..0fe52ba45 100644
--- a/api/apic/format/format.go
+++ b/api/apic/format/format.go
@@ -59,7 +59,7 @@ func doFormat(flags flag.FlagSet) error {
 			continue
 		}
 
-		api, errs := parser.Parse(string(f))
+		api, errs := parser.Parse(path, string(f))
 		if len(errs) > 0 {
 			fmt.Printf("Errors while parsing '%s':\n", path)
 			for i, e := range errs {
diff --git a/api/apic/validate/no_unreachables_test.go b/api/apic/validate/no_unreachables_test.go
index aeda86e09..57cb48577 100644
--- a/api/apic/validate/no_unreachables_test.go
+++ b/api/apic/validate/no_unreachables_test.go
@@ -25,7 +25,7 @@ import (
 )
 
 func compile(t *testing.T, source string) (*semantic.API, error) {
-	parsed, errs := parser.Parse(source)
+	parsed, errs := parser.Parse("no_unreachables_test.api", source)
 	if err := commands.CheckErrors(source, errs); err != nil {
 		return nil, err
 	}
diff --git a/api/parser/parser.go b/api/parser/parser.go
index f769a2212..f45b9dc02 100644
--- a/api/parser/parser.go
+++ b/api/parser/parser.go
@@ -26,11 +26,11 @@ import (
 // If the string is not syntactically valid, it will also return the
 // errors encountered. If errors are returned, the ast returned will be
 // the incomplete tree so far, and may not be structurally valid.
-func Parse(data string) (*ast.API, parse.ErrorList) {
+func Parse(filename, data string) (*ast.API, parse.ErrorList) {
 	var api *ast.API
 	parser := func(p *parse.Parser, cst *parse.Branch) {
 		api = requireAPI(p, cst)
 	}
-	errors := parse.Parse(parser, data, parse.NewSkip("//", "/*", "*/"))
+	errors := parse.Parse(parser, filename, data, parse.NewSkip("//", "/*", "*/"))
 	return api, errors
 }
diff --git a/api/parser/parser_test.go b/api/parser/parser_test.go
index b7dfa5a37..ac5ba2b36 100644
--- a/api/parser/parser_test.go
+++ b/api/parser/parser_test.go
@@ -76,14 +76,17 @@ func printCSTTree(t *testing.T, n parse.Node, hints map[parse.Node][]ast.Node, t
 }
 
 func TestParsedCST(t *testing.T) {
-	runes := make([]rune, 0, 0xffff) // needs to be big enough to not be resized
+	source := &parse.Source{
+		Filename: "parser_test.api",
+		Runes:    make([]rune, 0, 0xffff), // needs to be big enough to not be resized
+	}
 	B := func(n ...parse.Node) *parse.Branch { return &parse.Branch{Children: n} }
 	L := func(str string) *parse.Leaf {
 		r := []rune(str)
-		s, e := len(runes), len(runes)+len(r)
-		runes = append(append(runes, r...), '·')
+		s, e := len(source.Runes), len(source.Runes)+len(r)
+		source.Runes = append(append(source.Runes, r...), '·')
 		l := &parse.Leaf{}
-		l.SetToken(parse.Token{Runes: runes, Start: s, End: e})
+		l.SetToken(parse.Token{Source: source, Start: s, End: e})
 		return l
 	}
 
@@ -104,7 +107,7 @@ func TestParsedCST(t *testing.T) {
 			expected: B(B(B(B(L("const"), B(B(L("char")), L("*"))), L("const"), L("*")), L("a"))),
 		},
 	} {
-		api, errs := Parse(test.source)
+		api, errs := Parse("parser_test.api", test.source)
 		m := map[parse.Node][]ast.Node{api.Node(): {api}}
 		var traverse func(n ast.Node)
 		traverse = func(n ast.Node) {
diff --git a/api/resolver/resolve_test.go b/api/resolver/resolve_test.go
index 3d8c1ce64..e0037cd4b 100644
--- a/api/resolver/resolve_test.go
+++ b/api/resolver/resolve_test.go
@@ -59,7 +59,7 @@ cmd void foo() { a := A(1,2,3) i := a[3] }`,
 			errors: []string{"2:37: array index 3 is out of bounds for u32[3]"},
 		},
 	} {
-		astAPI, errs := parser.Parse(test.source)
+		astAPI, errs := parser.Parse("resolve_test.api", test.source)
 		if len(errs) > 0 {
 			t.Errorf("Testing '%s' - Unexpected parse errors: %v", test.name, errs)
 			continue
diff --git a/gfxapi/gles/glsl/preprocessor/constants.go b/gfxapi/gles/glsl/preprocessor/constants.go
index 040651885..d0a0778bb 100644
--- a/gfxapi/gles/glsl/preprocessor/constants.go
+++ b/gfxapi/gles/glsl/preprocessor/constants.go
@@ -37,12 +37,6 @@ type TokenInfo struct {
 	Cst        *parse.Leaf // Structure containing the preceeding and following whitespace.
 }
 
-func (ti *TokenInfo) setToken(t Token) {
-	ti.Token = t
-	str := []rune(t.String())
-	ti.Cst.SetToken(parse.Token{str, 0, len(str)})
-}
-
 // Keyword represents a language keyword token returned by the preprocessor.
 type Keyword struct {
 	word *string
diff --git a/gfxapi/gles/glsl/preprocessor/lexer.go b/gfxapi/gles/glsl/preprocessor/lexer.go
index 4ebafcca3..05b04a5aa 100644
--- a/gfxapi/gles/glsl/preprocessor/lexer.go
+++ b/gfxapi/gles/glsl/preprocessor/lexer.go
@@ -220,11 +220,11 @@ func processLineContinuations(input string) string {
 
 /////////////////////////// Lexer interface below /////////////////////////////
 
-func newLexer(input string) *lexer {
+func newLexer(filename, input string) *lexer {
 	input = processLineContinuations(input)
 	l := &lexer{
 		current: TokenInfo{Newline: true, Cst: &parse.Leaf{}},
-		reader:  parse.NewReader(input),
+		reader:  parse.NewReader(filename, input),
 	}
 	l.current.Cst.AddPrefix(l.skip(parse.SkipPrefix))
 	l.read()
diff --git a/gfxapi/gles/glsl/preprocessor/preprocessorImpl.go b/gfxapi/gles/glsl/preprocessor/preprocessorImpl.go
index 1ea1954b0..b13d05832 100644
--- a/gfxapi/gles/glsl/preprocessor/preprocessorImpl.go
+++ b/gfxapi/gles/glsl/preprocessor/preprocessorImpl.go
@@ -18,6 +18,7 @@ import (
 	"android.googlesource.com/platform/tools/gpu/gfxapi/gles/glsl/ast"
 	"android.googlesource.com/platform/tools/gpu/parse"
 	"bytes"
+	"fmt"
 )
 
 // ifEntry is a structure containing the data necessary for proper evaluation of #if*
@@ -356,7 +357,7 @@ func (p *preprocessorImpl) evaluateIf(args []TokenInfo) bool {
 	// append fake EOF
 	lastToken := args[len(args)-1].Cst.Token()
 	eof := &parse.Leaf{}
-	eof.SetToken(parse.Token{Runes: lastToken.Runes, Start: lastToken.End, End: lastToken.End})
+	eof.SetToken(parse.Token{Source: lastToken.Source, Start: lastToken.End, End: lastToken.End})
 	args = append(args, TokenInfo{Token: nil, Cst: eof})
 
 	var list []tokenExpansion
@@ -508,7 +509,7 @@ func addBuiltinMacro(macros map[string]macroDefinition, name string, expander ma
 
 func newPreprocessorImpl(data string, eval ExpressionEvaluator, file int) *preprocessorImpl {
 	p := &preprocessorImpl{
-		lexer:     newLexer(data),
+		lexer:     newLexer(fmt.Sprintf("File %v", file), data),
 		macros:    make(map[string]macroDefinition),
 		evaluator: eval,
 	}
diff --git a/parse/cst_test.go b/parse/cst_test.go
index 7d233f5b5..ab9a09453 100644
--- a/parse/cst_test.go
+++ b/parse/cst_test.go
@@ -126,8 +126,8 @@ func compareSeparators(t *testing.T, in string, expect, got Separator) {
 }
 
 func newStringToken(value string) Token {
-	runes := bytes.Runes([]byte(value))
-	return Token{Runes: runes, Start: 0, End: len(runes)}
+	source := &Source{Filename: "cst_test.api", Runes: bytes.Runes([]byte(value))}
+	return Token{Source: source, Start: 0, End: len(source.Runes)}
 }
 
 func nodeOf(v interface{}) Node {
diff --git a/parse/parser.go b/parse/parser.go
index 2692a98ad..972301ac2 100644
--- a/parse/parser.go
+++ b/parse/parser.go
@@ -37,11 +37,11 @@ type LeafParser func(p *Parser, cst *Leaf)
 // Given a root parse function, the input string and the Skip controller, it builds a
 // and initializes a Parser, runs the root using it, verifies it worked
 // correctly and then returns the errors generated if any.
-func Parse(root BranchParser, data string, skip Skip) []Error {
+func Parse(root BranchParser, filename, data string, skip Skip) []Error {
 	p := &Parser{
 		skip: skip,
 	}
-	p.setData(data)
+	p.setData(filename, data)
 	p.parse(root)
 	return p.Errors
 }
diff --git a/parse/parser_test.go b/parse/parser_test.go
index a39fc5a7e..c2dc46b62 100644
--- a/parse/parser_test.go
+++ b/parse/parser_test.go
@@ -121,7 +121,7 @@ func TestErrorLimit(t *testing.T) {
 				t.Fatalf("Parsing not terminated after %v errors", i)
 			}
 		}
-	}, "", NewSkip("//", "/*", "*/"))
+	}, "parser_test.api", "", NewSkip("//", "/*", "*/"))
 	if len(errs) != ParseErrorLimit {
 		t.Fatalf("Expected %v errors, got %v", ParseErrorLimit, len(errs))
 	}
@@ -138,7 +138,7 @@ func TestCursor(t *testing.T) {
 		content += " "
 	}
 	content += "@  \n  "
-	errs := Parse(root.parse, content, NewSkip("//", "/*", "*/"))
+	errs := Parse(root.parse, "parser_test.api", content, NewSkip("//", "/*", "*/"))
 	if len(errs) == 0 {
 		t.Fatalf("Expected errors")
 	}
@@ -161,7 +161,7 @@ func TestCustomPanic(t *testing.T) {
 			t.Fatalf("Expected custom panic recovery")
 		}
 	}()
-	Parse(func(p *Parser, _ *Branch) { panic(custom) }, "", NewSkip("//", "/*", "*/"))
+	Parse(func(p *Parser, _ *Branch) { panic(custom) }, "parser_test.api", "", NewSkip("//", "/*", "*/"))
 }
 
 func testParse(t *testing.T, content string, cst Node, ast *listNode) {
@@ -171,7 +171,7 @@ func testParse(t *testing.T, content string, cst Node, ast *listNode) {
 		gotCst = cst
 		root.parse(p, cst)
 	}
-	errs := Parse(rootParse, content, NewSkip("//", "/*", "*/"))
+	errs := Parse(rootParse, "parser_test.api", content, NewSkip("//", "/*", "*/"))
 	if len(errs) > 0 {
 		for _, e := range errs {
 			line, column := e.At.Token().Cursor()
@@ -198,7 +198,7 @@ func testParse(t *testing.T, content string, cst Node, ast *listNode) {
 }
 
 func testCustomFail(t *testing.T, content string, do BranchParser) {
-	errs := Parse(do, content, NewSkip("//", "/*", "*/"))
+	errs := Parse(do, "parser_test.api", content, NewSkip("//", "/*", "*/"))
 	if len(errs) == 0 {
 		t.Fatalf("Expected errors")
 	} else {
diff --git a/parse/reader.go b/parse/reader.go
index 241f15240..564ee3454 100644
--- a/parse/reader.go
+++ b/parse/reader.go
@@ -22,18 +22,20 @@ import (
 
 // Reader is the interface to an object that converts a rune array into tokens.
 type Reader struct {
-	runes  []rune // The string being parsed.
-	offset int    // The start of the current token.
-	cursor int    // The offset of the next unparsed rune.
+	Source *Source // The source being parsed.
+	runes  []rune  // The string being parsed.
+	offset int     // The start of the current token.
+	cursor int     // The offset of the next unparsed rune.
 }
 
-func (r *Reader) setData(data string) {
+func (r *Reader) setData(filename string, data string) {
 	r.runes = bytes.Runes([]byte(data))
+	r.Source = &Source{Filename: filename, Runes: r.runes}
 }
 
 // Token peeks at the current scanned token value. It does not consume anything.
 func (r *Reader) Token() Token {
-	return Token{Runes: r.runes, Start: r.offset, End: r.cursor}
+	return Token{Source: r.Source, Start: r.offset, End: r.cursor}
 }
 
 // Consume consumes the current token.
@@ -315,8 +317,8 @@ func (r *Reader) AlphaNumeric() bool {
 }
 
 // NewReader creates a new reader which reads from the supplied string.
-func NewReader(data string) *Reader {
+func NewReader(filename string, data string) *Reader {
 	r := &Reader{}
-	r.setData(data)
+	r.setData(filename, data)
 	return r
 }
diff --git a/parse/reader_test.go b/parse/reader_test.go
index bf40546b7..db39a5107 100644
--- a/parse/reader_test.go
+++ b/parse/reader_test.go
@@ -43,7 +43,7 @@ var numericTests = []struct {
 
 func TestNumeric(t *testing.T) {
 	for _, test := range numericTests {
-		r := NewReader(test.in)
+		r := NewReader("reader_test.api", test.in)
 		gotKind := r.Numeric()
 		gotTok := r.Token()
 
diff --git a/parse/source.go b/parse/source.go
new file mode 100644
index 000000000..c2ca291bc
--- /dev/null
+++ b/parse/source.go
@@ -0,0 +1,20 @@
+// Copyright (C) 2015 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parse
+
+type Source struct {
+	Filename string // The path which was used to load the source (if any).
+	Runes    []rune // The full content of the source file.
+}
diff --git a/parse/token.go b/parse/token.go
index d93ad7fe3..8150314a0 100644
--- a/parse/token.go
+++ b/parse/token.go
@@ -23,9 +23,9 @@ const RuneEOL = '\n'
 
 // A Token represents the smallest consumed unit input.
 type Token struct {
-	Runes []rune // The full rune array for the string this token is from.
-	Start int    // The start of the token in the full rune array.
-	End   int    // One past the end of the token.
+	Source *Source // The source object this token is from (including the full rune array).
+	Start  int     // The start of the token in the full rune array.
+	End    int     // One past the end of the token.
 }
 
 // Format implements fmt.Formatter writing the start end and value of the token.
@@ -35,10 +35,10 @@ func (t Token) Format(f fmt.State, c rune) {
 
 // String returns the string form of the rune range the token represents.
 func (t Token) String() string {
-	if t.Start >= t.End || len(t.Runes) == 0 {
+	if t.Start >= t.End || len(t.Source.Runes) == 0 {
 		return ""
 	}
-	return string(t.Runes[t.Start:t.End])
+	return string(t.Source.Runes[t.Start:t.End])
 }
 
 // Cursor is used to calculate the line and column of the start of the token.
@@ -47,7 +47,7 @@ func (t Token) String() string {
 func (t Token) Cursor() (line int, column int) {
 	line = 1
 	column = 1
-	for _, r := range t.Runes[:t.Start] {
+	for _, r := range t.Source.Runes[:t.Start] {
 		if r == RuneEOL {
 			line++
 			column = 0
diff --git a/tools/generate_gles_api/generate_api.go b/tools/generate_gles_api/generate_api.go
index ee143b8d9..f443c6c4e 100644
--- a/tools/generate_gles_api/generate_api.go
+++ b/tools/generate_gles_api/generate_api.go
@@ -375,7 +375,7 @@ func printCommand(out io.Writer, reg *Registry, cmd *Command, api KhronosAPI) {
 				if len(childs) > 0 {
 					start := apiCmd.Block.CST.Children[0].Token()
 					end := apiCmd.Block.CST.Children[len(childs)-1].Token()
-					fmt.Fprintf(out, "%s", string(start.Runes[start.End:end.Start]))
+					fmt.Fprintf(out, "%s", string(start.Source.Runes[start.End:end.Start]))
 					foundOldCode = true
 				}
 			}
@@ -408,7 +408,7 @@ func oldParamType(cmdName string, paramIndex int) (string, bool) {
 		for _, cmd := range oldApi.Commands {
 			if cmd.Name.Value == cmdName {
 				token := cmd.Parameters[paramIndex].Type.Node().Token()
-				return string(token.Runes[token.Start:token.End]), true
+				return string(token.Source.Runes[token.Start:token.End]), true
 			}
 		}
 	}
diff --git a/tools/generate_gles_api/main.go b/tools/generate_gles_api/main.go
index 4771814cc..bf48deb7f 100644
--- a/tools/generate_gles_api/main.go
+++ b/tools/generate_gles_api/main.go
@@ -41,7 +41,7 @@ func main() {
 		if err != nil {
 			panic(err)
 		}
-		api, errs := parser.Parse(string(f))
+		api, errs := parser.Parse(*oldApiPath, string(f))
 		if len(errs) > 0 {
 			for i, e := range errs {
 				fmt.Printf("%d: %v", i, e)
-- 
cgit v1.2.3