compile: optimize a+b statically for literal string/list/tuple (#97)

* compile: optimize a+b statically for literal string/list/tuple

Previously, the parser would fold two string literals "a"+"b" into a
fake string literal, to avoid quadratic runtime cost when a long chain
of strings is added (as often happens due to line length limits).
This made the syntax tree unfaithful to the source,
preventing the frontend from being used in a reformatting tool.

This change does the same optimization in the compiler,
and generalizes it to list and tuple expressions too:

   "a  b" + "c  d" => "a  bc  d"
   [a, b] + [c, d] => [a, b, c, d]
   (a, b) + (c, d) => (a, b, c, d)

It also creates the seam for later addition of the "accumulator"
optimization, in which n-ary non-literal additions such as a+b+c are
optimized to:
   acc = start(a)
   acc.add(b)
   acc.add(c)
   acc.done()
I evaluated this optimization but it doesn't yet pull its weight
on the kinds of builds I can do so far.

2 files changed, 3 insertions, 21 deletions

diff --git a/syntax/parse.go b/syntax/parse.go
index e7d3106..ed8aeaf 100644
--- a/syntax/parse.go
+++ b/syntax/parse.go
@@ -11,7 +11,7 @@ package syntax
 // package.  Verify that error positions are correct using the
 // chunkedfile mechanism.
 
-import "log"
+import log "log"
 
 // Enable this flag to print the token stream and log.Fatal on the first error.
 const debug = false
@@ -579,7 +579,7 @@ func (p *parser) parseBinopExpr(prec int) Expr {
 		op := p.tok
 		pos := p.nextToken()
 		y := p.parseTestPrec(opprec + 1)
-		x = makeBinaryExpr(op, pos, x, y)
+		x = &BinaryExpr{OpPos: pos, Op: op, X: x, Y: y}
 	}
 }
 
@@ -613,24 +613,6 @@ func init() {
 	}
 }
 
-func makeBinaryExpr(op Token, pos Position, x, y Expr) Expr {
-	// Concatenate literal strings during parsing.
-	if op == PLUS {
-		if x, ok := x.(*Literal); ok && x.Token == STRING {
-			if y, ok := y.(*Literal); ok && y.Token == STRING {
-				// The Span of this synthetic node will be wrong.
-				return &Literal{
-					Token:    STRING,
-					TokenPos: x.TokenPos,
-					Raw:      x.Raw + " + " + y.Raw, // ugh
-					Value:    x.Value.(string) + y.Value.(string),
-				}
-			}
-		}
-	}
-	return &BinaryExpr{OpPos: pos, Op: op, X: x, Y: y}
-}
-
 // primary_with_suffix = primary
 //                     | primary '.' IDENT
 //                     | primary slice_suffix
diff --git a/syntax/parse_test.go b/syntax/parse_test.go
index d511d3f..bee9388 100644
--- a/syntax/parse_test.go
+++ b/syntax/parse_test.go
@@ -85,7 +85,7 @@ func TestExprParseTrees(t *testing.T) {
 		{`-1 + +2`,
 			`(BinaryExpr X=(UnaryExpr Op=- X=1) Op=+ Y=(UnaryExpr Op=+ X=2))`},
 		{`"foo" + "bar"`,
-			`"foobar"`}, // concatenated
+			`(BinaryExpr X="foo" Op=+ Y="bar")`},
 		{`-1 * 2`, // prec(unary -) > prec(binary *)
 			`(BinaryExpr X=(UnaryExpr Op=- X=1) Op=* Y=2)`},
 		{`-x[i]`, // prec(unary -) < prec(x[i])