diff options
| -rw-r--r-- | ast.go | 4 | ||||
| -rw-r--r-- | buf.go | 210 | ||||
| -rw-r--r-- | buf_test.go (renamed from ioutil_test.go) | 0 | ||||
| -rw-r--r-- | eval.go | 36 | ||||
| -rw-r--r-- | evalcmd.go | 4 | ||||
| -rw-r--r-- | expr.go | 12 | ||||
| -rw-r--r-- | fileutil.go | 5 | ||||
| -rw-r--r-- | func.go | 217 | ||||
| -rw-r--r-- | func_test.go | 6 | ||||
| -rw-r--r-- | ioutil.go | 147 | ||||
| -rw-r--r-- | shellutil.go | 36 | ||||
| -rw-r--r-- | var.go | 16 |
12 files changed, 392 insertions, 301 deletions
@@ -50,8 +50,8 @@ func (ast *assignAST) evalRHS(ev *Evaluator, lhs string) (Var, error) { case tmpval: return &simpleVar{value: v.String(), origin: origin}, nil default: - var buf buffer - buf.resetSpace() + var buf evalBuffer + buf.resetSep() err := v.Eval(&buf, ev) if err != nil { return nil, err @@ -0,0 +1,210 @@ +// Copyright 2015 Google Inc. All rights reserved +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kati + +import ( + "io" + "sync" +) + +var ( + ebufFree = sync.Pool{ + New: func() interface{} { return new(evalBuffer) }, + } + wbufFree = sync.Pool{ + New: func() interface{} { return new(wordBuffer) }, + } +) + +func writeByte(w io.Writer, b byte) error { + if bw, ok := w.(io.ByteWriter); ok { + return bw.WriteByte(b) + } + _, err := w.Write([]byte{b}) + return err +} + +// use io.WriteString to stringWrite. + +type ssvWriter struct { + io.Writer + sep bool +} + +func (w *ssvWriter) writeWord(word []byte) { + if w.sep { + writeByte(w.Writer, ' ') + } + w.sep = true + w.Writer.Write(word) +} + +func (w *ssvWriter) writeWordString(word string) { + if w.sep { + writeByte(w.Writer, ' ') + } + w.sep = true + io.WriteString(w.Writer, word) +} + +func (w *ssvWriter) resetSep() { + w.sep = false +} + +type buffer struct { + buf []byte + bootstrap [64]byte // memory to hold first slice +} + +func (b *buffer) Write(data []byte) (int, error) { + b.buf = append(b.buf, data...) + return len(data), nil +} + +func (b *buffer) WriteByte(c byte) error { + b.buf = append(b.buf, c) + return nil +} + +func (b *buffer) WriteString(s string) (int, error) { + b.buf = append(b.buf, []byte(s)...) + return len(s), nil +} + +func (b *buffer) Bytes() []byte { return b.buf } +func (b *buffer) Len() int { return len(b.buf) } +func (b *buffer) String() string { return string(b.buf) } + +func (b *buffer) Reset() { + if b.buf == nil { + b.buf = b.bootstrap[:0] + } + b.buf = b.buf[:0] +} + +type evalBuffer struct { + buffer + ssvWriter + args [][]byte +} + +func newEbuf() *evalBuffer { + buf := ebufFree.Get().(*evalBuffer) + buf.Reset() + return buf +} + +func (buf *evalBuffer) release() { + if cap(buf.Bytes()) > 1024 { + return + } + buf.Reset() + buf.args = buf.args[:0] + ebufFree.Put(buf) +} + +func (b *evalBuffer) Reset() { + b.buffer.Reset() + b.resetSep() +} + +func (b *evalBuffer) resetSep() { + if b.ssvWriter.Writer == nil { + b.ssvWriter.Writer = &b.buffer + } + b.ssvWriter.resetSep() +} + +type wordBuffer struct { + buf buffer + words [][]byte +} + +func newWbuf() *wordBuffer { + buf := wbufFree.Get().(*wordBuffer) + buf.Reset() + return buf +} + +func (buf *wordBuffer) release() { + if cap(buf.Bytes()) > 1024 { + return + } + buf.Reset() + wbufFree.Put(buf) +} + +func (wb *wordBuffer) Write(data []byte) (int, error) { + if len(data) == 0 { + return 0, nil + } + off := len(wb.buf.buf) + var cont bool + if !isWhitespace(rune(data[0])) && len(wb.buf.buf) > 0 { + cont = !isWhitespace(rune(wb.buf.buf[off-1])) + } + ws := newWordScanner(data) + for ws.Scan() { + if cont { + word := wb.words[len(wb.words)-1] + wb.words = wb.words[:len(wb.words)-1] + wb.buf.buf = wb.buf.buf[:len(wb.buf.buf)-len(word)] + var w []byte + w = append(w, word...) + w = append(w, ws.Bytes()...) + wb.writeWord(w) + cont = false + continue + } + wb.writeWord(ws.Bytes()) + } + if isWhitespace(rune(data[len(data)-1])) { + wb.buf.buf = append(wb.buf.buf, ' ') + } + return len(data), nil +} + +func (wb *wordBuffer) WriteByte(c byte) error { + _, err := wb.Write([]byte{c}) + return err +} + +func (wb *wordBuffer) WriteString(s string) (int, error) { + return wb.Write([]byte(s)) +} + +func (wb *wordBuffer) writeWord(word []byte) { + if len(wb.buf.buf) > 0 { + wb.buf.buf = append(wb.buf.buf, ' ') + } + off := len(wb.buf.buf) + wb.buf.buf = append(wb.buf.buf, word...) + wb.words = append(wb.words, wb.buf.buf[off:off+len(word)]) +} + +func (wb *wordBuffer) writeWordString(word string) { + wb.writeWord([]byte(word)) +} + +func (wb *wordBuffer) Reset() { + wb.buf.Reset() + wb.words = nil +} + +func (wb *wordBuffer) resetSep() {} + +func (wb *wordBuffer) Bytes() []byte { + return wb.buf.Bytes() +} diff --git a/ioutil_test.go b/buf_test.go index 8ee14a0..8ee14a0 100644 --- a/ioutil_test.go +++ b/buf_test.go @@ -171,10 +171,10 @@ func NewEvaluator(vars map[string]Var) *Evaluator { } } -func (ev *Evaluator) args(buf *buffer, args ...Value) ([][]byte, error) { +func (ev *Evaluator) args(buf *evalBuffer, args ...Value) ([][]byte, error) { pos := make([]int, 0, len(args)) for _, arg := range args { - buf.resetSpace() + buf.resetSep() err := arg.Eval(buf, ev) if err != nil { return nil, err @@ -217,13 +217,13 @@ func (ev *Evaluator) evalAssignAST(ast *assignAST) (string, Var, error) { case tmpval: lhs = string(v) default: - buf := newBuf() + buf := newEbuf() err := v.Eval(buf, ev) if err != nil { return "", nil, err } lhs = string(trimSpaceBytes(buf.Bytes())) - freeBuf(buf) + buf.release() } rhs, err := ast.evalRHS(ev, lhs) if err != nil { @@ -257,13 +257,13 @@ func (ev *Evaluator) evalMaybeRule(ast *maybeRuleAST) error { logf("maybe rule %s: %q assign:%v", ev.srcpos, ast.expr, ast.assign) - abuf := newBuf() + abuf := newEbuf() aexpr := toExpr(ast.expr) var rhs expr semi := ast.semi for i, v := range aexpr { - var buf buffer - buf.resetSpace() + var buf evalBuffer + buf.resetSep() err := v.Eval(&buf, ev) if err != nil { return err @@ -302,7 +302,7 @@ func (ev *Evaluator) evalMaybeRule(ast *maybeRuleAST) error { if err != nil { return ast.error(err) } - freeBuf(abuf) + abuf.release() if LogFlag { logf("rule %q assign:%v rhs:%v=> outputs:%q, inputs:%q", ast.expr, ast.assign, rhs, r.outputs, r.inputs) } @@ -397,8 +397,8 @@ func (ev *Evaluator) lookupVarInCurrentScope(name string) Var { // Only for a few special uses such as getting SHELL and handling // export/unexport. func (ev *Evaluator) EvaluateVar(name string) (string, error) { - var buf buffer - buf.resetSpace() + var buf evalBuffer + buf.resetSep() err := ev.LookupVar(name).Eval(&buf, ev) if err != nil { return "", err @@ -437,8 +437,8 @@ func (ev *Evaluator) evalInclude(ast *includeAST) error { if err != nil { return ast.errorf("parse failed: %q: %v", ast.expr, err) } - var buf buffer - buf.resetSpace() + var buf evalBuffer + buf.resetSep() err = v.Eval(&buf, ev) if err != nil { return ast.errorf("%v", err) @@ -492,7 +492,7 @@ func (ev *Evaluator) evalIf(iast *ifAST) error { switch iast.op { case "ifdef", "ifndef": expr := iast.lhs - buf := newBuf() + buf := newEbuf() err := expr.Eval(buf, ev) if err != nil { return iast.errorf("%v\n expr:%s", err, expr) @@ -505,7 +505,7 @@ func (ev *Evaluator) evalIf(iast *ifAST) error { } value := buf.String() val := buf.Len() - freeBuf(buf) + buf.release() isTrue = (val > 0) == (iast.op == "ifdef") if LogFlag { logf("%s lhs=%q value=%q => %t", iast.op, iast.lhs, value, isTrue) @@ -513,14 +513,14 @@ func (ev *Evaluator) evalIf(iast *ifAST) error { case "ifeq", "ifneq": lexpr := iast.lhs rexpr := iast.rhs - buf := newBuf() + buf := newEbuf() params, err := ev.args(buf, lexpr, rexpr) if err != nil { return iast.errorf("%v\n (%s,%s)", err, lexpr, rexpr) } lhs := string(params[0]) rhs := string(params[1]) - freeBuf(buf) + buf.release() isTrue = (lhs == rhs) == (iast.op == "ifeq") if LogFlag { logf("%s lhs=%q %q rhs=%q %q => %t", iast.op, iast.lhs, lhs, iast.rhs, rhs, isTrue) @@ -552,8 +552,8 @@ func (ev *Evaluator) evalExport(ast *exportAST) error { if err != nil { return ast.errorf("failed to parse: %q: %v", string(ast.expr), err) } - var buf buffer - buf.resetSpace() + var buf evalBuffer + buf.resetSep() err = v.Eval(&buf, ev) if err != nil { return ast.errorf("%v\n expr:%s", err, v) @@ -239,13 +239,13 @@ func (r runner) eval(ev *Evaluator, s string) ([]runner, error) { if err != nil { return nil, ev.errorf("parse cmd %q: %v", r.cmd, err) } - buf := newBuf() + buf := newEbuf() err = expr.Eval(buf, ev) if err != nil { return nil, err } cmds := buf.String() - freeBuf(buf) + buf.release() logf("evalcmd: %q => %q", r.cmd, cmds) var runners []runner for _, cmd := range strings.Split(cmds, "\n") { @@ -35,7 +35,7 @@ type evalWriter interface { io.Writer writeWord([]byte) writeWordString(string) - resetSpace() + resetSep() } // Value is an interface for value. @@ -92,7 +92,7 @@ func (e expr) String() string { func (e expr) Eval(w evalWriter, ev *Evaluator) error { for _, v := range e { - w.resetSpace() + w.resetSep() err := v.Eval(w, ev) if err != nil { return err @@ -153,13 +153,13 @@ func (v *varref) String() string { func (v *varref) Eval(w evalWriter, ev *Evaluator) error { te := traceEvent.begin("var", v, traceEventMain) - buf := newBuf() + buf := newEbuf() err := v.varname.Eval(buf, ev) if err != nil { return err } vv := ev.LookupVar(buf.String()) - freeBuf(buf) + buf.release() err = vv.Eval(w, ev) if err != nil { return err @@ -234,7 +234,7 @@ func (v varsubst) String() string { func (v varsubst) Eval(w evalWriter, ev *Evaluator) error { te := traceEvent.begin("varsubst", v, traceEventMain) - buf := newBuf() + buf := newEbuf() params, err := ev.args(buf, v.varname, v.pat, v.subst) if err != nil { return err @@ -249,7 +249,7 @@ func (v varsubst) Eval(w evalWriter, ev *Evaluator) error { return err } vals := splitSpaces(buf.String()) - freeBuf(buf) + buf.release() space := false for _, val := range vals { if space { diff --git a/fileutil.go b/fileutil.go index 610fd73..fd721ef 100644 --- a/fileutil.go +++ b/fileutil.go @@ -38,8 +38,8 @@ func existsInVPATH(ev *Evaluator, target string) (string, bool) { } // TODO(ukai): support vpath directive (pattern vpath). // TODO(ukai): ok to cache vpath value? - var wb wordBuffer - err := vpath.Eval(&wb, ev) + wb := newWbuf() + err := vpath.Eval(wb, ev) if err != nil { return target, false } @@ -53,5 +53,6 @@ func existsInVPATH(ev *Evaluator, target string) (string, bool) { } } } + wb.release() return target, false } @@ -172,7 +172,7 @@ func (f *funcSubst) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() fargs, err := ev.args(abuf, f.args[1:]...) if err != nil { return err @@ -188,7 +188,7 @@ func (f *funcSubst) Eval(w evalWriter, ev *Evaluator) error { } else { w.Write(bytes.Replace(text, from, to, -1)) } - freeBuf(abuf) + abuf.release() stats.add("funcbody", "subst", t) return nil } @@ -201,13 +201,13 @@ func (f *funcPatsubst) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() fargs, err := ev.args(abuf, f.args[1], f.args[2]) if err != nil { return err } - var wb wordBuffer - err = f.args[3].Eval(&wb, ev) + wb := newWbuf() + err = f.args[3].Eval(wb, ev) if err != nil { return err } @@ -224,7 +224,8 @@ func (f *funcPatsubst) Eval(w evalWriter, ev *Evaluator) error { } w.writeWord(sword) } - freeBuf(abuf) + abuf.release() + wb.release() stats.add("funcbody", "patsubst", t) return nil } @@ -237,8 +238,8 @@ func (f *funcStrip) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var wb wordBuffer - err = f.args[1].Eval(&wb, ev) + wb := newWbuf() + err = f.args[1].Eval(wb, ev) if err != nil { return err } @@ -246,6 +247,7 @@ func (f *funcStrip) Eval(w evalWriter, ev *Evaluator) error { for _, word := range wb.words { w.writeWord(word) } + wb.release() stats.add("funcbody", "strip", t) return nil } @@ -258,7 +260,7 @@ func (f *funcFindstring) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() fargs, err := ev.args(abuf, f.args[1:]...) if err != nil { return err @@ -269,7 +271,7 @@ func (f *funcFindstring) Eval(w evalWriter, ev *Evaluator) error { if bytes.Index(text, find) >= 0 { w.Write(find) } - freeBuf(abuf) + abuf.release() stats.add("funcbody", "findstring", t) return nil } @@ -282,13 +284,13 @@ func (f *funcFilter) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var patternsBuffer wordBuffer - err = f.args[1].Eval(&patternsBuffer, ev) + patternsBuffer := newWbuf() + err = f.args[1].Eval(patternsBuffer, ev) if err != nil { return err } - var textBuffer wordBuffer - err = f.args[2].Eval(&textBuffer, ev) + textBuffer := newWbuf() + err = f.args[2].Eval(textBuffer, ev) if err != nil { return err } @@ -300,6 +302,8 @@ func (f *funcFilter) Eval(w evalWriter, ev *Evaluator) error { } } } + patternsBuffer.release() + textBuffer.release() stats.add("funcbody", "filter", t) return nil } @@ -312,13 +316,13 @@ func (f *funcFilterOut) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var patternsBuffer wordBuffer - err = f.args[1].Eval(&patternsBuffer, ev) + patternsBuffer := newWbuf() + err = f.args[1].Eval(patternsBuffer, ev) if err != nil { return err } - var textBuffer wordBuffer - err = f.args[2].Eval(&textBuffer, ev) + textBuffer := newWbuf() + err = f.args[2].Eval(textBuffer, ev) if err != nil { return err } @@ -332,6 +336,8 @@ Loop: } w.writeWord(text) } + patternsBuffer.release() + textBuffer.release() stats.add("funcbody", "filter-out", t) return err } @@ -344,8 +350,8 @@ func (f *funcSort) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var wb wordBuffer - err = f.args[1].Eval(&wb, ev) + wb := newWbuf() + err = f.args[1].Eval(wb, ev) if err != nil { return err } @@ -354,6 +360,7 @@ func (f *funcSort) Eval(w evalWriter, ev *Evaluator) error { for _, tok := range wb.words { toks = append(toks, string(tok)) } + wb.release() sort.Strings(toks) // Remove duplicate words. @@ -377,13 +384,13 @@ func (f *funcWord) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() err = f.args[1].Eval(abuf, ev) if err != nil { return err } v := string(trimSpaceBytes(abuf.Bytes())) - freeBuf(abuf) + abuf.release() index, ok := numericValueForFunc(v) if !ok { return ev.errorf(`*** non-numeric first argument to "word" function: %q.`, v) @@ -391,8 +398,8 @@ func (f *funcWord) Eval(w evalWriter, ev *Evaluator) error { if index == 0 { return ev.errorf(`*** first argument to "word" function must be greater than 0.`) } - var wb wordBuffer - err = f.args[2].Eval(&wb, ev) + wb := newWbuf() + err = f.args[2].Eval(wb, ev) if err != nil { return err } @@ -400,6 +407,7 @@ func (f *funcWord) Eval(w evalWriter, ev *Evaluator) error { if index-1 < len(wb.words) { w.writeWord(wb.words[index-1]) } + wb.release() stats.add("funcbody", "word", t) return err } @@ -412,7 +420,7 @@ func (f *funcWordlist) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() fargs, err := ev.args(abuf, f.args[1], f.args[2]) if err != nil { return err @@ -431,10 +439,10 @@ func (f *funcWordlist) Eval(w evalWriter, ev *Evaluator) error { if !ok { return ev.errorf(`*** non-numeric second argument to "wordlist" function: %q.`, v) } - freeBuf(abuf) + abuf.release() - var wb wordBuffer - err = f.args[3].Eval(&wb, ev) + wb := newWbuf() + err = f.args[3].Eval(wb, ev) if err != nil { return err } @@ -443,6 +451,7 @@ func (f *funcWordlist) Eval(w evalWriter, ev *Evaluator) error { w.writeWord(word) } } + wb.release() stats.add("funcbody", "wordlist", t) return nil } @@ -455,13 +464,14 @@ func (f *funcWords) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var wb wordBuffer - err = f.args[1].Eval(&wb, ev) + wb := newWbuf() + err = f.args[1].Eval(wb, ev) if err != nil { return err } t := time.Now() n := len(wb.words) + wb.release() w.writeWordString(strconv.Itoa(n)) stats.add("funcbody", "words", t) return nil @@ -475,8 +485,8 @@ func (f *funcFirstword) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var wb wordBuffer - err = f.args[1].Eval(&wb, ev) + wb := newWbuf() + err = f.args[1].Eval(wb, ev) if err != nil { return err } @@ -484,6 +494,7 @@ func (f *funcFirstword) Eval(w evalWriter, ev *Evaluator) error { if len(wb.words) > 0 { w.writeWord(wb.words[0]) } + wb.release() stats.add("funcbody", "firstword", t) return nil } @@ -496,8 +507,8 @@ func (f *funcLastword) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var wb wordBuffer - err = f.args[1].Eval(&wb, ev) + wb := newWbuf() + err = f.args[1].Eval(wb, ev) if err != nil { return err } @@ -505,6 +516,7 @@ func (f *funcLastword) Eval(w evalWriter, ev *Evaluator) error { if len(wb.words) > 0 { w.writeWord(wb.words[len(wb.words)-1]) } + wb.release() stats.add("funcbody", "lastword", t) return err } @@ -519,13 +531,13 @@ func (f *funcJoin) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var wb1 wordBuffer - err = f.args[1].Eval(&wb1, ev) + wb1 := newWbuf() + err = f.args[1].Eval(wb1, ev) if err != nil { return err } - var wb2 wordBuffer - err = f.args[2].Eval(&wb2, ev) + wb2 := newWbuf() + err = f.args[2].Eval(wb2, ev) if err != nil { return err } @@ -540,6 +552,8 @@ func (f *funcJoin) Eval(w evalWriter, ev *Evaluator) error { } w.writeWord(word) } + wb1.release() + wb2.release() stats.add("funcbody", "join", t) return nil } @@ -552,8 +566,8 @@ func (f *funcWildcard) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var wb wordBuffer - err = f.args[1].Eval(&wb, ev) + wb := newWbuf() + err = f.args[1].Eval(wb, ev) if err != nil { return err } @@ -563,6 +577,7 @@ func (f *funcWildcard) Eval(w evalWriter, ev *Evaluator) error { io.WriteString(w, "$(/bin/ls -d ") w.Write(wb.Bytes()) io.WriteString(w, " 2> /dev/null)") + wb.release() traceEvent.end(te) return nil } @@ -574,6 +589,7 @@ func (f *funcWildcard) Eval(w evalWriter, ev *Evaluator) error { return err } } + wb.release() traceEvent.end(te) stats.add("funcbody", "wildcard", t) return nil @@ -587,8 +603,8 @@ func (f *funcDir) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var wb wordBuffer - err = f.args[1].Eval(&wb, ev) + wb := newWbuf() + err = f.args[1].Eval(wb, ev) if err != nil { return err } @@ -601,6 +617,7 @@ func (f *funcDir) Eval(w evalWriter, ev *Evaluator) error { } w.writeWordString(name + string(filepath.Separator)) } + wb.release() stats.add("funcbody", "dir", t) return nil } @@ -613,8 +630,8 @@ func (f *funcNotdir) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var wb wordBuffer - err = f.args[1].Eval(&wb, ev) + wb := newWbuf() + err = f.args[1].Eval(wb, ev) if err != nil { return err } @@ -627,6 +644,7 @@ func (f *funcNotdir) Eval(w evalWriter, ev *Evaluator) error { } w.writeWordString(filepath.Base(name)) } + wb.release() stats.add("funcbody", "notdir", t) return nil } @@ -639,8 +657,8 @@ func (f *funcSuffix) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var wb wordBuffer - err = f.args[1].Eval(&wb, ev) + wb := newWbuf() + err = f.args[1].Eval(wb, ev) if err != nil { return err } @@ -652,6 +670,7 @@ func (f *funcSuffix) Eval(w evalWriter, ev *Evaluator) error { w.writeWordString(e) } } + wb.release() stats.add("funcbody", "suffix", t) return err } @@ -664,8 +683,8 @@ func (f *funcBasename) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var wb wordBuffer - err = f.args[1].Eval(&wb, ev) + wb := newWbuf() + err = f.args[1].Eval(wb, ev) if err != nil { return err } @@ -675,6 +694,7 @@ func (f *funcBasename) Eval(w evalWriter, ev *Evaluator) error { e := stripExt(tok) w.writeWordString(e) } + wb.release() stats.add("funcbody", "basename", t) return nil } @@ -687,13 +707,13 @@ func (f *funcAddsuffix) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() err = f.args[1].Eval(abuf, ev) if err != nil { return err } - var wb wordBuffer - err = f.args[2].Eval(&wb, ev) + wb := newWbuf() + err = f.args[2].Eval(wb, ev) if err != nil { return err } @@ -705,7 +725,8 @@ func (f *funcAddsuffix) Eval(w evalWriter, ev *Evaluator) error { name = append(name, suf...) w.writeWord(name) } - freeBuf(abuf) + wb.release() + abuf.release() stats.add("funcbody", "addsuffix", t) return err } @@ -718,14 +739,14 @@ func (f *funcAddprefix) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() err = f.args[1].Eval(abuf, ev) if err != nil { return err } pre := abuf.Bytes() - var wb wordBuffer - err = f.args[2].Eval(&wb, ev) + wb := newWbuf() + err = f.args[2].Eval(wb, ev) if err != nil { return err } @@ -736,7 +757,8 @@ func (f *funcAddprefix) Eval(w evalWriter, ev *Evaluator) error { name = append(name, word...) w.writeWord(name) } - freeBuf(abuf) + wb.release() + abuf.release() stats.add("funcbody", "addprefix", t) return err } @@ -754,8 +776,8 @@ func (f *funcRealpath) Eval(w evalWriter, ev *Evaluator) error { ev.hasIO = true return nil } - var wb wordBuffer - err = f.args[1].Eval(&wb, ev) + wb := newWbuf() + err = f.args[1].Eval(wb, ev) if err != nil { return err } @@ -774,6 +796,7 @@ func (f *funcRealpath) Eval(w evalWriter, ev *Evaluator) error { } w.writeWordString(name) } + wb.release() stats.add("funcbody", "realpath", t) return err } @@ -786,8 +809,8 @@ func (f *funcAbspath) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - var wb wordBuffer - err = f.args[1].Eval(&wb, ev) + wb := newWbuf() + err = f.args[1].Eval(wb, ev) if err != nil { return err } @@ -801,6 +824,7 @@ func (f *funcAbspath) Eval(w evalWriter, ev *Evaluator) error { } w.writeWordString(name) } + wb.release() stats.add("funcbody", "abspath", t) return nil } @@ -814,16 +838,16 @@ func (f *funcIf) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() err = f.args[1].Eval(abuf, ev) if err != nil { return err } if len(abuf.Bytes()) != 0 { - freeBuf(abuf) + abuf.release() return f.args[2].Eval(w, ev) } - freeBuf(abuf) + abuf.release() if len(f.args) > 3 { return f.args[3].Eval(w, ev) } @@ -838,7 +862,7 @@ func (f *funcAnd) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return nil } - abuf := newBuf() + abuf := newEbuf() var cond []byte for _, arg := range f.args[1:] { abuf.Reset() @@ -848,12 +872,12 @@ func (f *funcAnd) Eval(w evalWriter, ev *Evaluator) error { } cond = abuf.Bytes() if len(cond) == 0 { - freeBuf(abuf) + abuf.release() return nil } } w.Write(cond) - freeBuf(abuf) + abuf.release() return nil } @@ -865,7 +889,7 @@ func (f *funcOr) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() for _, arg := range f.args[1:] { abuf.Reset() err = arg.Eval(abuf, ev) @@ -875,11 +899,11 @@ func (f *funcOr) Eval(w evalWriter, ev *Evaluator) error { cond := abuf.Bytes() if len(cond) != 0 { w.Write(cond) - freeBuf(abuf) + abuf.release() return nil } } - freeBuf(abuf) + abuf.release() return nil } @@ -910,7 +934,7 @@ func (f *funcShell) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() err = f.args[1].Eval(abuf, ev) if err != nil { return err @@ -922,11 +946,11 @@ func (f *funcShell) Eval(w evalWriter, ev *Evaluator) error { w.Write(abuf.Bytes()) writeByte(w, ')') traceEvent.end(te) - freeBuf(abuf) + abuf.release() return nil } arg := abuf.String() - freeBuf(abuf) + abuf.release() shellVar, err := ev.EvaluateVar("SHELL") if err != nil { return err @@ -985,7 +1009,7 @@ type funcCall struct{ fclosure } func (f *funcCall) Arity() int { return 0 } func (f *funcCall) Eval(w evalWriter, ev *Evaluator) error { - abuf := newBuf() + abuf := newEbuf() fargs, err := ev.args(abuf, f.args[1:]...) if err != nil { return err @@ -1028,7 +1052,7 @@ func (f *funcCall) Eval(w evalWriter, ev *Evaluator) error { if LogFlag { logf("call %q variable %q return %q", f.args[1], variable, buf.Bytes()) } - freeBuf(abuf) + abuf.release() return nil } @@ -1041,13 +1065,13 @@ func (f *funcValue) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() err = f.args[1].Eval(abuf, ev) if err != nil { return err } v := ev.LookupVar(abuf.String()) - freeBuf(abuf) + abuf.release() io.WriteString(w, v.String()) return nil } @@ -1061,7 +1085,7 @@ func (f *funcEval) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() err = f.args[1].Eval(abuf, ev) if err != nil { return err @@ -1079,7 +1103,7 @@ func (f *funcEval) Eval(w evalWriter, ev *Evaluator) error { return err } } - freeBuf(abuf) + abuf.release() return nil } @@ -1194,8 +1218,8 @@ func (f *funcEvalAssign) String() string { } func (f *funcEvalAssign) Eval(w evalWriter, ev *Evaluator) error { - var abuf buffer - abuf.resetSpace() + var abuf evalBuffer + abuf.resetSep() err := f.rhs.Eval(&abuf, ev) if err != nil { return err @@ -1211,13 +1235,13 @@ func (f *funcEvalAssign) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return ev.errorf("eval assign error: %q: %v", f.String(), err) } - vbuf := newBuf() + vbuf := newEbuf() err = exp.Eval(vbuf, ev) if err != nil { return err } rvalue = &simpleVar{value: vbuf.String(), origin: "file"} - freeBuf(vbuf) + vbuf.release() case "=": rvalue = &recursiveVar{expr: tmpval(rhs), origin: "file"} case "+=": @@ -1271,13 +1295,13 @@ func (f *funcOrigin) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() err = f.args[1].Eval(abuf, ev) if err != nil { return err } v := ev.LookupVar(abuf.String()) - freeBuf(abuf) + abuf.release() io.WriteString(w, v.Origin()) return nil } @@ -1291,13 +1315,13 @@ func (f *funcFlavor) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() err = f.args[1].Eval(abuf, ev) if err != nil { return err } v := ev.LookupVar(abuf.String()) - freeBuf(abuf) + abuf.release() io.WriteString(w, v.Flavor()) return nil } @@ -1316,13 +1340,13 @@ func (f *funcInfo) Eval(w evalWriter, ev *Evaluator) error { ev.hasIO = true return nil } - abuf := newBuf() + abuf := newEbuf() err = f.args[1].Eval(abuf, ev) if err != nil { return err } fmt.Printf("%s\n", abuf.String()) - freeBuf(abuf) + abuf.release() return nil } @@ -1339,13 +1363,13 @@ func (f *funcWarning) Eval(w evalWriter, ev *Evaluator) error { ev.hasIO = true return nil } - abuf := newBuf() + abuf := newEbuf() err = f.args[1].Eval(abuf, ev) if err != nil { return err } fmt.Printf("%s: %s\n", ev.srcpos, abuf.String()) - freeBuf(abuf) + abuf.release() return nil } @@ -1362,8 +1386,8 @@ func (f *funcError) Eval(w evalWriter, ev *Evaluator) error { ev.hasIO = true return nil } - var abuf buffer - abuf.resetSpace() + var abuf evalBuffer + abuf.resetSep() err = f.args[1].Eval(&abuf, ev) if err != nil { return err @@ -1381,15 +1405,15 @@ func (f *funcForeach) Eval(w evalWriter, ev *Evaluator) error { if err != nil { return err } - abuf := newBuf() + abuf := newEbuf() err = f.args[1].Eval(abuf, ev) if err != nil { return err } varname := string(abuf.Bytes()) - freeBuf(abuf) - var wb wordBuffer - err = f.args[2].Eval(&wb, ev) + abuf.release() + wb := newWbuf() + err = f.args[2].Eval(wb, ev) if err != nil { return err } @@ -1407,6 +1431,7 @@ func (f *funcForeach) Eval(w evalWriter, ev *Evaluator) error { } space = true } + wb.release() av := ev.LookupVar(varname) if _, ok := av.(*automaticVar); ok { ev.outVars.Assign(varname, ov) diff --git a/func_test.go b/func_test.go index 8f3b81a..b56291e 100644 --- a/func_test.go +++ b/func_test.go @@ -26,7 +26,7 @@ func BenchmarkFuncStrip(b *testing.B) { }, } ev := NewEvaluator(make(map[string]Var)) - var buf buffer + var buf evalBuffer b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -45,7 +45,7 @@ func BenchmarkFuncSort(b *testing.B) { }, } ev := NewEvaluator(make(map[string]Var)) - var buf buffer + var buf evalBuffer b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -66,7 +66,7 @@ func BenchmarkFuncPatsubst(b *testing.B) { }, } ev := NewEvaluator(make(map[string]Var)) - var buf buffer + var buf evalBuffer b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/ioutil.go b/ioutil.go deleted file mode 100644 index 8f89f6f..0000000 --- a/ioutil.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2015 Google Inc. All rights reserved -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package kati - -import ( - "bytes" - "io" - "sync" -) - -var bufFree = sync.Pool{ - New: func() interface{} { return new(buffer) }, -} - -func writeByte(w io.Writer, b byte) error { - if bw, ok := w.(io.ByteWriter); ok { - return bw.WriteByte(b) - } - _, err := w.Write([]byte{b}) - return err -} - -// use io.WriteString to stringWrite. - -type ssvWriter struct { - io.Writer - space bool -} - -func (w *ssvWriter) writeWord(word []byte) { - if w.space { - writeByte(w.Writer, ' ') - } - w.space = true - w.Writer.Write(word) -} - -func (w *ssvWriter) writeWordString(word string) { - if w.space { - writeByte(w.Writer, ' ') - } - w.space = true - io.WriteString(w.Writer, word) -} - -func (w *ssvWriter) resetSpace() { - w.space = false -} - -type buffer struct { - bytes.Buffer - ssvWriter - args [][]byte -} - -func newBuf() *buffer { - buf := bufFree.Get().(*buffer) - buf.Reset() - return buf -} - -func freeBuf(buf *buffer) { - if cap(buf.Bytes()) > 1024 { - return - } - buf.Reset() - buf.args = buf.args[:0] - bufFree.Put(buf) -} - -func (b *buffer) Reset() { - b.Buffer.Reset() - b.resetSpace() -} - -func (b *buffer) resetSpace() { - if b.ssvWriter.Writer == nil { - b.ssvWriter.Writer = &b.Buffer - } - b.ssvWriter.resetSpace() -} - -type wordBuffer struct { - words [][]byte - cont bool -} - -func (wb *wordBuffer) Write(data []byte) (int, error) { - if len(data) == 0 { - return 0, nil - } - if isWhitespace(rune(data[0])) { - wb.cont = false - } - ws := newWordScanner(data) - for ws.Scan() { - if wb.cont { - word := wb.words[len(wb.words)-1] - word = append(word, ws.Bytes()...) - wb.words[len(wb.words)-1] = word - wb.cont = false - continue - } - wb.writeWord(ws.Bytes()) - } - if !isWhitespace(rune(data[len(data)-1])) { - wb.cont = true - } - return len(data), nil -} - -func (wb *wordBuffer) writeWord(word []byte) { - var w []byte - w = append(w, word...) - wb.words = append(wb.words, w) -} - -func (wb *wordBuffer) writeWordString(word string) { - wb.writeWord([]byte(word)) -} - -func (wb *wordBuffer) resetSpace() {} - -func (wb *wordBuffer) Bytes() []byte { - var sp bool - var b []byte - for _, word := range wb.words { - if sp { - b = append(b, ' ') - } - b = append(b, word...) - sp = true - } - return b -} diff --git a/shellutil.go b/shellutil.go index aed1dfe..2275a93 100644 --- a/shellutil.go +++ b/shellutil.go @@ -256,14 +256,14 @@ func rot13(buf []byte) { } func (f *funcShellAndroidRot13) Eval(w evalWriter, ev *Evaluator) error { - abuf := newBuf() + abuf := newEbuf() fargs, err := ev.args(abuf, f.v) if err != nil { return err } rot13(fargs[0]) w.Write(fargs[0]) - freeBuf(abuf) + abuf.release() return nil } @@ -273,13 +273,13 @@ type funcShellAndroidFindFileInDir struct { } func (f *funcShellAndroidFindFileInDir) Eval(w evalWriter, ev *Evaluator) error { - abuf := newBuf() + abuf := newEbuf() fargs, err := ev.args(abuf, f.dir) if err != nil { return err } dir := string(trimSpaceBytes(fargs[0])) - freeBuf(abuf) + abuf.release() logf("shellAndroidFindFileInDir %s => %s", f.dir.String(), dir) if strings.Contains(dir, "..") { logf("shellAndroidFindFileInDir contains ..: call original shell") @@ -301,15 +301,15 @@ type funcShellAndroidFindExtFilesUnder struct { } func (f *funcShellAndroidFindExtFilesUnder) Eval(w evalWriter, ev *Evaluator) error { - abuf := newBuf() + abuf := newEbuf() err := f.chdir.Eval(abuf, ev) if err != nil { return err } chdir := string(trimSpaceBytes(abuf.Bytes())) - freeBuf(abuf) - var wb wordBuffer - err = f.roots.Eval(&wb, ev) + abuf.release() + wb := newWbuf() + err = f.roots.Eval(wb, ev) if err != nil { return err } @@ -322,6 +322,7 @@ func (f *funcShellAndroidFindExtFilesUnder) Eval(w evalWriter, ev *Evaluator) er } roots = append(roots, root) } + wb.release() logf("shellAndroidFindExtFilesUnder %s,%s => %s,%s", f.chdir.String(), f.roots.String(), chdir, roots) if strings.Contains(chdir, "..") || hasDotDot { logf("shellAndroidFindExtFilesUnder contains ..: call original shell") @@ -331,16 +332,16 @@ func (f *funcShellAndroidFindExtFilesUnder) Eval(w evalWriter, ev *Evaluator) er logf("shellAndroidFindExtFilesUnder androidFindCache is not ready: call original shell") return f.funcShell.Eval(w, ev) } - buf := newBuf() + buf := newEbuf() for _, root := range roots { if !androidFindCache.findExtFilesUnder(buf, chdir, root, f.ext) { - freeBuf(buf) + buf.release() logf("shellAndroidFindExtFilesUnder androidFindCache couldn't handle: call original shell") return f.funcShell.Eval(w, ev) } } w.Write(buf.Bytes()) - freeBuf(buf) + buf.release() return nil } @@ -350,13 +351,13 @@ type funcShellAndroidFindJavaResourceFileGroup struct { } func (f *funcShellAndroidFindJavaResourceFileGroup) Eval(w evalWriter, ev *Evaluator) error { - abuf := newBuf() + abuf := newEbuf() fargs, err := ev.args(abuf, f.dir) if err != nil { return err } dir := string(trimSpaceBytes(fargs[0])) - freeBuf(abuf) + abuf.release() logf("shellAndroidFindJavaResourceFileGroup %s => %s", f.dir.String(), dir) if strings.Contains(dir, "..") { logf("shellAndroidFindJavaResourceFileGroup contains ..: call original shell") @@ -383,7 +384,7 @@ func (f *funcShellAndroidFindleaves) Eval(w evalWriter, ev *Evaluator) error { logf("shellAndroidFindleaves androidFindCache is not ready: call original shell") return f.funcShell.Eval(w, ev) } - abuf := newBuf() + abuf := newEbuf() var params []Value params = append(params, f.name) params = append(params, f.prunes...) @@ -396,10 +397,10 @@ func (f *funcShellAndroidFindleaves) Eval(w evalWriter, ev *Evaluator) error { for _, arg := range fargs[1:] { prunes = append(prunes, string(trimSpaceBytes(arg))) } - freeBuf(abuf) + abuf.release() - var wb wordBuffer - err = f.dirlist.Eval(&wb, ev) + wb := newWbuf() + err = f.dirlist.Eval(wb, ev) if err != nil { return err } @@ -412,6 +413,7 @@ func (f *funcShellAndroidFindleaves) Eval(w evalWriter, ev *Evaluator) error { } dirs = append(dirs, dir) } + wb.release() for _, dir := range dirs { androidFindCache.findleaves(w, dir, name, prunes, f.mindepth) @@ -119,7 +119,7 @@ func (v *simpleVar) Append(ev *Evaluator, s string) (Var, error) { if err != nil { return nil, err } - abuf := newBuf() + abuf := newEbuf() io.WriteString(abuf, v.value) writeByte(abuf, ' ') err = val.Eval(abuf, ev) @@ -127,12 +127,12 @@ func (v *simpleVar) Append(ev *Evaluator, s string) (Var, error) { return nil, err } v.value = abuf.String() - freeBuf(abuf) + abuf.release() return v, nil } func (v *simpleVar) AppendVar(ev *Evaluator, val Value) (Var, error) { - abuf := newBuf() + abuf := newEbuf() io.WriteString(abuf, v.value) writeByte(abuf, ' ') err := val.Eval(abuf, ev) @@ -140,7 +140,7 @@ func (v *simpleVar) AppendVar(ev *Evaluator, val Value) (Var, error) { return nil, err } v.value = abuf.String() - freeBuf(abuf) + abuf.release() return v, nil } @@ -169,10 +169,10 @@ func (v *automaticVar) Append(ev *Evaluator, s string) (Var, error) { if err != nil { return nil, err } - var buf buffer + var buf evalBuffer buf.Write(v.value) buf.WriteByte(' ') - buf.resetSpace() + buf.resetSep() err = val.Eval(&buf, ev) if err != nil { return nil, err @@ -184,10 +184,10 @@ func (v *automaticVar) Append(ev *Evaluator, s string) (Var, error) { } func (v *automaticVar) AppendVar(ev *Evaluator, val Value) (Var, error) { - var buf buffer + var buf evalBuffer buf.Write(v.value) buf.WriteByte(' ') - buf.resetSpace() + buf.resetSep() err := val.Eval(&buf, ev) if err != nil { return nil, err |