diff options
| -rw-r--r-- | eval.go | 41 | ||||
| -rw-r--r-- | func.go | 52 | ||||
| -rw-r--r-- | main.go | 3 | ||||
| -rw-r--r-- | parser.go | 101 | ||||
| -rw-r--r-- | pathutil.go | 18 | ||||
| -rw-r--r-- | stats.go | 44 | ||||
| -rw-r--r-- | symtab.go | 19 |
7 files changed, 159 insertions, 119 deletions
@@ -18,7 +18,7 @@ import ( "bytes" "crypto/sha1" "fmt" - "io/ioutil" + "os" "path/filepath" "strings" ) @@ -272,21 +272,11 @@ func (ev *Evaluator) EvaluateVar(name string) string { return buf.String() } -func (ev *Evaluator) evalIncludeFile(fname string, c []byte) error { +func (ev *Evaluator) evalIncludeFile(fname string, mk Makefile) error { te := traceEvent.begin("include", literal(fname), traceEventMain) defer func() { traceEvent.end(te) }() - mk, ok, err := LookupMakefileCache(fname) - if !ok { - if katiLogFlag { - Logf("Reading makefile %q", fname) - } - mk, err = ParseMakefile(c, fname) - } - if err != nil { - return err - } makefileList := ev.outVars.Lookup("MAKEFILE_LIST") makefileList = makefileList.Append(ev, mk.filename) ev.outVars.Assign("MAKEFILE_LIST", makefileList) @@ -297,19 +287,18 @@ func (ev *Evaluator) evalIncludeFile(fname string, c []byte) error { return nil } -func (ev *Evaluator) updateReadMakefile(fn string, c []byte, st FileState) { +func (ev *Evaluator) updateReadMakefile(fn string, hash [sha1.Size]byte, st FileState) { if !useCache { return } - h := sha1.Sum(c) rm, present := ev.readMks[fn] if present { switch rm.State { case FileExists: if st != FileExists { Warn(ev.filename, ev.lineno, "%s was removed after the previous read", fn) - } else if !bytes.Equal(h[:], rm.Hash[:]) { + } else if !bytes.Equal(hash[:], rm.Hash[:]) { Warn(ev.filename, ev.lineno, "%s was modified after the previous read", fn) ev.readMks[fn].State = FileInconsistent } @@ -322,12 +311,12 @@ func (ev *Evaluator) updateReadMakefile(fn string, c []byte, st FileState) { case FileInconsistent: return } - } else { - ev.readMks[fn] = &ReadMakefile{ - Filename: fn, - Hash: h, - State: st, - } + return + } + ev.readMks[fn] = &ReadMakefile{ + Filename: fn, + Hash: hash, + State: st, } } @@ -363,17 +352,17 @@ func (ev *Evaluator) evalInclude(ast *IncludeAST) { if ignoreOptionalInclude != "" && ast.op == "-include" && matchPattern(fn, ignoreOptionalInclude) { continue } - c, err := ioutil.ReadFile(fn) - if err != nil { + mk, hash, err := makefileCache.parse(fn) + if os.IsNotExist(err) { if ast.op == "include" { Error(ev.filename, ev.lineno, fmt.Sprintf("%v\nNOTE: kati does not support generating missing makefiles", err)) } else { - ev.updateReadMakefile(fn, nil, FileNotExists) + ev.updateReadMakefile(fn, hash, FileNotExists) continue } } - ev.updateReadMakefile(fn, c, FileExists) - err = ev.evalIncludeFile(fn, c) + ev.updateReadMakefile(fn, hash, FileExists) + err = ev.evalIncludeFile(fn, mk) if err != nil { panic(err) } @@ -168,7 +168,7 @@ func (f *funcSubst) Eval(w io.Writer, ev *Evaluator) { Logf("subst from:%q to:%q text:%q", from, to, text) w.Write(bytes.Replace(text, from, to, -1)) freeBuf(abuf) - addStats("funcbody", "subst", t) + stats.add("funcbody", "subst", t) } type funcPatsubst struct{ fclosure } @@ -196,7 +196,7 @@ func (f *funcPatsubst) Eval(w io.Writer, ev *Evaluator) { space = true } freeBuf(abuf) - addStats("funcbody", "patsubst", t) + stats.add("funcbody", "patsubst", t) } type funcStrip struct{ fclosure } @@ -217,7 +217,7 @@ func (f *funcStrip) Eval(w io.Writer, ev *Evaluator) { space = true } freeBuf(abuf) - addStats("funcbody", "strip", t) + stats.add("funcbody", "strip", t) } type funcFindstring struct{ fclosure } @@ -234,7 +234,7 @@ func (f *funcFindstring) Eval(w io.Writer, ev *Evaluator) { w.Write(find) } freeBuf(abuf) - addStats("funcbody", "findstring", t) + stats.add("funcbody", "findstring", t) } type funcFilter struct{ fclosure } @@ -261,7 +261,7 @@ func (f *funcFilter) Eval(w io.Writer, ev *Evaluator) { } } freeBuf(abuf) - addStats("funcbody", "filter", t) + stats.add("funcbody", "filter", t) } type funcFilterOut struct{ fclosure } @@ -290,7 +290,7 @@ Loop: sw.Write(text) } freeBuf(abuf) - addStats("funcbody", "filter-out", t) + stats.add("funcbody", "filter-out", t) } type funcSort struct{ fclosure } @@ -321,7 +321,7 @@ func (f *funcSort) Eval(w io.Writer, ev *Evaluator) { io.WriteString(w, tok) prev = tok } - addStats("funcbody", "sort", t) + stats.add("funcbody", "sort", t) } type funcWord struct{ fclosure } @@ -349,7 +349,7 @@ func (f *funcWord) Eval(w io.Writer, ev *Evaluator) { } } freeBuf(abuf) - addStats("funcbody", "word", t) + stats.add("funcbody", "word", t) } type funcWordlist struct{ fclosure } @@ -384,7 +384,7 @@ func (f *funcWordlist) Eval(w io.Writer, ev *Evaluator) { } } freeBuf(abuf) - addStats("funcbody", "wordlist", t) + stats.add("funcbody", "wordlist", t) } type funcWords struct{ fclosure } @@ -402,7 +402,7 @@ func (f *funcWords) Eval(w io.Writer, ev *Evaluator) { } freeBuf(abuf) io.WriteString(w, strconv.Itoa(n)) - addStats("funcbody", "words", t) + stats.add("funcbody", "words", t) } type funcFirstword struct{ fclosure } @@ -418,7 +418,7 @@ func (f *funcFirstword) Eval(w io.Writer, ev *Evaluator) { w.Write(ws.Bytes()) } freeBuf(abuf) - addStats("funcbody", "firstword", t) + stats.add("funcbody", "firstword", t) } type funcLastword struct{ fclosure } @@ -438,7 +438,7 @@ func (f *funcLastword) Eval(w io.Writer, ev *Evaluator) { w.Write(lw) } freeBuf(abuf) - addStats("funcbody", "lastword", t) + stats.add("funcbody", "lastword", t) } // https://www.gnu.org/software/make/manual/html_node/File-Name-Functions.html#File-Name-Functions @@ -463,7 +463,7 @@ func (f *funcJoin) Eval(w io.Writer, ev *Evaluator) { w.Write(ws2.Bytes()) } freeBuf(abuf) - addStats("funcbody", "join", t) + stats.add("funcbody", "join", t) } type funcWildcard struct{ fclosure } @@ -492,7 +492,7 @@ func (f *funcWildcard) Eval(w io.Writer, ev *Evaluator) { } traceEvent.end(te) freeBuf(abuf) - addStats("funcbody", "wildcard", t) + stats.add("funcbody", "wildcard", t) } type funcDir struct{ fclosure } @@ -514,7 +514,7 @@ func (f *funcDir) Eval(w io.Writer, ev *Evaluator) { sw.WriteString(filepath.Dir(string(name)) + string(filepath.Separator)) } freeBuf(abuf) - addStats("funcbody", "dir", t) + stats.add("funcbody", "dir", t) } type funcNotdir struct{ fclosure } @@ -536,7 +536,7 @@ func (f *funcNotdir) Eval(w io.Writer, ev *Evaluator) { sw.WriteString(filepath.Base(name)) } freeBuf(abuf) - addStats("funcbody", "notdir", t) + stats.add("funcbody", "notdir", t) } type funcSuffix struct{ fclosure } @@ -557,7 +557,7 @@ func (f *funcSuffix) Eval(w io.Writer, ev *Evaluator) { } } freeBuf(abuf) - addStats("funcbody", "suffix", t) + stats.add("funcbody", "suffix", t) } type funcBasename struct{ fclosure } @@ -576,7 +576,7 @@ func (f *funcBasename) Eval(w io.Writer, ev *Evaluator) { sw.WriteString(e) } freeBuf(abuf) - addStats("funcbody", "basename", t) + stats.add("funcbody", "basename", t) } type funcAddsuffix struct{ fclosure } @@ -596,7 +596,7 @@ func (f *funcAddsuffix) Eval(w io.Writer, ev *Evaluator) { w.Write(suf) } freeBuf(abuf) - addStats("funcbody", "addsuffix", t) + stats.add("funcbody", "addsuffix", t) } type funcAddprefix struct{ fclosure } @@ -616,7 +616,7 @@ func (f *funcAddprefix) Eval(w io.Writer, ev *Evaluator) { w.Write(ws.Bytes()) } freeBuf(abuf) - addStats("funcbody", "addprefix", t) + stats.add("funcbody", "addprefix", t) } type funcRealpath struct{ fclosure } @@ -649,7 +649,7 @@ func (f *funcRealpath) Eval(w io.Writer, ev *Evaluator) { sw.WriteString(name) } freeBuf(abuf) - addStats("funcbody", "realpath", t) + stats.add("funcbody", "realpath", t) } type funcAbspath struct{ fclosure } @@ -672,7 +672,7 @@ func (f *funcAbspath) Eval(w io.Writer, ev *Evaluator) { sw.WriteString(name) } freeBuf(abuf) - addStats("funcbody", "abspath", t) + stats.add("funcbody", "abspath", t) } // http://www.gnu.org/software/make/manual/make.html#Conditional-Functions @@ -736,11 +736,6 @@ func (f *funcOr) Eval(w io.Writer, ev *Evaluator) { // http://www.gnu.org/software/make/manual/make.html#Shell-Function type funcShell struct{ fclosure } -var ( - shellFuncTime time.Duration - shellFuncCount int -) - func (f *funcShell) Arity() int { return 1 } // A hack for Android build. We need to evaluate things like $((3+4)) @@ -786,8 +781,7 @@ func (f *funcShell) Eval(w io.Writer, ev *Evaluator) { } te := traceEvent.begin("shell", literal(arg), traceEventMain) out, err := cmd.Output() - shellFuncTime += time.Since(te.t) - shellFuncCount++ + shellStats.add(time.Since(te.t)) if err != nil { Logf("$(shell %q) failed: %q", arg, err) } @@ -242,7 +242,7 @@ func getDepGraph(clvars []string, targets []string) *DepGraph { vars.Merge(er.vars) LogStats("eval time: %q", time.Since(startTime)) - LogStats("shell func time: %q %d", shellFuncTime, shellFuncCount) + LogStats("shell func time: %q %d", shellStats.duration, shellStats.count) startTime = time.Now() db := NewDepBuilder(er, vars) @@ -333,7 +333,6 @@ func main() { } clvars, targets := parseCommandLine() - InitMakefileCache() g := getDepGraph(clvars, targets) nodes := g.nodes @@ -22,10 +22,13 @@ package main import ( "bufio" "bytes" + "crypto/sha1" "fmt" "io" + "io/ioutil" "os" "strings" + "sync" "time" ) @@ -681,29 +684,6 @@ func ParseMakefileFd(filename string, f *os.File) (Makefile, error) { return parser.parse() } -/* -func ParseMakefile(filename string) (Makefile, error) { - Logf("ParseMakefile %q", filename) - f, err := os.Open(filename) - if err != nil { - return Makefile{}, err - } - defer f.Close() - return ParseMakefileFd(filename, f) -} - -func ParseDefaultMakefile() (Makefile, string, error) { - candidates := []string{"GNUmakefile", "makefile", "Makefile"} - for _, filename := range candidates { - if exists(filename) { - mk, err := ParseMakefile(filename) - return mk, filename, err - } - } - return Makefile{}, "", errors.New("no targets specified and no makefile found.") -} -*/ - func GetDefaultMakefile() string { candidates := []string{"GNUmakefile", "makefile", "Makefile"} for _, filename := range candidates { @@ -731,41 +711,70 @@ func ParseMakefileBytes(s []byte, name string, lineno int) (Makefile, error) { return parseMakefileReader(bytes.NewReader(s), name, lineno) } -type MakefileCache struct { - mk Makefile - err error - ts int64 +type mkCacheEntry struct { + mk Makefile + hash [sha1.Size]byte + err error + ts int64 } -var makefileCache map[string]MakefileCache +type makefileCacheT struct { + mu sync.Mutex + mk map[string]mkCacheEntry +} -func InitMakefileCache() { - if makefileCache == nil { - makefileCache = make(map[string]MakefileCache) - } +var makefileCache = &makefileCacheT{ + mk: make(map[string]mkCacheEntry), } -func LookupMakefileCache(filename string) (Makefile, bool, error) { - c, present := makefileCache[filename] +func (mc *makefileCacheT) lookup(filename string) (Makefile, [sha1.Size]byte, bool, error) { + var hash [sha1.Size]byte + mc.mu.Lock() + c, present := mc.mk[filename] + mc.mu.Unlock() if !present { - return Makefile{}, false, nil + return Makefile{}, hash, false, nil } ts := getTimestamp(filename) if ts < 0 || ts >= c.ts { - return Makefile{}, false, nil + return Makefile{}, hash, false, nil } - Logf("Cache hit for %q", filename) - return c.mk, true, c.err + return c.mk, c.hash, true, c.err +} + +func (mc *makefileCacheT) parse(filename string) (Makefile, [sha1.Size]byte, error) { + Logf("parse Makefile %q", filename) + mk, hash, ok, err := makefileCache.lookup(filename) + if ok { + if katiLogFlag { + Logf("makefile cache hit for %q", filename) + } + return mk, hash, err + } + if katiLogFlag { + Logf("reading makefile %q", filename) + } + c, err := ioutil.ReadFile(filename) + if err != nil { + return Makefile{}, hash, err + } + hash = sha1.Sum(c) + mk, err = ParseMakefile(c, filename) + if err != nil { + return Makefile{}, hash, err + } + makefileCache.mu.Lock() + makefileCache.mk[filename] = mkCacheEntry{ + mk: mk, + hash: hash, + err: err, + ts: time.Now().Unix(), + } + makefileCache.mu.Unlock() + return mk, hash, err } func ParseMakefile(s []byte, filename string) (Makefile, error) { - Logf("ParseMakefile %q", filename) parser := newParser(bytes.NewReader(s), filename) - mk, err := parser.parse() - makefileCache[filename] = MakefileCache{ - mk: mk, - err: err, - ts: time.Now().Unix(), - } - return mk, err + return parser.parse() } diff --git a/pathutil.go b/pathutil.go index 00eb8e2..38e613f 100644 --- a/pathutil.go +++ b/pathutil.go @@ -27,7 +27,14 @@ import ( "time" ) -var wildcardCache = make(map[string][]string) +type wildcardCacheT struct { + mu sync.Mutex + m map[string][]string +} + +var wildcardCache = &wildcardCacheT{ + m: make(map[string][]string), +} func wildcardGlob(pat string) []string { // TODO(ukai): use find cache for glob if exists. @@ -93,7 +100,10 @@ func wildcardGlob(pat string) []string { func wildcard(sw *ssvWriter, pat string) { if useWildcardCache { // TODO(ukai): make sure it didn't chdir? - if files, ok := wildcardCache[pat]; ok { + wildcardCache.mu.Lock() + files, ok := wildcardCache.m[pat] + wildcardCache.mu.Unlock() + if ok { for _, file := range files { sw.WriteString(file) } @@ -105,7 +115,9 @@ func wildcard(sw *ssvWriter, pat string) { sw.WriteString(file) } if useWildcardCache { - wildcardCache[pat] = files + wildcardCache.mu.Lock() + wildcardCache.m[pat] = files + wildcardCache.mu.Unlock() } } @@ -103,7 +103,7 @@ func (t *traceEventT) end(e event) { t.emit("E", e, time.Since(t.t0)) } } - addStats(e.name, e.v, e.t) + stats.add(e.name, e.v, e.t) } type statsData struct { @@ -113,21 +113,30 @@ type statsData struct { Total time.Duration } -var stats = map[string]statsData{} +type statsT struct { + mu sync.Mutex + data map[string]statsData +} + +var stats = &statsT{ + data: make(map[string]statsData), +} -func addStats(name, v string, t time.Time) { +func (s *statsT) add(name, v string, t time.Time) { if !katiEvalStatsFlag { return } d := time.Since(t) key := fmt.Sprintf("%s:%s", name, v) - s := stats[key] - if d > s.Longest { - s.Longest = d + s.mu.Lock() + sd := s.data[key] + if d > sd.Longest { + sd.Longest = d } - s.Total += d - s.Count++ - stats[key] = s + sd.Total += d + sd.Count++ + s.data[key] = sd + s.mu.Unlock() } func dumpStats() { @@ -135,7 +144,7 @@ func dumpStats() { return } var sv byTotalTime - for k, v := range stats { + for k, v := range stats.data { v.Name = k sv = append(sv, v) } @@ -153,3 +162,18 @@ func (b byTotalTime) Swap(i, j int) { b[i], b[j] = b[j], b[i] } func (b byTotalTime) Less(i, j int) bool { return b[i].Total > b[j].Total } + +type shellStatsT struct { + mu sync.Mutex + duration time.Duration + count int +} + +var shellStats = &shellStatsT{} + +func (s *shellStatsT) add(d time.Duration) { + s.mu.Lock() + s.duration += d + s.count++ + s.mu.Unlock() +} @@ -14,13 +14,26 @@ package main -var symtab = make(map[string]string) +import "sync" + +type symtabT struct { + mu sync.Mutex + m map[string]string +} + +var symtab = &symtabT{ + m: make(map[string]string), +} func intern(s string) string { - if v, ok := symtab[s]; ok { + symtab.mu.Lock() + v, ok := symtab.m[s] + if ok { + symtab.mu.Unlock() return v } - symtab[s] = s + symtab.m[s] = s + symtab.mu.Unlock() return s } |