// Copyright 2015 Google Inc. All rights reserved // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package kati import ( "bytes" "fmt" "io" "path/filepath" "strings" "sync" "time" ) // Executor manages execution of makefile rules. type Executor struct { rules map[string]*rule implicitRules []*rule suffixRules map[string][]*rule firstRule *rule shell string vars Vars varsLock sync.Mutex // target -> Job, nil means the target is currently being processed. done map[string]*job wm *workerManager currentOutput string currentInputs []string currentStem string trace []string buildCnt int alreadyDoneCnt int noRuleCnt int upToDateCnt int runCommandCnt int } type autoVar struct{ ex *Executor } func (v autoVar) Flavor() string { return "undefined" } func (v autoVar) Origin() string { return "automatic" } func (v autoVar) IsDefined() bool { return true } func (v autoVar) Append(*Evaluator, string) (Var, error) { return nil, fmt.Errorf("cannot append to autovar") } func (v autoVar) AppendVar(*Evaluator, Value) (Var, error) { return nil, fmt.Errorf("cannot append to autovar") } func (v autoVar) serialize() serializableVar { return serializableVar{Type: ""} } func (v autoVar) dump(d *dumpbuf) { d.err = fmt.Errorf("cannot dump auto var: %v", v) } type autoAtVar struct{ autoVar } func (v autoAtVar) Eval(w io.Writer, ev *Evaluator) error { fmt.Fprint(w, v.ex.currentOutput) return nil } func (v autoAtVar) String() string { return "$*" } type autoLessVar struct{ autoVar } func (v autoLessVar) Eval(w io.Writer, ev *Evaluator) error { if len(v.ex.currentInputs) > 0 { fmt.Fprint(w, v.ex.currentInputs[0]) } return nil } func (v autoLessVar) String() string { return "$<" } type autoHatVar struct{ autoVar } func (v autoHatVar) Eval(w io.Writer, ev *Evaluator) error { var uniqueInputs []string seen := make(map[string]bool) for _, input := range v.ex.currentInputs { if !seen[input] { seen[input] = true uniqueInputs = append(uniqueInputs, input) } } fmt.Fprint(w, strings.Join(uniqueInputs, " ")) return nil } func (v autoHatVar) String() string { return "$^" } type autoPlusVar struct{ autoVar } func (v autoPlusVar) Eval(w io.Writer, ev *Evaluator) error { fmt.Fprint(w, strings.Join(v.ex.currentInputs, " ")) return nil } func (v autoPlusVar) String() string { return "$+" } type autoStarVar struct{ autoVar } func (v autoStarVar) Eval(w io.Writer, ev *Evaluator) error { // TODO: Use currentStem. See auto_stem_var.mk fmt.Fprint(w, stripExt(v.ex.currentOutput)) return nil } func (v autoStarVar) String() string { return "$*" } type autoSuffixDVar struct { autoVar v Var } func (v autoSuffixDVar) Eval(w io.Writer, ev *Evaluator) error { var buf bytes.Buffer err := v.v.Eval(&buf, ev) if err != nil { return err } ws := newWordScanner(buf.Bytes()) sw := ssvWriter{w: w} for ws.Scan() { sw.WriteString(filepath.Dir(string(ws.Bytes()))) } return nil } func (v autoSuffixDVar) String() string { return v.v.String() + "D" } type autoSuffixFVar struct { autoVar v Var } func (v autoSuffixFVar) Eval(w io.Writer, ev *Evaluator) error { var buf bytes.Buffer err := v.v.Eval(&buf, ev) if err != nil { return err } ws := newWordScanner(buf.Bytes()) sw := ssvWriter{w: w} for ws.Scan() { sw.WriteString(filepath.Base(string(ws.Bytes()))) } return nil } func (v autoSuffixFVar) String() string { return v.v.String() + "F" } func (ex *Executor) makeJobs(n *DepNode, neededBy *job) error { output := n.Output if neededBy != nil { logf("MakeJob: %s for %s", output, neededBy.n.Output) } ex.buildCnt++ if ex.buildCnt%100 == 0 { ex.reportStats() } j, present := ex.done[output] if present { if j == nil { if !n.IsPhony { fmt.Printf("Circular %s <- %s dependency dropped.\n", neededBy.n.Output, n.Output) } if neededBy != nil { neededBy.numDeps-- } } else { logf("%s already done: %d", j.n.Output, j.outputTs) if neededBy != nil { ex.wm.ReportNewDep(j, neededBy) } } return nil } j = &job{ n: n, ex: ex, numDeps: len(n.Deps), depsTs: int64(-1), } if neededBy != nil { j.parents = append(j.parents, neededBy) } ex.done[output] = nil // We iterate n.Deps twice. In the first run, we may modify // numDeps. There will be a race if we do so after the first // ex.makeJobs(d, j). var deps []*DepNode for _, d := range n.Deps { if d.IsOrderOnly && exists(d.Output) { j.numDeps-- continue } deps = append(deps, d) } logf("new: %s (%d)", j.n.Output, j.numDeps) for _, d := range deps { ex.trace = append(ex.trace, d.Output) err := ex.makeJobs(d, j) ex.trace = ex.trace[0 : len(ex.trace)-1] if err != nil { return err } } ex.done[output] = j return ex.wm.PostJob(j) } func (ex *Executor) reportStats() { if !LogFlag && !PeriodicStatsFlag { return } logStats("build=%d alreadyDone=%d noRule=%d, upToDate=%d runCommand=%d", ex.buildCnt, ex.alreadyDoneCnt, ex.noRuleCnt, ex.upToDateCnt, ex.runCommandCnt) if len(ex.trace) > 1 { logStats("trace=%q", ex.trace) } } // ExecutorOpt is an option for Executor. type ExecutorOpt struct { NumJobs int ParaPath string } // NewExecutor creates new Executor. func NewExecutor(vars Vars, opt *ExecutorOpt) (*Executor, error) { if opt == nil { opt = &ExecutorOpt{NumJobs: 1} } if opt.NumJobs < 1 { opt.NumJobs = 1 } wm, err := newWorkerManager(opt.NumJobs, opt.ParaPath) if err != nil { return nil, err } ex := &Executor{ rules: make(map[string]*rule), suffixRules: make(map[string][]*rule), done: make(map[string]*job), vars: vars, wm: wm, } // TODO: We should move this to somewhere around evalCmd so that // we can handle SHELL in target specific variables. ev := NewEvaluator(ex.vars) ex.shell, err = ev.EvaluateVar("SHELL") if err != nil { ex.shell = "/bin/sh" } for k, v := range map[string]Var{ "@": autoAtVar{autoVar: autoVar{ex: ex}}, "<": autoLessVar{autoVar: autoVar{ex: ex}}, "^": autoHatVar{autoVar: autoVar{ex: ex}}, "+": autoPlusVar{autoVar: autoVar{ex: ex}}, "*": autoStarVar{autoVar: autoVar{ex: ex}}, } { ex.vars[k] = v ex.vars[k+"D"] = autoSuffixDVar{v: v} ex.vars[k+"F"] = autoSuffixFVar{v: v} } return ex, nil } // Exec executes to build roots. func (ex *Executor) Exec(roots []*DepNode) error { startTime := time.Now() for _, root := range roots { err := ex.makeJobs(root, nil) if err != nil { break } } err := ex.wm.Wait() logStats("exec time: %q", time.Since(startTime)) return err } func (ex *Executor) createRunners(n *DepNode, avoidIO bool) ([]runner, bool, error) { var runners []runner if len(n.Cmds) == 0 { return runners, false, nil } var restores []func() defer func() { for i := len(restores) - 1; i >= 0; i-- { restores[i]() } }() ex.varsLock.Lock() restores = append(restores, func() { ex.varsLock.Unlock() }) // For automatic variables. ex.currentOutput = n.Output ex.currentInputs = n.ActualInputs for k, v := range n.TargetSpecificVars { restores = append(restores, ex.vars.save(k)) ex.vars[k] = v logf("tsv: %s=%s", k, v) } ev := NewEvaluator(ex.vars) ev.avoidIO = avoidIO ev.filename = n.Filename ev.lineno = n.Lineno logf("Building: %s cmds:%q", n.Output, n.Cmds) r := runner{ output: n.Output, echo: true, shell: ex.shell, } for _, cmd := range n.Cmds { rr, err := evalCmd(ev, r, cmd) if err != nil { return nil, false, err } for _, r := range rr { if len(r.cmd) != 0 { runners = append(runners, r) } } } return runners, ev.hasIO, nil } func evalCommands(nodes []*DepNode, vars Vars) error { ioCnt := 0 ex, err := NewExecutor(vars, nil) if err != nil { return err } for i, n := range nodes { runners, hasIO, err := ex.createRunners(n, true) if err != nil { return err } if hasIO { ioCnt++ if ioCnt%100 == 0 { logStats("%d/%d rules have IO", ioCnt, i+1) } continue } n.Cmds = []string{} n.TargetSpecificVars = make(Vars) for _, r := range runners { cmd := r.cmd // TODO: Do not preserve the effect of dryRunFlag. if r.echo { cmd = "@" + cmd } if r.ignoreError { cmd = "-" + cmd } n.Cmds = append(n.Cmds, cmd) } } err = ex.wm.Wait() logStats("%d/%d rules have IO", ioCnt, len(nodes)) return err }