aboutsummaryrefslogtreecommitdiff
path: root/dashboard/cmd/builder
diff options
context:
space:
mode:
Diffstat (limited to 'dashboard/cmd/builder')
-rw-r--r--dashboard/cmd/builder/bench.go256
-rw-r--r--dashboard/cmd/builder/doc.go58
-rw-r--r--dashboard/cmd/builder/env.go299
-rw-r--r--dashboard/cmd/builder/exec.go99
-rw-r--r--dashboard/cmd/builder/filemutex_flock.go66
-rw-r--r--dashboard/cmd/builder/filemutex_local.go27
-rw-r--r--dashboard/cmd/builder/filemutex_windows.go105
-rw-r--r--dashboard/cmd/builder/http.go225
-rw-r--r--dashboard/cmd/builder/main.go679
-rw-r--r--dashboard/cmd/builder/vcs.go225
10 files changed, 2039 insertions, 0 deletions
diff --git a/dashboard/cmd/builder/bench.go b/dashboard/cmd/builder/bench.go
new file mode 100644
index 0000000..a9a59ce
--- /dev/null
+++ b/dashboard/cmd/builder/bench.go
@@ -0,0 +1,256 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "strconv"
+ "strings"
+)
+
+// benchHash benchmarks a single commit.
+func (b *Builder) benchHash(hash string, benchs []string) error {
+ if *verbose {
+ log.Println(b.name, "benchmarking", hash)
+ }
+
+ res := &PerfResult{Hash: hash, Benchmark: "meta-done"}
+
+ // Create place in which to do work.
+ workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
+ // Prepare a workpath if we don't have one we can reuse.
+ update := false
+ if b.lastWorkpath != workpath {
+ if err := os.Mkdir(workpath, mkdirPerm); err != nil {
+ return err
+ }
+ buildLog, _, err := b.buildRepoOnHash(workpath, hash, makeCmd)
+ if err != nil {
+ removePath(workpath)
+ // record failure
+ res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
+ return b.recordPerfResult(res)
+ }
+ b.lastWorkpath = workpath
+ update = true
+ }
+
+ // Build the benchmark binary.
+ benchBin, buildLog, err := b.buildBenchmark(workpath, update)
+ if err != nil {
+ // record failure
+ res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
+ return b.recordPerfResult(res)
+ }
+
+ benchmark, procs, affinity, last := chooseBenchmark(benchBin, benchs)
+ if benchmark != "" {
+ res.Benchmark = fmt.Sprintf("%v-%v", benchmark, procs)
+ res.Metrics, res.Artifacts, res.OK = b.executeBenchmark(workpath, hash, benchBin, benchmark, procs, affinity)
+ if err = b.recordPerfResult(res); err != nil {
+ return fmt.Errorf("recordResult: %s", err)
+ }
+ }
+
+ if last {
+ // All benchmarks have beed executed, don't need workpath anymore.
+ removePath(b.lastWorkpath)
+ b.lastWorkpath = ""
+ // Notify the app.
+ res = &PerfResult{Hash: hash, Benchmark: "meta-done", OK: true}
+ if err = b.recordPerfResult(res); err != nil {
+ return fmt.Errorf("recordResult: %s", err)
+ }
+ }
+
+ return nil
+}
+
+// buildBenchmark builds the benchmark binary.
+func (b *Builder) buildBenchmark(workpath string, update bool) (benchBin, log string, err error) {
+ goroot := filepath.Join(workpath, "go")
+ gobin := filepath.Join(goroot, "bin", "go") + exeExt
+ gopath := filepath.Join(*buildroot, "gopath")
+ env := append([]string{
+ "GOROOT=" + goroot,
+ "GOPATH=" + gopath},
+ b.envv()...)
+ // First, download without installing.
+ args := []string{"get", "-d"}
+ if update {
+ args = append(args, "-u")
+ }
+ args = append(args, *benchPath)
+ var buildlog bytes.Buffer
+ runOpts := []runOpt{runTimeout(*buildTimeout), runEnv(env), allOutput(&buildlog), runDir(workpath)}
+ err = run(exec.Command(gobin, args...), runOpts...)
+ if err != nil {
+ fmt.Fprintf(&buildlog, "go get -d %s failed: %s", *benchPath, err)
+ return "", buildlog.String(), err
+ }
+ // Then, build into workpath.
+ benchBin = filepath.Join(workpath, "benchbin") + exeExt
+ args = []string{"build", "-o", benchBin, *benchPath}
+ buildlog.Reset()
+ err = run(exec.Command(gobin, args...), runOpts...)
+ if err != nil {
+ fmt.Fprintf(&buildlog, "go build %s failed: %s", *benchPath, err)
+ return "", buildlog.String(), err
+ }
+ return benchBin, "", nil
+}
+
+// chooseBenchmark chooses the next benchmark to run
+// based on the list of available benchmarks, already executed benchmarks
+// and -benchcpu list.
+func chooseBenchmark(benchBin string, doneBenchs []string) (bench string, procs, affinity int, last bool) {
+ var out bytes.Buffer
+ err := run(exec.Command(benchBin), allOutput(&out))
+ if err != nil {
+ log.Printf("Failed to query benchmark list: %v\n%s", err, &out)
+ last = true
+ return
+ }
+ outStr := out.String()
+ nlIdx := strings.Index(outStr, "\n")
+ if nlIdx < 0 {
+ log.Printf("Failed to parse benchmark list (no new line): %s", outStr)
+ last = true
+ return
+ }
+ localBenchs := strings.Split(outStr[:nlIdx], ",")
+ benchsMap := make(map[string]bool)
+ for _, b := range doneBenchs {
+ benchsMap[b] = true
+ }
+ cnt := 0
+ // We want to run all benchmarks with GOMAXPROCS=1 first.
+ for i, procs1 := range benchCPU {
+ for _, bench1 := range localBenchs {
+ if benchsMap[fmt.Sprintf("%v-%v", bench1, procs1)] {
+ continue
+ }
+ cnt++
+ if cnt == 1 {
+ bench = bench1
+ procs = procs1
+ if i < len(benchAffinity) {
+ affinity = benchAffinity[i]
+ }
+ }
+ }
+ }
+ last = cnt <= 1
+ return
+}
+
+// executeBenchmark runs a single benchmark and parses its output.
+func (b *Builder) executeBenchmark(workpath, hash, benchBin, bench string, procs, affinity int) (metrics []PerfMetric, artifacts []PerfArtifact, ok bool) {
+ // Benchmarks runs mutually exclusive with other activities.
+ benchMutex.RUnlock()
+ defer benchMutex.RLock()
+ benchMutex.Lock()
+ defer benchMutex.Unlock()
+
+ log.Printf("%v executing benchmark %v-%v on %v", b.name, bench, procs, hash)
+
+ // The benchmark executes 'go build'/'go tool',
+ // so we need properly setup env.
+ env := append([]string{
+ "GOROOT=" + filepath.Join(workpath, "go"),
+ "PATH=" + filepath.Join(workpath, "go", "bin") + string(os.PathListSeparator) + os.Getenv("PATH"),
+ "GODEBUG=gctrace=1", // since Go1.2
+ "GOGCTRACE=1", // before Go1.2
+ fmt.Sprintf("GOMAXPROCS=%v", procs)},
+ b.envv()...)
+ args := []string{
+ "-bench", bench,
+ "-benchmem", strconv.Itoa(*benchMem),
+ "-benchtime", benchTime.String(),
+ "-benchnum", strconv.Itoa(*benchNum),
+ "-tmpdir", workpath}
+ if affinity != 0 {
+ args = append(args, "-affinity", strconv.Itoa(affinity))
+ }
+ benchlog := new(bytes.Buffer)
+ err := run(exec.Command(benchBin, args...), runEnv(env), allOutput(benchlog), runDir(workpath))
+ if strip := benchlog.Len() - 512<<10; strip > 0 {
+ // Leave the last 512K, that part contains metrics.
+ benchlog = bytes.NewBuffer(benchlog.Bytes()[strip:])
+ }
+ artifacts = []PerfArtifact{{Type: "log", Body: benchlog.String()}}
+ if err != nil {
+ if err != nil {
+ log.Printf("Failed to execute benchmark '%v': %v", bench, err)
+ ok = false
+ }
+ return
+ }
+
+ metrics1, artifacts1, err := parseBenchmarkOutput(benchlog)
+ if err != nil {
+ log.Printf("Failed to parse benchmark output: %v", err)
+ ok = false
+ return
+ }
+ metrics = metrics1
+ artifacts = append(artifacts, artifacts1...)
+ ok = true
+ return
+}
+
+// parseBenchmarkOutput fetches metrics and artifacts from benchmark output.
+func parseBenchmarkOutput(out io.Reader) (metrics []PerfMetric, artifacts []PerfArtifact, err error) {
+ s := bufio.NewScanner(out)
+ metricRe := regexp.MustCompile("^GOPERF-METRIC:([a-z,0-9,-]+)=([0-9]+)$")
+ fileRe := regexp.MustCompile("^GOPERF-FILE:([a-z,0-9,-]+)=(.+)$")
+ for s.Scan() {
+ ln := s.Text()
+ if ss := metricRe.FindStringSubmatch(ln); ss != nil {
+ var v uint64
+ v, err = strconv.ParseUint(ss[2], 10, 64)
+ if err != nil {
+ err = fmt.Errorf("Failed to parse metric '%v=%v': %v", ss[1], ss[2], err)
+ return
+ }
+ metrics = append(metrics, PerfMetric{Type: ss[1], Val: v})
+ } else if ss := fileRe.FindStringSubmatch(ln); ss != nil {
+ var buf []byte
+ buf, err = ioutil.ReadFile(ss[2])
+ if err != nil {
+ err = fmt.Errorf("Failed to read file '%v': %v", ss[2], err)
+ return
+ }
+ artifacts = append(artifacts, PerfArtifact{ss[1], string(buf)})
+ }
+ }
+ return
+}
+
+// needsBenchmarking determines whether the commit needs benchmarking.
+func needsBenchmarking(log *HgLog) bool {
+ // Do not benchmark branch commits, they are usually not interesting
+ // and fall out of the trunk succession.
+ if log.Branch != "" {
+ return false
+ }
+ // Do not benchmark commits that do not touch source files (e.g. CONTRIBUTORS).
+ for _, f := range strings.Split(log.Files, " ") {
+ if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) &&
+ !strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") {
+ return true
+ }
+ }
+ return false
+}
diff --git a/dashboard/cmd/builder/doc.go b/dashboard/cmd/builder/doc.go
new file mode 100644
index 0000000..15b7252
--- /dev/null
+++ b/dashboard/cmd/builder/doc.go
@@ -0,0 +1,58 @@
+// Copyright 2010 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+/*
+
+Go Builder is a continuous build client for the Go project.
+It integrates with the Go Dashboard AppEngine application.
+
+Go Builder is intended to run continuously as a background process.
+
+It periodically pulls updates from the Go Mercurial repository.
+
+When a newer revision is found, Go Builder creates a clone of the repository,
+runs all.bash, and reports build success or failure to the Go Dashboard.
+
+For a release revision (a change description that matches "release.YYYY-MM-DD"),
+Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
+Go Google Code project's downloads section.
+
+Usage:
+
+ gobuilder goos-goarch...
+
+ Several goos-goarch combinations can be provided, and the builder will
+ build them in serial.
+
+Optional flags:
+
+ -dashboard="godashboard.appspot.com": Go Dashboard Host
+ The location of the Go Dashboard application to which Go Builder will
+ report its results.
+
+ -release: Build and deliver binary release archive
+
+ -rev=N: Build revision N and exit
+
+ -cmd="./all.bash": Build command (specify absolute or relative to go/src)
+
+ -v: Verbose logging
+
+ -external: External package builder mode (will not report Go build
+ state to dashboard or issue releases)
+
+The key file should be located at $HOME/.gobuildkey or, for a builder-specific
+key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64).
+
+The build key file is a text file of the format:
+
+ godashboard-key
+ googlecode-username
+ googlecode-password
+
+If the Google Code credentials are not provided the archival step
+will be skipped.
+
+*/
+package main // import "golang.org/x/tools/dashboard/cmd/builder"
diff --git a/dashboard/cmd/builder/env.go b/dashboard/cmd/builder/env.go
new file mode 100644
index 0000000..7261229
--- /dev/null
+++ b/dashboard/cmd/builder/env.go
@@ -0,0 +1,299 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "runtime"
+ "strings"
+
+ "golang.org/x/tools/go/vcs"
+)
+
+// builderEnv represents the environment that a Builder will run tests in.
+type builderEnv interface {
+ // setup sets up the builder environment and returns the directory to run the buildCmd in.
+ setup(repo *Repo, workpath, hash string, envv []string) (string, error)
+}
+
+// goEnv represents the builderEnv for the main Go repo.
+type goEnv struct {
+ goos, goarch string
+}
+
+func (b *Builder) envv() []string {
+ if runtime.GOOS == "windows" {
+ return b.envvWindows()
+ }
+
+ var e []string
+ if *buildTool == "go" {
+ e = []string{
+ "GOOS=" + b.goos,
+ "GOARCH=" + b.goarch,
+ "GOROOT_FINAL=/usr/local/go",
+ }
+ switch b.goos {
+ case "android", "nacl":
+ // Cross compile.
+ default:
+ // If we are building, for example, linux/386 on a linux/amd64 machine we want to
+ // make sure that the whole build is done as a if this were compiled on a real
+ // linux/386 machine. In other words, we want to not do a cross compilation build.
+ // To do this we set GOHOSTOS and GOHOSTARCH to override the detection in make.bash.
+ //
+ // The exception to this rule is when we are doing nacl/android builds. These are by
+ // definition always cross compilation, and we have support built into cmd/go to be
+ // able to handle this case.
+ e = append(e, "GOHOSTOS="+b.goos, "GOHOSTARCH="+b.goarch)
+ }
+ }
+
+ for _, k := range extraEnv() {
+ if s, ok := getenvOk(k); ok {
+ e = append(e, k+"="+s)
+ }
+ }
+ return e
+}
+
+func (b *Builder) envvWindows() []string {
+ var start map[string]string
+ if *buildTool == "go" {
+ start = map[string]string{
+ "GOOS": b.goos,
+ "GOHOSTOS": b.goos,
+ "GOARCH": b.goarch,
+ "GOHOSTARCH": b.goarch,
+ "GOROOT_FINAL": `c:\go`,
+ "GOBUILDEXIT": "1", // exit all.bat with completion status.
+ }
+ }
+
+ for _, name := range extraEnv() {
+ if s, ok := getenvOk(name); ok {
+ start[name] = s
+ }
+ }
+ if b.goos == "windows" {
+ switch b.goarch {
+ case "amd64":
+ start["PATH"] = `c:\TDM-GCC-64\bin;` + start["PATH"]
+ case "386":
+ start["PATH"] = `c:\TDM-GCC-32\bin;` + start["PATH"]
+ }
+ }
+ skip := map[string]bool{
+ "GOBIN": true,
+ "GOPATH": true,
+ "GOROOT": true,
+ "INCLUDE": true,
+ "LIB": true,
+ }
+ var e []string
+ for name, v := range start {
+ e = append(e, name+"="+v)
+ skip[name] = true
+ }
+ for _, kv := range os.Environ() {
+ s := strings.SplitN(kv, "=", 2)
+ name := strings.ToUpper(s[0])
+ switch {
+ case name == "":
+ // variables, like "=C:=C:\", just copy them
+ e = append(e, kv)
+ case !skip[name]:
+ e = append(e, kv)
+ skip[name] = true
+ }
+ }
+ return e
+}
+
+// setup for a goEnv clones the main go repo to workpath/go at the provided hash
+// and returns the path workpath/go/src, the location of all go build scripts.
+func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
+ goworkpath := filepath.Join(workpath, "go")
+ if err := repo.Export(goworkpath, hash); err != nil {
+ return "", fmt.Errorf("error exporting repository: %s", err)
+ }
+ return filepath.Join(goworkpath, "src"), nil
+}
+
+// gccgoEnv represents the builderEnv for the gccgo compiler.
+type gccgoEnv struct{}
+
+// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
+// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
+// replaced with the updated sources in the gofrontend repo and gcc gets
+// gets configured and built in workpath/gcc-objdir. The path to
+// workpath/gcc-objdir is returned.
+func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
+ gccpath := filepath.Join(repo.Path, "gcc")
+
+ // get a handle to Git vcs.Cmd for pulling down GCC from the mirror.
+ git := vcs.ByCmd("git")
+
+ // only pull down gcc if we don't have a local copy.
+ if _, err := os.Stat(gccpath); err != nil {
+ if err := timeout(*cmdTimeout, func() error {
+ // pull down a working copy of GCC.
+
+ cloneCmd := []string{
+ "clone",
+ // This is just a guess since there are ~6000 commits to
+ // GCC per year. It's likely there will be enough history
+ // to cross-reference the Gofrontend commit against GCC.
+ // The disadvantage would be if the commit being built is more than
+ // a year old; in this case, the user should make a clone that has
+ // the full history.
+ "--depth", "6000",
+ // We only care about the master branch.
+ "--branch", "master", "--single-branch",
+ *gccPath,
+ }
+
+ // Clone Kind Clone Time(Dry run) Clone Size
+ // ---------------------------------------------------------------
+ // Full Clone 10 - 15 min 2.2 GiB
+ // Master Branch 2 - 3 min 1.5 GiB
+ // Full Clone(shallow) 1 min 900 MiB
+ // Master Branch(shallow) 40 sec 900 MiB
+ //
+ // The shallow clones have the same size, which is expected,
+ // but the full shallow clone will only have 6000 commits
+ // spread across all branches. There are ~50 branches.
+ return run(exec.Command("git", cloneCmd...), runEnv(envv), allOutput(os.Stdout), runDir(repo.Path))
+ }); err != nil {
+ return "", err
+ }
+ }
+
+ if err := git.Download(gccpath); err != nil {
+ return "", err
+ }
+
+ // get the modified files for this commit.
+
+ var buf bytes.Buffer
+ if err := run(exec.Command("hg", "status", "--no-status", "--change", hash),
+ allOutput(&buf), runDir(repo.Path), runEnv(envv)); err != nil {
+ return "", fmt.Errorf("Failed to find the modified files for %s: %s", hash, err)
+ }
+ modifiedFiles := strings.Split(buf.String(), "\n")
+ var isMirrored bool
+ for _, f := range modifiedFiles {
+ if strings.HasPrefix(f, "go/") || strings.HasPrefix(f, "libgo/") {
+ isMirrored = true
+ break
+ }
+ }
+
+ // use git log to find the corresponding commit to sync to in the gcc mirror.
+ // If the files modified in the gofrontend are mirrored to gcc, we expect a
+ // commit with a similar description in the gcc mirror. If the files modified are
+ // not mirrored, e.g. in support/, we can sync to the most recent gcc commit that
+ // occurred before those files were modified to verify gccgo's status at that point.
+ logCmd := []string{
+ "log",
+ "-1",
+ "--format=%H",
+ }
+ var errMsg string
+ if isMirrored {
+ commitDesc, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{desc|firstline|escape}")
+ if err != nil {
+ return "", err
+ }
+
+ quotedDesc := regexp.QuoteMeta(string(commitDesc))
+ logCmd = append(logCmd, "--grep", quotedDesc, "--regexp-ignore-case", "--extended-regexp")
+ errMsg = fmt.Sprintf("Failed to find a commit with a similar description to '%s'", string(commitDesc))
+ } else {
+ commitDate, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{date|rfc3339date}")
+ if err != nil {
+ return "", err
+ }
+
+ logCmd = append(logCmd, "--before", string(commitDate))
+ errMsg = fmt.Sprintf("Failed to find a commit before '%s'", string(commitDate))
+ }
+
+ buf.Reset()
+ if err := run(exec.Command("git", logCmd...), runEnv(envv), allOutput(&buf), runDir(gccpath)); err != nil {
+ return "", fmt.Errorf("%s: %s", errMsg, err)
+ }
+ gccRev := buf.String()
+ if gccRev == "" {
+ return "", fmt.Errorf(errMsg)
+ }
+
+ // checkout gccRev
+ // TODO(cmang): Fix this to work in parallel mode.
+ if err := run(exec.Command("git", "reset", "--hard", strings.TrimSpace(gccRev)), runEnv(envv), runDir(gccpath)); err != nil {
+ return "", fmt.Errorf("Failed to checkout commit at revision %s: %s", gccRev, err)
+ }
+
+ // make objdir to work in
+ gccobjdir := filepath.Join(workpath, "gcc-objdir")
+ if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
+ return "", err
+ }
+
+ // configure GCC with substituted gofrontend and libgo
+ if err := run(exec.Command(filepath.Join(gccpath, "configure"),
+ "--enable-languages=c,c++,go",
+ "--disable-bootstrap",
+ "--disable-multilib",
+ ), runEnv(envv), runDir(gccobjdir)); err != nil {
+ return "", fmt.Errorf("Failed to configure GCC: %v", err)
+ }
+
+ // build gcc
+ if err := run(exec.Command("make", *gccOpts), runTimeout(*buildTimeout), runEnv(envv), runDir(gccobjdir)); err != nil {
+ return "", fmt.Errorf("Failed to build GCC: %s", err)
+ }
+
+ return gccobjdir, nil
+}
+
+func getenvOk(k string) (v string, ok bool) {
+ v = os.Getenv(k)
+ if v != "" {
+ return v, true
+ }
+ keq := k + "="
+ for _, kv := range os.Environ() {
+ if kv == keq {
+ return "", true
+ }
+ }
+ return "", false
+}
+
+// extraEnv returns environment variables that need to be copied from
+// the gobuilder's environment to the envv of its subprocesses.
+func extraEnv() []string {
+ extra := []string{
+ "GOARM",
+ "GO386",
+ "GOROOT_BOOTSTRAP", // See https://golang.org/s/go15bootstrap
+ "CGO_ENABLED",
+ "CC",
+ "CC_FOR_TARGET",
+ "PATH",
+ "TMPDIR",
+ "USER",
+ }
+ if runtime.GOOS == "plan9" {
+ extra = append(extra, "objtype", "cputype", "path")
+ }
+ return extra
+}
diff --git a/dashboard/cmd/builder/exec.go b/dashboard/cmd/builder/exec.go
new file mode 100644
index 0000000..1b46ed1
--- /dev/null
+++ b/dashboard/cmd/builder/exec.go
@@ -0,0 +1,99 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "fmt"
+ "io"
+ "log"
+ "os/exec"
+ "time"
+)
+
+// run runs a command with optional arguments.
+func run(cmd *exec.Cmd, opts ...runOpt) error {
+ a := runArgs{cmd, *cmdTimeout}
+ for _, opt := range opts {
+ opt.modArgs(&a)
+ }
+ if *verbose {
+ log.Printf("running %v in %v", a.cmd.Args, a.cmd.Dir)
+ }
+ if err := cmd.Start(); err != nil {
+ log.Printf("failed to start command %v: %v", a.cmd.Args, err)
+ return err
+ }
+ err := timeout(a.timeout, cmd.Wait)
+ if _, ok := err.(timeoutError); ok {
+ cmd.Process.Kill()
+ }
+ return err
+}
+
+// Zero or more runOpts can be passed to run to modify the command
+// before it's run.
+type runOpt interface {
+ modArgs(*runArgs)
+}
+
+// allOutput sends both stdout and stderr to w.
+func allOutput(w io.Writer) optFunc {
+ return func(a *runArgs) {
+ a.cmd.Stdout = w
+ a.cmd.Stderr = w
+ }
+}
+
+func runTimeout(timeout time.Duration) optFunc {
+ return func(a *runArgs) {
+ a.timeout = timeout
+ }
+}
+
+func runDir(dir string) optFunc {
+ return func(a *runArgs) {
+ a.cmd.Dir = dir
+ }
+}
+
+func runEnv(env []string) optFunc {
+ return func(a *runArgs) {
+ a.cmd.Env = env
+ }
+}
+
+// timeout runs f and returns its error value, or if the function does not
+// complete before the provided duration it returns a timeout error.
+func timeout(d time.Duration, f func() error) error {
+ errc := make(chan error, 1)
+ go func() {
+ errc <- f()
+ }()
+ t := time.NewTimer(d)
+ defer t.Stop()
+ select {
+ case <-t.C:
+ return timeoutError(d)
+ case err := <-errc:
+ return err
+ }
+}
+
+type timeoutError time.Duration
+
+func (e timeoutError) Error() string {
+ return fmt.Sprintf("timed out after %v", time.Duration(e))
+}
+
+// optFunc implements runOpt with a function, like http.HandlerFunc.
+type optFunc func(*runArgs)
+
+func (f optFunc) modArgs(a *runArgs) { f(a) }
+
+// internal detail to exec.go:
+type runArgs struct {
+ cmd *exec.Cmd
+ timeout time.Duration
+}
diff --git a/dashboard/cmd/builder/filemutex_flock.go b/dashboard/cmd/builder/filemutex_flock.go
new file mode 100644
index 0000000..68851b8
--- /dev/null
+++ b/dashboard/cmd/builder/filemutex_flock.go
@@ -0,0 +1,66 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin dragonfly freebsd linux netbsd openbsd
+
+package main
+
+import (
+ "sync"
+ "syscall"
+)
+
+// FileMutex is similar to sync.RWMutex, but also synchronizes across processes.
+// This implementation is based on flock syscall.
+type FileMutex struct {
+ mu sync.RWMutex
+ fd int
+}
+
+func MakeFileMutex(filename string) *FileMutex {
+ if filename == "" {
+ return &FileMutex{fd: -1}
+ }
+ fd, err := syscall.Open(filename, syscall.O_CREAT|syscall.O_RDONLY, mkdirPerm)
+ if err != nil {
+ panic(err)
+ }
+ return &FileMutex{fd: fd}
+}
+
+func (m *FileMutex) Lock() {
+ m.mu.Lock()
+ if m.fd != -1 {
+ if err := syscall.Flock(m.fd, syscall.LOCK_EX); err != nil {
+ panic(err)
+ }
+ }
+}
+
+func (m *FileMutex) Unlock() {
+ if m.fd != -1 {
+ if err := syscall.Flock(m.fd, syscall.LOCK_UN); err != nil {
+ panic(err)
+ }
+ }
+ m.mu.Unlock()
+}
+
+func (m *FileMutex) RLock() {
+ m.mu.RLock()
+ if m.fd != -1 {
+ if err := syscall.Flock(m.fd, syscall.LOCK_SH); err != nil {
+ panic(err)
+ }
+ }
+}
+
+func (m *FileMutex) RUnlock() {
+ if m.fd != -1 {
+ if err := syscall.Flock(m.fd, syscall.LOCK_UN); err != nil {
+ panic(err)
+ }
+ }
+ m.mu.RUnlock()
+}
diff --git a/dashboard/cmd/builder/filemutex_local.go b/dashboard/cmd/builder/filemutex_local.go
new file mode 100644
index 0000000..68cfb62
--- /dev/null
+++ b/dashboard/cmd/builder/filemutex_local.go
@@ -0,0 +1,27 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build nacl plan9 solaris
+
+package main
+
+import (
+ "log"
+ "sync"
+)
+
+// FileMutex is similar to sync.RWMutex, but also synchronizes across processes.
+// This implementation is a fallback that does not actually provide inter-process synchronization.
+type FileMutex struct {
+ sync.RWMutex
+}
+
+func MakeFileMutex(filename string) *FileMutex {
+ return &FileMutex{}
+}
+
+func init() {
+ log.Printf("WARNING: using fake file mutex." +
+ " Don't run more than one of these at once!!!")
+}
diff --git a/dashboard/cmd/builder/filemutex_windows.go b/dashboard/cmd/builder/filemutex_windows.go
new file mode 100644
index 0000000..1f058b2
--- /dev/null
+++ b/dashboard/cmd/builder/filemutex_windows.go
@@ -0,0 +1,105 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "sync"
+ "syscall"
+ "unsafe"
+)
+
+var (
+ modkernel32 = syscall.NewLazyDLL("kernel32.dll")
+ procLockFileEx = modkernel32.NewProc("LockFileEx")
+ procUnlockFileEx = modkernel32.NewProc("UnlockFileEx")
+)
+
+const (
+ INVALID_FILE_HANDLE = ^syscall.Handle(0)
+ LOCKFILE_EXCLUSIVE_LOCK = 2
+)
+
+func lockFileEx(h syscall.Handle, flags, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) {
+ r1, _, e1 := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(h), uintptr(flags), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)))
+ if r1 == 0 {
+ if e1 != 0 {
+ err = error(e1)
+ } else {
+ err = syscall.EINVAL
+ }
+ }
+ return
+}
+
+func unlockFileEx(h syscall.Handle, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) {
+ r1, _, e1 := syscall.Syscall6(procUnlockFileEx.Addr(), 5, uintptr(h), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)), 0)
+ if r1 == 0 {
+ if e1 != 0 {
+ err = error(e1)
+ } else {
+ err = syscall.EINVAL
+ }
+ }
+ return
+}
+
+// FileMutex is similar to sync.RWMutex, but also synchronizes across processes.
+// This implementation is based on flock syscall.
+type FileMutex struct {
+ mu sync.RWMutex
+ fd syscall.Handle
+}
+
+func MakeFileMutex(filename string) *FileMutex {
+ if filename == "" {
+ return &FileMutex{fd: INVALID_FILE_HANDLE}
+ }
+ fd, err := syscall.CreateFile(&(syscall.StringToUTF16(filename)[0]), syscall.GENERIC_READ|syscall.GENERIC_WRITE,
+ syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, nil, syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0)
+ if err != nil {
+ panic(err)
+ }
+ return &FileMutex{fd: fd}
+}
+
+func (m *FileMutex) Lock() {
+ m.mu.Lock()
+ if m.fd != INVALID_FILE_HANDLE {
+ var ol syscall.Overlapped
+ if err := lockFileEx(m.fd, LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &ol); err != nil {
+ panic(err)
+ }
+ }
+}
+
+func (m *FileMutex) Unlock() {
+ if m.fd != INVALID_FILE_HANDLE {
+ var ol syscall.Overlapped
+ if err := unlockFileEx(m.fd, 0, 1, 0, &ol); err != nil {
+ panic(err)
+ }
+ }
+ m.mu.Unlock()
+}
+
+func (m *FileMutex) RLock() {
+ m.mu.RLock()
+ if m.fd != INVALID_FILE_HANDLE {
+ var ol syscall.Overlapped
+ if err := lockFileEx(m.fd, 0, 0, 1, 0, &ol); err != nil {
+ panic(err)
+ }
+ }
+}
+
+func (m *FileMutex) RUnlock() {
+ if m.fd != INVALID_FILE_HANDLE {
+ var ol syscall.Overlapped
+ if err := unlockFileEx(m.fd, 0, 1, 0, &ol); err != nil {
+ panic(err)
+ }
+ }
+ m.mu.RUnlock()
+}
diff --git a/dashboard/cmd/builder/http.go b/dashboard/cmd/builder/http.go
new file mode 100644
index 0000000..8d0923c
--- /dev/null
+++ b/dashboard/cmd/builder/http.go
@@ -0,0 +1,225 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "net/url"
+ "time"
+)
+
+const builderVersion = 1 // keep in sync with dashboard/app/build/handler.go
+
+type obj map[string]interface{}
+
+// dash runs the given method and command on the dashboard.
+// If args is non-nil it is encoded as the URL query string.
+// If req is non-nil it is JSON-encoded and passed as the body of the HTTP POST.
+// If resp is non-nil the server's response is decoded into the value pointed
+// to by resp (resp must be a pointer).
+func dash(meth, cmd string, args url.Values, req, resp interface{}) error {
+ argsCopy := url.Values{"version": {fmt.Sprint(builderVersion)}}
+ for k, v := range args {
+ if k == "version" {
+ panic(`dash: reserved args key: "version"`)
+ }
+ argsCopy[k] = v
+ }
+ var r *http.Response
+ var err error
+ if *verbose {
+ log.Println("dash <-", meth, cmd, argsCopy, req)
+ }
+ cmd = *dashboard + "/" + cmd + "?" + argsCopy.Encode()
+ switch meth {
+ case "GET":
+ if req != nil {
+ log.Panicf("%s to %s with req", meth, cmd)
+ }
+ r, err = http.Get(cmd)
+ case "POST":
+ var body io.Reader
+ if req != nil {
+ b, err := json.Marshal(req)
+ if err != nil {
+ return err
+ }
+ body = bytes.NewBuffer(b)
+ }
+ r, err = http.Post(cmd, "text/json", body)
+ default:
+ log.Panicf("%s: invalid method %q", cmd, meth)
+ panic("invalid method: " + meth)
+ }
+ if err != nil {
+ return err
+ }
+ defer r.Body.Close()
+ if r.StatusCode != http.StatusOK {
+ return fmt.Errorf("bad http response: %v", r.Status)
+ }
+ body := new(bytes.Buffer)
+ if _, err := body.ReadFrom(r.Body); err != nil {
+ return err
+ }
+
+ // Read JSON-encoded Response into provided resp
+ // and return an error if present.
+ var result = struct {
+ Response interface{}
+ Error string
+ }{
+ // Put the provided resp in here as it can be a pointer to
+ // some value we should unmarshal into.
+ Response: resp,
+ }
+ if err = json.Unmarshal(body.Bytes(), &result); err != nil {
+ log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err)
+ return err
+ }
+ if *verbose {
+ log.Println("dash ->", result)
+ }
+ if result.Error != "" {
+ return errors.New(result.Error)
+ }
+
+ return nil
+}
+
+// todo returns the next hash to build or benchmark.
+func (b *Builder) todo(kinds []string, pkg, goHash string) (kind, rev string, benchs []string, err error) {
+ args := url.Values{
+ "builder": {b.name},
+ "packagePath": {pkg},
+ "goHash": {goHash},
+ }
+ for _, k := range kinds {
+ args.Add("kind", k)
+ }
+ var resp *struct {
+ Kind string
+ Data struct {
+ Hash string
+ PerfResults []string
+ }
+ }
+ if err = dash("GET", "todo", args, nil, &resp); err != nil {
+ return
+ }
+ if resp == nil {
+ return
+ }
+ if *verbose {
+ fmt.Printf("dash resp: %+v\n", *resp)
+ }
+ for _, k := range kinds {
+ if k == resp.Kind {
+ return resp.Kind, resp.Data.Hash, resp.Data.PerfResults, nil
+ }
+ }
+ err = fmt.Errorf("expecting Kinds %q, got %q", kinds, resp.Kind)
+ return
+}
+
+// recordResult sends build results to the dashboard
+func (b *Builder) recordResult(ok bool, pkg, hash, goHash, buildLog string, runTime time.Duration) error {
+ if !*report {
+ return nil
+ }
+ req := obj{
+ "Builder": b.name,
+ "PackagePath": pkg,
+ "Hash": hash,
+ "GoHash": goHash,
+ "OK": ok,
+ "Log": buildLog,
+ "RunTime": runTime,
+ }
+ args := url.Values{"key": {b.key}, "builder": {b.name}}
+ return dash("POST", "result", args, req, nil)
+}
+
+// Result of running a single benchmark on a single commit.
+type PerfResult struct {
+ Builder string
+ Benchmark string
+ Hash string
+ OK bool
+ Metrics []PerfMetric
+ Artifacts []PerfArtifact
+}
+
+type PerfMetric struct {
+ Type string
+ Val uint64
+}
+
+type PerfArtifact struct {
+ Type string
+ Body string
+}
+
+// recordPerfResult sends benchmarking results to the dashboard
+func (b *Builder) recordPerfResult(req *PerfResult) error {
+ if !*report {
+ return nil
+ }
+ req.Builder = b.name
+ args := url.Values{"key": {b.key}, "builder": {b.name}}
+ return dash("POST", "perf-result", args, req, nil)
+}
+
+func postCommit(key, pkg string, l *HgLog) error {
+ if !*report {
+ return nil
+ }
+ t, err := time.Parse(time.RFC3339, l.Date)
+ if err != nil {
+ return fmt.Errorf("parsing %q: %v", l.Date, t)
+ }
+ return dash("POST", "commit", url.Values{"key": {key}}, obj{
+ "PackagePath": pkg,
+ "Hash": l.Hash,
+ "ParentHash": l.Parent,
+ "Time": t.Format(time.RFC3339),
+ "User": l.Author,
+ "Desc": l.Desc,
+ "NeedsBenchmarking": l.bench,
+ }, nil)
+}
+
+func dashboardCommit(pkg, hash string) bool {
+ err := dash("GET", "commit", url.Values{
+ "packagePath": {pkg},
+ "hash": {hash},
+ }, nil, nil)
+ return err == nil
+}
+
+func dashboardPackages(kind string) []string {
+ args := url.Values{"kind": []string{kind}}
+ var resp []struct {
+ Path string
+ }
+ if err := dash("GET", "packages", args, nil, &resp); err != nil {
+ log.Println("dashboardPackages:", err)
+ return nil
+ }
+ if *verbose {
+ fmt.Printf("dash resp: %+v\n", resp)
+ }
+ var pkgs []string
+ for _, r := range resp {
+ pkgs = append(pkgs, r.Path)
+ }
+ return pkgs
+}
diff --git a/dashboard/cmd/builder/main.go b/dashboard/cmd/builder/main.go
new file mode 100644
index 0000000..9e7c1ed
--- /dev/null
+++ b/dashboard/cmd/builder/main.go
@@ -0,0 +1,679 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "net"
+ "net/http"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "regexp"
+ "runtime"
+ "strconv"
+ "strings"
+ "time"
+
+ "golang.org/x/tools/go/vcs"
+)
+
+const (
+ codeProject = "go"
+ codePyScript = "misc/dashboard/googlecode_upload.py"
+ gofrontendImportPath = "code.google.com/p/gofrontend"
+ mkdirPerm = 0750
+ waitInterval = 30 * time.Second // time to wait before checking for new revs
+ pkgBuildInterval = 24 * time.Hour // rebuild packages every 24 hours
+)
+
+type Builder struct {
+ goroot *Repo
+ name string
+ goos, goarch string
+ key string
+ env builderEnv
+ // Last benchmarking workpath. We reuse it, if do successive benchmarks on the same commit.
+ lastWorkpath string
+}
+
+var (
+ doBuild = flag.Bool("build", true, "Build and test packages")
+ doBench = flag.Bool("bench", false, "Run benchmarks")
+ buildroot = flag.String("buildroot", defaultBuildRoot(), "Directory under which to build")
+ dashboard = flag.String("dashboard", "https://build.golang.org", "Dashboard app base path")
+ buildRelease = flag.Bool("release", false, "Build and upload binary release archives")
+ buildRevision = flag.String("rev", "", "Build specified revision and exit")
+ buildCmd = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)")
+ buildTool = flag.String("tool", "go", "Tool to build.")
+ gcPath = flag.String("gcpath", "go.googlesource.com/go", "Path to download gc from")
+ gccPath = flag.String("gccpath", "https://github.com/mirrors/gcc.git", "Path to download gcc from")
+ gccOpts = flag.String("gccopts", "", "Command-line options to pass to `make` when building gccgo")
+ benchPath = flag.String("benchpath", "golang.org/x/benchmarks/bench", "Path to download benchmarks from")
+ failAll = flag.Bool("fail", false, "fail all builds")
+ parallel = flag.Bool("parallel", false, "Build multiple targets in parallel")
+ buildTimeout = flag.Duration("buildTimeout", 60*time.Minute, "Maximum time to wait for builds and tests")
+ cmdTimeout = flag.Duration("cmdTimeout", 10*time.Minute, "Maximum time to wait for an external command")
+ benchNum = flag.Int("benchnum", 5, "Run each benchmark that many times")
+ benchTime = flag.Duration("benchtime", 5*time.Second, "Benchmarking time for a single benchmark run")
+ benchMem = flag.Int("benchmem", 64, "Approx RSS value to aim at in benchmarks, in MB")
+ fileLock = flag.String("filelock", "", "File to lock around benchmaring (synchronizes several builders)")
+ verbose = flag.Bool("v", false, "verbose")
+ report = flag.Bool("report", true, "whether to report results to the dashboard")
+)
+
+var (
+ binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`)
+ releaseRe = regexp.MustCompile(`^release\.r[0-9\-.]+`)
+ allCmd = "all" + suffix
+ makeCmd = "make" + suffix
+ raceCmd = "race" + suffix
+ cleanCmd = "clean" + suffix
+ suffix = defaultSuffix()
+ exeExt = defaultExeExt()
+
+ benchCPU = CpuList([]int{1})
+ benchAffinity = CpuList([]int{})
+ benchMutex *FileMutex // Isolates benchmarks from other activities
+)
+
+// CpuList is used as flag.Value for -benchcpu flag.
+type CpuList []int
+
+func (cl *CpuList) String() string {
+ str := ""
+ for _, cpu := range *cl {
+ if str == "" {
+ str = strconv.Itoa(cpu)
+ } else {
+ str += fmt.Sprintf(",%v", cpu)
+ }
+ }
+ return str
+}
+
+func (cl *CpuList) Set(str string) error {
+ *cl = []int{}
+ for _, val := range strings.Split(str, ",") {
+ val = strings.TrimSpace(val)
+ if val == "" {
+ continue
+ }
+ cpu, err := strconv.Atoi(val)
+ if err != nil || cpu <= 0 {
+ return fmt.Errorf("%v is a bad value for GOMAXPROCS", val)
+ }
+ *cl = append(*cl, cpu)
+ }
+ if len(*cl) == 0 {
+ *cl = append(*cl, 1)
+ }
+ return nil
+}
+
+func main() {
+ flag.Var(&benchCPU, "benchcpu", "Comma-delimited list of GOMAXPROCS values for benchmarking")
+ flag.Var(&benchAffinity, "benchaffinity", "Comma-delimited list of affinity values for benchmarking")
+ flag.Usage = func() {
+ fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0])
+ flag.PrintDefaults()
+ os.Exit(2)
+ }
+ flag.Parse()
+ if len(flag.Args()) == 0 {
+ flag.Usage()
+ }
+
+ vcs.ShowCmd = *verbose
+ vcs.Verbose = *verbose
+
+ benchMutex = MakeFileMutex(*fileLock)
+
+ rr, err := repoForTool()
+ if err != nil {
+ log.Fatal("Error finding repository:", err)
+ }
+ rootPath := filepath.Join(*buildroot, "goroot")
+ goroot := &Repo{
+ Path: rootPath,
+ Master: rr,
+ }
+
+ // set up work environment, use existing environment if possible
+ if goroot.Exists() || *failAll {
+ log.Print("Found old workspace, will use it")
+ } else {
+ if err := os.RemoveAll(*buildroot); err != nil {
+ log.Fatalf("Error removing build root (%s): %s", *buildroot, err)
+ }
+ if err := os.Mkdir(*buildroot, mkdirPerm); err != nil {
+ log.Fatalf("Error making build root (%s): %s", *buildroot, err)
+ }
+ var err error
+ goroot, err = RemoteRepo(goroot.Master.Root, rootPath)
+ if err != nil {
+ log.Fatalf("Error creating repository with url (%s): %s", goroot.Master.Root, err)
+ }
+
+ goroot, err = goroot.Clone(goroot.Path, "")
+ if err != nil {
+ log.Fatal("Error cloning repository:", err)
+ }
+ }
+
+ // set up builders
+ builders := make([]*Builder, len(flag.Args()))
+ for i, name := range flag.Args() {
+ b, err := NewBuilder(goroot, name)
+ if err != nil {
+ log.Fatal(err)
+ }
+ builders[i] = b
+ }
+
+ if *failAll {
+ failMode(builders)
+ return
+ }
+
+ // if specified, build revision and return
+ if *buildRevision != "" {
+ hash, err := goroot.FullHash(*buildRevision)
+ if err != nil {
+ log.Fatal("Error finding revision: ", err)
+ }
+ var exitErr error
+ for _, b := range builders {
+ if err := b.buildHash(hash); err != nil {
+ log.Println(err)
+ exitErr = err
+ }
+ }
+ if exitErr != nil && !*report {
+ // This mode (-report=false) is used for
+ // testing Docker images, making sure the
+ // environment is correctly configured. For
+ // testing, we want a non-zero exit status, as
+ // returned by log.Fatal:
+ log.Fatal("Build error.")
+ }
+ return
+ }
+
+ if !*doBuild && !*doBench {
+ fmt.Fprintf(os.Stderr, "Nothing to do, exiting (specify either -build or -bench or both)\n")
+ os.Exit(2)
+ }
+
+ // go continuous build mode
+ // check for new commits and build them
+ benchMutex.RLock()
+ for {
+ built := false
+ t := time.Now()
+ if *parallel {
+ done := make(chan bool)
+ for _, b := range builders {
+ go func(b *Builder) {
+ done <- b.buildOrBench()
+ }(b)
+ }
+ for _ = range builders {
+ built = <-done || built
+ }
+ } else {
+ for _, b := range builders {
+ built = b.buildOrBench() || built
+ }
+ }
+ // sleep if there was nothing to build
+ benchMutex.RUnlock()
+ if !built {
+ time.Sleep(waitInterval)
+ }
+ benchMutex.RLock()
+ // sleep if we're looping too fast.
+ dt := time.Now().Sub(t)
+ if dt < waitInterval {
+ time.Sleep(waitInterval - dt)
+ }
+ }
+}
+
+// go continuous fail mode
+// check for new commits and FAIL them
+func failMode(builders []*Builder) {
+ for {
+ built := false
+ for _, b := range builders {
+ built = b.failBuild() || built
+ }
+ // stop if there was nothing to fail
+ if !built {
+ break
+ }
+ }
+}
+
+func NewBuilder(goroot *Repo, name string) (*Builder, error) {
+ b := &Builder{
+ goroot: goroot,
+ name: name,
+ }
+
+ // get builderEnv for this tool
+ var err error
+ if b.env, err = b.builderEnv(name); err != nil {
+ return nil, err
+ }
+ if *report {
+ err = b.setKey()
+ }
+ return b, err
+}
+
+func (b *Builder) setKey() error {
+ // read keys from keyfile
+ fn := ""
+ switch runtime.GOOS {
+ case "plan9":
+ fn = os.Getenv("home")
+ case "windows":
+ fn = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
+ default:
+ fn = os.Getenv("HOME")
+ }
+ fn = filepath.Join(fn, ".gobuildkey")
+ if s := fn + "-" + b.name; isFile(s) { // builder-specific file
+ fn = s
+ }
+ c, err := ioutil.ReadFile(fn)
+ if err != nil {
+ // If the on-disk file doesn't exist, also try the
+ // Google Compute Engine metadata.
+ if v := gceProjectMetadata("buildkey-" + b.name); v != "" {
+ b.key = v
+ return nil
+ }
+ return fmt.Errorf("readKeys %s (%s): %s", b.name, fn, err)
+ }
+ b.key = string(bytes.TrimSpace(bytes.SplitN(c, []byte("\n"), 2)[0]))
+ return nil
+}
+
+func gceProjectMetadata(attr string) string {
+ client := &http.Client{
+ Transport: &http.Transport{
+ Dial: (&net.Dialer{
+ Timeout: 750 * time.Millisecond,
+ KeepAlive: 30 * time.Second,
+ }).Dial,
+ ResponseHeaderTimeout: 750 * time.Millisecond,
+ },
+ }
+ req, _ := http.NewRequest("GET", "http://metadata.google.internal/computeMetadata/v1/project/attributes/"+attr, nil)
+ req.Header.Set("Metadata-Flavor", "Google")
+ res, err := client.Do(req)
+ if err != nil {
+ return ""
+ }
+ defer res.Body.Close()
+ if res.StatusCode != 200 {
+ return ""
+ }
+ slurp, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ return ""
+ }
+ return string(bytes.TrimSpace(slurp))
+}
+
+// builderEnv returns the builderEnv for this buildTool.
+func (b *Builder) builderEnv(name string) (builderEnv, error) {
+ // get goos/goarch from builder string
+ s := strings.SplitN(b.name, "-", 3)
+ if len(s) < 2 {
+ return nil, fmt.Errorf("unsupported builder form: %s", name)
+ }
+ b.goos = s[0]
+ b.goarch = s[1]
+
+ switch *buildTool {
+ case "go":
+ return &goEnv{
+ goos: s[0],
+ goarch: s[1],
+ }, nil
+ case "gccgo":
+ return &gccgoEnv{}, nil
+ default:
+ return nil, fmt.Errorf("unsupported build tool: %s", *buildTool)
+ }
+}
+
+// buildCmd returns the build command to invoke.
+// Builders which contain the string '-race' in their
+// name will override *buildCmd and return raceCmd.
+func (b *Builder) buildCmd() string {
+ if strings.Contains(b.name, "-race") {
+ return raceCmd
+ }
+ return *buildCmd
+}
+
+// buildOrBench checks for a new commit for this builder
+// and builds or benchmarks it if one is found.
+// It returns true if a build/benchmark was attempted.
+func (b *Builder) buildOrBench() bool {
+ var kinds []string
+ if *doBuild {
+ kinds = append(kinds, "build-go-commit")
+ }
+ if *doBench {
+ kinds = append(kinds, "benchmark-go-commit")
+ }
+ kind, hash, benchs, err := b.todo(kinds, "", "")
+ if err != nil {
+ log.Println(err)
+ return false
+ }
+ if hash == "" {
+ return false
+ }
+ switch kind {
+ case "build-go-commit":
+ if err := b.buildHash(hash); err != nil {
+ log.Println(err)
+ }
+ return true
+ case "benchmark-go-commit":
+ if err := b.benchHash(hash, benchs); err != nil {
+ log.Println(err)
+ }
+ return true
+ default:
+ log.Printf("Unknown todo kind %v", kind)
+ return false
+ }
+}
+
+func (b *Builder) buildHash(hash string) error {
+ log.Println(b.name, "building", hash)
+
+ // create place in which to do work
+ workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
+ if err := os.Mkdir(workpath, mkdirPerm); err != nil {
+ if err2 := removePath(workpath); err2 != nil {
+ return err
+ }
+ if err := os.Mkdir(workpath, mkdirPerm); err != nil {
+ return err
+ }
+ }
+ defer removePath(workpath)
+
+ buildLog, runTime, err := b.buildRepoOnHash(workpath, hash, b.buildCmd())
+ if err != nil {
+ log.Printf("%s failed at %v: %v", b.name, hash, err)
+ // record failure
+ return b.recordResult(false, "", hash, "", buildLog, runTime)
+ }
+
+ // record success
+ if err = b.recordResult(true, "", hash, "", "", runTime); err != nil {
+ return fmt.Errorf("recordResult: %s", err)
+ }
+
+ if *buildTool == "go" {
+ // build sub-repositories
+ goRoot := filepath.Join(workpath, *buildTool)
+ goPath := workpath
+ b.buildSubrepos(goRoot, goPath, hash)
+ }
+
+ return nil
+}
+
+// buildRepoOnHash clones repo into workpath and builds it.
+func (b *Builder) buildRepoOnHash(workpath, hash, cmd string) (buildLog string, runTime time.Duration, err error) {
+ // Delete the previous workdir, if necessary
+ // (benchmarking code can execute several benchmarks in the same workpath).
+ if b.lastWorkpath != "" {
+ if b.lastWorkpath == workpath {
+ panic("workpath already exists: " + workpath)
+ }
+ removePath(b.lastWorkpath)
+ b.lastWorkpath = ""
+ }
+
+ // pull before cloning to ensure we have the revision
+ if err = b.goroot.Pull(); err != nil {
+ buildLog = err.Error()
+ return
+ }
+
+ // set up builder's environment.
+ srcDir, err := b.env.setup(b.goroot, workpath, hash, b.envv())
+ if err != nil {
+ buildLog = err.Error()
+ return
+ }
+
+ // build
+ var buildbuf bytes.Buffer
+ logfile := filepath.Join(workpath, "build.log")
+ f, err := os.Create(logfile)
+ if err != nil {
+ return err.Error(), 0, err
+ }
+ defer f.Close()
+ w := io.MultiWriter(f, &buildbuf)
+
+ // go's build command is a script relative to the srcDir, whereas
+ // gccgo's build command is usually "make check-go" in the srcDir.
+ if *buildTool == "go" {
+ if !filepath.IsAbs(cmd) {
+ cmd = filepath.Join(srcDir, cmd)
+ }
+ }
+
+ // naive splitting of command from its arguments:
+ args := strings.Split(cmd, " ")
+ c := exec.Command(args[0], args[1:]...)
+ c.Dir = srcDir
+ c.Env = b.envv()
+ if *verbose {
+ c.Stdout = io.MultiWriter(os.Stdout, w)
+ c.Stderr = io.MultiWriter(os.Stderr, w)
+ } else {
+ c.Stdout = w
+ c.Stderr = w
+ }
+
+ startTime := time.Now()
+ err = run(c, runTimeout(*buildTimeout))
+ runTime = time.Since(startTime)
+ if err != nil {
+ fmt.Fprintf(w, "Build complete, duration %v. Result: error: %v\n", runTime, err)
+ } else {
+ fmt.Fprintf(w, "Build complete, duration %v. Result: success\n", runTime)
+ }
+ return buildbuf.String(), runTime, err
+}
+
+// failBuild checks for a new commit for this builder
+// and fails it if one is found.
+// It returns true if a build was "attempted".
+func (b *Builder) failBuild() bool {
+ _, hash, _, err := b.todo([]string{"build-go-commit"}, "", "")
+ if err != nil {
+ log.Println(err)
+ return false
+ }
+ if hash == "" {
+ return false
+ }
+
+ log.Printf("fail %s %s\n", b.name, hash)
+
+ if err := b.recordResult(false, "", hash, "", "auto-fail mode run by "+os.Getenv("USER"), 0); err != nil {
+ log.Print(err)
+ }
+ return true
+}
+
+func (b *Builder) buildSubrepos(goRoot, goPath, goHash string) {
+ for _, pkg := range dashboardPackages("subrepo") {
+ // get the latest todo for this package
+ _, hash, _, err := b.todo([]string{"build-package"}, pkg, goHash)
+ if err != nil {
+ log.Printf("buildSubrepos %s: %v", pkg, err)
+ continue
+ }
+ if hash == "" {
+ continue
+ }
+
+ // build the package
+ if *verbose {
+ log.Printf("buildSubrepos %s: building %q", pkg, hash)
+ }
+ buildLog, err := b.buildSubrepo(goRoot, goPath, pkg, hash)
+ if err != nil {
+ if buildLog == "" {
+ buildLog = err.Error()
+ }
+ log.Printf("buildSubrepos %s: %v", pkg, err)
+ }
+
+ // record the result
+ err = b.recordResult(err == nil, pkg, hash, goHash, buildLog, 0)
+ if err != nil {
+ log.Printf("buildSubrepos %s: %v", pkg, err)
+ }
+ }
+}
+
+// buildSubrepo fetches the given package, updates it to the specified hash,
+// and runs 'go test -short pkg/...'. It returns the build log and any error.
+func (b *Builder) buildSubrepo(goRoot, goPath, pkg, hash string) (string, error) {
+ goTool := filepath.Join(goRoot, "bin", "go") + exeExt
+ env := append(b.envv(), "GOROOT="+goRoot, "GOPATH="+goPath)
+
+ // add $GOROOT/bin and $GOPATH/bin to PATH
+ for i, e := range env {
+ const p = "PATH="
+ if !strings.HasPrefix(e, p) {
+ continue
+ }
+ sep := string(os.PathListSeparator)
+ env[i] = p + filepath.Join(goRoot, "bin") + sep + filepath.Join(goPath, "bin") + sep + e[len(p):]
+ }
+
+ // HACK: check out to new sub-repo location instead of old location.
+ pkg = strings.Replace(pkg, "code.google.com/p/go.", "golang.org/x/", 1)
+
+ // fetch package and dependencies
+ var outbuf bytes.Buffer
+ err := run(exec.Command(goTool, "get", "-d", pkg+"/..."), runEnv(env), allOutput(&outbuf), runDir(goPath))
+ if err != nil {
+ return outbuf.String(), err
+ }
+ outbuf.Reset()
+
+ // hg update to the specified hash
+ pkgmaster, err := vcs.RepoRootForImportPath(pkg, *verbose)
+ if err != nil {
+ return "", fmt.Errorf("Error finding subrepo (%s): %s", pkg, err)
+ }
+ repo := &Repo{
+ Path: filepath.Join(goPath, "src", pkg),
+ Master: pkgmaster,
+ }
+ if err := repo.UpdateTo(hash); err != nil {
+ return "", err
+ }
+
+ // test the package
+ err = run(exec.Command(goTool, "test", "-short", pkg+"/..."),
+ runTimeout(*buildTimeout), runEnv(env), allOutput(&outbuf), runDir(goPath))
+ return outbuf.String(), err
+}
+
+// repoForTool returns the correct RepoRoot for the buildTool, or an error if
+// the tool is unknown.
+func repoForTool() (*vcs.RepoRoot, error) {
+ switch *buildTool {
+ case "go":
+ return vcs.RepoRootForImportPath(*gcPath, *verbose)
+ case "gccgo":
+ return vcs.RepoRootForImportPath(gofrontendImportPath, *verbose)
+ default:
+ return nil, fmt.Errorf("unknown build tool: %s", *buildTool)
+ }
+}
+
+func isDirectory(name string) bool {
+ s, err := os.Stat(name)
+ return err == nil && s.IsDir()
+}
+
+func isFile(name string) bool {
+ s, err := os.Stat(name)
+ return err == nil && !s.IsDir()
+}
+
+// defaultSuffix returns file extension used for command files in
+// current os environment.
+func defaultSuffix() string {
+ switch runtime.GOOS {
+ case "windows":
+ return ".bat"
+ case "plan9":
+ return ".rc"
+ default:
+ return ".bash"
+ }
+}
+
+func defaultExeExt() string {
+ switch runtime.GOOS {
+ case "windows":
+ return ".exe"
+ default:
+ return ""
+ }
+}
+
+// defaultBuildRoot returns default buildroot directory.
+func defaultBuildRoot() string {
+ var d string
+ if runtime.GOOS == "windows" {
+ // will use c:\, otherwise absolute paths become too long
+ // during builder run, see http://golang.org/issue/3358.
+ d = `c:\`
+ } else {
+ d = os.TempDir()
+ }
+ return filepath.Join(d, "gobuilder")
+}
+
+// removePath is a more robust version of os.RemoveAll.
+// On windows, if remove fails (which can happen if test/benchmark timeouts
+// and keeps some files open) it tries to rename the dir.
+func removePath(path string) error {
+ if err := os.RemoveAll(path); err != nil {
+ if runtime.GOOS == "windows" {
+ err = os.Rename(path, filepath.Clean(path)+"_remove_me")
+ }
+ return err
+ }
+ return nil
+}
diff --git a/dashboard/cmd/builder/vcs.go b/dashboard/cmd/builder/vcs.go
new file mode 100644
index 0000000..2139a90
--- /dev/null
+++ b/dashboard/cmd/builder/vcs.go
@@ -0,0 +1,225 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+ "bytes"
+ "encoding/xml"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "sync"
+
+ "golang.org/x/tools/go/vcs"
+)
+
+// Repo represents a mercurial repository.
+type Repo struct {
+ Path string
+ Master *vcs.RepoRoot
+ sync.Mutex
+}
+
+// RemoteRepo constructs a *Repo representing a remote repository.
+func RemoteRepo(url, path string) (*Repo, error) {
+ rr, err := vcs.RepoRootForImportPath(url, *verbose)
+ if err != nil {
+ return nil, err
+ }
+ return &Repo{
+ Path: path,
+ Master: rr,
+ }, nil
+}
+
+// Clone clones the current Repo to a new destination
+// returning a new *Repo if successful.
+func (r *Repo) Clone(path, rev string) (*Repo, error) {
+ r.Lock()
+ defer r.Unlock()
+
+ err := timeout(*cmdTimeout, func() error {
+ downloadPath := r.Path
+ if !r.Exists() {
+ downloadPath = r.Master.Repo
+ }
+ if rev == "" {
+ return r.Master.VCS.Create(path, downloadPath)
+ }
+ return r.Master.VCS.CreateAtRev(path, downloadPath, rev)
+ })
+ if err != nil {
+ return nil, err
+ }
+ return &Repo{
+ Path: path,
+ Master: r.Master,
+ }, nil
+}
+
+// Export exports the current Repo at revision rev to a new destination.
+func (r *Repo) Export(path, rev string) error {
+ // TODO(adg,cmang): implement Export in go/vcs
+ _, err := r.Clone(path, rev)
+ return err
+}
+
+// UpdateTo updates the working copy of this Repo to the
+// supplied revision.
+func (r *Repo) UpdateTo(hash string) error {
+ r.Lock()
+ defer r.Unlock()
+
+ if r.Master.VCS.Cmd == "git" {
+ cmd := exec.Command("git", "reset", "--hard", hash)
+ var log bytes.Buffer
+ err := run(cmd, runTimeout(*cmdTimeout), runDir(r.Path), allOutput(&log))
+ if err != nil {
+ return fmt.Errorf("Error running git update -C %v: %v ; output=%s", hash, err, log.Bytes())
+ }
+ return nil
+ }
+
+ // Else go down three more levels of abstractions, at
+ // least two of which are broken for git.
+ return timeout(*cmdTimeout, func() error {
+ return r.Master.VCS.TagSync(r.Path, hash)
+ })
+}
+
+// Exists reports whether this Repo represents a valid Mecurial repository.
+func (r *Repo) Exists() bool {
+ fi, err := os.Stat(filepath.Join(r.Path, "."+r.Master.VCS.Cmd))
+ if err != nil {
+ return false
+ }
+ return fi.IsDir()
+}
+
+// Pull pulls changes from the default path, that is, the path
+// this Repo was cloned from.
+func (r *Repo) Pull() error {
+ r.Lock()
+ defer r.Unlock()
+
+ return timeout(*cmdTimeout, func() error {
+ return r.Master.VCS.Download(r.Path)
+ })
+}
+
+// Log returns the changelog for this repository.
+func (r *Repo) Log() ([]HgLog, error) {
+ if err := r.Pull(); err != nil {
+ return nil, err
+ }
+ r.Lock()
+ defer r.Unlock()
+
+ var logStruct struct {
+ Log []HgLog
+ }
+ err := timeout(*cmdTimeout, func() error {
+ data, err := r.Master.VCS.Log(r.Path, xmlLogTemplate)
+ if err != nil {
+ return err
+ }
+
+ // We have a commit with description that contains 0x1b byte.
+ // Mercurial does not escape it, but xml.Unmarshal does not accept it.
+ data = bytes.Replace(data, []byte{0x1b}, []byte{'?'}, -1)
+
+ err = xml.Unmarshal([]byte("<Top>"+string(data)+"</Top>"), &logStruct)
+ if err != nil {
+ return fmt.Errorf("unmarshal %s log: %v", r.Master.VCS, err)
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ for i, log := range logStruct.Log {
+ // Let's pretend there can be only one parent.
+ if log.Parent != "" && strings.Contains(log.Parent, " ") {
+ logStruct.Log[i].Parent = strings.Split(log.Parent, " ")[0]
+ }
+ }
+ return logStruct.Log, nil
+}
+
+// FullHash returns the full hash for the given Git or Mercurial revision.
+func (r *Repo) FullHash(rev string) (string, error) {
+ r.Lock()
+ defer r.Unlock()
+
+ var hash string
+ err := timeout(*cmdTimeout, func() error {
+ var data []byte
+ // Avoid the vcs package for git, since it's broken
+ // for git, and and we're trying to remove levels of
+ // abstraction which are increasingly getting
+ // difficult to navigate.
+ if r.Master.VCS.Cmd == "git" {
+ cmd := exec.Command("git", "rev-parse", rev)
+ var out bytes.Buffer
+ err := run(cmd, runTimeout(*cmdTimeout), runDir(r.Path), allOutput(&out))
+ data = out.Bytes()
+ if err != nil {
+ return fmt.Errorf("Failed to find FullHash of %q; git rev-parse: %v, %s", rev, err, data)
+ }
+ } else {
+ var err error
+ data, err = r.Master.VCS.LogAtRev(r.Path, rev, "{node}")
+ if err != nil {
+ return err
+ }
+ }
+ s := strings.TrimSpace(string(data))
+ if s == "" {
+ return fmt.Errorf("cannot find revision")
+ }
+ if len(s) != 40 { // correct for both hg and git
+ return fmt.Errorf("%s returned invalid hash: %s", r.Master.VCS, s)
+ }
+ hash = s
+ return nil
+ })
+ if err != nil {
+ return "", err
+ }
+ return hash, nil
+}
+
+// HgLog represents a single Mercurial revision.
+type HgLog struct {
+ Hash string
+ Author string
+ Date string
+ Desc string
+ Parent string
+ Branch string
+ Files string
+
+ // Internal metadata
+ added bool
+ bench bool // needs to be benchmarked?
+}
+
+// xmlLogTemplate is a template to pass to Mercurial to make
+// hg log print the log in valid XML for parsing with xml.Unmarshal.
+// Can not escape branches and files, because it crashes python with:
+// AttributeError: 'NoneType' object has no attribute 'replace'
+const xmlLogTemplate = `
+ <Log>
+ <Hash>{node|escape}</Hash>
+ <Parent>{p1node}</Parent>
+ <Author>{author|escape}</Author>
+ <Date>{date|rfc3339date}</Date>
+ <Desc>{desc|escape}</Desc>
+ <Branch>{branches}</Branch>
+ <Files>{files}</Files>
+ </Log>
+`
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-02 22:03:36 +0000