aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndrew Gerrand <adg@golang.org>2015-01-21 17:32:48 +1100
committerAndrew Gerrand <adg@golang.org>2015-01-21 06:38:18 +0000
commitcb6b359e69e54fe0afde51176ac97c38f6095fb1 (patch)
treea4318960aa10d94ee614edeb03e8a974de851817
parentb34c44b0e229d347ba41e25702f82e305f572b2b (diff)
downloadtools-cb6b359e69e54fe0afde51176ac97c38f6095fb1.tar.gz
dashboard: delete after move to the golang.org/x/build repository
Change-Id: I04335eb643d342fa7de1f39fc3fc26b1543a60e1 Reviewed-on: https://go-review.googlesource.com/3078 Reviewed-by: Andrew Gerrand <adg@golang.org>
-rw-r--r--dashboard/README40
-rw-r--r--dashboard/app/app.yaml21
-rw-r--r--dashboard/app/build/build.go946
-rw-r--r--dashboard/app/build/dash.go201
-rw-r--r--dashboard/app/build/handler.go994
-rw-r--r--dashboard/app/build/init.go47
-rw-r--r--dashboard/app/build/notify.go378
-rw-r--r--dashboard/app/build/notify.txt9
-rw-r--r--dashboard/app/build/perf.go313
-rw-r--r--dashboard/app/build/perf_changes.go282
-rw-r--r--dashboard/app/build/perf_changes.html89
-rw-r--r--dashboard/app/build/perf_detail.go219
-rw-r--r--dashboard/app/build/perf_detail.html101
-rw-r--r--dashboard/app/build/perf_graph.go268
-rw-r--r--dashboard/app/build/perf_graph.html120
-rw-r--r--dashboard/app/build/perf_learn.go186
-rw-r--r--dashboard/app/build/perf_learn.html45
-rw-r--r--dashboard/app/build/perf_notify.txt13
-rw-r--r--dashboard/app/build/test.go378
-rw-r--r--dashboard/app/build/ui.go605
-rw-r--r--dashboard/app/build/ui.html212
-rw-r--r--dashboard/app/build/update.go117
-rw-r--r--dashboard/app/cache/cache.go86
-rw-r--r--dashboard/app/cron.yaml6
-rw-r--r--dashboard/app/index.yaml54
-rw-r--r--dashboard/app/key/key.go64
-rw-r--r--dashboard/app/static/status_alert.gifbin570 -> 0 bytes
-rw-r--r--dashboard/app/static/status_good.gifbin328 -> 0 bytes
-rw-r--r--dashboard/app/static/style.css308
-rw-r--r--dashboard/auth/auth.go49
-rw-r--r--dashboard/builders.go226
-rw-r--r--dashboard/builders_test.go22
-rw-r--r--dashboard/buildlet/buildletclient.go222
-rw-r--r--dashboard/buildlet/gce.go315
-rw-r--r--dashboard/buildlet/keypair.go132
-rw-r--r--dashboard/cmd/builder/bench.go256
-rw-r--r--dashboard/cmd/builder/doc.go58
-rw-r--r--dashboard/cmd/builder/env.go299
-rw-r--r--dashboard/cmd/builder/exec.go99
-rw-r--r--dashboard/cmd/builder/filemutex_flock.go66
-rw-r--r--dashboard/cmd/builder/filemutex_local.go27
-rw-r--r--dashboard/cmd/builder/filemutex_windows.go105
-rw-r--r--dashboard/cmd/builder/http.go225
-rw-r--r--dashboard/cmd/builder/main.go679
-rw-r--r--dashboard/cmd/builder/vcs.go225
-rw-r--r--dashboard/cmd/buildlet/.gitignore5
-rw-r--r--dashboard/cmd/buildlet/Makefile31
-rw-r--r--dashboard/cmd/buildlet/README12
-rw-r--r--dashboard/cmd/buildlet/buildlet.go654
-rw-r--r--dashboard/cmd/buildlet/stage0/Makefile3
-rw-r--r--dashboard/cmd/buildlet/stage0/stage0.go78
-rw-r--r--dashboard/cmd/coordinator/.gitignore3
-rw-r--r--dashboard/cmd/coordinator/Makefile9
-rw-r--r--dashboard/cmd/coordinator/buildongce/create.go299
-rw-r--r--dashboard/cmd/coordinator/coordinator.go1220
-rw-r--r--dashboard/cmd/gomote/auth.go87
-rw-r--r--dashboard/cmd/gomote/create.go77
-rw-r--r--dashboard/cmd/gomote/destroy.go81
-rw-r--r--dashboard/cmd/gomote/get.go44
-rw-r--r--dashboard/cmd/gomote/gomote.go106
-rw-r--r--dashboard/cmd/gomote/list.go68
-rw-r--r--dashboard/cmd/gomote/put.go84
-rw-r--r--dashboard/cmd/gomote/run.go47
-rw-r--r--dashboard/cmd/retrybuilds/retrybuilds.go238
-rw-r--r--dashboard/cmd/upload/upload.go117
-rw-r--r--dashboard/env/commit-watcher/Dockerfile16
-rw-r--r--dashboard/env/commit-watcher/Makefile9
-rwxr-xr-xdashboard/env/commit-watcher/scripts/build-commit-watcher.sh20
-rwxr-xr-xdashboard/env/commit-watcher/scripts/install-apt-deps.sh11
-rw-r--r--dashboard/env/linux-x86-base/Dockerfile19
-rw-r--r--dashboard/env/linux-x86-base/Makefile15
-rw-r--r--dashboard/env/linux-x86-base/README11
-rwxr-xr-xdashboard/env/linux-x86-base/scripts/build-go-builder.sh24
-rwxr-xr-xdashboard/env/linux-x86-base/scripts/install-apt-deps.sh18
-rw-r--r--dashboard/env/linux-x86-clang/Dockerfile23
-rw-r--r--dashboard/env/linux-x86-clang/Makefile15
-rwxr-xr-xdashboard/env/linux-x86-clang/scripts/build-go-builder.sh24
-rwxr-xr-xdashboard/env/linux-x86-clang/scripts/install-apt-deps.sh22
-rw-r--r--dashboard/env/linux-x86-clang/sources/clang-deps.list3
-rw-r--r--dashboard/env/linux-x86-gccgo/Dockerfile19
-rw-r--r--dashboard/env/linux-x86-gccgo/Makefile15
-rw-r--r--dashboard/env/linux-x86-gccgo/README6
-rwxr-xr-xdashboard/env/linux-x86-gccgo/scripts/install-apt-deps.sh20
-rwxr-xr-xdashboard/env/linux-x86-gccgo/scripts/install-gccgo-builder.sh7
-rwxr-xr-xdashboard/env/linux-x86-gccgo/scripts/install-gold.sh9
-rw-r--r--dashboard/env/linux-x86-nacl/Dockerfile27
-rw-r--r--dashboard/env/linux-x86-nacl/Makefile12
-rw-r--r--dashboard/env/linux-x86-nacl/README6
-rwxr-xr-xdashboard/env/linux-x86-nacl/build-command.pl13
-rwxr-xr-xdashboard/env/linux-x86-nacl/scripts/build-go-builder.sh30
-rwxr-xr-xdashboard/env/linux-x86-nacl/scripts/install-apt-deps.sh14
-rw-r--r--dashboard/env/linux-x86-sid/Dockerfile17
-rw-r--r--dashboard/env/linux-x86-sid/Makefile15
-rwxr-xr-xdashboard/env/linux-x86-sid/scripts/build-go-builder.sh24
-rwxr-xr-xdashboard/env/linux-x86-sid/scripts/install-apt-deps.sh18
-rw-r--r--dashboard/env/openbsd-amd64/.gitignore8
-rw-r--r--dashboard/env/openbsd-amd64/README20
-rwxr-xr-xdashboard/env/openbsd-amd64/make.bash227
-rw-r--r--dashboard/env/plan9-386/.gitignore2
-rw-r--r--dashboard/env/plan9-386/README21
-rwxr-xr-xdashboard/env/plan9-386/make.bash236
-rw-r--r--dashboard/env/plan9-386/plan9-gce.iso.sha11
-rw-r--r--dashboard/env/windows/README70
-rw-r--r--dashboard/types/types.go40
-rw-r--r--dashboard/watcher/watcher.go775
105 files changed, 0 insertions, 14252 deletions
diff --git a/dashboard/README b/dashboard/README
deleted file mode 100644
index 7731947..0000000
--- a/dashboard/README
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-The files in these directories constitute the continuous builder:
-
-app/: a.k.a the "dashboard"; the App Engine code that runs http://build.golang.org/
-
-cmd/:
-
- buildlet/: HTTP server that runs on a VM and is told what to write to disk
- and what command to run. This is cross-compiled to different architectures
- and is the first program run when a builder VM comes up. It then
- is contacted by the coordinator to do a build. Not all builders use
- the buildlet (at least not yet).
-
- builder/: gobuilder, a Go continuous build client. The original Go builder program.
-
- coordinator/: daemon that runs on CoreOS on Google Compute Engine and manages
- builds using Docker containers and/or VMs as needed.
-
- retrybuilds/: a Go client program to delete build results from the dashboard (app)
-
- upload/: a Go program to upload to Google Cloud Storage. used by Makefiles elsewhere.
-
- watcher/: a daemon that watches for new commits to the Go repository and
- its sub-repositories, and notifies the dashboard of those commits.
-
-env/: configuration files describing the environment of builders and related
- binaries. Many builders are still configured ad-hoc, without a hermetic
- environment.
-
-types/: a Go package contain common types used by other pieces.
-
-
-If you wish to run a Go builder, please email golang-dev@googlegroups.com first.
-There is documentation at https://golang.org/wiki/DashboardBuilders but
-depending on the type of builder, we may want to run it ourselves, after you
-prepare an environment description (resulting in a VM image) of it. See the env
-directory.
diff --git a/dashboard/app/app.yaml b/dashboard/app/app.yaml
deleted file mode 100644
index 35fb75b..0000000
--- a/dashboard/app/app.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-# Update with
-# google_appengine/appcfg.py [-V build-test] update .
-#
-# Using -V build-test will run as build-test.golang.org.
-
-application: golang-org
-version: build
-runtime: go
-api_version: go1
-
-handlers:
-- url: /static
- static_dir: static
-- url: /(|gccgo/|hg/)log/.+
- script: _go_app
-- url: /(|gccgo/|hg/)(|building|clear-results|commit|packages|result|perf-result|tag|todo|perf|perfdetail|perfgraph|updatebenchmark)
- script: _go_app
-- url: /(|gccgo/|hg/)(init|buildtest|key|perflearn|_ah/queue/go/delay)
- script: _go_app
- login: admin
-
diff --git a/dashboard/app/build/build.go b/dashboard/app/build/build.go
deleted file mode 100644
index 3b52a7c..0000000
--- a/dashboard/app/build/build.go
+++ /dev/null
@@ -1,946 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package build
-
-import (
- "bytes"
- "compress/gzip"
- "crypto/sha1"
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "net/http"
- "sort"
- "strconv"
- "strings"
- "time"
-
- "appengine"
- "appengine/datastore"
-
- "cache"
-)
-
-const (
- maxDatastoreStringLen = 500
- PerfRunLength = 1024
-)
-
-// A Package describes a package that is listed on the dashboard.
-type Package struct {
- Kind string // "subrepo", "external", or empty for the main Go tree
- Name string
- Path string // (empty for the main Go tree)
- NextNum int // Num of the next head Commit
-}
-
-func (p *Package) String() string {
- return fmt.Sprintf("%s: %q", p.Path, p.Name)
-}
-
-func (p *Package) Key(c appengine.Context) *datastore.Key {
- key := p.Path
- if key == "" {
- key = "go"
- }
- return datastore.NewKey(c, "Package", key, 0, nil)
-}
-
-// LastCommit returns the most recent Commit for this Package.
-func (p *Package) LastCommit(c appengine.Context) (*Commit, error) {
- var commits []*Commit
- _, err := datastore.NewQuery("Commit").
- Ancestor(p.Key(c)).
- Order("-Time").
- Limit(1).
- GetAll(c, &commits)
- if err != nil {
- return nil, err
- }
- if len(commits) != 1 {
- return nil, datastore.ErrNoSuchEntity
- }
- return commits[0], nil
-}
-
-// GetPackage fetches a Package by path from the datastore.
-func GetPackage(c appengine.Context, path string) (*Package, error) {
- p := &Package{Path: path}
- err := datastore.Get(c, p.Key(c), p)
- if err == datastore.ErrNoSuchEntity {
- return nil, fmt.Errorf("package %q not found", path)
- }
- return p, err
-}
-
-type builderAndGoHash struct {
- builder, goHash string
-}
-
-// A Commit describes an individual commit in a package.
-//
-// Each Commit entity is a descendant of its associated Package entity.
-// In other words, all Commits with the same PackagePath belong to the same
-// datastore entity group.
-type Commit struct {
- PackagePath string // (empty for main repo commits)
- Hash string
- ParentHash string
- Num int // Internal monotonic counter unique to this package.
-
- User string
- Desc string `datastore:",noindex"`
- Time time.Time
- NeedsBenchmarking bool
- TryPatch bool
- Branch string
-
- // ResultData is the Data string of each build Result for this Commit.
- // For non-Go commits, only the Results for the current Go tip, weekly,
- // and release Tags are stored here. This is purely de-normalized data.
- // The complete data set is stored in Result entities.
- ResultData []string `datastore:",noindex"`
-
- // PerfResults holds a set of “builder|benchmark” tuples denoting
- // what benchmarks have been executed on the commit.
- PerfResults []string `datastore:",noindex"`
-
- FailNotificationSent bool
-
- buildingURLs map[builderAndGoHash]string
-}
-
-func (com *Commit) Key(c appengine.Context) *datastore.Key {
- if com.Hash == "" {
- panic("tried Key on Commit with empty Hash")
- }
- p := Package{Path: com.PackagePath}
- key := com.PackagePath + "|" + com.Hash
- return datastore.NewKey(c, "Commit", key, 0, p.Key(c))
-}
-
-func (c *Commit) Valid() error {
- if !validHash(c.Hash) {
- return errors.New("invalid Hash")
- }
- if c.ParentHash != "" && !validHash(c.ParentHash) { // empty is OK
- return errors.New("invalid ParentHash")
- }
- return nil
-}
-
-func putCommit(c appengine.Context, com *Commit) error {
- if err := com.Valid(); err != nil {
- return fmt.Errorf("putting Commit: %v", err)
- }
- if com.Num == 0 && com.ParentHash != "0000" { // 0000 is used in tests
- return fmt.Errorf("putting Commit: invalid Num (must be > 0)")
- }
- if _, err := datastore.Put(c, com.Key(c), com); err != nil {
- return fmt.Errorf("putting Commit: %v", err)
- }
- return nil
-}
-
-// each result line is approx 105 bytes. This constant is a tradeoff between
-// build history and the AppEngine datastore limit of 1mb.
-const maxResults = 1000
-
-// AddResult adds the denormalized Result data to the Commit's ResultData field.
-// It must be called from inside a datastore transaction.
-func (com *Commit) AddResult(c appengine.Context, r *Result) error {
- if err := datastore.Get(c, com.Key(c), com); err != nil {
- return fmt.Errorf("getting Commit: %v", err)
- }
-
- var resultExists bool
- for i, s := range com.ResultData {
- // if there already exists result data for this builder at com, overwrite it.
- if strings.HasPrefix(s, r.Builder+"|") && strings.HasSuffix(s, "|"+r.GoHash) {
- resultExists = true
- com.ResultData[i] = r.Data()
- }
- }
- if !resultExists {
- // otherwise, add the new result data for this builder.
- com.ResultData = trim(append(com.ResultData, r.Data()), maxResults)
- }
- return putCommit(c, com)
-}
-
-// removeResult removes the denormalized Result data from the ResultData field
-// for the given builder and go hash.
-// It must be called from within the datastore transaction that gets and puts
-// the Commit. Note this is slightly different to AddResult, above.
-func (com *Commit) RemoveResult(r *Result) {
- var rd []string
- for _, s := range com.ResultData {
- if strings.HasPrefix(s, r.Builder+"|") && strings.HasSuffix(s, "|"+r.GoHash) {
- continue
- }
- rd = append(rd, s)
- }
- com.ResultData = rd
-}
-
-// AddPerfResult remembers that the builder has run the benchmark on the commit.
-// It must be called from inside a datastore transaction.
-func (com *Commit) AddPerfResult(c appengine.Context, builder, benchmark string) error {
- if err := datastore.Get(c, com.Key(c), com); err != nil {
- return fmt.Errorf("getting Commit: %v", err)
- }
- if !com.NeedsBenchmarking {
- return fmt.Errorf("trying to add perf result to Commit(%v) that does not require benchmarking", com.Hash)
- }
- s := builder + "|" + benchmark
- for _, v := range com.PerfResults {
- if v == s {
- return nil
- }
- }
- com.PerfResults = append(com.PerfResults, s)
- return putCommit(c, com)
-}
-
-func trim(s []string, n int) []string {
- l := min(len(s), n)
- return s[len(s)-l:]
-}
-
-func min(a, b int) int {
- if a < b {
- return a
- }
- return b
-}
-
-// Result returns the build Result for this Commit for the given builder/goHash.
-func (c *Commit) Result(builder, goHash string) *Result {
- for _, r := range c.ResultData {
- if !strings.HasPrefix(r, builder) {
- // Avoid strings.SplitN alloc in the common case.
- continue
- }
- p := strings.SplitN(r, "|", 4)
- if len(p) != 4 || p[0] != builder || p[3] != goHash {
- continue
- }
- return partsToResult(c, p)
- }
- if u, ok := c.buildingURLs[builderAndGoHash{builder, goHash}]; ok {
- return &Result{
- Builder: builder,
- BuildingURL: u,
- Hash: c.Hash,
- GoHash: goHash,
- }
- }
- return nil
-}
-
-// Results returns the build Results for this Commit.
-func (c *Commit) Results() (results []*Result) {
- for _, r := range c.ResultData {
- p := strings.SplitN(r, "|", 4)
- if len(p) != 4 {
- continue
- }
- results = append(results, partsToResult(c, p))
- }
- return
-}
-
-func (c *Commit) ResultGoHashes() []string {
- // For the main repo, just return the empty string
- // (there's no corresponding main repo hash for a main repo Commit).
- // This function is only really useful for sub-repos.
- if c.PackagePath == "" {
- return []string{""}
- }
- var hashes []string
- for _, r := range c.ResultData {
- p := strings.SplitN(r, "|", 4)
- if len(p) != 4 {
- continue
- }
- // Append only new results (use linear scan to preserve order).
- if !contains(hashes, p[3]) {
- hashes = append(hashes, p[3])
- }
- }
- // Return results in reverse order (newest first).
- reverse(hashes)
- return hashes
-}
-
-func contains(t []string, s string) bool {
- for _, s2 := range t {
- if s2 == s {
- return true
- }
- }
- return false
-}
-
-func reverse(s []string) {
- for i := 0; i < len(s)/2; i++ {
- j := len(s) - i - 1
- s[i], s[j] = s[j], s[i]
- }
-}
-
-// A CommitRun provides summary information for commits [StartCommitNum, StartCommitNum + PerfRunLength).
-// Descendant of Package.
-type CommitRun struct {
- PackagePath string // (empty for main repo commits)
- StartCommitNum int
- Hash []string `datastore:",noindex"`
- User []string `datastore:",noindex"`
- Desc []string `datastore:",noindex"` // Only first line.
- Time []time.Time `datastore:",noindex"`
- NeedsBenchmarking []bool `datastore:",noindex"`
-}
-
-func (cr *CommitRun) Key(c appengine.Context) *datastore.Key {
- p := Package{Path: cr.PackagePath}
- key := strconv.Itoa(cr.StartCommitNum)
- return datastore.NewKey(c, "CommitRun", key, 0, p.Key(c))
-}
-
-// GetCommitRun loads and returns CommitRun that contains information
-// for commit commitNum.
-func GetCommitRun(c appengine.Context, commitNum int) (*CommitRun, error) {
- cr := &CommitRun{StartCommitNum: commitNum / PerfRunLength * PerfRunLength}
- err := datastore.Get(c, cr.Key(c), cr)
- if err != nil && err != datastore.ErrNoSuchEntity {
- return nil, fmt.Errorf("getting CommitRun: %v", err)
- }
- if len(cr.Hash) != PerfRunLength {
- cr.Hash = make([]string, PerfRunLength)
- cr.User = make([]string, PerfRunLength)
- cr.Desc = make([]string, PerfRunLength)
- cr.Time = make([]time.Time, PerfRunLength)
- cr.NeedsBenchmarking = make([]bool, PerfRunLength)
- }
- return cr, nil
-}
-
-func (cr *CommitRun) AddCommit(c appengine.Context, com *Commit) error {
- if com.Num < cr.StartCommitNum || com.Num >= cr.StartCommitNum+PerfRunLength {
- return fmt.Errorf("AddCommit: commit num %v out of range [%v, %v)",
- com.Num, cr.StartCommitNum, cr.StartCommitNum+PerfRunLength)
- }
- i := com.Num - cr.StartCommitNum
- // Be careful with string lengths,
- // we need to fit 1024 commits into 1 MB.
- cr.Hash[i] = com.Hash
- cr.User[i] = shortDesc(com.User)
- cr.Desc[i] = shortDesc(com.Desc)
- cr.Time[i] = com.Time
- cr.NeedsBenchmarking[i] = com.NeedsBenchmarking
- if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
- return fmt.Errorf("putting CommitRun: %v", err)
- }
- return nil
-}
-
-// GetCommits returns [startCommitNum, startCommitNum+n) commits.
-// Commits information is partial (obtained from CommitRun),
-// do not store them back into datastore.
-func GetCommits(c appengine.Context, startCommitNum, n int) ([]*Commit, error) {
- if startCommitNum < 0 || n <= 0 {
- return nil, fmt.Errorf("GetCommits: invalid args (%v, %v)", startCommitNum, n)
- }
-
- p := &Package{}
- t := datastore.NewQuery("CommitRun").
- Ancestor(p.Key(c)).
- Filter("StartCommitNum >=", startCommitNum/PerfRunLength*PerfRunLength).
- Order("StartCommitNum").
- Limit(100).
- Run(c)
-
- res := make([]*Commit, n)
- for {
- cr := new(CommitRun)
- _, err := t.Next(cr)
- if err == datastore.Done {
- break
- }
- if err != nil {
- return nil, err
- }
- if cr.StartCommitNum >= startCommitNum+n {
- break
- }
- // Calculate start index for copying.
- i := 0
- if cr.StartCommitNum < startCommitNum {
- i = startCommitNum - cr.StartCommitNum
- }
- // Calculate end index for copying.
- e := PerfRunLength
- if cr.StartCommitNum+e > startCommitNum+n {
- e = startCommitNum + n - cr.StartCommitNum
- }
- for ; i < e; i++ {
- com := new(Commit)
- com.Hash = cr.Hash[i]
- com.User = cr.User[i]
- com.Desc = cr.Desc[i]
- com.Time = cr.Time[i]
- com.NeedsBenchmarking = cr.NeedsBenchmarking[i]
- res[cr.StartCommitNum-startCommitNum+i] = com
- }
- if e != PerfRunLength {
- break
- }
- }
- return res, nil
-}
-
-// partsToResult converts a Commit and ResultData substrings to a Result.
-func partsToResult(c *Commit, p []string) *Result {
- return &Result{
- Builder: p[0],
- Hash: c.Hash,
- PackagePath: c.PackagePath,
- GoHash: p[3],
- OK: p[1] == "true",
- LogHash: p[2],
- }
-}
-
-// A Result describes a build result for a Commit on an OS/architecture.
-//
-// Each Result entity is a descendant of its associated Package entity.
-type Result struct {
- PackagePath string // (empty for Go commits)
- Builder string // "os-arch[-note]"
- Hash string
-
- // The Go Commit this was built against (empty for Go commits).
- GoHash string
-
- BuildingURL string `datastore:"-"` // non-empty if currently building
- OK bool
- Log string `datastore:"-"` // for JSON unmarshaling only
- LogHash string `datastore:",noindex"` // Key to the Log record.
-
- RunTime int64 // time to build+test in nanoseconds
-}
-
-func (r *Result) Key(c appengine.Context) *datastore.Key {
- p := Package{Path: r.PackagePath}
- key := r.Builder + "|" + r.PackagePath + "|" + r.Hash + "|" + r.GoHash
- return datastore.NewKey(c, "Result", key, 0, p.Key(c))
-}
-
-func (r *Result) Valid() error {
- if !validHash(r.Hash) {
- return errors.New("invalid Hash")
- }
- if r.PackagePath != "" && !validHash(r.GoHash) {
- return errors.New("invalid GoHash")
- }
- return nil
-}
-
-// Data returns the Result in string format
-// to be stored in Commit's ResultData field.
-func (r *Result) Data() string {
- return fmt.Sprintf("%v|%v|%v|%v", r.Builder, r.OK, r.LogHash, r.GoHash)
-}
-
-// A PerfResult describes all benchmarking result for a Commit.
-// Descendant of Package.
-type PerfResult struct {
- PackagePath string
- CommitHash string
- CommitNum int
- Data []string `datastore:",noindex"` // "builder|benchmark|ok|metric1=val1|metric2=val2|file:log=hash|file:cpuprof=hash"
-
- // Local cache with parsed Data.
- // Maps builder->benchmark->ParsedPerfResult.
- parsedData map[string]map[string]*ParsedPerfResult
-}
-
-type ParsedPerfResult struct {
- OK bool
- Metrics map[string]uint64
- Artifacts map[string]string
-}
-
-func (r *PerfResult) Key(c appengine.Context) *datastore.Key {
- p := Package{Path: r.PackagePath}
- key := r.CommitHash
- return datastore.NewKey(c, "PerfResult", key, 0, p.Key(c))
-}
-
-// AddResult add the benchmarking result to r.
-// Existing result for the same builder/benchmark is replaced if already exists.
-// Returns whether the result was already present.
-func (r *PerfResult) AddResult(req *PerfRequest) bool {
- present := false
- str := fmt.Sprintf("%v|%v|", req.Builder, req.Benchmark)
- for i, s := range r.Data {
- if strings.HasPrefix(s, str) {
- present = true
- last := len(r.Data) - 1
- r.Data[i] = r.Data[last]
- r.Data = r.Data[:last]
- break
- }
- }
- ok := "ok"
- if !req.OK {
- ok = "false"
- }
- str += ok
- for _, m := range req.Metrics {
- str += fmt.Sprintf("|%v=%v", m.Type, m.Val)
- }
- for _, a := range req.Artifacts {
- str += fmt.Sprintf("|file:%v=%v", a.Type, a.Body)
- }
- r.Data = append(r.Data, str)
- r.parsedData = nil
- return present
-}
-
-func (r *PerfResult) ParseData() map[string]map[string]*ParsedPerfResult {
- if r.parsedData != nil {
- return r.parsedData
- }
- res := make(map[string]map[string]*ParsedPerfResult)
- for _, str := range r.Data {
- ss := strings.Split(str, "|")
- builder := ss[0]
- bench := ss[1]
- ok := ss[2]
- m := res[builder]
- if m == nil {
- m = make(map[string]*ParsedPerfResult)
- res[builder] = m
- }
- var p ParsedPerfResult
- p.OK = ok == "ok"
- p.Metrics = make(map[string]uint64)
- p.Artifacts = make(map[string]string)
- for _, entry := range ss[3:] {
- if strings.HasPrefix(entry, "file:") {
- ss1 := strings.Split(entry[len("file:"):], "=")
- p.Artifacts[ss1[0]] = ss1[1]
- } else {
- ss1 := strings.Split(entry, "=")
- val, _ := strconv.ParseUint(ss1[1], 10, 64)
- p.Metrics[ss1[0]] = val
- }
- }
- m[bench] = &p
- }
- r.parsedData = res
- return res
-}
-
-// A PerfMetricRun entity holds a set of metric values for builder/benchmark/metric
-// for commits [StartCommitNum, StartCommitNum + PerfRunLength).
-// Descendant of Package.
-type PerfMetricRun struct {
- PackagePath string
- Builder string
- Benchmark string
- Metric string // e.g. realtime, cputime, gc-pause
- StartCommitNum int
- Vals []int64 `datastore:",noindex"`
-}
-
-func (m *PerfMetricRun) Key(c appengine.Context) *datastore.Key {
- p := Package{Path: m.PackagePath}
- key := m.Builder + "|" + m.Benchmark + "|" + m.Metric + "|" + strconv.Itoa(m.StartCommitNum)
- return datastore.NewKey(c, "PerfMetricRun", key, 0, p.Key(c))
-}
-
-// GetPerfMetricRun loads and returns PerfMetricRun that contains information
-// for commit commitNum.
-func GetPerfMetricRun(c appengine.Context, builder, benchmark, metric string, commitNum int) (*PerfMetricRun, error) {
- startCommitNum := commitNum / PerfRunLength * PerfRunLength
- m := &PerfMetricRun{Builder: builder, Benchmark: benchmark, Metric: metric, StartCommitNum: startCommitNum}
- err := datastore.Get(c, m.Key(c), m)
- if err != nil && err != datastore.ErrNoSuchEntity {
- return nil, fmt.Errorf("getting PerfMetricRun: %v", err)
- }
- if len(m.Vals) != PerfRunLength {
- m.Vals = make([]int64, PerfRunLength)
- }
- return m, nil
-}
-
-func (m *PerfMetricRun) AddMetric(c appengine.Context, commitNum int, v uint64) error {
- if commitNum < m.StartCommitNum || commitNum >= m.StartCommitNum+PerfRunLength {
- return fmt.Errorf("AddMetric: CommitNum %v out of range [%v, %v)",
- commitNum, m.StartCommitNum, m.StartCommitNum+PerfRunLength)
- }
- m.Vals[commitNum-m.StartCommitNum] = int64(v)
- if _, err := datastore.Put(c, m.Key(c), m); err != nil {
- return fmt.Errorf("putting PerfMetricRun: %v", err)
- }
- return nil
-}
-
-// GetPerfMetricsForCommits returns perf metrics for builder/benchmark/metric
-// and commits [startCommitNum, startCommitNum+n).
-func GetPerfMetricsForCommits(c appengine.Context, builder, benchmark, metric string, startCommitNum, n int) ([]uint64, error) {
- if startCommitNum < 0 || n <= 0 {
- return nil, fmt.Errorf("GetPerfMetricsForCommits: invalid args (%v, %v)", startCommitNum, n)
- }
-
- p := &Package{}
- t := datastore.NewQuery("PerfMetricRun").
- Ancestor(p.Key(c)).
- Filter("Builder =", builder).
- Filter("Benchmark =", benchmark).
- Filter("Metric =", metric).
- Filter("StartCommitNum >=", startCommitNum/PerfRunLength*PerfRunLength).
- Order("StartCommitNum").
- Limit(100).
- Run(c)
-
- res := make([]uint64, n)
- for {
- metrics := new(PerfMetricRun)
- _, err := t.Next(metrics)
- if err == datastore.Done {
- break
- }
- if err != nil {
- return nil, err
- }
- if metrics.StartCommitNum >= startCommitNum+n {
- break
- }
- // Calculate start index for copying.
- i := 0
- if metrics.StartCommitNum < startCommitNum {
- i = startCommitNum - metrics.StartCommitNum
- }
- // Calculate end index for copying.
- e := PerfRunLength
- if metrics.StartCommitNum+e > startCommitNum+n {
- e = startCommitNum + n - metrics.StartCommitNum
- }
- for ; i < e; i++ {
- res[metrics.StartCommitNum-startCommitNum+i] = uint64(metrics.Vals[i])
- }
- if e != PerfRunLength {
- break
- }
- }
- return res, nil
-}
-
-// PerfConfig holds read-mostly configuration related to benchmarking.
-// There is only one PerfConfig entity.
-type PerfConfig struct {
- BuilderBench []string `datastore:",noindex"` // "builder|benchmark" pairs
- BuilderProcs []string `datastore:",noindex"` // "builder|proc" pairs
- BenchMetric []string `datastore:",noindex"` // "benchmark|metric" pairs
- NoiseLevels []string `datastore:",noindex"` // "builder|benchmark|metric1=noise1|metric2=noise2"
-
- // Local cache of "builder|benchmark|metric" -> noise.
- noise map[string]float64
-}
-
-func PerfConfigKey(c appengine.Context) *datastore.Key {
- p := Package{}
- return datastore.NewKey(c, "PerfConfig", "PerfConfig", 0, p.Key(c))
-}
-
-const perfConfigCacheKey = "perf-config"
-
-func GetPerfConfig(c appengine.Context, r *http.Request) (*PerfConfig, error) {
- pc := new(PerfConfig)
- now := cache.Now(c)
- if cache.Get(r, now, perfConfigCacheKey, pc) {
- return pc, nil
- }
- err := datastore.Get(c, PerfConfigKey(c), pc)
- if err != nil && err != datastore.ErrNoSuchEntity {
- return nil, fmt.Errorf("GetPerfConfig: %v", err)
- }
- cache.Set(r, now, perfConfigCacheKey, pc)
- return pc, nil
-}
-
-func (pc *PerfConfig) NoiseLevel(builder, benchmark, metric string) float64 {
- if pc.noise == nil {
- pc.noise = make(map[string]float64)
- for _, str := range pc.NoiseLevels {
- split := strings.Split(str, "|")
- builderBench := split[0] + "|" + split[1]
- for _, entry := range split[2:] {
- metricValue := strings.Split(entry, "=")
- noise, _ := strconv.ParseFloat(metricValue[1], 64)
- pc.noise[builderBench+"|"+metricValue[0]] = noise
- }
- }
- }
- me := fmt.Sprintf("%v|%v|%v", builder, benchmark, metric)
- n := pc.noise[me]
- if n == 0 {
- // Use a very conservative value
- // until we have learned the real noise level.
- n = 200
- }
- return n
-}
-
-// UpdatePerfConfig updates the PerfConfig entity with results of benchmarking.
-// Returns whether it's a benchmark that we have not yet seem on the builder.
-func UpdatePerfConfig(c appengine.Context, r *http.Request, req *PerfRequest) (newBenchmark bool, err error) {
- pc, err := GetPerfConfig(c, r)
- if err != nil {
- return false, err
- }
-
- modified := false
- add := func(arr *[]string, str string) {
- for _, s := range *arr {
- if s == str {
- return
- }
- }
- *arr = append(*arr, str)
- modified = true
- return
- }
-
- BenchProcs := strings.Split(req.Benchmark, "-")
- benchmark := BenchProcs[0]
- procs := "1"
- if len(BenchProcs) > 1 {
- procs = BenchProcs[1]
- }
-
- add(&pc.BuilderBench, req.Builder+"|"+benchmark)
- newBenchmark = modified
- add(&pc.BuilderProcs, req.Builder+"|"+procs)
- for _, m := range req.Metrics {
- add(&pc.BenchMetric, benchmark+"|"+m.Type)
- }
-
- if modified {
- if _, err := datastore.Put(c, PerfConfigKey(c), pc); err != nil {
- return false, fmt.Errorf("putting PerfConfig: %v", err)
- }
- cache.Tick(c)
- }
- return newBenchmark, nil
-}
-
-type MetricList []string
-
-func (l MetricList) Len() int {
- return len(l)
-}
-
-func (l MetricList) Less(i, j int) bool {
- bi := strings.HasPrefix(l[i], "build-") || strings.HasPrefix(l[i], "binary-")
- bj := strings.HasPrefix(l[j], "build-") || strings.HasPrefix(l[j], "binary-")
- if bi == bj {
- return l[i] < l[j]
- }
- return !bi
-}
-
-func (l MetricList) Swap(i, j int) {
- l[i], l[j] = l[j], l[i]
-}
-
-func collectList(all []string, idx int, second string) (res []string) {
- m := make(map[string]bool)
- for _, str := range all {
- ss := strings.Split(str, "|")
- v := ss[idx]
- v2 := ss[1-idx]
- if (second == "" || second == v2) && !m[v] {
- m[v] = true
- res = append(res, v)
- }
- }
- sort.Sort(MetricList(res))
- return res
-}
-
-func (pc *PerfConfig) BuildersForBenchmark(bench string) []string {
- return collectList(pc.BuilderBench, 0, bench)
-}
-
-func (pc *PerfConfig) BenchmarksForBuilder(builder string) []string {
- return collectList(pc.BuilderBench, 1, builder)
-}
-
-func (pc *PerfConfig) MetricsForBenchmark(bench string) []string {
- return collectList(pc.BenchMetric, 1, bench)
-}
-
-func (pc *PerfConfig) BenchmarkProcList() (res []string) {
- bl := pc.BenchmarksForBuilder("")
- pl := pc.ProcList("")
- for _, b := range bl {
- for _, p := range pl {
- res = append(res, fmt.Sprintf("%v-%v", b, p))
- }
- }
- return res
-}
-
-func (pc *PerfConfig) ProcList(builder string) []int {
- ss := collectList(pc.BuilderProcs, 1, builder)
- var procs []int
- for _, s := range ss {
- p, _ := strconv.ParseInt(s, 10, 32)
- procs = append(procs, int(p))
- }
- sort.Ints(procs)
- return procs
-}
-
-// A PerfTodo contains outstanding commits for benchmarking for a builder.
-// Descendant of Package.
-type PerfTodo struct {
- PackagePath string // (empty for main repo commits)
- Builder string
- CommitNums []int `datastore:",noindex"` // LIFO queue of commits to benchmark.
-}
-
-func (todo *PerfTodo) Key(c appengine.Context) *datastore.Key {
- p := Package{Path: todo.PackagePath}
- key := todo.Builder
- return datastore.NewKey(c, "PerfTodo", key, 0, p.Key(c))
-}
-
-// AddCommitToPerfTodo adds the commit to all existing PerfTodo entities.
-func AddCommitToPerfTodo(c appengine.Context, com *Commit) error {
- var todos []*PerfTodo
- _, err := datastore.NewQuery("PerfTodo").
- Ancestor((&Package{}).Key(c)).
- GetAll(c, &todos)
- if err != nil {
- return fmt.Errorf("fetching PerfTodo's: %v", err)
- }
- for _, todo := range todos {
- todo.CommitNums = append(todo.CommitNums, com.Num)
- _, err = datastore.Put(c, todo.Key(c), todo)
- if err != nil {
- return fmt.Errorf("updating PerfTodo: %v", err)
- }
- }
- return nil
-}
-
-// A Log is a gzip-compressed log file stored under the SHA1 hash of the
-// uncompressed log text.
-type Log struct {
- CompressedLog []byte
-}
-
-func (l *Log) Text() ([]byte, error) {
- d, err := gzip.NewReader(bytes.NewBuffer(l.CompressedLog))
- if err != nil {
- return nil, fmt.Errorf("reading log data: %v", err)
- }
- b, err := ioutil.ReadAll(d)
- if err != nil {
- return nil, fmt.Errorf("reading log data: %v", err)
- }
- return b, nil
-}
-
-func PutLog(c appengine.Context, text string) (hash string, err error) {
- h := sha1.New()
- io.WriteString(h, text)
- b := new(bytes.Buffer)
- z, _ := gzip.NewWriterLevel(b, gzip.BestCompression)
- io.WriteString(z, text)
- z.Close()
- hash = fmt.Sprintf("%x", h.Sum(nil))
- key := datastore.NewKey(c, "Log", hash, 0, nil)
- _, err = datastore.Put(c, key, &Log{b.Bytes()})
- return
-}
-
-// A Tag is used to keep track of the most recent Go weekly and release tags.
-// Typically there will be one Tag entity for each kind of hg tag.
-type Tag struct {
- Kind string // "weekly", "release", or "tip"
- Name string // the tag itself (for example: "release.r60")
- Hash string
-}
-
-func (t *Tag) Key(c appengine.Context) *datastore.Key {
- p := &Package{}
- return datastore.NewKey(c, "Tag", t.Kind, 0, p.Key(c))
-}
-
-func (t *Tag) Valid() error {
- if t.Kind != "weekly" && t.Kind != "release" && t.Kind != "tip" {
- return errors.New("invalid Kind")
- }
- if !validHash(t.Hash) {
- return errors.New("invalid Hash")
- }
- return nil
-}
-
-// Commit returns the Commit that corresponds with this Tag.
-func (t *Tag) Commit(c appengine.Context) (*Commit, error) {
- com := &Commit{Hash: t.Hash}
- err := datastore.Get(c, com.Key(c), com)
- return com, err
-}
-
-// GetTag fetches a Tag by name from the datastore.
-func GetTag(c appengine.Context, tag string) (*Tag, error) {
- t := &Tag{Kind: tag}
- if err := datastore.Get(c, t.Key(c), t); err != nil {
- if err == datastore.ErrNoSuchEntity {
- return nil, errors.New("tag not found: " + tag)
- }
- return nil, err
- }
- if err := t.Valid(); err != nil {
- return nil, err
- }
- return t, nil
-}
-
-// Packages returns packages of the specified kind.
-// Kind must be one of "external" or "subrepo".
-func Packages(c appengine.Context, kind string) ([]*Package, error) {
- switch kind {
- case "external", "subrepo":
- default:
- return nil, errors.New(`kind must be one of "external" or "subrepo"`)
- }
- var pkgs []*Package
- q := datastore.NewQuery("Package").Filter("Kind=", kind)
- for t := q.Run(c); ; {
- pkg := new(Package)
- _, err := t.Next(pkg)
- if err == datastore.Done {
- break
- } else if err != nil {
- return nil, err
- }
- if pkg.Path != "" {
- pkgs = append(pkgs, pkg)
- }
- }
- return pkgs, nil
-}
diff --git a/dashboard/app/build/dash.go b/dashboard/app/build/dash.go
deleted file mode 100644
index 0441326..0000000
--- a/dashboard/app/build/dash.go
+++ /dev/null
@@ -1,201 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package build
-
-import (
- "net/http"
- "strings"
-
- "appengine"
-)
-
-func handleFunc(path string, h http.HandlerFunc) {
- for _, d := range dashboards {
- http.HandleFunc(d.Prefix+path, h)
- }
-}
-
-// Dashboard describes a unique build dashboard.
-type Dashboard struct {
- Name string // This dashboard's name (eg, "Go")
- Namespace string // This dashboard's namespace (eg, "" (default), "Git")
- Prefix string // The path prefix (no trailing /)
- Packages []*Package // The project's packages to build
-}
-
-// dashboardForRequest returns the appropriate dashboard for a given URL path.
-func dashboardForRequest(r *http.Request) *Dashboard {
- if strings.HasPrefix(r.URL.Path, gccgoDash.Prefix) {
- return gccgoDash
- }
- if strings.HasPrefix(r.URL.Path, hgDash.Prefix) {
- return hgDash
- }
- return goDash
-}
-
-// Context returns a namespaced context for this dashboard, or panics if it
-// fails to create a new context.
-func (d *Dashboard) Context(c appengine.Context) appengine.Context {
- if d.Namespace == "" {
- return c
- }
- n, err := appengine.Namespace(c, d.Namespace)
- if err != nil {
- panic(err)
- }
- return n
-}
-
-// the currently known dashboards.
-var dashboards = []*Dashboard{goDash, hgDash, gccgoDash}
-
-// hgDash is the dashboard for the old Mercural Go repository.
-var hgDash = &Dashboard{
- Name: "Mercurial",
- Namespace: "", // Used to be the default.
- Prefix: "/hg",
- Packages: hgPackages,
-}
-
-// hgPackages is a list of all of the packages
-// built by the old Mercurial Go repository.
-var hgPackages = []*Package{
- {
- Kind: "go",
- Name: "Go",
- },
- {
- Kind: "subrepo",
- Name: "go.blog",
- Path: "code.google.com/p/go.blog",
- },
- {
- Kind: "subrepo",
- Name: "go.codereview",
- Path: "code.google.com/p/go.codereview",
- },
- {
- Kind: "subrepo",
- Name: "go.crypto",
- Path: "code.google.com/p/go.crypto",
- },
- {
- Kind: "subrepo",
- Name: "go.exp",
- Path: "code.google.com/p/go.exp",
- },
- {
- Kind: "subrepo",
- Name: "go.image",
- Path: "code.google.com/p/go.image",
- },
- {
- Kind: "subrepo",
- Name: "go.net",
- Path: "code.google.com/p/go.net",
- },
- {
- Kind: "subrepo",
- Name: "go.sys",
- Path: "code.google.com/p/go.sys",
- },
- {
- Kind: "subrepo",
- Name: "go.talks",
- Path: "code.google.com/p/go.talks",
- },
- {
- Kind: "subrepo",
- Name: "go.tools",
- Path: "code.google.com/p/go.tools",
- },
-}
-
-// goDash is the dashboard for the main go repository.
-var goDash = &Dashboard{
- Name: "Go",
- Namespace: "Git",
- Prefix: "",
- Packages: goPackages,
-}
-
-// goPackages is a list of all of the packages built by the main go repository.
-var goPackages = []*Package{
- {
- Kind: "go",
- Name: "Go",
- },
- {
- Kind: "subrepo",
- Name: "blog",
- Path: "golang.org/x/blog",
- },
- {
- Kind: "subrepo",
- Name: "crypto",
- Path: "golang.org/x/crypto",
- },
- {
- Kind: "subrepo",
- Name: "exp",
- Path: "golang.org/x/exp",
- },
- {
- Kind: "subrepo",
- Name: "image",
- Path: "golang.org/x/image",
- },
- {
- Kind: "subrepo",
- Name: "mobile",
- Path: "golang.org/x/mobile",
- },
- {
- Kind: "subrepo",
- Name: "net",
- Path: "golang.org/x/net",
- },
- {
- Kind: "subrepo",
- Name: "review",
- Path: "golang.org/x/review",
- },
- {
- Kind: "subrepo",
- Name: "sys",
- Path: "golang.org/x/sys",
- },
- {
- Kind: "subrepo",
- Name: "talks",
- Path: "golang.org/x/talks",
- },
- {
- Kind: "subrepo",
- Name: "text",
- Path: "golang.org/x/text",
- },
- {
- Kind: "subrepo",
- Name: "tools",
- Path: "golang.org/x/tools",
- },
-}
-
-// gccgoDash is the dashboard for gccgo.
-var gccgoDash = &Dashboard{
- Name: "Gccgo",
- Namespace: "Gccgo",
- Prefix: "/gccgo",
- Packages: []*Package{
- {
- Kind: "gccgo",
- Name: "Gccgo",
- },
- },
-}
diff --git a/dashboard/app/build/handler.go b/dashboard/app/build/handler.go
deleted file mode 100644
index 5b7c7cb..0000000
--- a/dashboard/app/build/handler.go
+++ /dev/null
@@ -1,994 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package build
-
-import (
- "bytes"
- "crypto/hmac"
- "crypto/md5"
- "encoding/json"
- "errors"
- "fmt"
- "io/ioutil"
- "net/http"
- "strconv"
- "strings"
- "time"
- "unicode/utf8"
-
- "appengine"
- "appengine/datastore"
- "appengine/memcache"
-
- "cache"
- "key"
-)
-
-const (
- commitsPerPage = 30
- watcherVersion = 3 // must match dashboard/watcher/watcher.go
- builderVersion = 1 // must match dashboard/builder/http.go
-)
-
-// commitHandler retrieves commit data or records a new commit.
-//
-// For GET requests it returns a Commit value for the specified
-// packagePath and hash.
-//
-// For POST requests it reads a JSON-encoded Commit value from the request
-// body and creates a new Commit entity. It also updates the "tip" Tag for
-// each new commit at tip.
-//
-// This handler is used by a gobuilder process in -commit mode.
-func commitHandler(r *http.Request) (interface{}, error) {
- c := contextForRequest(r)
- com := new(Commit)
-
- if r.Method == "GET" {
- com.PackagePath = r.FormValue("packagePath")
- com.Hash = r.FormValue("hash")
- err := datastore.Get(c, com.Key(c), com)
- if com.Num == 0 && com.Desc == "" {
- // Perf builder might have written an incomplete Commit.
- // Pretend it doesn't exist, so that we can get complete details.
- err = datastore.ErrNoSuchEntity
- }
- if err != nil {
- if err == datastore.ErrNoSuchEntity {
- // This error string is special.
- // The commit watcher expects it.
- // Do not change it.
- return nil, errors.New("Commit not found")
- }
- return nil, fmt.Errorf("getting Commit: %v", err)
- }
- if com.Num == 0 {
- // Corrupt state which shouldn't happen but does.
- // Return an error so builders' commit loops will
- // be willing to retry submitting this commit.
- return nil, errors.New("in datastore with zero Num")
- }
- if com.Desc == "" || com.User == "" {
- // Also shouldn't happen, but at least happened
- // once on a single commit when trying to fix data
- // in the datastore viewer UI?
- return nil, errors.New("missing field")
- }
- // Strip potentially large and unnecessary fields.
- com.ResultData = nil
- com.PerfResults = nil
- return com, nil
- }
- if r.Method != "POST" {
- return nil, errBadMethod(r.Method)
- }
- if !isMasterKey(c, r.FormValue("key")) {
- return nil, errors.New("can only POST commits with master key")
- }
-
- // For now, the commit watcher doesn't support gccgo.
- // TODO(adg,cmang): remove this exception when gccgo is supported.
- if dashboardForRequest(r) != gccgoDash {
- v, _ := strconv.Atoi(r.FormValue("version"))
- if v != watcherVersion {
- return nil, fmt.Errorf("rejecting POST from commit watcher; need version %v", watcherVersion)
- }
- }
-
- // POST request
- body, err := ioutil.ReadAll(r.Body)
- r.Body.Close()
- if err != nil {
- return nil, fmt.Errorf("reading Body: %v", err)
- }
- if !bytes.Contains(body, needsBenchmarkingBytes) {
- c.Warningf("old builder detected at %v", r.RemoteAddr)
- return nil, fmt.Errorf("rejecting old builder request, body does not contain %s: %q", needsBenchmarkingBytes, body)
- }
- if err := json.Unmarshal(body, com); err != nil {
- return nil, fmt.Errorf("unmarshaling body %q: %v", body, err)
- }
- com.Desc = limitStringLength(com.Desc, maxDatastoreStringLen)
- if err := com.Valid(); err != nil {
- return nil, fmt.Errorf("validating Commit: %v", err)
- }
- defer cache.Tick(c)
- tx := func(c appengine.Context) error {
- return addCommit(c, com)
- }
- return nil, datastore.RunInTransaction(c, tx, nil)
-}
-
-var needsBenchmarkingBytes = []byte(`"NeedsBenchmarking"`)
-
-// addCommit adds the Commit entity to the datastore and updates the tip Tag.
-// It must be run inside a datastore transaction.
-func addCommit(c appengine.Context, com *Commit) error {
- var ec Commit // existing commit
- isUpdate := false
- err := datastore.Get(c, com.Key(c), &ec)
- if err != nil && err != datastore.ErrNoSuchEntity {
- return fmt.Errorf("getting Commit: %v", err)
- }
- if err == nil {
- // Commit already in the datastore. Any fields different?
- // If not, don't do anything.
- changes := (com.Num != 0 && com.Num != ec.Num) ||
- com.ParentHash != ec.ParentHash ||
- com.Desc != ec.Desc ||
- com.User != ec.User ||
- !com.Time.Equal(ec.Time)
- if !changes {
- return nil
- }
- ec.ParentHash = com.ParentHash
- ec.Desc = com.Desc
- ec.User = com.User
- if !com.Time.IsZero() {
- ec.Time = com.Time
- }
- if com.Num != 0 {
- ec.Num = com.Num
- }
- isUpdate = true
- com = &ec
- }
- p, err := GetPackage(c, com.PackagePath)
- if err != nil {
- return fmt.Errorf("GetPackage: %v", err)
- }
- if com.Num == 0 {
- // get the next commit number
- com.Num = p.NextNum
- p.NextNum++
- if _, err := datastore.Put(c, p.Key(c), p); err != nil {
- return fmt.Errorf("putting Package: %v", err)
- }
- } else if com.Num >= p.NextNum {
- p.NextNum = com.Num + 1
- if _, err := datastore.Put(c, p.Key(c), p); err != nil {
- return fmt.Errorf("putting Package: %v", err)
- }
- }
- // if this isn't the first Commit test the parent commit exists.
- // The all zeros are returned by hg's p1node template for parentless commits.
- if com.ParentHash != "" && com.ParentHash != "0000000000000000000000000000000000000000" && com.ParentHash != "0000" {
- n, err := datastore.NewQuery("Commit").
- Filter("Hash =", com.ParentHash).
- Ancestor(p.Key(c)).
- Count(c)
- if err != nil {
- return fmt.Errorf("testing for parent Commit: %v", err)
- }
- if n == 0 {
- return errors.New("parent commit not found")
- }
- } else if com.Num != 1 {
- // This is the first commit; fail if it is not number 1.
- // (This will happen if we try to upload a new/different repo
- // where there is already commit data. A bad thing to do.)
- return errors.New("this package already has a first commit; aborting")
- }
- // update the tip Tag if this is the Go repo and this isn't on a release branch
- if p.Path == "" && !strings.HasPrefix(com.Desc, "[") && !isUpdate {
- t := &Tag{Kind: "tip", Hash: com.Hash}
- if _, err = datastore.Put(c, t.Key(c), t); err != nil {
- return fmt.Errorf("putting Tag: %v", err)
- }
- }
- // put the Commit
- if err = putCommit(c, com); err != nil {
- return err
- }
- if com.NeedsBenchmarking {
- // add to CommitRun
- cr, err := GetCommitRun(c, com.Num)
- if err != nil {
- return err
- }
- if err = cr.AddCommit(c, com); err != nil {
- return err
- }
- // create PerfResult
- res := &PerfResult{CommitHash: com.Hash, CommitNum: com.Num}
- if _, err := datastore.Put(c, res.Key(c), res); err != nil {
- return fmt.Errorf("putting PerfResult: %v", err)
- }
- // Update perf todo if necessary.
- if err = AddCommitToPerfTodo(c, com); err != nil {
- return err
- }
- }
- return nil
-}
-
-// tagHandler records a new tag. It reads a JSON-encoded Tag value from the
-// request body and updates the Tag entity for the Kind of tag provided.
-//
-// This handler is used by a gobuilder process in -commit mode.
-func tagHandler(r *http.Request) (interface{}, error) {
- if r.Method != "POST" {
- return nil, errBadMethod(r.Method)
- }
-
- t := new(Tag)
- defer r.Body.Close()
- if err := json.NewDecoder(r.Body).Decode(t); err != nil {
- return nil, err
- }
- if err := t.Valid(); err != nil {
- return nil, err
- }
- c := contextForRequest(r)
- defer cache.Tick(c)
- _, err := datastore.Put(c, t.Key(c), t)
- return nil, err
-}
-
-// Todo is a todoHandler response.
-type Todo struct {
- Kind string // "build-go-commit" or "build-package"
- Data interface{}
-}
-
-// todoHandler returns the next action to be performed by a builder.
-// It expects "builder" and "kind" query parameters and returns a *Todo value.
-// Multiple "kind" parameters may be specified.
-func todoHandler(r *http.Request) (interface{}, error) {
- c := contextForRequest(r)
- now := cache.Now(c)
- key := "build-todo-" + r.Form.Encode()
- var todo *Todo
- if cache.Get(r, now, key, &todo) {
- return todo, nil
- }
- var err error
- builder := r.FormValue("builder")
- for _, kind := range r.Form["kind"] {
- var com *Commit
- switch kind {
- case "build-go-commit":
- com, err = buildTodo(c, builder, "", "")
- if com != nil {
- com.PerfResults = []string{}
- }
- case "build-package":
- packagePath := r.FormValue("packagePath")
- goHash := r.FormValue("goHash")
- com, err = buildTodo(c, builder, packagePath, goHash)
- if com != nil {
- com.PerfResults = []string{}
- }
- case "benchmark-go-commit":
- com, err = perfTodo(c, builder)
- }
- if com != nil || err != nil {
- if com != nil {
- // ResultData can be large and not needed on builder.
- com.ResultData = []string{}
- }
- todo = &Todo{Kind: kind, Data: com}
- break
- }
- }
- if err == nil {
- cache.Set(r, now, key, todo)
- }
- return todo, err
-}
-
-// buildTodo returns the next Commit to be built (or nil if none available).
-//
-// If packagePath and goHash are empty, it scans the first 20 Go Commits in
-// Num-descending order and returns the first one it finds that doesn't have a
-// Result for this builder.
-//
-// If provided with non-empty packagePath and goHash args, it scans the first
-// 20 Commits in Num-descending order for the specified packagePath and
-// returns the first that doesn't have a Result for this builder and goHash.
-func buildTodo(c appengine.Context, builder, packagePath, goHash string) (*Commit, error) {
- p, err := GetPackage(c, packagePath)
- if err != nil {
- return nil, err
- }
-
- t := datastore.NewQuery("Commit").
- Ancestor(p.Key(c)).
- Limit(commitsPerPage).
- Order("-Num").
- Run(c)
- for {
- com := new(Commit)
- if _, err := t.Next(com); err == datastore.Done {
- break
- } else if err != nil {
- return nil, err
- }
- if com.Result(builder, goHash) == nil {
- return com, nil
- }
- }
-
- // Nothing left to do if this is a package (not the Go tree).
- if packagePath != "" {
- return nil, nil
- }
-
- // If there are no Go tree commits left to build,
- // see if there are any subrepo commits that need to be built at tip.
- // If so, ask the builder to build a go tree at the tip commit.
- // TODO(adg): do the same for "weekly" and "release" tags.
-
- tag, err := GetTag(c, "tip")
- if err != nil {
- return nil, err
- }
-
- // Check that this Go commit builds OK for this builder.
- // If not, don't re-build as the subrepos will never get built anyway.
- com, err := tag.Commit(c)
- if err != nil {
- return nil, err
- }
- if r := com.Result(builder, ""); r != nil && !r.OK {
- return nil, nil
- }
-
- pkgs, err := Packages(c, "subrepo")
- if err != nil {
- return nil, err
- }
- for _, pkg := range pkgs {
- com, err := pkg.LastCommit(c)
- if err != nil {
- c.Warningf("%v: no Commit found: %v", pkg, err)
- continue
- }
- if com.Result(builder, tag.Hash) == nil {
- return tag.Commit(c)
- }
- }
-
- return nil, nil
-}
-
-// perfTodo returns the next Commit to be benchmarked (or nil if none available).
-func perfTodo(c appengine.Context, builder string) (*Commit, error) {
- p := &Package{}
- todo := &PerfTodo{Builder: builder}
- err := datastore.Get(c, todo.Key(c), todo)
- if err != nil && err != datastore.ErrNoSuchEntity {
- return nil, fmt.Errorf("fetching PerfTodo: %v", err)
- }
- if err == datastore.ErrNoSuchEntity {
- todo, err = buildPerfTodo(c, builder)
- if err != nil {
- return nil, err
- }
- }
- if len(todo.CommitNums) == 0 {
- return nil, nil
- }
-
- // Have commit to benchmark, fetch it.
- num := todo.CommitNums[len(todo.CommitNums)-1]
- t := datastore.NewQuery("Commit").
- Ancestor(p.Key(c)).
- Filter("Num =", num).
- Limit(1).
- Run(c)
- com := new(Commit)
- if _, err := t.Next(com); err != nil {
- return nil, err
- }
- if !com.NeedsBenchmarking {
- return nil, fmt.Errorf("commit from perf todo queue is not intended for benchmarking")
- }
-
- // Remove benchmarks from other builders.
- var benchs []string
- for _, b := range com.PerfResults {
- bb := strings.Split(b, "|")
- if bb[0] == builder && bb[1] != "meta-done" {
- benchs = append(benchs, bb[1])
- }
- }
- com.PerfResults = benchs
-
- return com, nil
-}
-
-// buildPerfTodo creates PerfTodo for the builder with all commits. In a transaction.
-func buildPerfTodo(c appengine.Context, builder string) (*PerfTodo, error) {
- todo := &PerfTodo{Builder: builder}
- tx := func(c appengine.Context) error {
- err := datastore.Get(c, todo.Key(c), todo)
- if err != nil && err != datastore.ErrNoSuchEntity {
- return fmt.Errorf("fetching PerfTodo: %v", err)
- }
- if err == nil {
- return nil
- }
- t := datastore.NewQuery("CommitRun").
- Ancestor((&Package{}).Key(c)).
- Order("-StartCommitNum").
- Run(c)
- var nums []int
- var releaseNums []int
- loop:
- for {
- cr := new(CommitRun)
- if _, err := t.Next(cr); err == datastore.Done {
- break
- } else if err != nil {
- return fmt.Errorf("scanning commit runs for perf todo: %v", err)
- }
- for i := len(cr.Hash) - 1; i >= 0; i-- {
- if !cr.NeedsBenchmarking[i] || cr.Hash[i] == "" {
- continue // There's nothing to see here. Move along.
- }
- num := cr.StartCommitNum + i
- for k, v := range knownTags {
- // Releases are benchmarked first, because they are important (and there are few of them).
- if cr.Hash[i] == v {
- releaseNums = append(releaseNums, num)
- if k == "go1" {
- break loop // Point of no benchmark: test/bench/shootout: update timing.log to Go 1.
- }
- }
- }
- nums = append(nums, num)
- }
- }
- todo.CommitNums = orderPerfTodo(nums)
- todo.CommitNums = append(todo.CommitNums, releaseNums...)
- if _, err = datastore.Put(c, todo.Key(c), todo); err != nil {
- return fmt.Errorf("putting PerfTodo: %v", err)
- }
- return nil
- }
- return todo, datastore.RunInTransaction(c, tx, nil)
-}
-
-func removeCommitFromPerfTodo(c appengine.Context, builder string, num int) error {
- todo := &PerfTodo{Builder: builder}
- err := datastore.Get(c, todo.Key(c), todo)
- if err != nil && err != datastore.ErrNoSuchEntity {
- return fmt.Errorf("fetching PerfTodo: %v", err)
- }
- if err == datastore.ErrNoSuchEntity {
- return nil
- }
- for i := len(todo.CommitNums) - 1; i >= 0; i-- {
- if todo.CommitNums[i] == num {
- for ; i < len(todo.CommitNums)-1; i++ {
- todo.CommitNums[i] = todo.CommitNums[i+1]
- }
- todo.CommitNums = todo.CommitNums[:i]
- _, err = datastore.Put(c, todo.Key(c), todo)
- if err != nil {
- return fmt.Errorf("putting PerfTodo: %v", err)
- }
- break
- }
- }
- return nil
-}
-
-// packagesHandler returns a list of the non-Go Packages monitored
-// by the dashboard.
-func packagesHandler(r *http.Request) (interface{}, error) {
- kind := r.FormValue("kind")
- c := contextForRequest(r)
- now := cache.Now(c)
- key := "build-packages-" + kind
- var p []*Package
- if cache.Get(r, now, key, &p) {
- return p, nil
- }
- p, err := Packages(c, kind)
- if err != nil {
- return nil, err
- }
- cache.Set(r, now, key, p)
- return p, nil
-}
-
-// buildingHandler records that a build is in progress.
-// The data is only stored in memcache and with a timeout. It's assumed
-// that the build system will periodically refresh this if the build
-// is slow.
-func buildingHandler(r *http.Request) (interface{}, error) {
- if r.Method != "POST" {
- return nil, errBadMethod(r.Method)
- }
- c := contextForRequest(r)
- key := fmt.Sprintf("building|%s|%s|%s", r.FormValue("hash"), r.FormValue("gohash"), r.FormValue("builder"))
- err := memcache.Set(c, &memcache.Item{
- Key: key,
- Value: []byte(r.FormValue("url")),
- Expiration: 15 * time.Minute,
- })
- if err != nil {
- return nil, err
- }
- return map[string]interface{}{
- "key": key,
- }, nil
-}
-
-// resultHandler records a build result.
-// It reads a JSON-encoded Result value from the request body,
-// creates a new Result entity, and updates the relevant Commit entity.
-// If the Log field is not empty, resultHandler creates a new Log entity
-// and updates the LogHash field before putting the Commit entity.
-func resultHandler(r *http.Request) (interface{}, error) {
- if r.Method != "POST" {
- return nil, errBadMethod(r.Method)
- }
-
- // For now, the gccgo builders are using the old stuff.
- // TODO(adg,cmang): remove this exception when gccgo is updated.
- if dashboardForRequest(r) != gccgoDash {
- v, _ := strconv.Atoi(r.FormValue("version"))
- if v != builderVersion {
- return nil, fmt.Errorf("rejecting POST from builder; need version %v", builderVersion)
- }
- }
-
- c := contextForRequest(r)
- res := new(Result)
- defer r.Body.Close()
- if err := json.NewDecoder(r.Body).Decode(res); err != nil {
- return nil, fmt.Errorf("decoding Body: %v", err)
- }
- if err := res.Valid(); err != nil {
- return nil, fmt.Errorf("validating Result: %v", err)
- }
- defer cache.Tick(c)
- // store the Log text if supplied
- if len(res.Log) > 0 {
- hash, err := PutLog(c, res.Log)
- if err != nil {
- return nil, fmt.Errorf("putting Log: %v", err)
- }
- res.LogHash = hash
- }
- tx := func(c appengine.Context) error {
- // check Package exists
- if _, err := GetPackage(c, res.PackagePath); err != nil {
- return fmt.Errorf("GetPackage: %v", err)
- }
- // put Result
- if _, err := datastore.Put(c, res.Key(c), res); err != nil {
- return fmt.Errorf("putting Result: %v", err)
- }
- // add Result to Commit
- com := &Commit{PackagePath: res.PackagePath, Hash: res.Hash}
- if err := com.AddResult(c, res); err != nil {
- return fmt.Errorf("AddResult: %v", err)
- }
- // Send build failure notifications, if necessary.
- // Note this must run after the call AddResult, which
- // populates the Commit's ResultData field.
- return notifyOnFailure(c, com, res.Builder)
- }
- return nil, datastore.RunInTransaction(c, tx, nil)
-}
-
-// perf-result request payload
-type PerfRequest struct {
- Builder string
- Benchmark string
- Hash string
- OK bool
- Metrics []PerfMetric
- Artifacts []PerfArtifact
-}
-
-type PerfMetric struct {
- Type string
- Val uint64
-}
-
-type PerfArtifact struct {
- Type string
- Body string
-}
-
-// perfResultHandler records a becnhmarking result.
-func perfResultHandler(r *http.Request) (interface{}, error) {
- defer r.Body.Close()
- if r.Method != "POST" {
- return nil, errBadMethod(r.Method)
- }
-
- req := new(PerfRequest)
- if err := json.NewDecoder(r.Body).Decode(req); err != nil {
- return nil, fmt.Errorf("decoding Body: %v", err)
- }
-
- c := contextForRequest(r)
- defer cache.Tick(c)
-
- // store the text files if supplied
- for i, a := range req.Artifacts {
- hash, err := PutLog(c, a.Body)
- if err != nil {
- return nil, fmt.Errorf("putting Log: %v", err)
- }
- req.Artifacts[i].Body = hash
- }
- tx := func(c appengine.Context) error {
- return addPerfResult(c, r, req)
- }
- return nil, datastore.RunInTransaction(c, tx, nil)
-}
-
-// addPerfResult creates PerfResult and updates Commit, PerfTodo,
-// PerfMetricRun and PerfConfig.
-// MUST be called from inside a transaction.
-func addPerfResult(c appengine.Context, r *http.Request, req *PerfRequest) error {
- // check Package exists
- p, err := GetPackage(c, "")
- if err != nil {
- return fmt.Errorf("GetPackage: %v", err)
- }
- // add result to Commit
- com := &Commit{Hash: req.Hash}
- if err := com.AddPerfResult(c, req.Builder, req.Benchmark); err != nil {
- return fmt.Errorf("AddPerfResult: %v", err)
- }
-
- // add the result to PerfResult
- res := &PerfResult{CommitHash: req.Hash}
- if err := datastore.Get(c, res.Key(c), res); err != nil {
- return fmt.Errorf("getting PerfResult: %v", err)
- }
- present := res.AddResult(req)
- if _, err := datastore.Put(c, res.Key(c), res); err != nil {
- return fmt.Errorf("putting PerfResult: %v", err)
- }
-
- // Meta-done denotes that there are no benchmarks left.
- if req.Benchmark == "meta-done" {
- // Don't send duplicate emails for the same commit/builder.
- // And don't send emails about too old commits.
- if !present && com.Num >= p.NextNum-commitsPerPage {
- if err := checkPerfChanges(c, r, com, req.Builder, res); err != nil {
- return err
- }
- }
- if err := removeCommitFromPerfTodo(c, req.Builder, com.Num); err != nil {
- return nil
- }
- return nil
- }
-
- // update PerfConfig
- newBenchmark, err := UpdatePerfConfig(c, r, req)
- if err != nil {
- return fmt.Errorf("updating PerfConfig: %v", err)
- }
- if newBenchmark {
- // If this is a new benchmark on the builder, delete PerfTodo.
- // It will be recreated later with all commits again.
- todo := &PerfTodo{Builder: req.Builder}
- err = datastore.Delete(c, todo.Key(c))
- if err != nil && err != datastore.ErrNoSuchEntity {
- return fmt.Errorf("deleting PerfTodo: %v", err)
- }
- }
-
- // add perf metrics
- for _, metric := range req.Metrics {
- m, err := GetPerfMetricRun(c, req.Builder, req.Benchmark, metric.Type, com.Num)
- if err != nil {
- return fmt.Errorf("GetPerfMetrics: %v", err)
- }
- if err = m.AddMetric(c, com.Num, metric.Val); err != nil {
- return fmt.Errorf("AddMetric: %v", err)
- }
- }
-
- return nil
-}
-
-// MUST be called from inside a transaction.
-func checkPerfChanges(c appengine.Context, r *http.Request, com *Commit, builder string, res *PerfResult) error {
- pc, err := GetPerfConfig(c, r)
- if err != nil {
- return err
- }
-
- results := res.ParseData()[builder]
- rcNewer := MakePerfResultCache(c, com, true)
- rcOlder := MakePerfResultCache(c, com, false)
-
- // Check whether we need to send failure notification email.
- if results["meta-done"].OK {
- // This one is successful, see if the next is failed.
- nextRes, err := rcNewer.Next(com.Num)
- if err != nil {
- return err
- }
- if nextRes != nil && isPerfFailed(nextRes, builder) {
- sendPerfFailMail(c, builder, nextRes)
- }
- } else {
- // This one is failed, see if the previous is successful.
- prevRes, err := rcOlder.Next(com.Num)
- if err != nil {
- return err
- }
- if prevRes != nil && !isPerfFailed(prevRes, builder) {
- sendPerfFailMail(c, builder, res)
- }
- }
-
- // Now see if there are any performance changes.
- // Find the previous and the next results for performance comparison.
- prevRes, err := rcOlder.NextForComparison(com.Num, builder)
- if err != nil {
- return err
- }
- nextRes, err := rcNewer.NextForComparison(com.Num, builder)
- if err != nil {
- return err
- }
- if results["meta-done"].OK {
- // This one is successful, compare with a previous one.
- if prevRes != nil {
- if err := comparePerfResults(c, pc, builder, prevRes, res); err != nil {
- return err
- }
- }
- // Compare a next one with the current.
- if nextRes != nil {
- if err := comparePerfResults(c, pc, builder, res, nextRes); err != nil {
- return err
- }
- }
- } else {
- // This one is failed, compare a previous one with a next one.
- if prevRes != nil && nextRes != nil {
- if err := comparePerfResults(c, pc, builder, prevRes, nextRes); err != nil {
- return err
- }
- }
- }
-
- return nil
-}
-
-func comparePerfResults(c appengine.Context, pc *PerfConfig, builder string, prevRes, res *PerfResult) error {
- changes := significantPerfChanges(pc, builder, prevRes, res)
- if len(changes) == 0 {
- return nil
- }
- com := &Commit{Hash: res.CommitHash}
- if err := datastore.Get(c, com.Key(c), com); err != nil {
- return fmt.Errorf("getting commit %v: %v", com.Hash, err)
- }
- sendPerfMailLater.Call(c, com, prevRes.CommitHash, builder, changes) // add task to queue
- return nil
-}
-
-// logHandler displays log text for a given hash.
-// It handles paths like "/log/hash".
-func logHandler(w http.ResponseWriter, r *http.Request) {
- w.Header().Set("Content-type", "text/plain; charset=utf-8")
- c := contextForRequest(r)
- hash := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
- key := datastore.NewKey(c, "Log", hash, 0, nil)
- l := new(Log)
- if err := datastore.Get(c, key, l); err != nil {
- logErr(w, r, err)
- return
- }
- b, err := l.Text()
- if err != nil {
- logErr(w, r, err)
- return
- }
- w.Write(b)
-}
-
-// clearResultsHandler purges the last commitsPerPage results for the given builder.
-// It optionally takes a comma-separated list of specific hashes to clear.
-func clearResultsHandler(r *http.Request) (interface{}, error) {
- if r.Method != "POST" {
- return nil, errBadMethod(r.Method)
- }
- builder := r.FormValue("builder")
- if builder == "" {
- return nil, errors.New("must specify a builder")
- }
- clearAll := r.FormValue("hash") == ""
- hash := strings.Split(r.FormValue("hash"), ",")
-
- c := contextForRequest(r)
- defer cache.Tick(c)
- pkg := (&Package{}).Key(c) // TODO(adg): support clearing sub-repos
- err := datastore.RunInTransaction(c, func(c appengine.Context) error {
- var coms []*Commit
- keys, err := datastore.NewQuery("Commit").
- Ancestor(pkg).
- Order("-Num").
- Limit(commitsPerPage).
- GetAll(c, &coms)
- if err != nil {
- return err
- }
- var rKeys []*datastore.Key
- for _, com := range coms {
- if !(clearAll || contains(hash, com.Hash)) {
- continue
- }
- r := com.Result(builder, "")
- if r == nil {
- continue
- }
- com.RemoveResult(r)
- rKeys = append(rKeys, r.Key(c))
- }
- _, err = datastore.PutMulti(c, keys, coms)
- if err != nil {
- return err
- }
- return datastore.DeleteMulti(c, rKeys)
- }, nil)
- return nil, err
-}
-
-type dashHandler func(*http.Request) (interface{}, error)
-
-type dashResponse struct {
- Response interface{}
- Error string
-}
-
-// errBadMethod is returned by a dashHandler when
-// the request has an unsuitable method.
-type errBadMethod string
-
-func (e errBadMethod) Error() string {
- return "bad method: " + string(e)
-}
-
-// AuthHandler wraps a http.HandlerFunc with a handler that validates the
-// supplied key and builder query parameters.
-func AuthHandler(h dashHandler) http.HandlerFunc {
- return func(w http.ResponseWriter, r *http.Request) {
- c := contextForRequest(r)
-
- // Put the URL Query values into r.Form to avoid parsing the
- // request body when calling r.FormValue.
- r.Form = r.URL.Query()
-
- var err error
- var resp interface{}
-
- // Validate key query parameter for POST requests only.
- key := r.FormValue("key")
- builder := r.FormValue("builder")
- if r.Method == "POST" && !validKey(c, key, builder) {
- err = fmt.Errorf("invalid key %q for builder %q", key, builder)
- }
-
- // Call the original HandlerFunc and return the response.
- if err == nil {
- resp, err = h(r)
- }
-
- // Write JSON response.
- dashResp := &dashResponse{Response: resp}
- if err != nil {
- c.Errorf("%v", err)
- dashResp.Error = err.Error()
- }
- w.Header().Set("Content-Type", "application/json")
- if err = json.NewEncoder(w).Encode(dashResp); err != nil {
- c.Criticalf("encoding response: %v", err)
- }
- }
-}
-
-func keyHandler(w http.ResponseWriter, r *http.Request) {
- builder := r.FormValue("builder")
- if builder == "" {
- logErr(w, r, errors.New("must supply builder in query string"))
- return
- }
- c := contextForRequest(r)
- fmt.Fprint(w, builderKey(c, builder))
-}
-
-func init() {
- // admin handlers
- handleFunc("/init", initHandler)
- handleFunc("/key", keyHandler)
-
- // authenticated handlers
- handleFunc("/building", AuthHandler(buildingHandler))
- handleFunc("/clear-results", AuthHandler(clearResultsHandler))
- handleFunc("/commit", AuthHandler(commitHandler))
- handleFunc("/packages", AuthHandler(packagesHandler))
- handleFunc("/perf-result", AuthHandler(perfResultHandler))
- handleFunc("/result", AuthHandler(resultHandler))
- handleFunc("/tag", AuthHandler(tagHandler))
- handleFunc("/todo", AuthHandler(todoHandler))
-
- // public handlers
- handleFunc("/log/", logHandler)
-}
-
-func validHash(hash string) bool {
- // TODO(adg): correctly validate a hash
- return hash != ""
-}
-
-func validKey(c appengine.Context, key, builder string) bool {
- return isMasterKey(c, key) || key == builderKey(c, builder)
-}
-
-func isMasterKey(c appengine.Context, k string) bool {
- return appengine.IsDevAppServer() || k == key.Secret(c)
-}
-
-func builderKey(c appengine.Context, builder string) string {
- h := hmac.New(md5.New, []byte(key.Secret(c)))
- h.Write([]byte(builder))
- return fmt.Sprintf("%x", h.Sum(nil))
-}
-
-func logErr(w http.ResponseWriter, r *http.Request, err error) {
- contextForRequest(r).Errorf("Error: %v", err)
- w.WriteHeader(http.StatusInternalServerError)
- fmt.Fprint(w, "Error: ", err)
-}
-
-func contextForRequest(r *http.Request) appengine.Context {
- return dashboardForRequest(r).Context(appengine.NewContext(r))
-}
-
-// limitStringLength essentially does return s[:max],
-// but it ensures that we dot not split UTF-8 rune in half.
-// Otherwise appengine python scripts will break badly.
-func limitStringLength(s string, max int) string {
- if len(s) <= max {
- return s
- }
- for {
- s = s[:max]
- r, size := utf8.DecodeLastRuneInString(s)
- if r != utf8.RuneError || size != 1 {
- return s
- }
- max--
- }
-}
diff --git a/dashboard/app/build/init.go b/dashboard/app/build/init.go
deleted file mode 100644
index 9654d18..0000000
--- a/dashboard/app/build/init.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package build
-
-import (
- "fmt"
- "net/http"
-
- "appengine"
- "appengine/datastore"
-
- "cache"
- "key"
-)
-
-func initHandler(w http.ResponseWriter, r *http.Request) {
- d := dashboardForRequest(r)
- c := d.Context(appengine.NewContext(r))
- defer cache.Tick(c)
- for _, p := range d.Packages {
- err := datastore.Get(c, p.Key(c), new(Package))
- if _, ok := err.(*datastore.ErrFieldMismatch); ok {
- // Some fields have been removed, so it's okay to ignore this error.
- err = nil
- }
- if err == nil {
- continue
- } else if err != datastore.ErrNoSuchEntity {
- logErr(w, r, err)
- return
- }
- p.NextNum = 1 // So we can add the first commit.
- if _, err := datastore.Put(c, p.Key(c), p); err != nil {
- logErr(w, r, err)
- return
- }
- }
-
- // Create secret key.
- key.Secret(c)
-
- fmt.Fprint(w, "OK")
-}
diff --git a/dashboard/app/build/notify.go b/dashboard/app/build/notify.go
deleted file mode 100644
index 1a71dd2..0000000
--- a/dashboard/app/build/notify.go
+++ /dev/null
@@ -1,378 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package build
-
-import (
- "bytes"
- "encoding/gob"
- "errors"
- "fmt"
- "io/ioutil"
- "net/http"
- "net/url"
- "regexp"
- "runtime"
- "sort"
- "text/template"
-
- "appengine"
- "appengine/datastore"
- "appengine/delay"
- "appengine/mail"
- "appengine/urlfetch"
-)
-
-const (
- mailFrom = "builder@golang.org" // use this for sending any mail
- failMailTo = "golang-dev@googlegroups.com"
- domain = "build.golang.org"
- gobotBase = "http://research.swtch.com/gobot_codereview"
-)
-
-// ignoreFailure is a set of builders that we don't email about because
-// they are not yet production-ready.
-var ignoreFailure = map[string]bool{
- "dragonfly-386": true,
- "dragonfly-amd64": true,
- "freebsd-arm": true,
- "netbsd-amd64-bsiegert": true,
- "netbsd-arm-rpi": true,
- "plan9-amd64-aram": true,
-}
-
-// notifyOnFailure checks whether the supplied Commit or the subsequent
-// Commit (if present) breaks the build for this builder.
-// If either of those commits break the build an email notification is sent
-// from a delayed task. (We use a task because this way the mail won't be
-// sent if the enclosing datastore transaction fails.)
-//
-// This must be run in a datastore transaction, and the provided *Commit must
-// have been retrieved from the datastore within that transaction.
-func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
- if ignoreFailure[builder] {
- return nil
- }
-
- // TODO(adg): implement notifications for packages
- if com.PackagePath != "" {
- return nil
- }
-
- p := &Package{Path: com.PackagePath}
- var broken *Commit
- cr := com.Result(builder, "")
- if cr == nil {
- return fmt.Errorf("no result for %s/%s", com.Hash, builder)
- }
- q := datastore.NewQuery("Commit").Ancestor(p.Key(c))
- if cr.OK {
- // This commit is OK. Notify if next Commit is broken.
- next := new(Commit)
- q = q.Filter("ParentHash=", com.Hash)
- if err := firstMatch(c, q, next); err != nil {
- if err == datastore.ErrNoSuchEntity {
- // OK at tip, no notification necessary.
- return nil
- }
- return err
- }
- if nr := next.Result(builder, ""); nr != nil && !nr.OK {
- c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
- c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
- broken = next
- }
- } else {
- // This commit is broken. Notify if the previous Commit is OK.
- prev := new(Commit)
- q = q.Filter("Hash=", com.ParentHash)
- if err := firstMatch(c, q, prev); err != nil {
- if err == datastore.ErrNoSuchEntity {
- // No previous result, let the backfill of
- // this result trigger the notification.
- return nil
- }
- return err
- }
- if pr := prev.Result(builder, ""); pr != nil && pr.OK {
- c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
- c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
- broken = com
- }
- }
- if broken == nil {
- return nil
- }
- r := broken.Result(builder, "")
- if r == nil {
- return fmt.Errorf("finding result for %q: %+v", builder, com)
- }
- return commonNotify(c, broken, builder, r.LogHash)
-}
-
-// firstMatch executes the query q and loads the first entity into v.
-func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
- t := q.Limit(1).Run(c)
- _, err := t.Next(v)
- if err == datastore.Done {
- err = datastore.ErrNoSuchEntity
- }
- return err
-}
-
-var notifyLater = delay.Func("notify", notify)
-
-// notify tries to update the CL for the given Commit with a failure message.
-// If it doesn't succeed, it sends a failure email to golang-dev.
-func notify(c appengine.Context, com *Commit, builder, logHash string) {
- v := url.Values{"brokebuild": {builder}, "log": {logHash}}
- if !updateCL(c, com, v) {
- // Send a mail notification if the CL can't be found.
- sendFailMail(c, com, builder, logHash)
- }
-}
-
-// updateCL tells gobot to update the CL for the given Commit with
-// the provided query values.
-func updateCL(c appengine.Context, com *Commit, v url.Values) bool {
- cl, err := lookupCL(c, com)
- if err != nil {
- c.Errorf("could not find CL for %v: %v", com.Hash, err)
- return false
- }
- u := fmt.Sprintf("%v?cl=%v&%s", gobotBase, cl, v.Encode())
- r, err := urlfetch.Client(c).Post(u, "text/plain", nil)
- if err != nil {
- c.Errorf("could not update CL %v: %v", cl, err)
- return false
- }
- r.Body.Close()
- if r.StatusCode != http.StatusOK {
- c.Errorf("could not update CL %v: %v", cl, r.Status)
- return false
- }
- return true
-}
-
-var clURL = regexp.MustCompile(`https://codereview.appspot.com/([0-9]+)`)
-
-// lookupCL consults code.google.com for the full change description for the
-// provided Commit, and returns the relevant CL number.
-func lookupCL(c appengine.Context, com *Commit) (string, error) {
- url := "https://code.google.com/p/go/source/detail?r=" + com.Hash
- r, err := urlfetch.Client(c).Get(url)
- if err != nil {
- return "", err
- }
- defer r.Body.Close()
- if r.StatusCode != http.StatusOK {
- return "", fmt.Errorf("retrieving %v: %v", url, r.Status)
- }
- b, err := ioutil.ReadAll(r.Body)
- if err != nil {
- return "", err
- }
- m := clURL.FindAllSubmatch(b, -1)
- if m == nil {
- return "", errors.New("no CL URL found on changeset page")
- }
- // Return the last visible codereview URL on the page,
- // in case the change description refers to another CL.
- return string(m[len(m)-1][1]), nil
-}
-
-var sendFailMailTmpl = template.Must(template.New("notify.txt").
- Funcs(template.FuncMap(tmplFuncs)).
- ParseFiles("build/notify.txt"))
-
-func init() {
- gob.Register(&Commit{}) // for delay
-}
-
-var (
- sendPerfMailLater = delay.Func("sendPerfMail", sendPerfMailFunc)
- sendPerfMailTmpl = template.Must(
- template.New("perf_notify.txt").
- Funcs(template.FuncMap(tmplFuncs)).
- ParseFiles("build/perf_notify.txt"),
- )
-)
-
-// MUST be called from inside a transaction.
-func sendPerfFailMail(c appengine.Context, builder string, res *PerfResult) error {
- com := &Commit{Hash: res.CommitHash}
- if err := datastore.Get(c, com.Key(c), com); err != nil {
- return err
- }
- logHash := ""
- parsed := res.ParseData()
- for _, data := range parsed[builder] {
- if !data.OK {
- logHash = data.Artifacts["log"]
- break
- }
- }
- if logHash == "" {
- return fmt.Errorf("can not find failed result for commit %v on builder %v", com.Hash, builder)
- }
- return commonNotify(c, com, builder, logHash)
-}
-
-// commonNotify MUST!!! be called from within a transaction inside which
-// the provided Commit entity was retrieved from the datastore.
-func commonNotify(c appengine.Context, com *Commit, builder, logHash string) error {
- if com.Num == 0 || com.Desc == "" {
- stk := make([]byte, 10000)
- n := runtime.Stack(stk, false)
- stk = stk[:n]
- c.Errorf("refusing to notify with com=%+v\n%s", *com, string(stk))
- return fmt.Errorf("misuse of commonNotify")
- }
- if com.FailNotificationSent {
- return nil
- }
- c.Infof("%s is broken commit; notifying", com.Hash)
- notifyLater.Call(c, com, builder, logHash) // add task to queue
- com.FailNotificationSent = true
- return putCommit(c, com)
-}
-
-// sendFailMail sends a mail notification that the build failed on the
-// provided commit and builder.
-func sendFailMail(c appengine.Context, com *Commit, builder, logHash string) {
- // get Log
- k := datastore.NewKey(c, "Log", logHash, 0, nil)
- l := new(Log)
- if err := datastore.Get(c, k, l); err != nil {
- c.Errorf("finding Log record %v: %v", logHash, err)
- return
- }
- logText, err := l.Text()
- if err != nil {
- c.Errorf("unpacking Log record %v: %v", logHash, err)
- return
- }
-
- // prepare mail message
- var body bytes.Buffer
- err = sendFailMailTmpl.Execute(&body, map[string]interface{}{
- "Builder": builder, "Commit": com, "LogHash": logHash, "LogText": logText,
- "Hostname": domain,
- })
- if err != nil {
- c.Errorf("rendering mail template: %v", err)
- return
- }
- subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc))
- msg := &mail.Message{
- Sender: mailFrom,
- To: []string{failMailTo},
- ReplyTo: failMailTo,
- Subject: subject,
- Body: body.String(),
- }
-
- // send mail
- if err := mail.Send(c, msg); err != nil {
- c.Errorf("sending mail: %v", err)
- }
-}
-
-type PerfChangeBenchmark struct {
- Name string
- Metrics []*PerfChangeMetric
-}
-
-type PerfChangeMetric struct {
- Name string
- Old uint64
- New uint64
- Delta float64
-}
-
-type PerfChangeBenchmarkSlice []*PerfChangeBenchmark
-
-func (l PerfChangeBenchmarkSlice) Len() int { return len(l) }
-func (l PerfChangeBenchmarkSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
-func (l PerfChangeBenchmarkSlice) Less(i, j int) bool {
- b1, p1 := splitBench(l[i].Name)
- b2, p2 := splitBench(l[j].Name)
- if b1 != b2 {
- return b1 < b2
- }
- return p1 < p2
-}
-
-type PerfChangeMetricSlice []*PerfChangeMetric
-
-func (l PerfChangeMetricSlice) Len() int { return len(l) }
-func (l PerfChangeMetricSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
-func (l PerfChangeMetricSlice) Less(i, j int) bool { return l[i].Name < l[j].Name }
-
-func sendPerfMailFunc(c appengine.Context, com *Commit, prevCommitHash, builder string, changes []*PerfChange) {
- // Sort the changes into the right order.
- var benchmarks []*PerfChangeBenchmark
- for _, ch := range changes {
- // Find the benchmark.
- var b *PerfChangeBenchmark
- for _, b1 := range benchmarks {
- if b1.Name == ch.Bench {
- b = b1
- break
- }
- }
- if b == nil {
- b = &PerfChangeBenchmark{Name: ch.Bench}
- benchmarks = append(benchmarks, b)
- }
- b.Metrics = append(b.Metrics, &PerfChangeMetric{Name: ch.Metric, Old: ch.Old, New: ch.New, Delta: ch.Diff})
- }
- for _, b := range benchmarks {
- sort.Sort(PerfChangeMetricSlice(b.Metrics))
- }
- sort.Sort(PerfChangeBenchmarkSlice(benchmarks))
-
- u := fmt.Sprintf("http://%v/perfdetail?commit=%v&commit0=%v&kind=builder&builder=%v", domain, com.Hash, prevCommitHash, builder)
-
- // Prepare mail message (without Commit, for updateCL).
- var body bytes.Buffer
- err := sendPerfMailTmpl.Execute(&body, map[string]interface{}{
- "Builder": builder, "Hostname": domain, "Url": u, "Benchmarks": benchmarks,
- })
- if err != nil {
- c.Errorf("rendering perf mail template: %v", err)
- return
- }
-
- // First, try to update the CL.
- v := url.Values{"textmsg": {body.String()}}
- if updateCL(c, com, v) {
- return
- }
-
- // Otherwise, send mail (with Commit, for independent mail message).
- body.Reset()
- err = sendPerfMailTmpl.Execute(&body, map[string]interface{}{
- "Builder": builder, "Commit": com, "Hostname": domain, "Url": u, "Benchmarks": benchmarks,
- })
- if err != nil {
- c.Errorf("rendering perf mail template: %v", err)
- return
- }
- subject := fmt.Sprintf("Perf changes on %s by %s", builder, shortDesc(com.Desc))
- msg := &mail.Message{
- Sender: mailFrom,
- To: []string{failMailTo},
- ReplyTo: failMailTo,
- Subject: subject,
- Body: body.String(),
- }
-
- // send mail
- if err := mail.Send(c, msg); err != nil {
- c.Errorf("sending mail: %v", err)
- }
-}
diff --git a/dashboard/app/build/notify.txt b/dashboard/app/build/notify.txt
deleted file mode 100644
index 438d5e8..0000000
--- a/dashboard/app/build/notify.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-Change {{shortHash .Commit.Hash}} broke the {{.Builder}} build:
-http://{{.Hostname}}/log/{{.LogHash}}
-
-{{.Commit.Desc}}
-
-https://golang.org/change/{{shortHash .Commit.Hash}}
-
-$ tail -200 < log
-{{printf "%s" .LogText | tail 200}}
diff --git a/dashboard/app/build/perf.go b/dashboard/app/build/perf.go
deleted file mode 100644
index a8e542f..0000000
--- a/dashboard/app/build/perf.go
+++ /dev/null
@@ -1,313 +0,0 @@
-// Copyright 2014 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package build
-
-import (
- "fmt"
- "sort"
- "strconv"
- "strings"
-
- "appengine"
- "appengine/datastore"
-)
-
-var knownTags = map[string]string{
- "go1": "0051c7442fed9c888de6617fa9239a913904d96e",
- "go1.1": "d29da2ced72ba2cf48ed6a8f1ec4abc01e4c5bf1",
- "go1.2": "b1edf8faa5d6cbc50c6515785df9df9c19296564",
- "go1.3": "f153208c0a0e306bfca14f71ef11f09859ccabc8",
- "go1.4": "faa3ed1dc30e42771a68b6337dcf8be9518d5c07",
-}
-
-var lastRelease = "go1.4"
-
-func splitBench(benchProcs string) (string, int) {
- ss := strings.Split(benchProcs, "-")
- procs, _ := strconv.Atoi(ss[1])
- return ss[0], procs
-}
-
-func dashPerfCommits(c appengine.Context, page int) ([]*Commit, error) {
- q := datastore.NewQuery("Commit").
- Ancestor((&Package{}).Key(c)).
- Order("-Num").
- Filter("NeedsBenchmarking =", true).
- Limit(commitsPerPage).
- Offset(page * commitsPerPage)
- var commits []*Commit
- _, err := q.GetAll(c, &commits)
- if err == nil && len(commits) == 0 {
- err = fmt.Errorf("no commits")
- }
- return commits, err
-}
-
-func perfChangeStyle(pc *PerfConfig, v float64, builder, benchmark, metric string) string {
- noise := pc.NoiseLevel(builder, benchmark, metric)
- if isNoise(v, noise) {
- return "noise"
- }
- if v > 0 {
- return "bad"
- }
- return "good"
-}
-
-func isNoise(diff, noise float64) bool {
- rnoise := -100 * noise / (noise + 100)
- return diff < noise && diff > rnoise
-}
-
-func perfDiff(old, new uint64) float64 {
- return 100*float64(new)/float64(old) - 100
-}
-
-func isPerfFailed(res *PerfResult, builder string) bool {
- data := res.ParseData()[builder]
- return data != nil && data["meta-done"] != nil && !data["meta-done"].OK
-}
-
-// PerfResultCache caches a set of PerfResults so that it's easy to access them
-// without lots of duplicate accesses to datastore.
-// It allows to iterate over newer or older results for some base commit.
-type PerfResultCache struct {
- c appengine.Context
- newer bool
- iter *datastore.Iterator
- results map[int]*PerfResult
-}
-
-func MakePerfResultCache(c appengine.Context, com *Commit, newer bool) *PerfResultCache {
- p := &Package{}
- q := datastore.NewQuery("PerfResult").Ancestor(p.Key(c)).Limit(100)
- if newer {
- q = q.Filter("CommitNum >=", com.Num).Order("CommitNum")
- } else {
- q = q.Filter("CommitNum <=", com.Num).Order("-CommitNum")
- }
- rc := &PerfResultCache{c: c, newer: newer, iter: q.Run(c), results: make(map[int]*PerfResult)}
- return rc
-}
-
-func (rc *PerfResultCache) Get(commitNum int) *PerfResult {
- rc.Next(commitNum) // fetch the commit, if necessary
- return rc.results[commitNum]
-}
-
-// Next returns the next PerfResult for the commit commitNum.
-// It does not care whether the result has any data, failed or whatever.
-func (rc *PerfResultCache) Next(commitNum int) (*PerfResult, error) {
- // See if we have next result in the cache.
- next := -1
- for ci := range rc.results {
- if rc.newer {
- if ci > commitNum && (next == -1 || ci < next) {
- next = ci
- }
- } else {
- if ci < commitNum && (next == -1 || ci > next) {
- next = ci
- }
- }
- }
- if next != -1 {
- return rc.results[next], nil
- }
- // Fetch next result from datastore.
- res := new(PerfResult)
- _, err := rc.iter.Next(res)
- if err == datastore.Done {
- return nil, nil
- }
- if err != nil {
- return nil, fmt.Errorf("fetching perf results: %v", err)
- }
- if (rc.newer && res.CommitNum < commitNum) || (!rc.newer && res.CommitNum > commitNum) {
- rc.c.Errorf("PerfResultCache.Next: bad commit num")
- }
- rc.results[res.CommitNum] = res
- return res, nil
-}
-
-// NextForComparison returns PerfResult which we need to use for performance comprison.
-// It skips failed results, but does not skip results with no data.
-func (rc *PerfResultCache) NextForComparison(commitNum int, builder string) (*PerfResult, error) {
- for {
- res, err := rc.Next(commitNum)
- if err != nil {
- return nil, err
- }
- if res == nil {
- return nil, nil
- }
- if res.CommitNum == commitNum {
- continue
- }
- parsed := res.ParseData()
- if builder != "" {
- // Comparing for a particular builder.
- // This is used in perf_changes and in email notifications.
- b := parsed[builder]
- if b == nil || b["meta-done"] == nil {
- // No results yet, must not do the comparison.
- return nil, nil
- }
- if b["meta-done"].OK {
- // Have complete results, compare.
- return res, nil
- }
- } else {
- // Comparing for all builders, find a result with at least
- // one successful meta-done.
- // This is used in perf_detail.
- for _, benchs := range parsed {
- if data := benchs["meta-done"]; data != nil && data.OK {
- return res, nil
- }
- }
- }
- // Failed, try next result.
- commitNum = res.CommitNum
- }
-}
-
-type PerfChange struct {
- Builder string
- Bench string
- Metric string
- Old uint64
- New uint64
- Diff float64
-}
-
-func significantPerfChanges(pc *PerfConfig, builder string, prevRes, res *PerfResult) (changes []*PerfChange) {
- // First, collect all significant changes.
- for builder1, benchmarks1 := range res.ParseData() {
- if builder != "" && builder != builder1 {
- // This is not the builder you're looking for, Luke.
- continue
- }
- benchmarks0 := prevRes.ParseData()[builder1]
- if benchmarks0 == nil {
- continue
- }
- for benchmark, data1 := range benchmarks1 {
- data0 := benchmarks0[benchmark]
- if data0 == nil {
- continue
- }
- for metric, val := range data1.Metrics {
- val0 := data0.Metrics[metric]
- if val0 == 0 {
- continue
- }
- diff := perfDiff(val0, val)
- noise := pc.NoiseLevel(builder, benchmark, metric)
- if isNoise(diff, noise) {
- continue
- }
- ch := &PerfChange{Builder: builder, Bench: benchmark, Metric: metric, Old: val0, New: val, Diff: diff}
- changes = append(changes, ch)
- }
- }
- }
- // Then, strip non-repeatable changes (flakes).
- // The hypothesis is that a real change must show up with the majority of GOMAXPROCS values.
- majority := len(pc.ProcList(builder))/2 + 1
- cnt := make(map[string]int)
- for _, ch := range changes {
- b, _ := splitBench(ch.Bench)
- name := b + "|" + ch.Metric
- if ch.Diff < 0 {
- name += "--"
- }
- cnt[name] = cnt[name] + 1
- }
- for i := 0; i < len(changes); i++ {
- ch := changes[i]
- b, _ := splitBench(ch.Bench)
- name := b + "|" + ch.Metric
- if cnt[name] >= majority {
- continue
- }
- if cnt[name+"--"] >= majority {
- continue
- }
- // Remove flake.
- last := len(changes) - 1
- changes[i] = changes[last]
- changes = changes[:last]
- i--
- }
- return changes
-}
-
-// orderPerfTodo reorders commit nums for benchmarking todo.
-// The resulting order is somewhat tricky. We want 2 things:
-// 1. benchmark sequentially backwards (this provides information about most
-// recent changes, and allows to estimate noise levels)
-// 2. benchmark old commits in "scatter" order (this allows to quickly gather
-// brief information about thousands of old commits)
-// So this function interleaves the two orders.
-func orderPerfTodo(nums []int) []int {
- sort.Ints(nums)
- n := len(nums)
- pow2 := uint32(0) // next power-of-two that is >= n
- npow2 := 0
- for npow2 <= n {
- pow2++
- npow2 = 1 << pow2
- }
- res := make([]int, n)
- resPos := n - 1 // result array is filled backwards
- present := make([]bool, n) // denotes values that already present in result array
- for i0, i1 := n-1, 0; i0 >= 0 || i1 < npow2; {
- // i0 represents "benchmark sequentially backwards" sequence
- // find the next commit that is not yet present and add it
- for cnt := 0; cnt < 2; cnt++ {
- for ; i0 >= 0; i0-- {
- if !present[i0] {
- present[i0] = true
- res[resPos] = nums[i0]
- resPos--
- i0--
- break
- }
- }
- }
- // i1 represents "scatter order" sequence
- // find the next commit that is not yet present and add it
- for ; i1 < npow2; i1++ {
- // do the "recursive split-ordering" trick
- idx := 0 // bitwise reverse of i1
- for j := uint32(0); j <= pow2; j++ {
- if (i1 & (1 << j)) != 0 {
- idx = idx | (1 << (pow2 - j - 1))
- }
- }
- if idx < n && !present[idx] {
- present[idx] = true
- res[resPos] = nums[idx]
- resPos--
- i1++
- break
- }
- }
- }
- // The above can't possibly be correct. Do dump check.
- res2 := make([]int, n)
- copy(res2, res)
- sort.Ints(res2)
- for i := range res2 {
- if res2[i] != nums[i] {
- panic(fmt.Sprintf("diff at %v: expect %v, want %v\nwas: %v\n become: %v",
- i, nums[i], res2[i], nums, res2))
- }
- }
- return res
-}
diff --git a/dashboard/app/build/perf_changes.go b/dashboard/app/build/perf_changes.go
deleted file mode 100644
index f52f211..0000000
--- a/dashboard/app/build/perf_changes.go
+++ /dev/null
@@ -1,282 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package build
-
-import (
- "bytes"
- "fmt"
- "html/template"
- "net/http"
- "sort"
- "strconv"
-
- "appengine"
- "appengine/datastore"
-)
-
-func init() {
- handleFunc("/perf", perfChangesHandler)
-}
-
-// perfSummaryHandler draws the main benchmarking page.
-func perfChangesHandler(w http.ResponseWriter, r *http.Request) {
- d := dashboardForRequest(r)
- c := d.Context(appengine.NewContext(r))
-
- page, _ := strconv.Atoi(r.FormValue("page"))
- if page < 0 {
- page = 0
- }
-
- pc, err := GetPerfConfig(c, r)
- if err != nil {
- logErr(w, r, err)
- return
- }
-
- commits, err := dashPerfCommits(c, page)
- if err != nil {
- logErr(w, r, err)
- return
- }
-
- // Fetch PerfResult's for the commits.
- var uiCommits []*perfChangesCommit
- rc := MakePerfResultCache(c, commits[0], false)
-
- // But first compare tip with the last release.
- if page == 0 {
- res0 := &PerfResult{CommitHash: knownTags[lastRelease]}
- if err := datastore.Get(c, res0.Key(c), res0); err != nil && err != datastore.ErrNoSuchEntity {
- logErr(w, r, fmt.Errorf("getting PerfResult: %v", err))
- return
- }
- if err != datastore.ErrNoSuchEntity {
- uiCom, err := handleOneCommit(pc, commits[0], rc, res0)
- if err != nil {
- logErr(w, r, err)
- return
- }
- uiCom.IsSummary = true
- uiCom.ParentHash = lastRelease
- uiCommits = append(uiCommits, uiCom)
- }
- }
-
- for _, com := range commits {
- uiCom, err := handleOneCommit(pc, com, rc, nil)
- if err != nil {
- logErr(w, r, err)
- return
- }
- uiCommits = append(uiCommits, uiCom)
- }
-
- p := &Pagination{}
- if len(commits) == commitsPerPage {
- p.Next = page + 1
- }
- if page > 0 {
- p.Prev = page - 1
- p.HasPrev = true
- }
-
- data := &perfChangesData{d, p, uiCommits}
-
- var buf bytes.Buffer
- if err := perfChangesTemplate.Execute(&buf, data); err != nil {
- logErr(w, r, err)
- return
- }
-
- buf.WriteTo(w)
-}
-
-func handleOneCommit(pc *PerfConfig, com *Commit, rc *PerfResultCache, baseRes *PerfResult) (*perfChangesCommit, error) {
- uiCom := new(perfChangesCommit)
- uiCom.Commit = com
- res1 := rc.Get(com.Num)
- for builder, benchmarks1 := range res1.ParseData() {
- for benchmark, data1 := range benchmarks1 {
- if benchmark != "meta-done" || !data1.OK {
- uiCom.NumResults++
- }
- if !data1.OK {
- v := new(perfChangesChange)
- v.diff = 10000
- v.Style = "fail"
- v.Builder = builder
- v.Link = fmt.Sprintf("log/%v", data1.Artifacts["log"])
- v.Val = builder
- v.Hint = builder
- if benchmark != "meta-done" {
- v.Hint += "/" + benchmark
- }
- m := findMetric(uiCom, "failure")
- m.BadChanges = append(m.BadChanges, v)
- }
- }
- res0 := baseRes
- if res0 == nil {
- var err error
- res0, err = rc.NextForComparison(com.Num, builder)
- if err != nil {
- return nil, err
- }
- if res0 == nil {
- continue
- }
- }
- changes := significantPerfChanges(pc, builder, res0, res1)
- changes = dedupPerfChanges(changes)
- for _, ch := range changes {
- v := new(perfChangesChange)
- v.Builder = builder
- v.Benchmark, v.Procs = splitBench(ch.Bench)
- v.diff = ch.Diff
- v.Val = fmt.Sprintf("%+.2f%%", ch.Diff)
- v.Hint = fmt.Sprintf("%v/%v", builder, ch.Bench)
- v.Link = fmt.Sprintf("perfdetail?commit=%v&commit0=%v&builder=%v&benchmark=%v", com.Hash, res0.CommitHash, builder, v.Benchmark)
- m := findMetric(uiCom, ch.Metric)
- if v.diff > 0 {
- v.Style = "bad"
- m.BadChanges = append(m.BadChanges, v)
- } else {
- v.Style = "good"
- m.GoodChanges = append(m.GoodChanges, v)
- }
- }
- }
-
- // Sort metrics and changes.
- for _, m := range uiCom.Metrics {
- sort.Sort(m.GoodChanges)
- sort.Sort(m.BadChanges)
- }
- sort.Sort(uiCom.Metrics)
- // Need at least one metric for UI.
- if len(uiCom.Metrics) == 0 {
- uiCom.Metrics = append(uiCom.Metrics, &perfChangesMetric{})
- }
- uiCom.Metrics[0].First = true
- return uiCom, nil
-}
-
-// Find builder-procs with the maximum absolute diff for every benchmark-metric, drop the rest.
-func dedupPerfChanges(changes []*PerfChange) (deduped []*PerfChange) {
- maxDiff := make(map[string]float64)
- maxBench := make(map[string]string)
- // First, find the maximum.
- for _, ch := range changes {
- bench, _ := splitBench(ch.Bench)
- k := bench + "|" + ch.Metric
- v := ch.Diff
- if v < 0 {
- v = -v
- }
- if maxDiff[k] < v {
- maxDiff[k] = v
- maxBench[k] = ch.Builder + "|" + ch.Bench
- }
- }
- // Then, remove the rest.
- for _, ch := range changes {
- bench, _ := splitBench(ch.Bench)
- k := bench + "|" + ch.Metric
- if maxBench[k] == ch.Builder+"|"+ch.Bench {
- deduped = append(deduped, ch)
- }
- }
- return
-}
-
-func findMetric(c *perfChangesCommit, metric string) *perfChangesMetric {
- for _, m := range c.Metrics {
- if m.Name == metric {
- return m
- }
- }
- m := new(perfChangesMetric)
- m.Name = metric
- c.Metrics = append(c.Metrics, m)
- return m
-}
-
-type uiPerfConfig struct {
- Builders []uiPerfConfigElem
- Benchmarks []uiPerfConfigElem
- Metrics []uiPerfConfigElem
- Procs []uiPerfConfigElem
- CommitsFrom []uiPerfConfigElem
- CommitsTo []uiPerfConfigElem
-}
-
-type uiPerfConfigElem struct {
- Name string
- Selected bool
-}
-
-var perfChangesTemplate = template.Must(
- template.New("perf_changes.html").Funcs(tmplFuncs).ParseFiles("build/perf_changes.html"),
-)
-
-type perfChangesData struct {
- Dashboard *Dashboard
- Pagination *Pagination
- Commits []*perfChangesCommit
-}
-
-type perfChangesCommit struct {
- *Commit
- IsSummary bool
- NumResults int
- Metrics perfChangesMetricSlice
-}
-
-type perfChangesMetric struct {
- Name string
- First bool
- BadChanges perfChangesChangeSlice
- GoodChanges perfChangesChangeSlice
-}
-
-type perfChangesChange struct {
- Builder string
- Benchmark string
- Link string
- Hint string
- Style string
- Val string
- Procs int
- diff float64
-}
-
-type perfChangesMetricSlice []*perfChangesMetric
-
-func (l perfChangesMetricSlice) Len() int { return len(l) }
-func (l perfChangesMetricSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
-func (l perfChangesMetricSlice) Less(i, j int) bool {
- if l[i].Name == "failure" || l[j].Name == "failure" {
- return l[i].Name == "failure"
- }
- return l[i].Name < l[j].Name
-}
-
-type perfChangesChangeSlice []*perfChangesChange
-
-func (l perfChangesChangeSlice) Len() int { return len(l) }
-func (l perfChangesChangeSlice) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
-func (l perfChangesChangeSlice) Less(i, j int) bool {
- vi, vj := l[i].diff, l[j].diff
- if vi > 0 && vj > 0 {
- return vi > vj
- } else if vi < 0 && vj < 0 {
- return vi < vj
- } else {
- panic("comparing positive and negative diff")
- }
-}
diff --git a/dashboard/app/build/perf_changes.html b/dashboard/app/build/perf_changes.html
deleted file mode 100644
index 2aee6c4..0000000
--- a/dashboard/app/build/perf_changes.html
+++ /dev/null
@@ -1,89 +0,0 @@
-<!doctype html>
-<html>
-<head>
- <title>{{$.Dashboard.Name}} Dashboard</title>
- <link rel="stylesheet" href="/static/style.css"/>
-</head>
-<body>
- <header id="topbar">
- <h1>Go Dashboard</h1>
- <nav>
- <a href="{{$.Dashboard.Prefix}}/">Test</a>
- <a href="{{$.Dashboard.Prefix}}/perf">Perf</a>
- <a href="{{$.Dashboard.Prefix}}/perfgraph">Graphs</a>
- </nav>
- <div class="clear"></div>
- </header>
-
- <div class="page">
- <div class="build-container">
- <table class="build">
- <colgroup class="col-hash"></colgroup>
- <colgroup class="col-numresults"></colgroup>
- <colgroup class="col-metric"></colgroup>
- <colgroup class="col-result"></colgroup>
- <colgroup class="col-result"></colgroup>
- <colgroup class="col-user"></colgroup>
- <colgroup class="col-time"></colgroup>
- <colgroup class="col-desc"></colgroup>
- <tbody>
- {{range $c := $.Commits}}
- {{range $m := $c.Metrics}}
- {{if $m.First}}
- <tr class="row-commit">
- {{if $c.IsSummary}}
- <td class="hash">tip vs {{$c.ParentHash}}</td>
- {{else}}
- <td class="hash"><a href="{{repoURL $.Dashboard.Name $c.Hash ""}}">{{shortHash $c.Hash}}</a></td>
- {{end}}
- <td class="numresults">{{$c.NumResults}}</td>
- {{else}}
- <tr>
- <td class="user">&nbsp;</td>
- <td class="numresults">&nbsp;</td>
- {{end}}
- <td>{{$m.Name}}</td>
- <td>
- {{range $ch := $m.BadChanges}}
- <a class="{{$ch.Style}}" href="{{$ch.Link}}" title="{{$ch.Hint}}">{{$ch.Val}}</a> &nbsp;
- {{end}}
- </td>
- <td>
- {{range $ch := $m.GoodChanges}}
- <a class="{{$ch.Style}}" href="{{$ch.Link}}" title="{{$ch.Hint}}">{{$ch.Val}}</a> &nbsp;
- {{end}}
- </td>
- {{if $m.First}}
- <td class="user" title="{{$c.User}}">{{shortUser $c.User}}</td>
- <td class="time">{{$c.Time.Format "Mon 02 Jan 15:04"}}</td>
- <td class="desc" title="{{$c.Desc}}">{{shortDesc $c.Desc}}</td>
- {{else}}
- <td class="user">&nbsp;</td>
- <td class="time">&nbsp;</td>
- <td class="desc">&nbsp;</td>
- {{end}}
- </tr>
- {{end}}
- {{if $c.IsSummary}}
- <tr class="row-commit"><td>---</td></tr>
- {{end}}
- {{end}}
- </tbody>
- </table>
-
- {{with $.Pagination}}
- <div class="paginate">
- <nav>
- <a {{if .HasPrev}}href="?page={{.Prev}}"{{else}}class="inactive"{{end}}>newer</a>
- <a {{if .Next}}href="?page={{.Next}}"{{else}}class="inactive"{{end}}>older</a>
- <a {{if .HasPrev}}href="?"{{else}}class="inactive"{{end}}>latest</a>
- <a href="https://golang.org/wiki/PerfDashboard">Help</a>
- </nav>
- </div>
- {{end}}
-
- </div>
- <div class="clear"></div>
-</div>
-</body>
-</html>
diff --git a/dashboard/app/build/perf_detail.go b/dashboard/app/build/perf_detail.go
deleted file mode 100644
index 04d374a..0000000
--- a/dashboard/app/build/perf_detail.go
+++ /dev/null
@@ -1,219 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package build
-
-import (
- "bytes"
- "fmt"
- "html/template"
- "net/http"
- "sort"
- "strconv"
- "strings"
-
- "appengine"
- "appengine/datastore"
-)
-
-func init() {
- handleFunc("/perfdetail", perfDetailUIHandler)
-}
-
-func perfDetailUIHandler(w http.ResponseWriter, r *http.Request) {
- d := dashboardForRequest(r)
- c := d.Context(appengine.NewContext(r))
- pc, err := GetPerfConfig(c, r)
- if err != nil {
- logErr(w, r, err)
- return
- }
-
- kind := r.FormValue("kind")
- builder := r.FormValue("builder")
- benchmark := r.FormValue("benchmark")
- if kind == "" {
- kind = "benchmark"
- }
- if kind != "benchmark" && kind != "builder" {
- logErr(w, r, fmt.Errorf("unknown kind %s", kind))
- return
- }
-
- // Fetch the new commit.
- com1 := new(Commit)
- com1.Hash = r.FormValue("commit")
- if hash, ok := knownTags[com1.Hash]; ok {
- com1.Hash = hash
- }
- if err := datastore.Get(c, com1.Key(c), com1); err != nil {
- logErr(w, r, fmt.Errorf("failed to fetch commit %s: %v", com1.Hash, err))
- return
- }
- // Fetch the associated perf result.
- ress1 := &PerfResult{CommitHash: com1.Hash}
- if err := datastore.Get(c, ress1.Key(c), ress1); err != nil {
- logErr(w, r, fmt.Errorf("failed to fetch perf result %s: %v", com1.Hash, err))
- return
- }
-
- // Fetch the old commit.
- var ress0 *PerfResult
- com0 := new(Commit)
- com0.Hash = r.FormValue("commit0")
- if hash, ok := knownTags[com0.Hash]; ok {
- com0.Hash = hash
- }
- if com0.Hash != "" {
- // Have an exact commit hash, fetch directly.
- if err := datastore.Get(c, com0.Key(c), com0); err != nil {
- logErr(w, r, fmt.Errorf("failed to fetch commit %s: %v", com0.Hash, err))
- return
- }
- ress0 = &PerfResult{CommitHash: com0.Hash}
- if err := datastore.Get(c, ress0.Key(c), ress0); err != nil {
- logErr(w, r, fmt.Errorf("failed to fetch perf result for %s: %v", com0.Hash, err))
- return
- }
- } else {
- // Don't have the commit hash, find the previous commit to compare.
- rc := MakePerfResultCache(c, com1, false)
- ress0, err = rc.NextForComparison(com1.Num, "")
- if err != nil {
- logErr(w, r, err)
- return
- }
- if ress0 == nil {
- logErr(w, r, fmt.Errorf("no previous commit with results"))
- return
- }
- // Now that we know the right result, fetch the commit.
- com0.Hash = ress0.CommitHash
- if err := datastore.Get(c, com0.Key(c), com0); err != nil {
- logErr(w, r, fmt.Errorf("failed to fetch commit %s: %v", com0.Hash, err))
- return
- }
- }
-
- res0 := ress0.ParseData()
- res1 := ress1.ParseData()
- var benchmarks []*uiPerfDetailBenchmark
- var list []string
- if kind == "builder" {
- list = pc.BenchmarksForBuilder(builder)
- } else {
- list = pc.BuildersForBenchmark(benchmark)
- }
- for _, other := range list {
- if kind == "builder" {
- benchmark = other
- } else {
- builder = other
- }
- var procs []*uiPerfDetailProcs
- allProcs := pc.ProcList(builder)
- for _, p := range allProcs {
- BenchProcs := fmt.Sprintf("%v-%v", benchmark, p)
- if res0[builder] == nil || res0[builder][BenchProcs] == nil {
- continue
- }
- pp := &uiPerfDetailProcs{Procs: p}
- for metric, val := range res0[builder][BenchProcs].Metrics {
- var pm uiPerfDetailMetric
- pm.Name = metric
- pm.Val0 = fmt.Sprintf("%v", val)
- val1 := uint64(0)
- if res1[builder] != nil && res1[builder][BenchProcs] != nil {
- val1 = res1[builder][BenchProcs].Metrics[metric]
- }
- pm.Val1 = fmt.Sprintf("%v", val1)
- v0 := val
- v1 := val1
- valf := perfDiff(v0, v1)
- pm.Delta = fmt.Sprintf("%+.2f%%", valf)
- pm.Style = perfChangeStyle(pc, valf, builder, BenchProcs, pm.Name)
- pp.Metrics = append(pp.Metrics, pm)
- }
- sort.Sort(pp.Metrics)
- for artifact, hash := range res0[builder][BenchProcs].Artifacts {
- var pm uiPerfDetailMetric
- pm.Val0 = fmt.Sprintf("%v", artifact)
- pm.Link0 = fmt.Sprintf("log/%v", hash)
- pm.Val1 = fmt.Sprintf("%v", artifact)
- if res1[builder] != nil && res1[builder][BenchProcs] != nil && res1[builder][BenchProcs].Artifacts[artifact] != "" {
- pm.Link1 = fmt.Sprintf("log/%v", res1[builder][BenchProcs].Artifacts[artifact])
- }
- pp.Metrics = append(pp.Metrics, pm)
- }
- procs = append(procs, pp)
- }
- benchmarks = append(benchmarks, &uiPerfDetailBenchmark{other, procs})
- }
-
- cfg := new(uiPerfConfig)
- for _, v := range pc.BuildersForBenchmark("") {
- cfg.Builders = append(cfg.Builders, uiPerfConfigElem{v, v == builder})
- }
- for _, v := range pc.BenchmarksForBuilder("") {
- cfg.Benchmarks = append(cfg.Benchmarks, uiPerfConfigElem{v, v == benchmark})
- }
-
- data := &uiPerfDetailTemplateData{d, cfg, kind == "builder", com0, com1, benchmarks}
-
- var buf bytes.Buffer
- if err := uiPerfDetailTemplate.Execute(&buf, data); err != nil {
- logErr(w, r, err)
- return
- }
-
- buf.WriteTo(w)
-}
-
-func perfResultSplit(s string) (builder string, benchmark string, procs int) {
- s1 := strings.Split(s, "|")
- s2 := strings.Split(s1[1], "-")
- procs, _ = strconv.Atoi(s2[1])
- return s1[0], s2[0], procs
-}
-
-type uiPerfDetailTemplateData struct {
- Dashboard *Dashboard
- Config *uiPerfConfig
- KindBuilder bool
- Commit0 *Commit
- Commit1 *Commit
- Benchmarks []*uiPerfDetailBenchmark
-}
-
-type uiPerfDetailBenchmark struct {
- Name string
- Procs []*uiPerfDetailProcs
-}
-
-type uiPerfDetailProcs struct {
- Procs int
- Metrics uiPerfDetailMetrics
-}
-
-type uiPerfDetailMetric struct {
- Name string
- Val0 string
- Val1 string
- Link0 string
- Link1 string
- Delta string
- Style string
-}
-
-type uiPerfDetailMetrics []uiPerfDetailMetric
-
-func (l uiPerfDetailMetrics) Len() int { return len(l) }
-func (l uiPerfDetailMetrics) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
-func (l uiPerfDetailMetrics) Less(i, j int) bool { return l[i].Name < l[j].Name }
-
-var uiPerfDetailTemplate = template.Must(
- template.New("perf_detail.html").Funcs(tmplFuncs).ParseFiles("build/perf_detail.html"),
-)
diff --git a/dashboard/app/build/perf_detail.html b/dashboard/app/build/perf_detail.html
deleted file mode 100644
index 4db0c3d..0000000
--- a/dashboard/app/build/perf_detail.html
+++ /dev/null
@@ -1,101 +0,0 @@
-<!doctype html>
-<html>
-<head>
- <title>{{$.Dashboard.Name}} Dashboard</title>
- <link rel="stylesheet" href="/static/style.css"/>
- <script type="text/javascript">
- function kindBuilder() {
- document.getElementById('checkBuilder').checked = true;
- document.getElementById('controlBuilder').style.display='inline';
- document.getElementById('controlBenchmark').style.display='none';
- }
- function kindBenchmark() {
- document.getElementById('checkBenchmark').checked = true;
- document.getElementById('controlBenchmark').style.display='inline';
- document.getElementById('controlBuilder').style.display='none';
- }
- window.onload = {{if $.KindBuilder}} kindBuilder {{else}} kindBenchmark {{end}};
- </script>
-</head>
-<body>
- <header id="topbar">
- <h1>Go Dashboard</h1>
- <nav>
- <a href="{{$.Dashboard.Prefix}}/">Test</a>
- <a href="{{$.Dashboard.Prefix}}/perf">Perf</a>
- <a href="{{$.Dashboard.Prefix}}/perfgraph">Graphs</a>
- </nav>
- <div class="clear"></div>
- </header>
-
- <div class="page">
- <div class="diff-container">
- <div class="diff-meta">
- <form>
- <div><b>New: </b><input type="edit" name="commit" value="{{$.Commit1.Hash}}" /> {{shortUser $.Commit1.User}} {{$.Commit1.Time.Format "Mon 02 Jan 15:04"}} {{shortDesc $.Commit1.Desc}} </div>
- <div><b>Old: </b><input type="edit" name="commit0" value="{{$.Commit0.Hash}}" /> {{shortUser $.Commit0.User}} {{$.Commit0.Time.Format "Mon 02 Jan 15:04"}} {{shortDesc $.Commit0.Desc}} </div>
- <div>
- <input id="checkBuilder" type="radio" name="kind" value="builder" required onclick="kindBuilder()">builder</input>
- <input id="checkBenchmark" type="radio" name="kind" value="benchmark" required onclick="kindBenchmark()">benchmark</input>
- <select id="controlBuilder" name="builder">
- {{range $.Config.Builders}}
- <option {{if .Selected}}selected{{end}}>{{.Name}}</option>
- {{end}}
- </select>
- <select id="controlBenchmark" name="benchmark">
- {{range $.Config.Benchmarks}}
- <option {{if .Selected}}selected{{end}}>{{.Name}}</option>
- {{end}}
- </select>
- <input type="submit" value="Refresh" />
- <a href="https://golang.org/wiki/PerfDashboard">Help</a>
- </div>
- </form>
- </div>
- <p></p>
-
- {{range $b := $.Benchmarks}}
- <div class="diff-benchmark">
- <h2>{{$b.Name}}</h2>
- {{range $p := $b.Procs}}
- <div class="diff">
- <h1>GOMAXPROCS={{$p.Procs}}</h1>
- <table>
- <thead>
- <tr>
- <th>Metric</th>
- <th>old</th>
- <th>new</th>
- <th>delta</th>
- </tr>
- </thead>
- <tbody>
- {{range $m := $p.Metrics}}
- <tr>
- <td class="metric">{{$m.Name}}</td>
- {{if $m.Link0}}
- <td><a href="{{$.Dashboard.Prefix}}/{{$m.Link0}}">{{$m.Val0}}</td>
- {{else}}
- <td>{{$m.Val0}}</td>
- {{end}}
- {{if $m.Link1}}
- <td><a href="{{$.Dashboard.Prefix}}/{{$m.Link1}}">{{$m.Val1}}</td>
- {{else}}
- <td>{{$m.Val1}}</td>
- {{end}}
- <td class="result"><span class="{{$m.Style}}">{{$m.Delta}}</span></td>
- </tr>
- {{end}}
- </tbody>
- </table>
- </div>
- {{end}}
- </div>
- {{end}}
-
- <div class="clear"></div>
- </div>
- <div class="clear"></div>
- </div>
-</body>
-</html>
diff --git a/dashboard/app/build/perf_graph.go b/dashboard/app/build/perf_graph.go
deleted file mode 100644
index fbdff89..0000000
--- a/dashboard/app/build/perf_graph.go
+++ /dev/null
@@ -1,268 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package build
-
-import (
- "bytes"
- "fmt"
- "html/template"
- "net/http"
- "strconv"
-
- "appengine"
- "appengine/datastore"
-)
-
-func init() {
- handleFunc("/perfgraph", perfGraphHandler)
-}
-
-func perfGraphHandler(w http.ResponseWriter, r *http.Request) {
- d := dashboardForRequest(r)
- c := d.Context(appengine.NewContext(r))
- pc, err := GetPerfConfig(c, r)
- if err != nil {
- logErr(w, r, err)
- return
- }
- allBuilders := pc.BuildersForBenchmark("")
- allBenchmarks := pc.BenchmarksForBuilder("")
- allMetrics := pc.MetricsForBenchmark("")
- allProcs := pc.ProcList("")
- r.ParseForm()
- selBuilders := r.Form["builder"]
- selBenchmarks := r.Form["benchmark"]
- selMetrics := r.Form["metric"]
- selProcs := r.Form["procs"]
- if len(selBuilders) == 0 {
- selBuilders = append(selBuilders, allBuilders[0])
- }
- if len(selBenchmarks) == 0 {
- selBenchmarks = append(selBenchmarks, "json")
- }
- if len(selMetrics) == 0 {
- selMetrics = append(selMetrics, "time")
- }
- if len(selProcs) == 0 {
- selProcs = append(selProcs, "1")
- }
- commitFrom := r.FormValue("commit-from")
- if commitFrom == "" {
- commitFrom = lastRelease
- }
- commitTo := r.FormValue("commit-to")
- if commitTo == "" {
- commitTo = "tip"
- }
- // TODO(dvyukov): validate input
-
- // Figure out start and end commit from commitFrom/commitTo.
- startCommitNum := 0
- endCommitNum := 0
- {
- comFrom := &Commit{Hash: knownTags[commitFrom]}
- if err := datastore.Get(c, comFrom.Key(c), comFrom); err != nil {
- logErr(w, r, err)
- return
- }
- startCommitNum = comFrom.Num
-
- retry:
- if commitTo == "tip" {
- p, err := GetPackage(c, "")
- if err != nil {
- logErr(w, r, err)
- return
- }
- endCommitNum = p.NextNum
- } else {
- comTo := &Commit{Hash: knownTags[commitTo]}
- if err := datastore.Get(c, comTo.Key(c), comTo); err != nil {
- logErr(w, r, err)
- return
- }
- endCommitNum = comTo.Num + 1
- }
- if endCommitNum <= startCommitNum {
- // User probably selected from:go1.3 to:go1.2. Fix go1.2 to tip.
- if commitTo == "tip" {
- logErr(w, r, fmt.Errorf("no commits to display (%v-%v)", commitFrom, commitTo))
- return
- }
- commitTo = "tip"
- goto retry
- }
- }
- commitsToDisplay := endCommitNum - startCommitNum
-
- present := func(set []string, s string) bool {
- for _, s1 := range set {
- if s1 == s {
- return true
- }
- }
- return false
- }
-
- cfg := &uiPerfConfig{}
- for _, v := range allBuilders {
- cfg.Builders = append(cfg.Builders, uiPerfConfigElem{v, present(selBuilders, v)})
- }
- for _, v := range allBenchmarks {
- cfg.Benchmarks = append(cfg.Benchmarks, uiPerfConfigElem{v, present(selBenchmarks, v)})
- }
- for _, v := range allMetrics {
- cfg.Metrics = append(cfg.Metrics, uiPerfConfigElem{v, present(selMetrics, v)})
- }
- for _, v := range allProcs {
- cfg.Procs = append(cfg.Procs, uiPerfConfigElem{strconv.Itoa(v), present(selProcs, strconv.Itoa(v))})
- }
- for k := range knownTags {
- cfg.CommitsFrom = append(cfg.CommitsFrom, uiPerfConfigElem{k, commitFrom == k})
- }
- for k := range knownTags {
- cfg.CommitsTo = append(cfg.CommitsTo, uiPerfConfigElem{k, commitTo == k})
- }
- cfg.CommitsTo = append(cfg.CommitsTo, uiPerfConfigElem{"tip", commitTo == "tip"})
-
- var vals [][]float64
- var hints [][]string
- var annotations [][]string
- var certainty [][]bool
- var headers []string
- commits2, err := GetCommits(c, startCommitNum, commitsToDisplay)
- if err != nil {
- logErr(w, r, err)
- return
- }
- for _, builder := range selBuilders {
- for _, metric := range selMetrics {
- for _, benchmark := range selBenchmarks {
- for _, procs := range selProcs {
- benchProcs := fmt.Sprintf("%v-%v", benchmark, procs)
- vv, err := GetPerfMetricsForCommits(c, builder, benchProcs, metric, startCommitNum, commitsToDisplay)
- if err != nil {
- logErr(w, r, err)
- return
- }
- hasdata := false
- for _, v := range vv {
- if v != 0 {
- hasdata = true
- }
- }
- if hasdata {
- noise := pc.NoiseLevel(builder, benchProcs, metric)
- descBuilder := "/" + builder
- descBenchmark := "/" + benchProcs
- descMetric := "/" + metric
- if len(selBuilders) == 1 {
- descBuilder = ""
- }
- if len(selBenchmarks) == 1 && len(selProcs) == 1 {
- descBenchmark = ""
- }
- if len(selMetrics) == 1 && (len(selBuilders) > 1 || len(selBenchmarks) > 1 || len(selProcs) > 1) {
- descMetric = ""
- }
- desc := fmt.Sprintf("%v%v%v", descBuilder, descBenchmark, descMetric)[1:]
- hh := make([]string, commitsToDisplay)
- ann := make([]string, commitsToDisplay)
- valf := make([]float64, commitsToDisplay)
- cert := make([]bool, commitsToDisplay)
- firstval := uint64(0)
- lastval := uint64(0)
- for i, v := range vv {
- cert[i] = true
- if v == 0 {
- if lastval == 0 {
- continue
- }
- cert[i] = false
- v = lastval
- }
- if firstval == 0 {
- firstval = v
- }
- valf[i] = float64(v) / float64(firstval)
- if cert[i] {
- d := ""
- if lastval != 0 {
- diff := perfDiff(lastval, v)
- d = fmt.Sprintf(" (%+.02f%%)", diff)
- if !isNoise(diff, noise) {
- ann[i] = fmt.Sprintf("%+.02f%%", diff)
- }
- }
- hh[i] = fmt.Sprintf("%v%v", v, d)
- } else {
- hh[i] = "NO DATA"
- }
- lastval = v
- }
- vals = append(vals, valf)
- hints = append(hints, hh)
- annotations = append(annotations, ann)
- certainty = append(certainty, cert)
- headers = append(headers, desc)
- }
- }
- }
- }
- }
-
- var commits []perfGraphCommit
- if len(vals) != 0 && len(vals[0]) != 0 {
- idx := 0
- for i := range vals[0] {
- com := commits2[i]
- if com == nil || !com.NeedsBenchmarking {
- continue
- }
- c := perfGraphCommit{Id: idx, Name: fmt.Sprintf("%v (%v)", com.Desc, com.Time.Format("Jan 2, 2006 1:04"))}
- idx++
- for j := range vals {
- c.Vals = append(c.Vals, perfGraphValue{float64(vals[j][i]), certainty[j][i], hints[j][i], annotations[j][i]})
- }
- commits = append(commits, c)
- }
- }
-
- data := &perfGraphData{d, cfg, headers, commits}
-
- var buf bytes.Buffer
- if err := perfGraphTemplate.Execute(&buf, data); err != nil {
- logErr(w, r, err)
- return
- }
-
- buf.WriteTo(w)
-}
-
-var perfGraphTemplate = template.Must(
- template.New("perf_graph.html").ParseFiles("build/perf_graph.html"),
-)
-
-type perfGraphData struct {
- Dashboard *Dashboard
- Config *uiPerfConfig
- Headers []string
- Commits []perfGraphCommit
-}
-
-type perfGraphCommit struct {
- Id int
- Name string
- Vals []perfGraphValue
-}
-
-type perfGraphValue struct {
- Val float64
- Certainty bool
- Hint string
- Ann string
-}
diff --git a/dashboard/app/build/perf_graph.html b/dashboard/app/build/perf_graph.html
deleted file mode 100644
index a72190a..0000000
--- a/dashboard/app/build/perf_graph.html
+++ /dev/null
@@ -1,120 +0,0 @@
-<!doctype html>
-<html>
- <head>
- <title>{{$.Dashboard.Name}} Dashboard</title>
- <link rel="stylesheet" href="/static/style.css"/>
- <style>
- .graph-container { background: #eee; }
- </style>
-
- <script type="text/javascript" src="https://www.google.com/jsapi"></script>
- <script type="text/javascript">
- google.load("visualization", "1", {packages:["corechart"]});
- google.setOnLoadCallback(drawCharts);
- function drawCharts() {
- var data = new google.visualization.DataTable();
- data.addColumn({type: 'number', label: 'Commit'});
- data.addColumn({type: 'number'});
- data.addColumn({type: 'string', role: 'tooltip'});
- {{range $.Headers}}
- data.addColumn({type: 'number', label: '{{.}}'});
- data.addColumn({type: 'boolean', role: 'certainty'});
- data.addColumn({type: 'string', role: 'tooltip'});
- data.addColumn({type: 'string', role: 'annotation'});
- {{end}}
- data.addRows([
- {{range $.Commits}}
- [ {{.Id}}, 1, "{{.Name}}",
- {{range .Vals}}
- {{if .Val}}
- {{.Val}}, {{.Certainty}}, '{{.Hint}}', '{{.Ann}}',
- {{else}}
- ,,,,
- {{end}}
- {{end}}
- ],
- {{end}}
- ]);
- new google.visualization.LineChart(document.getElementById('graph_div')).
- draw(data, {
- width: "100%",
- height: 700,
- legend: {position: "bottom"},
- focusTarget: "category",
- hAxis: {textPosition: "none"},
- chartArea: {left: "10%", top: "5%", width: "85%", height:"80%"},
- explorer: {axis: 'horizontal', maxZoomIn: 0, maxZoomOut: 1, zoomDelta: 1.2, keepInBounds: true}
- })
- }
- </script>
-</head>
-<body>
-
- <header id="topbar">
- <h1>Go Dashboard</h1>
- <nav>
- <a href="{{$.Dashboard.Prefix}}/">Test</a>
- <a href="{{$.Dashboard.Prefix}}/perf">Perf</a>
- <a href="{{$.Dashboard.Prefix}}/perfgraph">Graphs</a>
- </nav>
- <div class="clear"></div>
- </header>
-
- <div class="page">
- <div id="graph_div" class="main-content graph-container">
- </div>
-
- <aside>
- <form>
- <div class="panel">
- <h1>Builders</h1>
- {{range $.Config.Builders}}
- <input type="checkbox" name="builder" value="{{.Name}}" {{if .Selected}}checked{{end}}>{{.Name}}</input><br>
- {{end}}
- </div>
-
- <div class="panel">
- <h1>Benchmarks</h1>
- {{range $.Config.Benchmarks}}
- <input type="checkbox" name="benchmark" value="{{.Name}}" {{if .Selected}}checked{{end}}>{{.Name}}</input><br>
- {{end}}
- </div>
-
- <div class="panel">
- <h1>Procs</h1>
- {{range $.Config.Procs}}
- <input type="checkbox" name="procs" value="{{.Name}}" {{if .Selected}}checked{{end}}>{{.Name}}</input><br>
- {{end}}
- </div>
-
- <div class="panel">
- <h1>Metrics</h1>
- {{range $.Config.Metrics}}
- <input type="checkbox" name="metric" value="{{.Name}}" {{if .Selected}}checked{{end}}>{{.Name}}</input><br>
- {{end}}
- </div>
-
- <div class="panel">
- <h1>Commits</h1>
- <b>From:</b>
- <select required name="commit-from">
- {{range $.Config.CommitsFrom}}
- <option {{if .Selected}}selected{{end}}>{{.Name}}</option>
- {{end}}
- </select>
- <b>To:</b>
- <select required name="commit-to">
- {{range $.Config.CommitsTo}}
- <option {{if .Selected}}selected{{end}}>{{.Name}}</option>
- {{end}}
- </select>
- </div>
-
- <input class="button" type="submit" value="Refresh" name="refresh"/>
- <a href="https://golang.org/wiki/PerfDashboard">Help</a>
- </form>
- </aside>
- <div class="clear"></div>
- </div>
-</body>
-</html>
diff --git a/dashboard/app/build/perf_learn.go b/dashboard/app/build/perf_learn.go
deleted file mode 100644
index d2a57bf..0000000
--- a/dashboard/app/build/perf_learn.go
+++ /dev/null
@@ -1,186 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package build
-
-import (
- "bytes"
- "fmt"
- "html/template"
- "net/http"
- "sort"
-
- "appengine"
- "appengine/datastore"
-)
-
-func init() {
- handleFunc("/perflearn", perfLearnHandler)
-}
-
-const (
- learnPercentile = 0.95
- learnSignalMultiplier = 1.1
- learnMinSignal = 0.5
-)
-
-func perfLearnHandler(w http.ResponseWriter, r *http.Request) {
- d := dashboardForRequest(r)
- c := d.Context(appengine.NewContext(r))
-
- pc, err := GetPerfConfig(c, r)
- if err != nil {
- logErr(w, r, err)
- return
- }
-
- p, err := GetPackage(c, "")
- if err != nil {
- logErr(w, r, err)
- return
- }
-
- update := r.FormValue("update") != ""
- noise := make(map[string]string)
-
- data := &perfLearnData{}
-
- commits, err := GetCommits(c, 0, p.NextNum)
- if err != nil {
- logErr(w, r, err)
- return
- }
-
- for _, builder := range pc.BuildersForBenchmark("") {
- for _, benchmark := range pc.BenchmarksForBuilder(builder) {
- for _, metric := range pc.MetricsForBenchmark(benchmark) {
- for _, procs := range pc.ProcList(builder) {
- values, err := GetPerfMetricsForCommits(c, builder, fmt.Sprintf("%v-%v", benchmark, procs), metric, 0, p.NextNum)
- if err != nil {
- logErr(w, r, err)
- return
- }
- var dd []float64
- last := uint64(0)
- for i, v := range values {
- if v == 0 {
- if com := commits[i]; com == nil || com.NeedsBenchmarking {
- last = 0
- }
- continue
- }
- if last != 0 {
- v1 := v
- if v1 < last {
- v1, last = last, v1
- }
- diff := float64(v1)/float64(last)*100 - 100
- dd = append(dd, diff)
- }
- last = v
- }
- if len(dd) == 0 {
- continue
- }
- sort.Float64s(dd)
-
- baseIdx := int(float64(len(dd)) * learnPercentile)
- baseVal := dd[baseIdx]
- signalVal := baseVal * learnSignalMultiplier
- if signalVal < learnMinSignal {
- signalVal = learnMinSignal
- }
- signalIdx := -1
- noiseNum := 0
- signalNum := 0
-
- var diffs []*perfLearnDiff
- for i, d := range dd {
- if d > 3*signalVal {
- d = 3 * signalVal
- }
- diffs = append(diffs, &perfLearnDiff{Num: i, Val: d})
- if signalIdx == -1 && d >= signalVal {
- signalIdx = i
- }
- if d < signalVal {
- noiseNum++
- } else {
- signalNum++
- }
- }
- diffs[baseIdx].Hint = "95%"
- if signalIdx != -1 {
- diffs[signalIdx].Hint = "signal"
- }
- diffs = diffs[len(diffs)*4/5:]
- name := fmt.Sprintf("%v/%v-%v/%v", builder, benchmark, procs, metric)
- data.Entries = append(data.Entries, &perfLearnEntry{len(data.Entries), name, baseVal, noiseNum, signalVal, signalNum, diffs})
-
- if len(dd) >= 100 || r.FormValue("force") != "" {
- nname := fmt.Sprintf("%v|%v-%v", builder, benchmark, procs)
- n := noise[nname] + fmt.Sprintf("|%v=%.2f", metric, signalVal)
- noise[nname] = n
- }
- }
- }
- }
- }
-
- if update {
- var noiseLevels []string
- for k, v := range noise {
- noiseLevels = append(noiseLevels, k+v)
- }
- tx := func(c appengine.Context) error {
- pc, err := GetPerfConfig(c, r)
- if err != nil {
- return err
- }
- pc.NoiseLevels = noiseLevels
- if _, err := datastore.Put(c, PerfConfigKey(c), pc); err != nil {
- return fmt.Errorf("putting PerfConfig: %v", err)
- }
- return nil
- }
- if err := datastore.RunInTransaction(c, tx, nil); err != nil {
- logErr(w, r, err)
- return
- }
- }
-
- var buf bytes.Buffer
- if err := perfLearnTemplate.Execute(&buf, data); err != nil {
- logErr(w, r, err)
- return
- }
-
- buf.WriteTo(w)
-}
-
-var perfLearnTemplate = template.Must(
- template.New("perf_learn.html").Funcs(tmplFuncs).ParseFiles("build/perf_learn.html"),
-)
-
-type perfLearnData struct {
- Entries []*perfLearnEntry
-}
-
-type perfLearnEntry struct {
- Num int
- Name string
- BaseVal float64
- NoiseNum int
- SignalVal float64
- SignalNum int
- Diffs []*perfLearnDiff
-}
-
-type perfLearnDiff struct {
- Num int
- Val float64
- Hint string
-}
diff --git a/dashboard/app/build/perf_learn.html b/dashboard/app/build/perf_learn.html
deleted file mode 100644
index 294e957..0000000
--- a/dashboard/app/build/perf_learn.html
+++ /dev/null
@@ -1,45 +0,0 @@
-<!doctype html>
-<html>
- <head>
- <script type="text/javascript" src="https://www.google.com/jsapi"></script>
- <script type="text/javascript">
- google.load("visualization", "1", {packages:["corechart"]});
- google.setOnLoadCallback(drawCharts);
- function drawCharts() {
- {
- {{range $ent := $.Entries}}
- var data = new google.visualization.DataTable();
- data.addColumn('number', 'idx');
- data.addColumn('number', '95%');
- data.addColumn({type: 'boolean', role: 'certainty'});
- data.addColumn('number', 'signal');
- data.addColumn({type: 'boolean', role: 'certainty'});
- data.addColumn('number', 'diff');
- data.addColumn({type: 'string', role: 'annotation'});
- data.addRows([
- {{range .Diffs}} [{{.Num}}, {{$ent.BaseVal}}, false, {{$ent.SignalVal}}, false, {{.Val}}, '{{.Hint}}'], {{end}}
- ]);
- new google.visualization.LineChart(document.getElementById('graph{{.Num}}')).
- draw(data, {
- width: 600,
- height: 200,
- legend: {position: "none"},
- vAxis: {minValue: 0},
- chartArea: {left: "10%", top: "1%", width: "90%", height:"95%"}
- }
- )
- {{end}}
- }
- }
- </script>
- </head>
-
- <body>
- {{range $.Entries}}
- <p>
- {{.Name}}: base={{printf "%.2f[%d]" .BaseVal .NoiseNum}} signal={{printf "%.2f[%d]" .SignalVal .SignalNum}}
- <div id="graph{{.Num}}" width="100px" height="100px"> </div>
- </p>
- {{end}}
- </body>
-</html>
diff --git a/dashboard/app/build/perf_notify.txt b/dashboard/app/build/perf_notify.txt
deleted file mode 100644
index c5e8ebe..0000000
--- a/dashboard/app/build/perf_notify.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-{{if .Commit}}Change {{shortHash .Commit.Hash}} caused perf changes on {{.Builder}}:
-
-{{.Commit.Desc}}
-
-http://code.google.com/p/go/source/detail?r={{shortHash .Commit.Hash}}
-{{else}}This changed caused perf changes on {{.Builder}}:
-{{end}}
-{{range $b := .Benchmarks}}
-{{printf "%-16s %12s %12s %10s" $b.Name "old" "new" "delta"}}
-{{range $m := $b.Metrics}}{{printf "%-16s %12v %12v %+10.2f" $m.Name $m.Old $m.New $m.Delta}}
-{{end}}{{end}}
-{{.Url}}
-
diff --git a/dashboard/app/build/test.go b/dashboard/app/build/test.go
deleted file mode 100644
index 228dfef..0000000
--- a/dashboard/app/build/test.go
+++ /dev/null
@@ -1,378 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package build
-
-// TODO(adg): test authentication
-// TODO(adg): refactor to use appengine/aetest instead
-
-import (
- "bytes"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "net/http"
- "net/http/httptest"
- "net/url"
- "strings"
- "time"
-
- "appengine"
- "appengine/datastore"
-)
-
-func init() {
- handleFunc("/buildtest", testHandler)
-}
-
-var testEntityKinds = []string{
- "Package",
- "Commit",
- "CommitRun",
- "Result",
- "PerfResult",
- "PerfMetricRun",
- "PerfConfig",
- "PerfTodo",
- "Log",
-}
-
-const testPkg = "golang.org/x/test"
-
-var testPackage = &Package{Name: "Test", Kind: "subrepo", Path: testPkg}
-
-var testPackages = []*Package{
- {Name: "Go", Path: ""},
- testPackage,
-}
-
-var tCommitTime = time.Now().Add(-time.Hour * 24 * 7)
-
-func tCommit(hash, parentHash, path string, bench bool) *Commit {
- tCommitTime.Add(time.Hour) // each commit should have a different time
- return &Commit{
- PackagePath: path,
- Hash: hash,
- ParentHash: parentHash,
- Time: tCommitTime,
- User: "adg",
- Desc: "change description " + hash,
- NeedsBenchmarking: bench,
- }
-}
-
-var testRequests = []struct {
- path string
- vals url.Values
- req interface{}
- res interface{}
-}{
- // Packages
- {"/packages", url.Values{"kind": {"subrepo"}}, nil, []*Package{testPackage}},
-
- // Go repo
- {"/commit", nil, tCommit("0001", "0000", "", true), nil},
- {"/commit", nil, tCommit("0002", "0001", "", false), nil},
- {"/commit", nil, tCommit("0003", "0002", "", true), nil},
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
- {"/result", nil, &Result{Builder: "linux-386", Hash: "0001", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
- {"/result", nil, &Result{Builder: "linux-386", Hash: "0002", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
-
- // Other builders, to test the UI.
- {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0001", OK: true}, nil},
- {"/result", nil, &Result{Builder: "linux-amd64-race", Hash: "0001", OK: true}, nil},
- {"/result", nil, &Result{Builder: "netbsd-386", Hash: "0001", OK: true}, nil},
- {"/result", nil, &Result{Builder: "plan9-386", Hash: "0001", OK: true}, nil},
- {"/result", nil, &Result{Builder: "windows-386", Hash: "0001", OK: true}, nil},
- {"/result", nil, &Result{Builder: "windows-amd64", Hash: "0001", OK: true}, nil},
- {"/result", nil, &Result{Builder: "windows-amd64-race", Hash: "0001", OK: true}, nil},
- {"/result", nil, &Result{Builder: "linux-amd64-temp", Hash: "0001", OK: true}, nil},
-
- // multiple builders
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
- {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0003", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0002"}}},
-
- // branches
- {"/commit", nil, tCommit("0004", "0003", "", false), nil},
- {"/commit", nil, tCommit("0005", "0002", "", false), nil},
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}},
- {"/result", nil, &Result{Builder: "linux-386", Hash: "0005", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0004"}}},
- {"/result", nil, &Result{Builder: "linux-386", Hash: "0004", OK: false}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0003"}}},
-
- // logs
- {"/result", nil, &Result{Builder: "linux-386", Hash: "0003", OK: false, Log: "test"}, nil},
- {"/log/a94a8fe5ccb19ba61c4c0873d391e987982fbbd3", nil, nil, "test"},
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, nil},
-
- // repeat failure (shouldn't re-send mail)
- {"/result", nil, &Result{Builder: "linux-386", Hash: "0003", OK: false, Log: "test"}, nil},
-
- // non-Go repos
- {"/commit", nil, tCommit("1001", "0000", testPkg, false), nil},
- {"/commit", nil, tCommit("1002", "1001", testPkg, false), nil},
- {"/commit", nil, tCommit("1003", "1002", testPkg, false), nil},
- {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1003"}}},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1003", GoHash: "0001", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1002"}}},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1002", GoHash: "0001", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1001"}}},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0001", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0001"}}, nil, nil},
- {"/todo", url.Values{"kind": {"build-package"}, "builder": {"linux-386"}, "packagePath": {testPkg}, "goHash": {"0002"}}, nil, &Todo{Kind: "build-package", Data: &Commit{Hash: "1003"}}},
-
- // re-build Go revision for stale subrepos
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0005", OK: false, Log: "boo"}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit"}, "builder": {"linux-386"}}, nil, nil},
-
- // benchmarks
- // build-go-commit must have precedence over benchmark-go-commit
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0005"}}},
- // drain build-go-commit todo
- {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0005", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0004"}}},
- {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0004", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0002"}}},
- {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0002", OK: true}, nil},
- // drain sub-repo todos
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1001", GoHash: "0005", OK: false}, nil},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1002", GoHash: "0005", OK: false}, nil},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1003", GoHash: "0005", OK: false}, nil},
- // now we must get benchmark todo
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{}}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "http", Hash: "0003", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{"http"}}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "json", Hash: "0003", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{"http", "json"}}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0003", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0001", PerfResults: []string{}}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "http", Hash: "0001", OK: true}, nil},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0001", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, nil},
- // create new commit, it must appear in todo
- {"/commit", nil, tCommit("0006", "0005", "", true), nil},
- // drain build-go-commit todo
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0006"}}},
- {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0006", OK: true}, nil},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1003", GoHash: "0006", OK: false}, nil},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1002", GoHash: "0006", OK: false}, nil},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1001", GoHash: "0006", OK: false}, nil},
- // now we must get benchmark todo
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0006", PerfResults: []string{}}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "http", Hash: "0006", OK: true}, nil},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0006", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, nil},
- // create new benchmark, all commits must re-appear in todo
- {"/commit", nil, tCommit("0007", "0006", "", true), nil},
- // drain build-go-commit todo
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0007"}}},
- {"/result", nil, &Result{Builder: "linux-amd64", Hash: "0007", OK: true}, nil},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1003", GoHash: "0007", OK: false}, nil},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1002", GoHash: "0007", OK: false}, nil},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-amd64", Hash: "1001", GoHash: "0007", OK: false}, nil},
- // now we must get benchmark todo
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0007", PerfResults: []string{}}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "bson", Hash: "0007", OK: true}, nil},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0007", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0007", PerfResults: []string{"bson"}}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0007", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0006", PerfResults: []string{"http"}}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0006", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0001", PerfResults: []string{"http"}}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0001", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003", PerfResults: []string{"http", "json"}}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-amd64", Benchmark: "meta-done", Hash: "0003", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-amd64"}}, nil, nil},
- // attach second builder
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "build-go-commit", Data: &Commit{Hash: "0007"}}},
- // drain build-go-commit todo
- {"/result", nil, &Result{Builder: "linux-386", Hash: "0007", OK: true}, nil},
- {"/result", nil, &Result{Builder: "linux-386", Hash: "0006", OK: true}, nil},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1003", GoHash: "0007", OK: false}, nil},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1002", GoHash: "0007", OK: false}, nil},
- {"/result", nil, &Result{PackagePath: testPkg, Builder: "linux-386", Hash: "1001", GoHash: "0007", OK: false}, nil},
- // now we must get benchmark todo
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0007"}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0007", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0006"}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0006", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0001"}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0001", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, &Todo{Kind: "benchmark-go-commit", Data: &Commit{Hash: "0003"}}},
- {"/perf-result", nil, &PerfRequest{Builder: "linux-386", Benchmark: "meta-done", Hash: "0003", OK: true}, nil},
- {"/todo", url.Values{"kind": {"build-go-commit", "benchmark-go-commit"}, "builder": {"linux-386"}}, nil, nil},
-}
-
-func testHandler(w http.ResponseWriter, r *http.Request) {
- if !appengine.IsDevAppServer() {
- fmt.Fprint(w, "These tests must be run under the dev_appserver.")
- return
- }
- c := appengine.NewContext(r)
- if err := nukeEntities(c, testEntityKinds); err != nil {
- logErr(w, r, err)
- return
- }
- if r.FormValue("nukeonly") != "" {
- fmt.Fprint(w, "OK")
- return
- }
-
- for _, p := range testPackages {
- if _, err := datastore.Put(c, p.Key(c), p); err != nil {
- logErr(w, r, err)
- return
- }
- }
-
- origReq := *r
- defer func() {
- // HACK: We need to clobber the original request (see below)
- // so make sure we fix it before exiting the handler.
- *r = origReq
- }()
- for i, t := range testRequests {
- c.Infof("running test %d %s vals='%q' req='%q' res='%q'", i, t.path, t.vals, t.req, t.res)
- errorf := func(format string, args ...interface{}) {
- fmt.Fprintf(w, "%d %s: ", i, t.path)
- fmt.Fprintf(w, format, args...)
- fmt.Fprintln(w)
- }
- var body io.ReadWriter
- if t.req != nil {
- body = new(bytes.Buffer)
- json.NewEncoder(body).Encode(t.req)
- }
- url := "http://" + domain + t.path
- if t.vals != nil {
- url += "?" + t.vals.Encode() + "&version=2"
- } else {
- url += "?version=2"
- }
- req, err := http.NewRequest("POST", url, body)
- if err != nil {
- logErr(w, r, err)
- return
- }
- if t.req != nil {
- req.Method = "POST"
- }
- req.Header = origReq.Header
- rec := httptest.NewRecorder()
-
- // Make the request
- *r = *req // HACK: App Engine uses the request pointer
- // as a map key to resolve Contexts.
- http.DefaultServeMux.ServeHTTP(rec, r)
-
- if rec.Code != 0 && rec.Code != 200 {
- errorf(rec.Body.String())
- return
- }
- c.Infof("response='%v'", rec.Body.String())
- resp := new(dashResponse)
-
- // If we're expecting a *Todo value,
- // prime the Response field with a Todo and a Commit inside it.
- if t.path == "/todo" {
- resp.Response = &Todo{Data: &Commit{}}
- }
-
- if strings.HasPrefix(t.path, "/log/") {
- resp.Response = rec.Body.String()
- } else {
- err := json.NewDecoder(rec.Body).Decode(resp)
- if err != nil {
- errorf("decoding response: %v", err)
- return
- }
- }
- if e, ok := t.res.(string); ok {
- g, ok := resp.Response.(string)
- if !ok {
- errorf("Response not string: %T", resp.Response)
- return
- }
- if g != e {
- errorf("response mismatch: got %q want %q", g, e)
- return
- }
- }
- if e, ok := t.res.(*Todo); ok {
- g, ok := resp.Response.(*Todo)
- if !ok {
- errorf("Response not *Todo: %T", resp.Response)
- return
- }
- if e.Data == nil && g.Data != nil {
- errorf("Response.Data should be nil, got: %v", g.Data)
- return
- }
- if g.Data == nil {
- errorf("Response.Data is nil, want: %v", e.Data)
- return
- }
- gd, ok := g.Data.(*Commit)
- if !ok {
- errorf("Response.Data not *Commit: %T", g.Data)
- return
- }
- if g.Kind != e.Kind {
- errorf("kind don't match: got %q, want %q", g.Kind, e.Kind)
- return
- }
- ed := e.Data.(*Commit)
- if ed.Hash != gd.Hash {
- errorf("hashes don't match: got %q, want %q", gd.Hash, ed.Hash)
- return
- }
- if len(gd.PerfResults) != len(ed.PerfResults) {
- errorf("result data len don't match: got %v, want %v", len(gd.PerfResults), len(ed.PerfResults))
- return
- }
- for i := range gd.PerfResults {
- if gd.PerfResults[i] != ed.PerfResults[i] {
- errorf("result data %v don't match: got %v, want %v", i, gd.PerfResults[i], ed.PerfResults[i])
- return
- }
- }
- }
- if t.res == nil && resp.Response != nil {
- errorf("response mismatch: got %q expected <nil>", resp.Response)
- return
- }
- }
- fmt.Fprint(w, "PASS\nYou should see only one mail notification (for 0003/linux-386) in the dev_appserver logs.")
-}
-
-func nukeEntities(c appengine.Context, kinds []string) error {
- if !appengine.IsDevAppServer() {
- return errors.New("can't nuke production data")
- }
- var keys []*datastore.Key
- for _, kind := range kinds {
- q := datastore.NewQuery(kind).KeysOnly()
- for t := q.Run(c); ; {
- k, err := t.Next(nil)
- if err == datastore.Done {
- break
- }
- if err != nil {
- return err
- }
- keys = append(keys, k)
- }
- }
- return datastore.DeleteMulti(c, keys)
-}
diff --git a/dashboard/app/build/ui.go b/dashboard/app/build/ui.go
deleted file mode 100644
index 76be930..0000000
--- a/dashboard/app/build/ui.go
+++ /dev/null
@@ -1,605 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// TODO(adg): packages at weekly/release
-// TODO(adg): some means to register new packages
-
-// +build appengine
-
-package build
-
-import (
- "bytes"
- "encoding/json"
- "errors"
- "fmt"
- "html/template"
- "net/http"
- "regexp"
- "sort"
- "strconv"
- "strings"
-
- "golang.org/x/tools/dashboard/types"
-
- "cache"
-
- "appengine"
- "appengine/datastore"
- "appengine/memcache"
-)
-
-func init() {
- handleFunc("/", uiHandler)
-}
-
-// uiHandler draws the build status page.
-func uiHandler(w http.ResponseWriter, r *http.Request) {
- d := dashboardForRequest(r)
- c := d.Context(appengine.NewContext(r))
- now := cache.Now(c)
- key := "build-ui"
-
- page, _ := strconv.Atoi(r.FormValue("page"))
- if page < 0 {
- page = 0
- }
- key += fmt.Sprintf("-page%v", page)
-
- branch := r.FormValue("branch")
- if branch != "" {
- key += "-branch-" + branch
- }
-
- repo := r.FormValue("repo")
- if repo != "" {
- key += "-repo-" + repo
- }
-
- var b []byte
- if cache.Get(r, now, key, &b) {
- w.Write(b)
- return
- }
-
- pkg := &Package{} // empty package is the main repository
- if repo != "" {
- var err error
- pkg, err = GetPackage(c, repo)
- if err != nil {
- logErr(w, r, err)
- return
- }
- }
- commits, err := dashCommits(c, pkg, page, branch)
- if err != nil {
- logErr(w, r, err)
- return
- }
- builders := commitBuilders(commits)
-
- var tipState *TagState
- if pkg.Kind == "" && page == 0 && (branch == "" || branch == "default") {
- // only show sub-repo state on first page of normal repo view
- tipState, err = TagStateByName(c, "tip")
- if err != nil {
- logErr(w, r, err)
- return
- }
- }
-
- p := &Pagination{}
- if len(commits) == commitsPerPage {
- p.Next = page + 1
- }
- if page > 0 {
- p.Prev = page - 1
- p.HasPrev = true
- }
- data := &uiTemplateData{d, pkg, commits, builders, tipState, p, branch}
- data.populateBuildingURLs(c)
-
- switch r.FormValue("mode") {
- case "failures":
- failuresHandler(w, r, data)
- return
- case "json":
- jsonHandler(w, r, data)
- return
- }
-
- var buf bytes.Buffer
- if err := uiTemplate.Execute(&buf, data); err != nil {
- logErr(w, r, err)
- return
- }
-
- cache.Set(r, now, key, buf.Bytes())
-
- buf.WriteTo(w)
-}
-
-// failuresHandler is https://build.golang.org/?mode=failures , where it outputs
-// one line per failure on the front page, in the form:
-// hash builder failure-url
-func failuresHandler(w http.ResponseWriter, r *http.Request, data *uiTemplateData) {
- w.Header().Set("Content-Type", "text/plain")
- d := dashboardForRequest(r)
- for _, c := range data.Commits {
- for _, b := range data.Builders {
- res := c.Result(b, "")
- if res == nil || res.OK || res.LogHash == "" {
- continue
- }
- url := fmt.Sprintf("https://%v%v/log/%v", r.Host, d.Prefix, res.LogHash)
- fmt.Fprintln(w, c.Hash, b, url)
- }
- }
-}
-
-// jsonHandler is https://build.golang.org/?mode=json
-// The output is a types.BuildStatus JSON object.
-func jsonHandler(w http.ResponseWriter, r *http.Request, data *uiTemplateData) {
- d := dashboardForRequest(r)
-
- // cell returns one of "" (no data), "ok", or a failure URL.
- cell := func(res *Result) string {
- switch {
- case res == nil:
- return ""
- case res.OK:
- return "ok"
- }
- return fmt.Sprintf("https://%v%v/log/%v", r.Host, d.Prefix, res.LogHash)
- }
-
- var res types.BuildStatus
- res.Builders = data.Builders
-
- // First the commits from the main section (the "go" repo)
- for _, c := range data.Commits {
- rev := types.BuildRevision{
- Repo: "go",
- Revision: c.Hash,
- Results: make([]string, len(data.Builders)),
- }
- for i, b := range data.Builders {
- rev.Results[i] = cell(c.Result(b, ""))
- }
- res.Revisions = append(res.Revisions, rev)
- }
-
- // Then the one commit each for the subrepos.
- // TODO(bradfitz): we'll probably want more than one later, for people looking at
- // the subrepo-specific build history pages. But for now this gets me some data
- // to make forward progress.
- tip := data.TipState // a TagState
- for _, pkgState := range tip.Packages {
- goRev := tip.Tag.Hash
- rev := types.BuildRevision{
- Repo: pkgState.Package.Name,
- Revision: pkgState.Commit.Hash,
- GoRevision: goRev,
- Results: make([]string, len(data.Builders)),
- }
- for i, b := range res.Builders {
- rev.Results[i] = cell(pkgState.Commit.Result(b, goRev))
- }
- res.Revisions = append(res.Revisions, rev)
- }
-
- v, _ := json.MarshalIndent(res, "", "\t")
- w.Header().Set("Content-Type", "text/json; charset=utf-8")
- w.Write(v)
-}
-
-type Pagination struct {
- Next, Prev int
- HasPrev bool
-}
-
-// dashCommits gets a slice of the latest Commits to the current dashboard.
-// If page > 0 it paginates by commitsPerPage.
-func dashCommits(c appengine.Context, pkg *Package, page int, branch string) ([]*Commit, error) {
- offset := page * commitsPerPage
- q := datastore.NewQuery("Commit").
- Ancestor(pkg.Key(c)).
- Order("-Num")
-
- var commits []*Commit
- if branch == "" {
- _, err := q.Limit(commitsPerPage).Offset(offset).
- GetAll(c, &commits)
- return commits, err
- }
-
- // Look for commits on a specific branch.
- for t, n := q.Run(c), 0; len(commits) < commitsPerPage && n < 1000; {
- var c Commit
- _, err := t.Next(&c)
- if err == datastore.Done {
- break
- }
- if err != nil {
- return nil, err
- }
- if !isBranchCommit(&c, branch) {
- continue
- }
- if n >= offset {
- commits = append(commits, &c)
- }
- n++
- }
- return commits, nil
-}
-
-// isBranchCommit reports whether the given commit is on the specified branch.
-// It does so by examining the commit description, so there will be some bad
-// matches where the branch commits do not begin with the "[branch]" prefix.
-func isBranchCommit(c *Commit, b string) bool {
- d := strings.TrimSpace(c.Desc)
- if b == "default" {
- return !strings.HasPrefix(d, "[")
- }
- return strings.HasPrefix(d, "["+b+"]")
-}
-
-// commitBuilders returns the names of the builders that provided
-// Results for the provided commits.
-func commitBuilders(commits []*Commit) []string {
- builders := make(map[string]bool)
- for _, commit := range commits {
- for _, r := range commit.Results() {
- builders[r.Builder] = true
- }
- }
- k := keys(builders)
- sort.Sort(builderOrder(k))
- return k
-}
-
-func keys(m map[string]bool) (s []string) {
- for k := range m {
- s = append(s, k)
- }
- sort.Strings(s)
- return
-}
-
-// builderOrder implements sort.Interface, sorting builder names
-// ("darwin-amd64", etc) first by builderPriority and then alphabetically.
-type builderOrder []string
-
-func (s builderOrder) Len() int { return len(s) }
-func (s builderOrder) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
-func (s builderOrder) Less(i, j int) bool {
- pi, pj := builderPriority(s[i]), builderPriority(s[j])
- if pi == pj {
- return s[i] < s[j]
- }
- return pi < pj
-}
-
-func builderPriority(builder string) (p int) {
- // Put -temp builders at the end, always.
- if strings.HasSuffix(builder, "-temp") {
- defer func() { p += 20 }()
- }
- // Group race builders together.
- if isRace(builder) {
- return 1
- }
- // If the OS has a specified priority, use it.
- if p, ok := osPriority[builderOS(builder)]; ok {
- return p
- }
- // The rest.
- return 10
-}
-
-func isRace(s string) bool {
- return strings.Contains(s, "-race-") || strings.HasSuffix(s, "-race")
-}
-
-func unsupported(builder string) bool {
- if strings.HasSuffix(builder, "-temp") {
- return true
- }
- return unsupportedOS(builderOS(builder))
-}
-
-func unsupportedOS(os string) bool {
- if os == "race" {
- return false
- }
- p, ok := osPriority[os]
- return !ok || p > 0
-}
-
-// Priorities for specific operating systems.
-var osPriority = map[string]int{
- "darwin": 0,
- "freebsd": 0,
- "linux": 0,
- "windows": 0,
- // race == 1
- "openbsd": 2,
- "netbsd": 3,
- "dragonfly": 4,
-}
-
-// TagState represents the state of all Packages at a Tag.
-type TagState struct {
- Tag *Commit
- Packages []*PackageState
-}
-
-// PackageState represents the state of a Package at a Tag.
-type PackageState struct {
- Package *Package
- Commit *Commit
-}
-
-// TagStateByName fetches the results for all Go subrepos at the specified Tag.
-func TagStateByName(c appengine.Context, name string) (*TagState, error) {
- tag, err := GetTag(c, name)
- if err != nil {
- return nil, err
- }
- pkgs, err := Packages(c, "subrepo")
- if err != nil {
- return nil, err
- }
- var st TagState
- for _, pkg := range pkgs {
- com, err := pkg.LastCommit(c)
- if err != nil {
- c.Warningf("%v: no Commit found: %v", pkg, err)
- continue
- }
- st.Packages = append(st.Packages, &PackageState{pkg, com})
- }
- st.Tag, err = tag.Commit(c)
- if err != nil {
- return nil, err
- }
- return &st, nil
-}
-
-type uiTemplateData struct {
- Dashboard *Dashboard
- Package *Package
- Commits []*Commit
- Builders []string
- TipState *TagState
- Pagination *Pagination
- Branch string
-}
-
-// populateBuildingURLs populates each commit in Commits' buildingURLs map with the
-// URLs of builds which are currently in progress.
-func (td *uiTemplateData) populateBuildingURLs(ctx appengine.Context) {
- // need are memcache keys: "building|<hash>|<gohash>|<builder>"
- // The hash is of the main "go" repo, or the subrepo commit hash.
- // The gohash is empty for the main repo, else it's the Go hash.
- var need []string
-
- commit := map[string]*Commit{} // commit hash -> Commit
-
- // TODO(bradfitz): this only populates the main repo, not subpackages currently.
- for _, b := range td.Builders {
- for _, c := range td.Commits {
- if c.Result(b, "") == nil {
- commit[c.Hash] = c
- need = append(need, "building|"+c.Hash+"||"+b)
- }
- }
- }
- if len(need) == 0 {
- return
- }
- m, err := memcache.GetMulti(ctx, need)
- if err != nil {
- // oh well. this is a cute non-critical feature anyway.
- ctx.Debugf("GetMulti of building keys: %v", err)
- return
- }
- for k, it := range m {
- f := strings.SplitN(k, "|", 4)
- if len(f) != 4 {
- continue
- }
- hash, goHash, builder := f[1], f[2], f[3]
- c, ok := commit[hash]
- if !ok {
- continue
- }
- m := c.buildingURLs
- if m == nil {
- m = make(map[builderAndGoHash]string)
- c.buildingURLs = m
- }
- m[builderAndGoHash{builder, goHash}] = string(it.Value)
- }
-
-}
-
-var uiTemplate = template.Must(
- template.New("ui.html").Funcs(tmplFuncs).ParseFiles("build/ui.html"),
-)
-
-var tmplFuncs = template.FuncMap{
- "buildDashboards": buildDashboards,
- "builderOS": builderOS,
- "builderSpans": builderSpans,
- "builderSubheading": builderSubheading,
- "builderTitle": builderTitle,
- "repoURL": repoURL,
- "shortDesc": shortDesc,
- "shortHash": shortHash,
- "shortUser": shortUser,
- "tail": tail,
- "unsupported": unsupported,
-}
-
-func splitDash(s string) (string, string) {
- i := strings.Index(s, "-")
- if i >= 0 {
- return s[:i], s[i+1:]
- }
- return s, ""
-}
-
-// builderOS returns the os tag for a builder string
-func builderOS(s string) string {
- os, _ := splitDash(s)
- return os
-}
-
-// builderOSOrRace returns the builder OS or, if it is a race builder, "race".
-func builderOSOrRace(s string) string {
- if isRace(s) {
- return "race"
- }
- return builderOS(s)
-}
-
-// builderArch returns the arch tag for a builder string
-func builderArch(s string) string {
- _, arch := splitDash(s)
- arch, _ = splitDash(arch) // chop third part
- return arch
-}
-
-// builderSubheading returns a short arch tag for a builder string
-// or, if it is a race builder, the builder OS.
-func builderSubheading(s string) string {
- if isRace(s) {
- return builderOS(s)
- }
- arch := builderArch(s)
- switch arch {
- case "amd64":
- return "x64"
- }
- return arch
-}
-
-// builderArchChar returns the architecture letter for a builder string
-func builderArchChar(s string) string {
- arch := builderArch(s)
- switch arch {
- case "386":
- return "8"
- case "amd64":
- return "6"
- case "arm":
- return "5"
- }
- return arch
-}
-
-type builderSpan struct {
- N int
- OS string
- Unsupported bool
-}
-
-// builderSpans creates a list of tags showing
-// the builder's operating system names, spanning
-// the appropriate number of columns.
-func builderSpans(s []string) []builderSpan {
- var sp []builderSpan
- for len(s) > 0 {
- i := 1
- os := builderOSOrRace(s[0])
- u := unsupportedOS(os) || strings.HasSuffix(s[0], "-temp")
- for i < len(s) && builderOSOrRace(s[i]) == os {
- i++
- }
- sp = append(sp, builderSpan{i, os, u})
- s = s[i:]
- }
- return sp
-}
-
-// builderTitle formats "linux-amd64-foo" as "linux amd64 foo".
-func builderTitle(s string) string {
- return strings.Replace(s, "-", " ", -1)
-}
-
-// buildDashboards returns the known public dashboards.
-func buildDashboards() []*Dashboard {
- return dashboards
-}
-
-// shortDesc returns the first line of a description.
-func shortDesc(desc string) string {
- if i := strings.Index(desc, "\n"); i != -1 {
- desc = desc[:i]
- }
- return limitStringLength(desc, 100)
-}
-
-// shortHash returns a short version of a hash.
-func shortHash(hash string) string {
- if len(hash) > 12 {
- hash = hash[:12]
- }
- return hash
-}
-
-// shortUser returns a shortened version of a user string.
-func shortUser(user string) string {
- if i, j := strings.Index(user, "<"), strings.Index(user, ">"); 0 <= i && i < j {
- user = user[i+1 : j]
- }
- if i := strings.Index(user, "@"); i >= 0 {
- return user[:i]
- }
- return user
-}
-
-// repoRe matches Google Code repositories and subrepositories (without paths).
-var repoRe = regexp.MustCompile(`^code\.google\.com/p/([a-z0-9\-]+)(\.[a-z0-9\-]+)?$`)
-
-// repoURL returns the URL of a change at a Google Code repository or subrepo.
-func repoURL(dashboard, hash, packagePath string) (string, error) {
- if packagePath == "" {
- if dashboard == "Gccgo" {
- return "https://code.google.com/p/gofrontend/source/detail?r=" + hash, nil
- }
- if dashboard == "Mercurial" {
- return "https://golang.org/change/" + hash, nil
- }
- // TODO(adg): use the above once /change/ points to git hashes
- return "https://go.googlesource.com/go/+/" + hash, nil
- }
-
- // TODO(adg): remove this old hg stuff, one day.
- if dashboard == "Mercurial" {
- m := repoRe.FindStringSubmatch(packagePath)
- if m == nil {
- return "", errors.New("unrecognized package: " + packagePath)
- }
- url := "https://code.google.com/p/" + m[1] + "/source/detail?r=" + hash
- if len(m) > 2 {
- url += "&repo=" + m[2][1:]
- }
- return url, nil
- }
-
- repo := strings.TrimPrefix(packagePath, "golang.org/x/")
- return "https://go.googlesource.com/" + repo + "/+/" + hash, nil
-}
-
-// tail returns the trailing n lines of s.
-func tail(n int, s string) string {
- lines := strings.Split(s, "\n")
- if len(lines) < n {
- return s
- }
- return strings.Join(lines[len(lines)-n:], "\n")
-}
diff --git a/dashboard/app/build/ui.html b/dashboard/app/build/ui.html
deleted file mode 100644
index d885e1c..0000000
--- a/dashboard/app/build/ui.html
+++ /dev/null
@@ -1,212 +0,0 @@
-<!DOCTYPE HTML>
-<html>
- <head>
- <title>{{$.Dashboard.Name}} Build Dashboard</title>
- <link rel="stylesheet" href="/static/style.css"/>
- <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.8.2/jquery.min.js"></script>
- <script>
- var showUnsupported = window.location.hash.substr(1) != "short";
- function redraw() {
- showUnsupported = !$("#showshort").prop('checked');
- $('.unsupported')[showUnsupported?'show':'hide']();
- window.location.hash = showUnsupported?'':'short';
- }
- $(document).ready(function() {
- $("#showshort").attr('checked', !showUnsupported).change(redraw);
- redraw();
- })
- </script>
- </head>
-
- <body>
- <header id="topbar">
- <h1>Go Dashboard</h1>
- <nav>
- <a href="{{$.Dashboard.Prefix}}/">Test</a>
- <a href="{{$.Dashboard.Prefix}}/perf">Perf</a>
- <a href="{{$.Dashboard.Prefix}}/perfgraph">Graphs</a>
- </nav>
- <div class="clear"></div>
- </header>
-
- <nav class="dashboards">
- {{range buildDashboards}}
- <a href="{{.Prefix}}/">{{.Name}}</a>
- {{end}}
- <label>
- <input type=checkbox id="showshort">
- show only <a href="http://golang.org/wiki/PortingPolicy">first-class ports</a>
- </label>
- </nav>
- {{with $.Package.Name}}<h2>{{.}}</h2>{{end}}
-
- <div class="page">
-
- {{if $.Commits}}
-
- <table class="build">
- <colgroup class="col-hash" {{if $.Package.Path}}span="2"{{end}}></colgroup>
- {{range $.Builders | builderSpans}}
- <colgroup class="col-result{{if .Unsupported}} unsupported{{end}}" span="{{.N}}"></colgroup>
- {{end}}
- <colgroup class="col-user"></colgroup>
- <colgroup class="col-time"></colgroup>
- <colgroup class="col-desc"></colgroup>
- <tr>
- <!-- extra row to make alternating colors use dark for first result -->
- </tr>
- <tr>
- {{if $.Package.Path}}
- <th colspan="2">revision</th>
- {{else}}
- <th>&nbsp;</th>
- {{end}}
- {{range $.Builders | builderSpans}}
- <th {{if .Unsupported}}class="unsupported"{{end}} colspan="{{.N}}">{{.OS}}</th>
- {{end}}
- <th></th>
- <th></th>
- <th></th>
- </tr>
- <tr>
- {{if $.Package.Path}}
- <th class="result arch">repo</th>
- <th class="result arch">{{$.Dashboard.Name}}</th>
- {{else}}
- <th>&nbsp;</th>
- {{end}}
- {{range $.Builders}}
- <th class="result arch{{if (unsupported .)}} unsupported{{end}}" title="{{.}}">{{builderSubheading .}}</th>
- {{end}}
- <th></th>
- <th></th>
- <th></th>
- </tr>
- {{range $c := $.Commits}}
- {{range $i, $h := $c.ResultGoHashes}}
- <tr class="commit">
- {{if $i}}
- <td>&nbsp;</td>
- {{else}}
- <td class="hash"><a href="{{repoURL $.Dashboard.Name $c.Hash $.Package.Path}}">{{shortHash $c.Hash}}</a></td>
- {{end}}
- {{if $h}}
- <td class="hash"><a href="{{repoURL $.Dashboard.Name $h ""}}">{{shortHash $h}}</a></td>
- {{end}}
- {{range $.Builders}}
- <td class="result{{if (unsupported .)}} unsupported{{end}}">
- {{with $c.Result . $h}}
- {{if .BuildingURL}}
- <a href="{{.BuildingURL}}"><img src="https://golang.org/favicon.ico" border=0></a>
- {{else if .OK}}
- <span class="ok">ok</span>
- {{else}}
- <a href="{{$.Dashboard.Prefix}}/log/{{.LogHash}}" class="fail">fail</a>
- {{end}}
- {{else}}
- &nbsp;
- {{end}}
- </td>
- {{end}}
- {{if $i}}
- <td>&nbsp;</td>
- <td>&nbsp;</td>
- <td>&nbsp;</td>
- {{else}}
- <td class="user" title="{{$c.User}}">{{shortUser $c.User}}</td>
- <td class="time">{{$c.Time.Format "Mon 02 Jan 15:04"}}</td>
- <td class="desc" title="{{$c.Desc}}">{{shortDesc $c.Desc}}</td>
- {{end}}
- </tr>
- {{end}}
- {{end}}
- </table>
-
- {{with $.Pagination}}
- <div class="paginate">
- <nav>
- <a {{if .HasPrev}}href="?{{with $.Package.Path}}repo={{.}}&{{end}}page={{.Prev}}{{with $.Branch}}&branch={{.}}{{end}}"{{else}}class="inactive"{{end}}>newer</a>
- <a {{if .Next}}href="?{{with $.Package.Path}}repo={{.}}&{{end}}page={{.Next}}{{with $.Branch}}&branch={{.}}{{end}}"{{else}}class="inactive"{{end}}>older</a>
- <a {{if .HasPrev}}href=".{{with $.Branch}}?branch={{.}}{{end}}"{{else}}class="inactive"{{end}}>latest</a>
- </nav>
- </div>
- {{end}}
-
- {{else}}
- <p>No commits to display. Hm.</p>
- {{end}}
-
- {{with $.TipState}}
- {{$goHash := .Tag.Hash}}
- {{if .Packages}}
- <h2>
- Sub-repositories at tip
- <small>(<a href="{{repoURL $.Dashboard.Name .Tag.Hash ""}}">{{shortHash .Tag.Hash}}</a>)</small>
- </h2>
-
- <table class="build">
- <colgroup class="col-package"></colgroup>
- <colgroup class="col-hash"></colgroup>
- {{range $.Builders | builderSpans}}
- <colgroup class="col-result{{if .Unsupported}} unsupported{{end}}" span="{{.N}}"></colgroup>
- {{end}}
- <colgroup class="col-user"></colgroup>
- <colgroup class="col-time"></colgroup>
- <colgroup class="col-desc"></colgroup>
- <tr>
- <!-- extra row to make alternating colors use dark for first result -->
- </tr>
- <tr>
- <th></th>
- <th></th>
- {{range $.Builders | builderSpans}}
- <th {{if .Unsupported}}class="unsupported"{{end}} colspan="{{.N}}">{{.OS}}</th>
- {{end}}
- <th></th>
- <th></th>
- <th></th>
- </tr>
- <tr>
- <th></th>
- <th></th>
- {{range $.Builders}}
- <th class="result arch{{if (unsupported .)}} unsupported{{end}}" title="{{.}}">{{builderSubheading .}}</th>
- {{end}}
- <th></th>
- <th></th>
- <th></th>
- </tr>
- {{range $pkg := .Packages}}
- <tr class="commit">
- <td><a title="{{.Package.Path}}" href="?repo={{.Package.Path}}">{{.Package.Name}}</a></td>
- <td class="hash">
- {{$h := $pkg.Commit.Hash}}
- <a href="{{repoURL $.Dashboard.Name $h $pkg.Commit.PackagePath}}">{{shortHash $h}}</a>
- </td>
- {{range $.Builders}}
- <td class="result{{if (unsupported .)}} unsupported{{end}}">
- {{with $pkg.Commit.Result . $goHash}}
- {{if .OK}}
- <span class="ok">ok</span>
- {{else}}
- <a href="{{$.Dashboard.Prefix}}/log/{{.LogHash}}" class="fail">fail</a>
- {{end}}
- {{else}}
- &nbsp;
- {{end}}
- </td>
- {{end}}
- {{with $pkg.Commit}}
- <td class="user" title="{{.User}}">{{shortUser .User}}</td>
- <td class="time">{{.Time.Format "Mon 02 Jan 15:04"}}</td>
- <td class="desc" title="{{.Desc}}">{{shortDesc .Desc}}</td>
- {{end}}
- </tr>
- {{end}}
- </table>
- {{end}}
- {{end}}
-
- </div>
- </body>
-</html>
diff --git a/dashboard/app/build/update.go b/dashboard/app/build/update.go
deleted file mode 100644
index 27b49ed..0000000
--- a/dashboard/app/build/update.go
+++ /dev/null
@@ -1,117 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package build
-
-import (
- "encoding/json"
- "fmt"
- "net/http"
-
- "appengine"
- "appengine/datastore"
-)
-
-func init() {
- handleFunc("/updatebenchmark", updateBenchmark)
-}
-
-func updateBenchmark(w http.ResponseWriter, r *http.Request) {
- if !appengine.IsDevAppServer() {
- fmt.Fprint(w, "Update must not run on real server.")
- return
- }
-
- if r.Method != "POST" {
- fmt.Fprintf(w, "bad request method")
- return
- }
-
- c := contextForRequest(r)
- if !validKey(c, r.FormValue("key"), r.FormValue("builder")) {
- fmt.Fprintf(w, "bad builder/key")
- return
- }
-
- defer r.Body.Close()
- var hashes []string
- if err := json.NewDecoder(r.Body).Decode(&hashes); err != nil {
- fmt.Fprintf(w, "failed to decode request: %v", err)
- return
- }
-
- ncommit := 0
- nrun := 0
- tx := func(c appengine.Context) error {
- var cr *CommitRun
- for _, hash := range hashes {
- // Update Commit.
- com := &Commit{Hash: hash}
- err := datastore.Get(c, com.Key(c), com)
- if err != nil && err != datastore.ErrNoSuchEntity {
- return fmt.Errorf("fetching Commit: %v", err)
- }
- if err == datastore.ErrNoSuchEntity {
- continue
- }
- com.NeedsBenchmarking = true
- com.PerfResults = nil
- if err := putCommit(c, com); err != nil {
- return err
- }
- ncommit++
-
- // create PerfResult
- res := &PerfResult{CommitHash: com.Hash, CommitNum: com.Num}
- err = datastore.Get(c, res.Key(c), res)
- if err != nil && err != datastore.ErrNoSuchEntity {
- return fmt.Errorf("fetching PerfResult: %v", err)
- }
- if err == datastore.ErrNoSuchEntity {
- if _, err := datastore.Put(c, res.Key(c), res); err != nil {
- return fmt.Errorf("putting PerfResult: %v", err)
- }
- }
-
- // Update CommitRun.
- if cr != nil && cr.StartCommitNum != com.Num/PerfRunLength*PerfRunLength {
- if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
- return fmt.Errorf("putting CommitRun: %v", err)
- }
- nrun++
- cr = nil
- }
- if cr == nil {
- var err error
- cr, err = GetCommitRun(c, com.Num)
- if err != nil {
- return fmt.Errorf("getting CommitRun: %v", err)
- }
- }
- if com.Num < cr.StartCommitNum || com.Num >= cr.StartCommitNum+PerfRunLength {
- return fmt.Errorf("commit num %v out of range [%v, %v)", com.Num, cr.StartCommitNum, cr.StartCommitNum+PerfRunLength)
- }
- idx := com.Num - cr.StartCommitNum
- cr.Hash[idx] = com.Hash
- cr.User[idx] = shortDesc(com.User)
- cr.Desc[idx] = shortDesc(com.Desc)
- cr.Time[idx] = com.Time
- cr.NeedsBenchmarking[idx] = com.NeedsBenchmarking
- }
- if cr != nil {
- if _, err := datastore.Put(c, cr.Key(c), cr); err != nil {
- return fmt.Errorf("putting CommitRun: %v", err)
- }
- nrun++
- }
- return nil
- }
- if err := datastore.RunInTransaction(c, tx, nil); err != nil {
- fmt.Fprintf(w, "failed to execute tx: %v", err)
- return
- }
- fmt.Fprintf(w, "OK (updated %v commits and %v commit runs)", ncommit, nrun)
-}
diff --git a/dashboard/app/cache/cache.go b/dashboard/app/cache/cache.go
deleted file mode 100644
index 4b57614..0000000
--- a/dashboard/app/cache/cache.go
+++ /dev/null
@@ -1,86 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package cache
-
-import (
- "fmt"
- "net/http"
- "time"
-
- "appengine"
- "appengine/memcache"
-)
-
-// TimeKey specifies the memcache entity that keeps the logical datastore time.
-var TimeKey = "cachetime"
-
-const (
- nocache = "nocache"
- expiry = 600 // 10 minutes
-)
-
-func newTime() uint64 { return uint64(time.Now().Unix()) << 32 }
-
-// Now returns the current logical datastore time to use for cache lookups.
-func Now(c appengine.Context) uint64 {
- t, err := memcache.Increment(c, TimeKey, 0, newTime())
- if err != nil {
- c.Errorf("cache.Now: %v", err)
- return 0
- }
- return t
-}
-
-// Tick sets the current logical datastore time to a never-before-used time
-// and returns that time. It should be called to invalidate the cache.
-func Tick(c appengine.Context) uint64 {
- t, err := memcache.Increment(c, TimeKey, 1, newTime())
- if err != nil {
- c.Errorf("cache.Tick: %v", err)
- return 0
- }
- return t
-}
-
-// Get fetches data for name at time now from memcache and unmarshals it into
-// value. It reports whether it found the cache record and logs any errors to
-// the admin console.
-func Get(r *http.Request, now uint64, name string, value interface{}) bool {
- if now == 0 || r.FormValue(nocache) != "" {
- return false
- }
- c := appengine.NewContext(r)
- key := fmt.Sprintf("%s.%d", name, now)
- _, err := memcache.JSON.Get(c, key, value)
- if err == nil {
- c.Debugf("cache hit %q", key)
- return true
- }
- c.Debugf("cache miss %q", key)
- if err != memcache.ErrCacheMiss {
- c.Errorf("get cache %q: %v", key, err)
- }
- return false
-}
-
-// Set puts value into memcache under name at time now.
-// It logs any errors to the admin console.
-func Set(r *http.Request, now uint64, name string, value interface{}) {
- if now == 0 || r.FormValue(nocache) != "" {
- return
- }
- c := appengine.NewContext(r)
- key := fmt.Sprintf("%s.%d", name, now)
- err := memcache.JSON.Set(c, &memcache.Item{
- Key: key,
- Object: value,
- Expiration: expiry,
- })
- if err != nil {
- c.Errorf("set cache %q: %v", key, err)
- }
-}
diff --git a/dashboard/app/cron.yaml b/dashboard/app/cron.yaml
deleted file mode 100644
index 6d56526..0000000
--- a/dashboard/app/cron.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-cron:
-- description: updates noise level for benchmarking results
- url: /perflearn?update=1
- schedule: every 24 hours
- target: build
-
diff --git a/dashboard/app/index.yaml b/dashboard/app/index.yaml
deleted file mode 100644
index 670a667..0000000
--- a/dashboard/app/index.yaml
+++ /dev/null
@@ -1,54 +0,0 @@
-indexes:
-
-- kind: Commit
- ancestor: yes
- properties:
- - name: Num
- direction: desc
-
-- kind: Commit
- ancestor: yes
- properties:
- - name: Time
- direction: desc
-
-- kind: Commit
- ancestor: yes
- properties:
- - name: NeedsBenchmarking
- - name: Num
- direction: desc
-
-- kind: CommitRun
- ancestor: yes
- properties:
- - name: StartCommitNum
- direction: desc
-
-- kind: PerfResult
- ancestor: yes
- properties:
- - name: CommitNum
- direction: desc
-
-- kind: PerfResult
- ancestor: yes
- properties:
- - name: CommitNum
- direction: asc
-
-- kind: CommitRun
- ancestor: yes
- properties:
- - name: StartCommitNum
- direction: asc
-
-- kind: PerfMetricRun
- ancestor: yes
- properties:
- - name: Builder
- - name: Benchmark
- - name: Metric
- - name: StartCommitNum
- direction: asc
-
diff --git a/dashboard/app/key/key.go b/dashboard/app/key/key.go
deleted file mode 100644
index e52554f..0000000
--- a/dashboard/app/key/key.go
+++ /dev/null
@@ -1,64 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build appengine
-
-package key
-
-import (
- "sync"
-
- "appengine"
- "appengine/datastore"
-)
-
-var theKey struct {
- sync.RWMutex
- builderKey
-}
-
-type builderKey struct {
- Secret string
-}
-
-func (k *builderKey) Key(c appengine.Context) *datastore.Key {
- return datastore.NewKey(c, "BuilderKey", "root", 0, nil)
-}
-
-func Secret(c appengine.Context) string {
- // check with rlock
- theKey.RLock()
- k := theKey.Secret
- theKey.RUnlock()
- if k != "" {
- return k
- }
-
- // prepare to fill; check with lock and keep lock
- theKey.Lock()
- defer theKey.Unlock()
- if theKey.Secret != "" {
- return theKey.Secret
- }
-
- // fill
- if err := datastore.Get(c, theKey.Key(c), &theKey.builderKey); err != nil {
- if err == datastore.ErrNoSuchEntity {
- // If the key is not stored in datastore, write it.
- // This only happens at the beginning of a new deployment.
- // The code is left here for SDK use and in case a fresh
- // deployment is ever needed. "gophers rule" is not the
- // real key.
- if !appengine.IsDevAppServer() {
- panic("lost key from datastore")
- }
- theKey.Secret = "gophers rule"
- datastore.Put(c, theKey.Key(c), &theKey.builderKey)
- return theKey.Secret
- }
- panic("cannot load builder key: " + err.Error())
- }
-
- return theKey.Secret
-}
diff --git a/dashboard/app/static/status_alert.gif b/dashboard/app/static/status_alert.gif
deleted file mode 100644
index 495d9d2..0000000
--- a/dashboard/app/static/status_alert.gif
+++ /dev/null
Binary files differ
diff --git a/dashboard/app/static/status_good.gif b/dashboard/app/static/status_good.gif
deleted file mode 100644
index ef9c5a8..0000000
--- a/dashboard/app/static/status_good.gif
+++ /dev/null
Binary files differ
diff --git a/dashboard/app/static/style.css b/dashboard/app/static/style.css
deleted file mode 100644
index ddf2129..0000000
--- a/dashboard/app/static/style.css
+++ /dev/null
@@ -1,308 +0,0 @@
-* { box-sizing: border-box; }
-
- .dashboards {
- padding: 0.5em;
- }
- .dashboards a {
- padding: 0.5em;
- background: #eee;
- color: blue;
- }
-
-body {
- margin: 0;
- font-family: sans-serif;
- padding: 0; margin: 0;
- color: #222;
-}
-
-.container {
- max-width: 900px;
- margin: 0 auto;
-}
-
-p, pre, ul, ol { margin: 20px; }
-
-h1, h2, h3, h4 {
- margin: 20px 0;
- padding: 0;
- color: #375EAB;
- font-weight: bold;
-}
-
-h1 { font-size: 24px; }
-h2 { font-size: 20px; }
-h3 { font-size: 20px; }
-h4 { font-size: 16px; }
-
-h2 { background: #E0EBF5; padding: 2px 5px; }
-h3, h4 { margin: 20px 5px; }
-
-dl, dd { font-size: 14px; }
-dl { margin: 20px; }
-dd { margin: 2px 20px; }
-
-.clear {
- clear: both;
-}
-
-.button {
- padding: 10px;
-
- color: #222;
- border: 1px solid #375EAB;
- background: #E0EBF5;
-
- border-radius: 5px;
-
- cursor: pointer;
-
- margin-left: 60px;
-}
-
-/* navigation bar */
-
-#topbar {
- padding: 10px 10px;
- background: #E0EBF5;
-}
-
-#topbar a {
- color: #222;
-}
-#topbar h1 {
- float: left;
- margin: 0;
- padding-top: 5px;
-}
-
-#topbar nav {
- float: left;
- margin-left: 20px;
-}
-#topbar nav a {
- display: inline-block;
- padding: 10px;
-
- margin: 0;
- margin-right: 5px;
-
- color: white;
- background: #375EAB;
-
- text-decoration: none;
- font-size: 16px;
-
- border: 1px solid #375EAB;
- -webkit-border-radius: 5px;
- -moz-border-radius: 5px;
- border-radius: 5px;
-}
-
-.page {
- margin-top: 20px;
-}
-
-/* settings panels */
-aside {
- margin-top: 5px;
-}
-
-.panel {
- border: 1px solid #aaa;
- border-radius: 5px;
- margin-bottom: 5px;
-}
-
-.panel h1 {
- font-size: 16px;
- margin: 0;
- padding: 2px 8px;
-}
-
-.panel select {
- padding: 5px;
- border: 0;
- width: 100%;
-}
-
-/* results table */
-
-table {
- margin: 5px;
- border-collapse: collapse;
- font-size: 11px;
-}
-
-table td, table th, table td, table th {
- vertical-align: top;
- padding: 2px 6px;
-}
-
-table tr:nth-child(2n+1) {
- background: #F4F4F4;
-}
-
-table thead tr {
- background: #fff !important;
-}
-
-/* build results */
-
-.build td, .build th, .packages td, .packages th {
- vertical-align: top;
- padding: 2px 4px;
- font-size: 10pt;
-}
-
-.build .hash {
- font-family: monospace;
- font-size: 9pt;
-}
-
-.build .result {
- text-align: center;
- width: 2em;
-}
-
-.build .col-hash, .build .col-result, .build .col-metric, .build .col-numresults {
- border-right: 1px solid #ccc;
-}
-
-.build .row-commit {
- border-top: 2px solid #ccc;
-}
-
-.build .arch {
- font-size: 83%;
- font-weight: normal;
-}
-
-.build .time {
- color: #666;
-}
-
-.build .ok {
- font-size: 83%;
-}
-
-.build .desc, .build .time, .build .user {
- white-space: nowrap;
-}
-
-.build .desc {
- text-align: left;
- max-width: 470px;
- overflow: hidden;
- text-overflow: ellipsis;
-}
-
-.good { text-decoration: none; color: #000000; border: 2px solid #00E700}
-.bad { text-decoration: none; text-shadow: 1px 1px 0 #000000; color: #FFFFFF; background: #E70000;}
-.noise { text-decoration: none; color: #888; }
-.fail { color: #C00; }
-
-/* pagination */
-
-.paginate nav {
- padding: 0.5em;
- margin: 10px 0;
-}
-
-.paginate nav a {
- padding: 0.5em;
- background: #E0EBF5;
- color: blue;
-
- -webkit-border-radius: 5px;
- -moz-border-radius: 5px;
- border-radius: 5px;
-}
-
-.paginate nav a.inactive {
- color: #888;
- cursor: default;
- text-decoration: none;
-}
-
-/* diffs */
-
-.diff-meta {
- font-family: monospace;
- margin-bottom: 10px;
-}
-
-.diff-container {
- padding: 10px;
-}
-
-.diff table .metric {
- font-weight: bold;
-}
-
-.diff {
- border: 1px solid #aaa;
- border-radius: 5px;
- margin-bottom: 5px;
- margin-right: 10px;
- float: left;
-}
-
-.diff h1 {
- font-size: 16px;
- margin: 0;
- padding: 2px 8px;
-}
-
-.diff-benchmark {
- clear: both;
- padding-top: 5px;
-}
-
-/* positioning elements */
-
-.page {
- position: relative;
- width: 100%;
-}
-
-aside {
- position: absolute;
- top: 0;
- left: 0;
- bottom: 0;
- width: 200px;
-}
-
-.main-content {
- position: absolute;
- top: 0;
- left: 210px;
- right: 5px;
- min-height: 200px;
- overflow: hidden;
-}
-
-@media only screen and (max-width: 900px) {
- aside {
- position: relative;
- display: block;
- width: auto;
- }
-
- .main-content {
- position: static;
- padding: 0;
- }
-
- aside .panel {
- float: left;
- width: auto;
- margin-right: 5px;
- }
- aside .button {
- float: left;
- margin: 0;
- }
-}
-
diff --git a/dashboard/auth/auth.go b/dashboard/auth/auth.go
deleted file mode 100644
index 35fa71d..0000000
--- a/dashboard/auth/auth.go
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-// Package auth contains shared code related to OAuth2 and obtaining
-// tokens for a project.
-package auth
-
-import (
- "fmt"
- "io/ioutil"
- "os"
- "path/filepath"
- "runtime"
-
- "golang.org/x/oauth2"
- "golang.org/x/oauth2/google"
-)
-
-func homedir() string {
- if runtime.GOOS == "windows" {
- return os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
- }
- return os.Getenv("HOME")
-}
-
-// ProjectTokenSource returns an OAuth2 TokenSource for the given Google Project ID.
-func ProjectTokenSource(proj string, scopes ...string) (oauth2.TokenSource, error) {
- // TODO(bradfitz): try different strategies too, like
- // three-legged flow if the service account doesn't exist, and
- // then cache the token file on disk somewhere. Or maybe that should be an
- // option, for environments without stdin/stdout available to the user.
- // We'll figure it out as needed.
- fileName := filepath.Join(homedir(), "keys", proj+".key.json")
- jsonConf, err := ioutil.ReadFile(fileName)
- if err != nil {
- if os.IsNotExist(err) {
- return nil, fmt.Errorf("Missing JSON key configuration. Download the Service Account JSON key from https://console.developers.google.com/project/%s/apiui/credential and place it at %s", proj, fileName)
- }
- return nil, err
- }
- conf, err := google.JWTConfigFromJSON(jsonConf, scopes...)
- if err != nil {
- return nil, fmt.Errorf("reading JSON config from %s: %v", fileName, err)
- }
- return conf.TokenSource(oauth2.NoContext), nil
-}
diff --git a/dashboard/builders.go b/dashboard/builders.go
deleted file mode 100644
index e238d2d..0000000
--- a/dashboard/builders.go
+++ /dev/null
@@ -1,226 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package dashboard contains shared configuration and logic used by various
-// pieces of the Go continuous build system.
-package dashboard
-
-import (
- "errors"
- "io/ioutil"
- "os"
- "strings"
-)
-
-// Builders are the different build configurations.
-// The keys are like "darwin-amd64" or "linux-386-387".
-// This map should not be modified by other packages.
-var Builders = map[string]BuildConfig{}
-
-// A BuildConfig describes how to run either a Docker-based or VM-based builder.
-type BuildConfig struct {
- // Name is the unique name of the builder, in the form of
- // "darwin-386" or "linux-amd64-race".
- Name string
-
- // VM-specific settings:
- VMImage string // e.g. "openbsd-amd64-56"
- machineType string // optional GCE instance type
-
- // Docker-specific settings: (used if VMImage == "")
- Image string // Docker image to use to build
- cmd string // optional -cmd flag (relative to go/src/)
- env []string // extra environment ("key=value") pairs
- dashURL string // url of the build dashboard
- tool string // the tool this configuration is for
-}
-
-func (c *BuildConfig) GOOS() string { return c.Name[:strings.Index(c.Name, "-")] }
-
-func (c *BuildConfig) GOARCH() string {
- arch := c.Name[strings.Index(c.Name, "-")+1:]
- i := strings.Index(arch, "-")
- if i == -1 {
- return arch
- }
- return arch[:i]
-}
-
-// AllScript returns the relative path to the operating system's script to
-// do the build and run its standard set of tests.
-// Example values are "src/all.bash", "src/all.bat", "src/all.rc".
-func (c *BuildConfig) AllScript() string {
- if strings.HasPrefix(c.Name, "windows-") {
- return "src/all.bat"
- }
- if strings.HasPrefix(c.Name, "plan9-") {
- return "src/all.rc"
- }
- // TODO(bradfitz): race.bash, etc, once the race builder runs
- // via the buildlet.
- return "src/all.bash"
-}
-
-func (c *BuildConfig) UsesDocker() bool { return c.VMImage == "" }
-func (c *BuildConfig) UsesVM() bool { return c.VMImage != "" }
-
-// MachineType returns the GCE machine type to use for this builder.
-func (c *BuildConfig) MachineType() string {
- if v := c.machineType; v != "" {
- return v
- }
- return "n1-highcpu-2"
-}
-
-// DockerRunArgs returns the arguments that go after "docker run" to execute
-// this docker image. The rev (git hash) is required. The builderKey is optional.
-// TODO(bradfitz): remove the builderKey being passed down, once the coordinator
-// does the reporting to the dashboard for docker builds too.
-func (conf BuildConfig) DockerRunArgs(rev, builderKey string) ([]string, error) {
- if !conf.UsesDocker() {
- return nil, errors.New("not a docker-based build")
- }
- var args []string
- if builderKey != "" {
- tmpKey := "/tmp/" + conf.Name + ".buildkey"
- if _, err := os.Stat(tmpKey); err != nil {
- if err := ioutil.WriteFile(tmpKey, []byte(builderKey), 0600); err != nil {
- return nil, err
- }
- }
- // Images may look for .gobuildkey in / or /root, so provide both.
- // TODO(adg): fix images that look in the wrong place.
- args = append(args, "-v", tmpKey+":/.gobuildkey")
- args = append(args, "-v", tmpKey+":/root/.gobuildkey")
- }
- for _, pair := range conf.env {
- args = append(args, "-e", pair)
- }
- if strings.HasPrefix(conf.Name, "linux-amd64") {
- args = append(args, "-e", "GOROOT_BOOTSTRAP=/go1.4-amd64/go")
- } else if strings.HasPrefix(conf.Name, "linux-386") {
- args = append(args, "-e", "GOROOT_BOOTSTRAP=/go1.4-386/go")
- }
- args = append(args,
- conf.Image,
- "/usr/local/bin/builder",
- "-rev="+rev,
- "-dashboard="+conf.dashURL,
- "-tool="+conf.tool,
- "-buildroot=/",
- "-v",
- )
- if conf.cmd != "" {
- args = append(args, "-cmd", conf.cmd)
- }
- args = append(args, conf.Name)
- return args, nil
-}
-
-func init() {
- addBuilder(BuildConfig{Name: "linux-386"})
- addBuilder(BuildConfig{Name: "linux-386-387", env: []string{"GO386=387"}})
- addBuilder(BuildConfig{Name: "linux-amd64"})
- addBuilder(BuildConfig{Name: "linux-amd64-nocgo", env: []string{"CGO_ENABLED=0", "USER=root"}})
- addBuilder(BuildConfig{Name: "linux-amd64-noopt", env: []string{"GO_GCFLAGS=-N -l"}})
- addBuilder(BuildConfig{Name: "linux-amd64-race"})
- addBuilder(BuildConfig{Name: "nacl-386"})
- addBuilder(BuildConfig{Name: "nacl-amd64p32"})
- addBuilder(BuildConfig{
- Name: "linux-amd64-gccgo",
- Image: "gobuilders/linux-x86-gccgo",
- cmd: "make RUNTESTFLAGS=\"--target_board=unix/-m64\" check-go -j16",
- dashURL: "https://build.golang.org/gccgo",
- tool: "gccgo",
- })
- addBuilder(BuildConfig{
- Name: "linux-386-gccgo",
- Image: "gobuilders/linux-x86-gccgo",
- cmd: "make RUNTESTFLAGS=\"--target_board=unix/-m32\" check-go -j16",
- dashURL: "https://build.golang.org/gccgo",
- tool: "gccgo",
- })
- addBuilder(BuildConfig{Name: "linux-386-sid", Image: "gobuilders/linux-x86-sid"})
- addBuilder(BuildConfig{Name: "linux-amd64-sid", Image: "gobuilders/linux-x86-sid"})
- addBuilder(BuildConfig{Name: "linux-386-clang", Image: "gobuilders/linux-x86-clang"})
- addBuilder(BuildConfig{Name: "linux-amd64-clang", Image: "gobuilders/linux-x86-clang"})
-
- // VMs:
- addBuilder(BuildConfig{
- Name: "openbsd-amd64-gce56",
- VMImage: "openbsd-amd64-56",
- machineType: "n1-highcpu-2",
- })
- addBuilder(BuildConfig{
- // It's named "partial" because the buildlet sets
- // GOTESTONLY=std to stop after the "go test std"
- // tests because it's so slow otherwise.
- // TODO(braditz): move that env variable to the
- // coordinator and into this config.
- Name: "plan9-386-gcepartial",
- VMImage: "plan9-386",
- // We *were* using n1-standard-1 because Plan 9 can only
- // reliably use a single CPU. Using 2 or 4 and we see
- // test failures. See:
- // https://golang.org/issue/8393
- // https://golang.org/issue/9491
- // n1-standard-1 has 3.6 GB of memory which is
- // overkill (userspace probably only sees 2GB anyway),
- // but it's the cheapest option. And plenty to keep
- // our ~250 MB of inputs+outputs in its ramfs.
- //
- // But the docs says "For the n1 series of machine
- // types, a virtual CPU is implemented as a single
- // hyperthread on a 2.6GHz Intel Sandy Bridge Xeon or
- // Intel Ivy Bridge Xeon (or newer) processor. This
- // means that the n1-standard-2 machine type will see
- // a whole physical core."
- //
- // ... so we use n1-highcpu-2 (1.80 RAM, still
- // plenty), just so we can get 1 whole core for the
- // single-core Plan 9. It will see 2 virtual cores and
- // only use 1, but we hope that 1 will be more powerful
- // and we'll stop timing out on tests.
- machineType: "n1-highcpu-2",
- })
-
-}
-
-func addBuilder(c BuildConfig) {
- if c.tool == "gccgo" {
- // TODO(cmang,bradfitz,adg): fix gccgo
- return
- }
- if c.Name == "" {
- panic("empty name")
- }
- if _, dup := Builders[c.Name]; dup {
- panic("dup name")
- }
- if c.dashURL == "" {
- c.dashURL = "https://build.golang.org"
- }
- if c.tool == "" {
- c.tool = "go"
- }
-
- if strings.HasPrefix(c.Name, "nacl-") {
- if c.Image == "" {
- c.Image = "gobuilders/linux-x86-nacl"
- }
- if c.cmd == "" {
- c.cmd = "/usr/local/bin/build-command.pl"
- }
- }
- if strings.HasPrefix(c.Name, "linux-") && c.Image == "" {
- c.Image = "gobuilders/linux-x86-base"
- }
- if c.Image == "" && c.VMImage == "" {
- panic("empty image and vmImage")
- }
- if c.Image != "" && c.VMImage != "" {
- panic("can't specify both image and vmImage")
- }
- Builders[c.Name] = c
-}
diff --git a/dashboard/builders_test.go b/dashboard/builders_test.go
deleted file mode 100644
index 96effe5..0000000
--- a/dashboard/builders_test.go
+++ /dev/null
@@ -1,22 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package dashboard
-
-import (
- "strings"
- "testing"
-)
-
-func TestOSARCHAccessors(t *testing.T) {
- valid := func(s string) bool { return s != "" && !strings.Contains(s, "-") }
- for _, conf := range Builders {
- os := conf.GOOS()
- arch := conf.GOARCH()
- osArch := os + "-" + arch
- if !valid(os) || !valid(arch) || !(conf.Name == osArch || strings.HasPrefix(conf.Name, osArch+"-")) {
- t.Errorf("OS+ARCH(%q) = %q, %q; invalid", conf.Name, os, arch)
- }
- }
-}
diff --git a/dashboard/buildlet/buildletclient.go b/dashboard/buildlet/buildletclient.go
deleted file mode 100644
index 53b2c6a..0000000
--- a/dashboard/buildlet/buildletclient.go
+++ /dev/null
@@ -1,222 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-// Package buildlet contains client tools for working with a buildlet
-// server.
-package buildlet // import "golang.org/x/tools/dashboard/buildlet"
-
-import (
- "errors"
- "fmt"
- "io"
- "io/ioutil"
- "net/http"
- "net/url"
- "strings"
-)
-
-// NewClient returns a *Client that will manipulate ipPort,
-// authenticated using the provided keypair.
-//
-// This constructor returns immediately without testing the host or auth.
-func NewClient(ipPort string, kp KeyPair) *Client {
- return &Client{
- ipPort: ipPort,
- tls: kp,
- password: kp.Password(),
- httpClient: &http.Client{
- Transport: &http.Transport{
- DialTLS: kp.tlsDialer(),
- },
- },
- }
-}
-
-// A Client interacts with a single buildlet.
-type Client struct {
- ipPort string
- tls KeyPair
- password string // basic auth password or empty for none
- httpClient *http.Client
-}
-
-// URL returns the buildlet's URL prefix, without a trailing slash.
-func (c *Client) URL() string {
- if !c.tls.IsZero() {
- return "https://" + strings.TrimSuffix(c.ipPort, ":443")
- }
- return "http://" + strings.TrimSuffix(c.ipPort, ":80")
-}
-
-func (c *Client) do(req *http.Request) (*http.Response, error) {
- if c.password != "" {
- req.SetBasicAuth("gomote", c.password)
- }
- return c.httpClient.Do(req)
-}
-
-// doOK sends the request and expects a 200 OK response.
-func (c *Client) doOK(req *http.Request) error {
- res, err := c.do(req)
- if err != nil {
- return err
- }
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
- return fmt.Errorf("%v; body: %s", res.Status, slurp)
- }
- return nil
-}
-
-// PutTar writes files to the remote buildlet, rooted at the relative
-// directory dir.
-// If dir is empty, they're placed at the root of the buildlet's work directory.
-// The dir is created if necessary.
-// The Reader must be of a tar.gz file.
-func (c *Client) PutTar(r io.Reader, dir string) error {
- req, err := http.NewRequest("PUT", c.URL()+"/writetgz?dir="+url.QueryEscape(dir), r)
- if err != nil {
- return err
- }
- return c.doOK(req)
-}
-
-// PutTarFromURL tells the buildlet to download the tar.gz file from tarURL
-// and write it to dir, a relative directory from the workdir.
-// If dir is empty, they're placed at the root of the buildlet's work directory.
-// The dir is created if necessary.
-// The url must be of a tar.gz file.
-func (c *Client) PutTarFromURL(tarURL, dir string) error {
- form := url.Values{
- "url": {tarURL},
- }
- req, err := http.NewRequest("POST", c.URL()+"/writetgz?dir="+url.QueryEscape(dir), strings.NewReader(form.Encode()))
- if err != nil {
- return err
- }
- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- return c.doOK(req)
-}
-
-// GetTar returns a .tar.gz stream of the given directory, relative to the buildlet's work dir.
-// The provided dir may be empty to get everything.
-func (c *Client) GetTar(dir string) (tgz io.ReadCloser, err error) {
- req, err := http.NewRequest("GET", c.URL()+"/tgz?dir="+url.QueryEscape(dir), nil)
- if err != nil {
- return nil, err
- }
- res, err := c.do(req)
- if err != nil {
- return nil, err
- }
- if res.StatusCode != http.StatusOK {
- slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
- res.Body.Close()
- return nil, fmt.Errorf("%v; body: %s", res.Status, slurp)
- }
- return res.Body, nil
-}
-
-// ExecOpts are options for a remote command invocation.
-type ExecOpts struct {
- // Output is the output of stdout and stderr.
- // If nil, the output is discarded.
- Output io.Writer
-
- // Args are the arguments to pass to the cmd given to Client.Exec.
- Args []string
-
- // SystemLevel controls whether the command is run outside of
- // the buildlet's environment.
- SystemLevel bool
-
- // OnStartExec is an optional hook that runs after the 200 OK
- // response from the buildlet, but before the output begins
- // writing to Output.
- OnStartExec func()
-}
-
-// Exec runs cmd on the buildlet.
-//
-// Two errors are returned: one is whether the command succeeded
-// remotely (remoteErr), and the second (execErr) is whether there
-// were system errors preventing the command from being started or
-// seen to completition. If execErr is non-nil, the remoteErr is
-// meaningless.
-func (c *Client) Exec(cmd string, opts ExecOpts) (remoteErr, execErr error) {
- var mode string
- if opts.SystemLevel {
- mode = "sys"
- }
- form := url.Values{
- "cmd": {cmd},
- "mode": {mode},
- "cmdArg": opts.Args,
- }
- req, err := http.NewRequest("POST", c.URL()+"/exec", strings.NewReader(form.Encode()))
- if err != nil {
- return nil, err
- }
- req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
- res, err := c.do(req)
- if err != nil {
- return nil, err
- }
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
- return nil, fmt.Errorf("buildlet: HTTP status %v: %s", res.Status, slurp)
- }
- condRun(opts.OnStartExec)
-
- // Stream the output:
- out := opts.Output
- if out == nil {
- out = ioutil.Discard
- }
- if _, err := io.Copy(out, res.Body); err != nil {
- return nil, fmt.Errorf("error copying response: %v", err)
- }
-
- // Don't record to the dashboard unless we heard the trailer from
- // the buildlet, otherwise it was probably some unrelated error
- // (like the VM being killed, or the buildlet crashing due to
- // e.g. https://golang.org/issue/9309, since we require a tip
- // build of the buildlet to get Trailers support)
- state := res.Trailer.Get("Process-State")
- if state == "" {
- return nil, errors.New("missing Process-State trailer from HTTP response; buildlet built with old (<= 1.4) Go?")
- }
- if state != "ok" {
- return errors.New(state), nil
- }
- return nil, nil
-}
-
-// Destroy shuts down the buildlet, destroying all state immediately.
-func (c *Client) Destroy() error {
- req, err := http.NewRequest("POST", c.URL()+"/halt", nil)
- if err != nil {
- return err
- }
- res, err := c.do(req)
- if err != nil {
- return err
- }
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
- return fmt.Errorf("buildlet: HTTP status %v: %s", res.Status, slurp)
- }
- return nil
-}
-
-func condRun(fn func()) {
- if fn != nil {
- fn()
- }
-}
diff --git a/dashboard/buildlet/gce.go b/dashboard/buildlet/gce.go
deleted file mode 100644
index 3bd1d92..0000000
--- a/dashboard/buildlet/gce.go
+++ /dev/null
@@ -1,315 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-package buildlet
-
-import (
- "crypto/tls"
- "errors"
- "fmt"
- "net/http"
- "strings"
- "time"
-
- "golang.org/x/oauth2"
- "golang.org/x/tools/dashboard"
- "google.golang.org/api/compute/v1"
-)
-
-// VMOpts control how new VMs are started.
-type VMOpts struct {
- // Zone is the GCE zone to create the VM in. Required.
- Zone string
-
- // ProjectID is the GCE project ID. Required.
- ProjectID string
-
- // TLS optionally specifies the TLS keypair to use.
- // If zero, http without auth is used.
- TLS KeyPair
-
- // Optional description of the VM.
- Description string
-
- // Optional metadata to put on the instance.
- Meta map[string]string
-
- // DeleteIn optionally specifies a duration at which
-
- // to delete the VM.
- DeleteIn time.Duration
-
- // OnInstanceRequested optionally specifies a hook to run synchronously
- // after the computeService.Instances.Insert call, but before
- // waiting for its operation to proceed.
- OnInstanceRequested func()
-
- // OnInstanceCreated optionally specifies a hook to run synchronously
- // after the instance operation succeeds.
- OnInstanceCreated func()
-
- // OnInstanceCreated optionally specifies a hook to run synchronously
- // after the computeService.Instances.Get call.
- OnGotInstanceInfo func()
-}
-
-// StartNewVM boots a new VM on GCE and returns a buildlet client
-// configured to speak to it.
-func StartNewVM(ts oauth2.TokenSource, instName, builderType string, opts VMOpts) (*Client, error) {
- computeService, _ := compute.New(oauth2.NewClient(oauth2.NoContext, ts))
-
- conf, ok := dashboard.Builders[builderType]
- if !ok {
- return nil, fmt.Errorf("invalid builder type %q", builderType)
- }
-
- zone := opts.Zone
- if zone == "" {
- // TODO: automatic? maybe that's not useful.
- // For now just return an error.
- return nil, errors.New("buildlet: missing required Zone option")
- }
- projectID := opts.ProjectID
- if projectID == "" {
- return nil, errors.New("buildlet: missing required ProjectID option")
- }
-
- prefix := "https://www.googleapis.com/compute/v1/projects/" + projectID
- machType := prefix + "/zones/" + zone + "/machineTypes/" + conf.MachineType()
-
- instance := &compute.Instance{
- Name: instName,
- Description: opts.Description,
- MachineType: machType,
- Disks: []*compute.AttachedDisk{
- {
- AutoDelete: true,
- Boot: true,
- Type: "PERSISTENT",
- InitializeParams: &compute.AttachedDiskInitializeParams{
- DiskName: instName,
- SourceImage: "https://www.googleapis.com/compute/v1/projects/" + projectID + "/global/images/" + conf.VMImage,
- DiskType: "https://www.googleapis.com/compute/v1/projects/" + projectID + "/zones/" + zone + "/diskTypes/pd-ssd",
- },
- },
- },
- Tags: &compute.Tags{
- // Warning: do NOT list "http-server" or "allow-ssh" (our
- // project's custom tag to allow ssh access) here; the
- // buildlet provides full remote code execution.
- // The https-server is authenticated, though.
- Items: []string{"https-server"},
- },
- Metadata: &compute.Metadata{},
- NetworkInterfaces: []*compute.NetworkInterface{
- &compute.NetworkInterface{
- AccessConfigs: []*compute.AccessConfig{
- &compute.AccessConfig{
- Type: "ONE_TO_ONE_NAT",
- Name: "External NAT",
- },
- },
- Network: prefix + "/global/networks/default",
- },
- },
- }
- addMeta := func(key, value string) {
- instance.Metadata.Items = append(instance.Metadata.Items, &compute.MetadataItems{
- Key: key,
- Value: value,
- })
- }
- // The buildlet-binary-url is the URL of the buildlet binary
- // which the VMs are configured to download at boot and run.
- // This lets us/ update the buildlet more easily than
- // rebuilding the whole VM image.
- addMeta("buildlet-binary-url",
- "http://storage.googleapis.com/go-builder-data/buildlet."+conf.GOOS()+"-"+conf.GOARCH())
- addMeta("builder-type", builderType)
- if !opts.TLS.IsZero() {
- addMeta("tls-cert", opts.TLS.CertPEM)
- addMeta("tls-key", opts.TLS.KeyPEM)
- addMeta("password", opts.TLS.Password())
- }
-
- if opts.DeleteIn != 0 {
- // In case the VM gets away from us (generally: if the
- // coordinator dies while a build is running), then we
- // set this attribute of when it should be killed so
- // we can kill it later when the coordinator is
- // restarted. The cleanUpOldVMs goroutine loop handles
- // that killing.
- addMeta("delete-at", fmt.Sprint(time.Now().Add(opts.DeleteIn).Unix()))
- }
-
- for k, v := range opts.Meta {
- addMeta(k, v)
- }
-
- op, err := computeService.Instances.Insert(projectID, zone, instance).Do()
- if err != nil {
- return nil, fmt.Errorf("Failed to create instance: %v", err)
- }
- condRun(opts.OnInstanceRequested)
- createOp := op.Name
-
- // Wait for instance create operation to succeed.
-OpLoop:
- for {
- time.Sleep(2 * time.Second)
- op, err := computeService.ZoneOperations.Get(projectID, zone, createOp).Do()
- if err != nil {
- return nil, fmt.Errorf("Failed to get op %s: %v", createOp, err)
- }
- switch op.Status {
- case "PENDING", "RUNNING":
- continue
- case "DONE":
- if op.Error != nil {
- for _, operr := range op.Error.Errors {
- return nil, fmt.Errorf("Error creating instance: %+v", operr)
- }
- return nil, errors.New("Failed to start.")
- }
- break OpLoop
- default:
- return nil, fmt.Errorf("Unknown create status %q: %+v", op.Status, op)
- }
- }
- condRun(opts.OnInstanceCreated)
-
- inst, err := computeService.Instances.Get(projectID, zone, instName).Do()
- if err != nil {
- return nil, fmt.Errorf("Error getting instance %s details after creation: %v", instName, err)
- }
-
- // Finds its internal and/or external IP addresses.
- intIP, extIP := instanceIPs(inst)
-
- // Wait for it to boot and its buildlet to come up.
- var buildletURL string
- var ipPort string
- if !opts.TLS.IsZero() {
- if extIP == "" {
- return nil, errors.New("didn't find its external IP address")
- }
- buildletURL = "https://" + extIP
- ipPort = extIP + ":443"
- } else {
- if intIP == "" {
- return nil, errors.New("didn't find its internal IP address")
- }
- buildletURL = "http://" + intIP
- ipPort = intIP + ":80"
- }
- condRun(opts.OnGotInstanceInfo)
-
- const timeout = 90 * time.Second
- var alive bool
- impatientClient := &http.Client{
- Timeout: 5 * time.Second,
- Transport: &http.Transport{
- TLSClientConfig: &tls.Config{
- InsecureSkipVerify: true,
- },
- },
- }
- deadline := time.Now().Add(timeout)
- try := 0
- for time.Now().Before(deadline) {
- try++
- res, err := impatientClient.Get(buildletURL)
- if err != nil {
- time.Sleep(1 * time.Second)
- continue
- }
- res.Body.Close()
- if res.StatusCode != 200 {
- return nil, fmt.Errorf("buildlet returned HTTP status code %d on try number %d", res.StatusCode, try)
- }
- alive = true
- break
- }
- if !alive {
- return nil, fmt.Errorf("buildlet didn't come up in %v", timeout)
- }
-
- return NewClient(ipPort, opts.TLS), nil
-}
-
-// DestroyVM sends a request to delete a VM. Actual VM description is
-// currently (2015-01-19) very slow for no good reason. This function
-// returns once it's been requested, not when it's done.
-func DestroyVM(ts oauth2.TokenSource, proj, zone, instance string) error {
- computeService, _ := compute.New(oauth2.NewClient(oauth2.NoContext, ts))
- _, err := computeService.Instances.Delete(proj, zone, instance).Do()
- return err
-}
-
-type VM struct {
- // Name is the name of the GCE VM instance.
- // For example, it's of the form "mote-bradfitz-plan9-386-foo",
- // and not "plan9-386-foo".
- Name string
- IPPort string
- TLS KeyPair
- Type string
-}
-
-// ListVMs lists all VMs.
-func ListVMs(ts oauth2.TokenSource, proj, zone string) ([]VM, error) {
- var vms []VM
- computeService, _ := compute.New(oauth2.NewClient(oauth2.NoContext, ts))
-
- // TODO(bradfitz): paging over results if more than 500
- list, err := computeService.Instances.List(proj, zone).Do()
- if err != nil {
- return nil, err
- }
- for _, inst := range list.Items {
- if inst.Metadata == nil {
- // Defensive. Not seen in practice.
- continue
- }
- meta := map[string]string{}
- for _, it := range inst.Metadata.Items {
- meta[it.Key] = it.Value
- }
- builderType := meta["builder-type"]
- if builderType == "" {
- continue
- }
- vm := VM{
- Name: inst.Name,
- Type: builderType,
- TLS: KeyPair{
- CertPEM: meta["tls-cert"],
- KeyPEM: meta["tls-key"],
- },
- }
- _, extIP := instanceIPs(inst)
- if extIP == "" || vm.TLS.IsZero() {
- continue
- }
- vm.IPPort = extIP + ":443"
- vms = append(vms, vm)
- }
- return vms, nil
-}
-
-func instanceIPs(inst *compute.Instance) (intIP, extIP string) {
- for _, iface := range inst.NetworkInterfaces {
- if strings.HasPrefix(iface.NetworkIP, "10.") {
- intIP = iface.NetworkIP
- }
- for _, accessConfig := range iface.AccessConfigs {
- if accessConfig.Type == "ONE_TO_ONE_NAT" {
- extIP = accessConfig.NatIP
- }
- }
- }
- return
-}
diff --git a/dashboard/buildlet/keypair.go b/dashboard/buildlet/keypair.go
deleted file mode 100644
index 3dc9439..0000000
--- a/dashboard/buildlet/keypair.go
+++ /dev/null
@@ -1,132 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-package buildlet
-
-import (
- "bytes"
- "crypto/rand"
- "crypto/rsa"
- "crypto/sha1"
- "crypto/tls"
- "crypto/x509"
- "crypto/x509/pkix"
- "encoding/pem"
- "errors"
- "fmt"
- "math/big"
- "net"
- "time"
-)
-
-// KeyPair is the TLS public certificate PEM file and its associated
-// private key PEM file that a builder will use for its HTTPS
-// server. The zero value means no HTTPs, which is used by the
-// coordinator for machines running within a firewall.
-type KeyPair struct {
- CertPEM string
- KeyPEM string
-}
-
-func (kp KeyPair) IsZero() bool { return kp == KeyPair{} }
-
-// Password returns the SHA1 of the KeyPEM. This is used as the HTTP
-// Basic Auth password.
-func (kp KeyPair) Password() string {
- if kp.KeyPEM != "" {
- return fmt.Sprintf("%x", sha1.Sum([]byte(kp.KeyPEM)))
- }
- return ""
-}
-
-// tlsDialer returns a TLS dialer for http.Transport.DialTLS that expects
-// exactly our TLS cert.
-func (kp KeyPair) tlsDialer() func(network, addr string) (net.Conn, error) {
- if kp.IsZero() {
- // Unused.
- return nil
- }
- wantCert, _ := tls.X509KeyPair([]byte(kp.CertPEM), []byte(kp.KeyPEM))
- var wantPubKey *rsa.PublicKey = &wantCert.PrivateKey.(*rsa.PrivateKey).PublicKey
-
- return func(network, addr string) (net.Conn, error) {
- if network != "tcp" {
- return nil, fmt.Errorf("unexpected network %q", network)
- }
- plainConn, err := net.Dial("tcp", addr)
- if err != nil {
- return nil, err
- }
- tlsConn := tls.Client(plainConn, &tls.Config{InsecureSkipVerify: true})
- if err := tlsConn.Handshake(); err != nil {
- return nil, err
- }
- certs := tlsConn.ConnectionState().PeerCertificates
- if len(certs) < 1 {
- return nil, errors.New("no server peer certificate")
- }
- cert := certs[0]
- peerPubRSA, ok := cert.PublicKey.(*rsa.PublicKey)
- if !ok {
- return nil, fmt.Errorf("peer cert was a %T; expected RSA", cert.PublicKey)
- }
- if peerPubRSA.N.Cmp(wantPubKey.N) != 0 {
- return nil, fmt.Errorf("unexpected TLS certificate")
- }
- return tlsConn, nil
- }
-}
-
-// NoKeyPair is used by the coordinator to speak http directly to buildlets,
-// inside their firewall, without TLS.
-var NoKeyPair = KeyPair{}
-
-func NewKeyPair() (KeyPair, error) {
- fail := func(err error) (KeyPair, error) { return KeyPair{}, err }
- failf := func(format string, args ...interface{}) (KeyPair, error) { return fail(fmt.Errorf(format, args...)) }
-
- priv, err := rsa.GenerateKey(rand.Reader, 2048)
- if err != nil {
- return failf("rsa.GenerateKey: %s", err)
- }
-
- notBefore := time.Now()
- notAfter := notBefore.Add(5 * 365 * 24 * time.Hour) // 5 years
-
- serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
- serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
- if err != nil {
- return failf("failed to generate serial number: %s", err)
- }
-
- template := x509.Certificate{
- SerialNumber: serialNumber,
- Subject: pkix.Name{
- Organization: []string{"Gopher Co"},
- },
- NotBefore: notBefore,
- NotAfter: notAfter,
-
- KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
- ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
- BasicConstraintsValid: true,
- DNSNames: []string{"localhost"},
- }
-
- derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
- if err != nil {
- return failf("Failed to create certificate: %s", err)
- }
-
- var certOut bytes.Buffer
- pem.Encode(&certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
- var keyOut bytes.Buffer
- pem.Encode(&keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
- return KeyPair{
- CertPEM: certOut.String(),
- KeyPEM: keyOut.String(),
- }, nil
-}
diff --git a/dashboard/cmd/builder/bench.go b/dashboard/cmd/builder/bench.go
deleted file mode 100644
index a9a59ce..0000000
--- a/dashboard/cmd/builder/bench.go
+++ /dev/null
@@ -1,256 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "bufio"
- "bytes"
- "fmt"
- "io"
- "io/ioutil"
- "log"
- "os"
- "os/exec"
- "path/filepath"
- "regexp"
- "strconv"
- "strings"
-)
-
-// benchHash benchmarks a single commit.
-func (b *Builder) benchHash(hash string, benchs []string) error {
- if *verbose {
- log.Println(b.name, "benchmarking", hash)
- }
-
- res := &PerfResult{Hash: hash, Benchmark: "meta-done"}
-
- // Create place in which to do work.
- workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
- // Prepare a workpath if we don't have one we can reuse.
- update := false
- if b.lastWorkpath != workpath {
- if err := os.Mkdir(workpath, mkdirPerm); err != nil {
- return err
- }
- buildLog, _, err := b.buildRepoOnHash(workpath, hash, makeCmd)
- if err != nil {
- removePath(workpath)
- // record failure
- res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
- return b.recordPerfResult(res)
- }
- b.lastWorkpath = workpath
- update = true
- }
-
- // Build the benchmark binary.
- benchBin, buildLog, err := b.buildBenchmark(workpath, update)
- if err != nil {
- // record failure
- res.Artifacts = append(res.Artifacts, PerfArtifact{"log", buildLog})
- return b.recordPerfResult(res)
- }
-
- benchmark, procs, affinity, last := chooseBenchmark(benchBin, benchs)
- if benchmark != "" {
- res.Benchmark = fmt.Sprintf("%v-%v", benchmark, procs)
- res.Metrics, res.Artifacts, res.OK = b.executeBenchmark(workpath, hash, benchBin, benchmark, procs, affinity)
- if err = b.recordPerfResult(res); err != nil {
- return fmt.Errorf("recordResult: %s", err)
- }
- }
-
- if last {
- // All benchmarks have beed executed, don't need workpath anymore.
- removePath(b.lastWorkpath)
- b.lastWorkpath = ""
- // Notify the app.
- res = &PerfResult{Hash: hash, Benchmark: "meta-done", OK: true}
- if err = b.recordPerfResult(res); err != nil {
- return fmt.Errorf("recordResult: %s", err)
- }
- }
-
- return nil
-}
-
-// buildBenchmark builds the benchmark binary.
-func (b *Builder) buildBenchmark(workpath string, update bool) (benchBin, log string, err error) {
- goroot := filepath.Join(workpath, "go")
- gobin := filepath.Join(goroot, "bin", "go") + exeExt
- gopath := filepath.Join(*buildroot, "gopath")
- env := append([]string{
- "GOROOT=" + goroot,
- "GOPATH=" + gopath},
- b.envv()...)
- // First, download without installing.
- args := []string{"get", "-d"}
- if update {
- args = append(args, "-u")
- }
- args = append(args, *benchPath)
- var buildlog bytes.Buffer
- runOpts := []runOpt{runTimeout(*buildTimeout), runEnv(env), allOutput(&buildlog), runDir(workpath)}
- err = run(exec.Command(gobin, args...), runOpts...)
- if err != nil {
- fmt.Fprintf(&buildlog, "go get -d %s failed: %s", *benchPath, err)
- return "", buildlog.String(), err
- }
- // Then, build into workpath.
- benchBin = filepath.Join(workpath, "benchbin") + exeExt
- args = []string{"build", "-o", benchBin, *benchPath}
- buildlog.Reset()
- err = run(exec.Command(gobin, args...), runOpts...)
- if err != nil {
- fmt.Fprintf(&buildlog, "go build %s failed: %s", *benchPath, err)
- return "", buildlog.String(), err
- }
- return benchBin, "", nil
-}
-
-// chooseBenchmark chooses the next benchmark to run
-// based on the list of available benchmarks, already executed benchmarks
-// and -benchcpu list.
-func chooseBenchmark(benchBin string, doneBenchs []string) (bench string, procs, affinity int, last bool) {
- var out bytes.Buffer
- err := run(exec.Command(benchBin), allOutput(&out))
- if err != nil {
- log.Printf("Failed to query benchmark list: %v\n%s", err, &out)
- last = true
- return
- }
- outStr := out.String()
- nlIdx := strings.Index(outStr, "\n")
- if nlIdx < 0 {
- log.Printf("Failed to parse benchmark list (no new line): %s", outStr)
- last = true
- return
- }
- localBenchs := strings.Split(outStr[:nlIdx], ",")
- benchsMap := make(map[string]bool)
- for _, b := range doneBenchs {
- benchsMap[b] = true
- }
- cnt := 0
- // We want to run all benchmarks with GOMAXPROCS=1 first.
- for i, procs1 := range benchCPU {
- for _, bench1 := range localBenchs {
- if benchsMap[fmt.Sprintf("%v-%v", bench1, procs1)] {
- continue
- }
- cnt++
- if cnt == 1 {
- bench = bench1
- procs = procs1
- if i < len(benchAffinity) {
- affinity = benchAffinity[i]
- }
- }
- }
- }
- last = cnt <= 1
- return
-}
-
-// executeBenchmark runs a single benchmark and parses its output.
-func (b *Builder) executeBenchmark(workpath, hash, benchBin, bench string, procs, affinity int) (metrics []PerfMetric, artifacts []PerfArtifact, ok bool) {
- // Benchmarks runs mutually exclusive with other activities.
- benchMutex.RUnlock()
- defer benchMutex.RLock()
- benchMutex.Lock()
- defer benchMutex.Unlock()
-
- log.Printf("%v executing benchmark %v-%v on %v", b.name, bench, procs, hash)
-
- // The benchmark executes 'go build'/'go tool',
- // so we need properly setup env.
- env := append([]string{
- "GOROOT=" + filepath.Join(workpath, "go"),
- "PATH=" + filepath.Join(workpath, "go", "bin") + string(os.PathListSeparator) + os.Getenv("PATH"),
- "GODEBUG=gctrace=1", // since Go1.2
- "GOGCTRACE=1", // before Go1.2
- fmt.Sprintf("GOMAXPROCS=%v", procs)},
- b.envv()...)
- args := []string{
- "-bench", bench,
- "-benchmem", strconv.Itoa(*benchMem),
- "-benchtime", benchTime.String(),
- "-benchnum", strconv.Itoa(*benchNum),
- "-tmpdir", workpath}
- if affinity != 0 {
- args = append(args, "-affinity", strconv.Itoa(affinity))
- }
- benchlog := new(bytes.Buffer)
- err := run(exec.Command(benchBin, args...), runEnv(env), allOutput(benchlog), runDir(workpath))
- if strip := benchlog.Len() - 512<<10; strip > 0 {
- // Leave the last 512K, that part contains metrics.
- benchlog = bytes.NewBuffer(benchlog.Bytes()[strip:])
- }
- artifacts = []PerfArtifact{{Type: "log", Body: benchlog.String()}}
- if err != nil {
- if err != nil {
- log.Printf("Failed to execute benchmark '%v': %v", bench, err)
- ok = false
- }
- return
- }
-
- metrics1, artifacts1, err := parseBenchmarkOutput(benchlog)
- if err != nil {
- log.Printf("Failed to parse benchmark output: %v", err)
- ok = false
- return
- }
- metrics = metrics1
- artifacts = append(artifacts, artifacts1...)
- ok = true
- return
-}
-
-// parseBenchmarkOutput fetches metrics and artifacts from benchmark output.
-func parseBenchmarkOutput(out io.Reader) (metrics []PerfMetric, artifacts []PerfArtifact, err error) {
- s := bufio.NewScanner(out)
- metricRe := regexp.MustCompile("^GOPERF-METRIC:([a-z,0-9,-]+)=([0-9]+)$")
- fileRe := regexp.MustCompile("^GOPERF-FILE:([a-z,0-9,-]+)=(.+)$")
- for s.Scan() {
- ln := s.Text()
- if ss := metricRe.FindStringSubmatch(ln); ss != nil {
- var v uint64
- v, err = strconv.ParseUint(ss[2], 10, 64)
- if err != nil {
- err = fmt.Errorf("Failed to parse metric '%v=%v': %v", ss[1], ss[2], err)
- return
- }
- metrics = append(metrics, PerfMetric{Type: ss[1], Val: v})
- } else if ss := fileRe.FindStringSubmatch(ln); ss != nil {
- var buf []byte
- buf, err = ioutil.ReadFile(ss[2])
- if err != nil {
- err = fmt.Errorf("Failed to read file '%v': %v", ss[2], err)
- return
- }
- artifacts = append(artifacts, PerfArtifact{ss[1], string(buf)})
- }
- }
- return
-}
-
-// needsBenchmarking determines whether the commit needs benchmarking.
-func needsBenchmarking(log *HgLog) bool {
- // Do not benchmark branch commits, they are usually not interesting
- // and fall out of the trunk succession.
- if log.Branch != "" {
- return false
- }
- // Do not benchmark commits that do not touch source files (e.g. CONTRIBUTORS).
- for _, f := range strings.Split(log.Files, " ") {
- if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) &&
- !strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") {
- return true
- }
- }
- return false
-}
diff --git a/dashboard/cmd/builder/doc.go b/dashboard/cmd/builder/doc.go
deleted file mode 100644
index 15b7252..0000000
--- a/dashboard/cmd/builder/doc.go
+++ /dev/null
@@ -1,58 +0,0 @@
-// Copyright 2010 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-/*
-
-Go Builder is a continuous build client for the Go project.
-It integrates with the Go Dashboard AppEngine application.
-
-Go Builder is intended to run continuously as a background process.
-
-It periodically pulls updates from the Go Mercurial repository.
-
-When a newer revision is found, Go Builder creates a clone of the repository,
-runs all.bash, and reports build success or failure to the Go Dashboard.
-
-For a release revision (a change description that matches "release.YYYY-MM-DD"),
-Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
-Go Google Code project's downloads section.
-
-Usage:
-
- gobuilder goos-goarch...
-
- Several goos-goarch combinations can be provided, and the builder will
- build them in serial.
-
-Optional flags:
-
- -dashboard="godashboard.appspot.com": Go Dashboard Host
- The location of the Go Dashboard application to which Go Builder will
- report its results.
-
- -release: Build and deliver binary release archive
-
- -rev=N: Build revision N and exit
-
- -cmd="./all.bash": Build command (specify absolute or relative to go/src)
-
- -v: Verbose logging
-
- -external: External package builder mode (will not report Go build
- state to dashboard or issue releases)
-
-The key file should be located at $HOME/.gobuildkey or, for a builder-specific
-key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64).
-
-The build key file is a text file of the format:
-
- godashboard-key
- googlecode-username
- googlecode-password
-
-If the Google Code credentials are not provided the archival step
-will be skipped.
-
-*/
-package main // import "golang.org/x/tools/dashboard/cmd/builder"
diff --git a/dashboard/cmd/builder/env.go b/dashboard/cmd/builder/env.go
deleted file mode 100644
index 7261229..0000000
--- a/dashboard/cmd/builder/env.go
+++ /dev/null
@@ -1,299 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "bytes"
- "fmt"
- "os"
- "os/exec"
- "path/filepath"
- "regexp"
- "runtime"
- "strings"
-
- "golang.org/x/tools/go/vcs"
-)
-
-// builderEnv represents the environment that a Builder will run tests in.
-type builderEnv interface {
- // setup sets up the builder environment and returns the directory to run the buildCmd in.
- setup(repo *Repo, workpath, hash string, envv []string) (string, error)
-}
-
-// goEnv represents the builderEnv for the main Go repo.
-type goEnv struct {
- goos, goarch string
-}
-
-func (b *Builder) envv() []string {
- if runtime.GOOS == "windows" {
- return b.envvWindows()
- }
-
- var e []string
- if *buildTool == "go" {
- e = []string{
- "GOOS=" + b.goos,
- "GOARCH=" + b.goarch,
- "GOROOT_FINAL=/usr/local/go",
- }
- switch b.goos {
- case "android", "nacl":
- // Cross compile.
- default:
- // If we are building, for example, linux/386 on a linux/amd64 machine we want to
- // make sure that the whole build is done as a if this were compiled on a real
- // linux/386 machine. In other words, we want to not do a cross compilation build.
- // To do this we set GOHOSTOS and GOHOSTARCH to override the detection in make.bash.
- //
- // The exception to this rule is when we are doing nacl/android builds. These are by
- // definition always cross compilation, and we have support built into cmd/go to be
- // able to handle this case.
- e = append(e, "GOHOSTOS="+b.goos, "GOHOSTARCH="+b.goarch)
- }
- }
-
- for _, k := range extraEnv() {
- if s, ok := getenvOk(k); ok {
- e = append(e, k+"="+s)
- }
- }
- return e
-}
-
-func (b *Builder) envvWindows() []string {
- var start map[string]string
- if *buildTool == "go" {
- start = map[string]string{
- "GOOS": b.goos,
- "GOHOSTOS": b.goos,
- "GOARCH": b.goarch,
- "GOHOSTARCH": b.goarch,
- "GOROOT_FINAL": `c:\go`,
- "GOBUILDEXIT": "1", // exit all.bat with completion status.
- }
- }
-
- for _, name := range extraEnv() {
- if s, ok := getenvOk(name); ok {
- start[name] = s
- }
- }
- if b.goos == "windows" {
- switch b.goarch {
- case "amd64":
- start["PATH"] = `c:\TDM-GCC-64\bin;` + start["PATH"]
- case "386":
- start["PATH"] = `c:\TDM-GCC-32\bin;` + start["PATH"]
- }
- }
- skip := map[string]bool{
- "GOBIN": true,
- "GOPATH": true,
- "GOROOT": true,
- "INCLUDE": true,
- "LIB": true,
- }
- var e []string
- for name, v := range start {
- e = append(e, name+"="+v)
- skip[name] = true
- }
- for _, kv := range os.Environ() {
- s := strings.SplitN(kv, "=", 2)
- name := strings.ToUpper(s[0])
- switch {
- case name == "":
- // variables, like "=C:=C:\", just copy them
- e = append(e, kv)
- case !skip[name]:
- e = append(e, kv)
- skip[name] = true
- }
- }
- return e
-}
-
-// setup for a goEnv clones the main go repo to workpath/go at the provided hash
-// and returns the path workpath/go/src, the location of all go build scripts.
-func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
- goworkpath := filepath.Join(workpath, "go")
- if err := repo.Export(goworkpath, hash); err != nil {
- return "", fmt.Errorf("error exporting repository: %s", err)
- }
- return filepath.Join(goworkpath, "src"), nil
-}
-
-// gccgoEnv represents the builderEnv for the gccgo compiler.
-type gccgoEnv struct{}
-
-// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
-// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
-// replaced with the updated sources in the gofrontend repo and gcc gets
-// gets configured and built in workpath/gcc-objdir. The path to
-// workpath/gcc-objdir is returned.
-func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
- gccpath := filepath.Join(repo.Path, "gcc")
-
- // get a handle to Git vcs.Cmd for pulling down GCC from the mirror.
- git := vcs.ByCmd("git")
-
- // only pull down gcc if we don't have a local copy.
- if _, err := os.Stat(gccpath); err != nil {
- if err := timeout(*cmdTimeout, func() error {
- // pull down a working copy of GCC.
-
- cloneCmd := []string{
- "clone",
- // This is just a guess since there are ~6000 commits to
- // GCC per year. It's likely there will be enough history
- // to cross-reference the Gofrontend commit against GCC.
- // The disadvantage would be if the commit being built is more than
- // a year old; in this case, the user should make a clone that has
- // the full history.
- "--depth", "6000",
- // We only care about the master branch.
- "--branch", "master", "--single-branch",
- *gccPath,
- }
-
- // Clone Kind Clone Time(Dry run) Clone Size
- // ---------------------------------------------------------------
- // Full Clone 10 - 15 min 2.2 GiB
- // Master Branch 2 - 3 min 1.5 GiB
- // Full Clone(shallow) 1 min 900 MiB
- // Master Branch(shallow) 40 sec 900 MiB
- //
- // The shallow clones have the same size, which is expected,
- // but the full shallow clone will only have 6000 commits
- // spread across all branches. There are ~50 branches.
- return run(exec.Command("git", cloneCmd...), runEnv(envv), allOutput(os.Stdout), runDir(repo.Path))
- }); err != nil {
- return "", err
- }
- }
-
- if err := git.Download(gccpath); err != nil {
- return "", err
- }
-
- // get the modified files for this commit.
-
- var buf bytes.Buffer
- if err := run(exec.Command("hg", "status", "--no-status", "--change", hash),
- allOutput(&buf), runDir(repo.Path), runEnv(envv)); err != nil {
- return "", fmt.Errorf("Failed to find the modified files for %s: %s", hash, err)
- }
- modifiedFiles := strings.Split(buf.String(), "\n")
- var isMirrored bool
- for _, f := range modifiedFiles {
- if strings.HasPrefix(f, "go/") || strings.HasPrefix(f, "libgo/") {
- isMirrored = true
- break
- }
- }
-
- // use git log to find the corresponding commit to sync to in the gcc mirror.
- // If the files modified in the gofrontend are mirrored to gcc, we expect a
- // commit with a similar description in the gcc mirror. If the files modified are
- // not mirrored, e.g. in support/, we can sync to the most recent gcc commit that
- // occurred before those files were modified to verify gccgo's status at that point.
- logCmd := []string{
- "log",
- "-1",
- "--format=%H",
- }
- var errMsg string
- if isMirrored {
- commitDesc, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{desc|firstline|escape}")
- if err != nil {
- return "", err
- }
-
- quotedDesc := regexp.QuoteMeta(string(commitDesc))
- logCmd = append(logCmd, "--grep", quotedDesc, "--regexp-ignore-case", "--extended-regexp")
- errMsg = fmt.Sprintf("Failed to find a commit with a similar description to '%s'", string(commitDesc))
- } else {
- commitDate, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{date|rfc3339date}")
- if err != nil {
- return "", err
- }
-
- logCmd = append(logCmd, "--before", string(commitDate))
- errMsg = fmt.Sprintf("Failed to find a commit before '%s'", string(commitDate))
- }
-
- buf.Reset()
- if err := run(exec.Command("git", logCmd...), runEnv(envv), allOutput(&buf), runDir(gccpath)); err != nil {
- return "", fmt.Errorf("%s: %s", errMsg, err)
- }
- gccRev := buf.String()
- if gccRev == "" {
- return "", fmt.Errorf(errMsg)
- }
-
- // checkout gccRev
- // TODO(cmang): Fix this to work in parallel mode.
- if err := run(exec.Command("git", "reset", "--hard", strings.TrimSpace(gccRev)), runEnv(envv), runDir(gccpath)); err != nil {
- return "", fmt.Errorf("Failed to checkout commit at revision %s: %s", gccRev, err)
- }
-
- // make objdir to work in
- gccobjdir := filepath.Join(workpath, "gcc-objdir")
- if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
- return "", err
- }
-
- // configure GCC with substituted gofrontend and libgo
- if err := run(exec.Command(filepath.Join(gccpath, "configure"),
- "--enable-languages=c,c++,go",
- "--disable-bootstrap",
- "--disable-multilib",
- ), runEnv(envv), runDir(gccobjdir)); err != nil {
- return "", fmt.Errorf("Failed to configure GCC: %v", err)
- }
-
- // build gcc
- if err := run(exec.Command("make", *gccOpts), runTimeout(*buildTimeout), runEnv(envv), runDir(gccobjdir)); err != nil {
- return "", fmt.Errorf("Failed to build GCC: %s", err)
- }
-
- return gccobjdir, nil
-}
-
-func getenvOk(k string) (v string, ok bool) {
- v = os.Getenv(k)
- if v != "" {
- return v, true
- }
- keq := k + "="
- for _, kv := range os.Environ() {
- if kv == keq {
- return "", true
- }
- }
- return "", false
-}
-
-// extraEnv returns environment variables that need to be copied from
-// the gobuilder's environment to the envv of its subprocesses.
-func extraEnv() []string {
- extra := []string{
- "GOARM",
- "GO386",
- "GOROOT_BOOTSTRAP", // See https://golang.org/s/go15bootstrap
- "CGO_ENABLED",
- "CC",
- "CC_FOR_TARGET",
- "PATH",
- "TMPDIR",
- "USER",
- }
- if runtime.GOOS == "plan9" {
- extra = append(extra, "objtype", "cputype", "path")
- }
- return extra
-}
diff --git a/dashboard/cmd/builder/exec.go b/dashboard/cmd/builder/exec.go
deleted file mode 100644
index 1b46ed1..0000000
--- a/dashboard/cmd/builder/exec.go
+++ /dev/null
@@ -1,99 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "fmt"
- "io"
- "log"
- "os/exec"
- "time"
-)
-
-// run runs a command with optional arguments.
-func run(cmd *exec.Cmd, opts ...runOpt) error {
- a := runArgs{cmd, *cmdTimeout}
- for _, opt := range opts {
- opt.modArgs(&a)
- }
- if *verbose {
- log.Printf("running %v in %v", a.cmd.Args, a.cmd.Dir)
- }
- if err := cmd.Start(); err != nil {
- log.Printf("failed to start command %v: %v", a.cmd.Args, err)
- return err
- }
- err := timeout(a.timeout, cmd.Wait)
- if _, ok := err.(timeoutError); ok {
- cmd.Process.Kill()
- }
- return err
-}
-
-// Zero or more runOpts can be passed to run to modify the command
-// before it's run.
-type runOpt interface {
- modArgs(*runArgs)
-}
-
-// allOutput sends both stdout and stderr to w.
-func allOutput(w io.Writer) optFunc {
- return func(a *runArgs) {
- a.cmd.Stdout = w
- a.cmd.Stderr = w
- }
-}
-
-func runTimeout(timeout time.Duration) optFunc {
- return func(a *runArgs) {
- a.timeout = timeout
- }
-}
-
-func runDir(dir string) optFunc {
- return func(a *runArgs) {
- a.cmd.Dir = dir
- }
-}
-
-func runEnv(env []string) optFunc {
- return func(a *runArgs) {
- a.cmd.Env = env
- }
-}
-
-// timeout runs f and returns its error value, or if the function does not
-// complete before the provided duration it returns a timeout error.
-func timeout(d time.Duration, f func() error) error {
- errc := make(chan error, 1)
- go func() {
- errc <- f()
- }()
- t := time.NewTimer(d)
- defer t.Stop()
- select {
- case <-t.C:
- return timeoutError(d)
- case err := <-errc:
- return err
- }
-}
-
-type timeoutError time.Duration
-
-func (e timeoutError) Error() string {
- return fmt.Sprintf("timed out after %v", time.Duration(e))
-}
-
-// optFunc implements runOpt with a function, like http.HandlerFunc.
-type optFunc func(*runArgs)
-
-func (f optFunc) modArgs(a *runArgs) { f(a) }
-
-// internal detail to exec.go:
-type runArgs struct {
- cmd *exec.Cmd
- timeout time.Duration
-}
diff --git a/dashboard/cmd/builder/filemutex_flock.go b/dashboard/cmd/builder/filemutex_flock.go
deleted file mode 100644
index 68851b8..0000000
--- a/dashboard/cmd/builder/filemutex_flock.go
+++ /dev/null
@@ -1,66 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin dragonfly freebsd linux netbsd openbsd
-
-package main
-
-import (
- "sync"
- "syscall"
-)
-
-// FileMutex is similar to sync.RWMutex, but also synchronizes across processes.
-// This implementation is based on flock syscall.
-type FileMutex struct {
- mu sync.RWMutex
- fd int
-}
-
-func MakeFileMutex(filename string) *FileMutex {
- if filename == "" {
- return &FileMutex{fd: -1}
- }
- fd, err := syscall.Open(filename, syscall.O_CREAT|syscall.O_RDONLY, mkdirPerm)
- if err != nil {
- panic(err)
- }
- return &FileMutex{fd: fd}
-}
-
-func (m *FileMutex) Lock() {
- m.mu.Lock()
- if m.fd != -1 {
- if err := syscall.Flock(m.fd, syscall.LOCK_EX); err != nil {
- panic(err)
- }
- }
-}
-
-func (m *FileMutex) Unlock() {
- if m.fd != -1 {
- if err := syscall.Flock(m.fd, syscall.LOCK_UN); err != nil {
- panic(err)
- }
- }
- m.mu.Unlock()
-}
-
-func (m *FileMutex) RLock() {
- m.mu.RLock()
- if m.fd != -1 {
- if err := syscall.Flock(m.fd, syscall.LOCK_SH); err != nil {
- panic(err)
- }
- }
-}
-
-func (m *FileMutex) RUnlock() {
- if m.fd != -1 {
- if err := syscall.Flock(m.fd, syscall.LOCK_UN); err != nil {
- panic(err)
- }
- }
- m.mu.RUnlock()
-}
diff --git a/dashboard/cmd/builder/filemutex_local.go b/dashboard/cmd/builder/filemutex_local.go
deleted file mode 100644
index 68cfb62..0000000
--- a/dashboard/cmd/builder/filemutex_local.go
+++ /dev/null
@@ -1,27 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build nacl plan9 solaris
-
-package main
-
-import (
- "log"
- "sync"
-)
-
-// FileMutex is similar to sync.RWMutex, but also synchronizes across processes.
-// This implementation is a fallback that does not actually provide inter-process synchronization.
-type FileMutex struct {
- sync.RWMutex
-}
-
-func MakeFileMutex(filename string) *FileMutex {
- return &FileMutex{}
-}
-
-func init() {
- log.Printf("WARNING: using fake file mutex." +
- " Don't run more than one of these at once!!!")
-}
diff --git a/dashboard/cmd/builder/filemutex_windows.go b/dashboard/cmd/builder/filemutex_windows.go
deleted file mode 100644
index 1f058b2..0000000
--- a/dashboard/cmd/builder/filemutex_windows.go
+++ /dev/null
@@ -1,105 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "sync"
- "syscall"
- "unsafe"
-)
-
-var (
- modkernel32 = syscall.NewLazyDLL("kernel32.dll")
- procLockFileEx = modkernel32.NewProc("LockFileEx")
- procUnlockFileEx = modkernel32.NewProc("UnlockFileEx")
-)
-
-const (
- INVALID_FILE_HANDLE = ^syscall.Handle(0)
- LOCKFILE_EXCLUSIVE_LOCK = 2
-)
-
-func lockFileEx(h syscall.Handle, flags, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) {
- r1, _, e1 := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(h), uintptr(flags), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)))
- if r1 == 0 {
- if e1 != 0 {
- err = error(e1)
- } else {
- err = syscall.EINVAL
- }
- }
- return
-}
-
-func unlockFileEx(h syscall.Handle, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) (err error) {
- r1, _, e1 := syscall.Syscall6(procUnlockFileEx.Addr(), 5, uintptr(h), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol)), 0)
- if r1 == 0 {
- if e1 != 0 {
- err = error(e1)
- } else {
- err = syscall.EINVAL
- }
- }
- return
-}
-
-// FileMutex is similar to sync.RWMutex, but also synchronizes across processes.
-// This implementation is based on flock syscall.
-type FileMutex struct {
- mu sync.RWMutex
- fd syscall.Handle
-}
-
-func MakeFileMutex(filename string) *FileMutex {
- if filename == "" {
- return &FileMutex{fd: INVALID_FILE_HANDLE}
- }
- fd, err := syscall.CreateFile(&(syscall.StringToUTF16(filename)[0]), syscall.GENERIC_READ|syscall.GENERIC_WRITE,
- syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE, nil, syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0)
- if err != nil {
- panic(err)
- }
- return &FileMutex{fd: fd}
-}
-
-func (m *FileMutex) Lock() {
- m.mu.Lock()
- if m.fd != INVALID_FILE_HANDLE {
- var ol syscall.Overlapped
- if err := lockFileEx(m.fd, LOCKFILE_EXCLUSIVE_LOCK, 0, 1, 0, &ol); err != nil {
- panic(err)
- }
- }
-}
-
-func (m *FileMutex) Unlock() {
- if m.fd != INVALID_FILE_HANDLE {
- var ol syscall.Overlapped
- if err := unlockFileEx(m.fd, 0, 1, 0, &ol); err != nil {
- panic(err)
- }
- }
- m.mu.Unlock()
-}
-
-func (m *FileMutex) RLock() {
- m.mu.RLock()
- if m.fd != INVALID_FILE_HANDLE {
- var ol syscall.Overlapped
- if err := lockFileEx(m.fd, 0, 0, 1, 0, &ol); err != nil {
- panic(err)
- }
- }
-}
-
-func (m *FileMutex) RUnlock() {
- if m.fd != INVALID_FILE_HANDLE {
- var ol syscall.Overlapped
- if err := unlockFileEx(m.fd, 0, 1, 0, &ol); err != nil {
- panic(err)
- }
- }
- m.mu.RUnlock()
-}
diff --git a/dashboard/cmd/builder/http.go b/dashboard/cmd/builder/http.go
deleted file mode 100644
index 8d0923c..0000000
--- a/dashboard/cmd/builder/http.go
+++ /dev/null
@@ -1,225 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "bytes"
- "encoding/json"
- "errors"
- "fmt"
- "io"
- "log"
- "net/http"
- "net/url"
- "time"
-)
-
-const builderVersion = 1 // keep in sync with dashboard/app/build/handler.go
-
-type obj map[string]interface{}
-
-// dash runs the given method and command on the dashboard.
-// If args is non-nil it is encoded as the URL query string.
-// If req is non-nil it is JSON-encoded and passed as the body of the HTTP POST.
-// If resp is non-nil the server's response is decoded into the value pointed
-// to by resp (resp must be a pointer).
-func dash(meth, cmd string, args url.Values, req, resp interface{}) error {
- argsCopy := url.Values{"version": {fmt.Sprint(builderVersion)}}
- for k, v := range args {
- if k == "version" {
- panic(`dash: reserved args key: "version"`)
- }
- argsCopy[k] = v
- }
- var r *http.Response
- var err error
- if *verbose {
- log.Println("dash <-", meth, cmd, argsCopy, req)
- }
- cmd = *dashboard + "/" + cmd + "?" + argsCopy.Encode()
- switch meth {
- case "GET":
- if req != nil {
- log.Panicf("%s to %s with req", meth, cmd)
- }
- r, err = http.Get(cmd)
- case "POST":
- var body io.Reader
- if req != nil {
- b, err := json.Marshal(req)
- if err != nil {
- return err
- }
- body = bytes.NewBuffer(b)
- }
- r, err = http.Post(cmd, "text/json", body)
- default:
- log.Panicf("%s: invalid method %q", cmd, meth)
- panic("invalid method: " + meth)
- }
- if err != nil {
- return err
- }
- defer r.Body.Close()
- if r.StatusCode != http.StatusOK {
- return fmt.Errorf("bad http response: %v", r.Status)
- }
- body := new(bytes.Buffer)
- if _, err := body.ReadFrom(r.Body); err != nil {
- return err
- }
-
- // Read JSON-encoded Response into provided resp
- // and return an error if present.
- var result = struct {
- Response interface{}
- Error string
- }{
- // Put the provided resp in here as it can be a pointer to
- // some value we should unmarshal into.
- Response: resp,
- }
- if err = json.Unmarshal(body.Bytes(), &result); err != nil {
- log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err)
- return err
- }
- if *verbose {
- log.Println("dash ->", result)
- }
- if result.Error != "" {
- return errors.New(result.Error)
- }
-
- return nil
-}
-
-// todo returns the next hash to build or benchmark.
-func (b *Builder) todo(kinds []string, pkg, goHash string) (kind, rev string, benchs []string, err error) {
- args := url.Values{
- "builder": {b.name},
- "packagePath": {pkg},
- "goHash": {goHash},
- }
- for _, k := range kinds {
- args.Add("kind", k)
- }
- var resp *struct {
- Kind string
- Data struct {
- Hash string
- PerfResults []string
- }
- }
- if err = dash("GET", "todo", args, nil, &resp); err != nil {
- return
- }
- if resp == nil {
- return
- }
- if *verbose {
- fmt.Printf("dash resp: %+v\n", *resp)
- }
- for _, k := range kinds {
- if k == resp.Kind {
- return resp.Kind, resp.Data.Hash, resp.Data.PerfResults, nil
- }
- }
- err = fmt.Errorf("expecting Kinds %q, got %q", kinds, resp.Kind)
- return
-}
-
-// recordResult sends build results to the dashboard
-func (b *Builder) recordResult(ok bool, pkg, hash, goHash, buildLog string, runTime time.Duration) error {
- if !*report {
- return nil
- }
- req := obj{
- "Builder": b.name,
- "PackagePath": pkg,
- "Hash": hash,
- "GoHash": goHash,
- "OK": ok,
- "Log": buildLog,
- "RunTime": runTime,
- }
- args := url.Values{"key": {b.key}, "builder": {b.name}}
- return dash("POST", "result", args, req, nil)
-}
-
-// Result of running a single benchmark on a single commit.
-type PerfResult struct {
- Builder string
- Benchmark string
- Hash string
- OK bool
- Metrics []PerfMetric
- Artifacts []PerfArtifact
-}
-
-type PerfMetric struct {
- Type string
- Val uint64
-}
-
-type PerfArtifact struct {
- Type string
- Body string
-}
-
-// recordPerfResult sends benchmarking results to the dashboard
-func (b *Builder) recordPerfResult(req *PerfResult) error {
- if !*report {
- return nil
- }
- req.Builder = b.name
- args := url.Values{"key": {b.key}, "builder": {b.name}}
- return dash("POST", "perf-result", args, req, nil)
-}
-
-func postCommit(key, pkg string, l *HgLog) error {
- if !*report {
- return nil
- }
- t, err := time.Parse(time.RFC3339, l.Date)
- if err != nil {
- return fmt.Errorf("parsing %q: %v", l.Date, t)
- }
- return dash("POST", "commit", url.Values{"key": {key}}, obj{
- "PackagePath": pkg,
- "Hash": l.Hash,
- "ParentHash": l.Parent,
- "Time": t.Format(time.RFC3339),
- "User": l.Author,
- "Desc": l.Desc,
- "NeedsBenchmarking": l.bench,
- }, nil)
-}
-
-func dashboardCommit(pkg, hash string) bool {
- err := dash("GET", "commit", url.Values{
- "packagePath": {pkg},
- "hash": {hash},
- }, nil, nil)
- return err == nil
-}
-
-func dashboardPackages(kind string) []string {
- args := url.Values{"kind": []string{kind}}
- var resp []struct {
- Path string
- }
- if err := dash("GET", "packages", args, nil, &resp); err != nil {
- log.Println("dashboardPackages:", err)
- return nil
- }
- if *verbose {
- fmt.Printf("dash resp: %+v\n", resp)
- }
- var pkgs []string
- for _, r := range resp {
- pkgs = append(pkgs, r.Path)
- }
- return pkgs
-}
diff --git a/dashboard/cmd/builder/main.go b/dashboard/cmd/builder/main.go
deleted file mode 100644
index 9e7c1ed..0000000
--- a/dashboard/cmd/builder/main.go
+++ /dev/null
@@ -1,679 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "bytes"
- "flag"
- "fmt"
- "io"
- "io/ioutil"
- "log"
- "net"
- "net/http"
- "os"
- "os/exec"
- "path/filepath"
- "regexp"
- "runtime"
- "strconv"
- "strings"
- "time"
-
- "golang.org/x/tools/go/vcs"
-)
-
-const (
- codeProject = "go"
- codePyScript = "misc/dashboard/googlecode_upload.py"
- gofrontendImportPath = "code.google.com/p/gofrontend"
- mkdirPerm = 0750
- waitInterval = 30 * time.Second // time to wait before checking for new revs
- pkgBuildInterval = 24 * time.Hour // rebuild packages every 24 hours
-)
-
-type Builder struct {
- goroot *Repo
- name string
- goos, goarch string
- key string
- env builderEnv
- // Last benchmarking workpath. We reuse it, if do successive benchmarks on the same commit.
- lastWorkpath string
-}
-
-var (
- doBuild = flag.Bool("build", true, "Build and test packages")
- doBench = flag.Bool("bench", false, "Run benchmarks")
- buildroot = flag.String("buildroot", defaultBuildRoot(), "Directory under which to build")
- dashboard = flag.String("dashboard", "https://build.golang.org", "Dashboard app base path")
- buildRelease = flag.Bool("release", false, "Build and upload binary release archives")
- buildRevision = flag.String("rev", "", "Build specified revision and exit")
- buildCmd = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)")
- buildTool = flag.String("tool", "go", "Tool to build.")
- gcPath = flag.String("gcpath", "go.googlesource.com/go", "Path to download gc from")
- gccPath = flag.String("gccpath", "https://github.com/mirrors/gcc.git", "Path to download gcc from")
- gccOpts = flag.String("gccopts", "", "Command-line options to pass to `make` when building gccgo")
- benchPath = flag.String("benchpath", "golang.org/x/benchmarks/bench", "Path to download benchmarks from")
- failAll = flag.Bool("fail", false, "fail all builds")
- parallel = flag.Bool("parallel", false, "Build multiple targets in parallel")
- buildTimeout = flag.Duration("buildTimeout", 60*time.Minute, "Maximum time to wait for builds and tests")
- cmdTimeout = flag.Duration("cmdTimeout", 10*time.Minute, "Maximum time to wait for an external command")
- benchNum = flag.Int("benchnum", 5, "Run each benchmark that many times")
- benchTime = flag.Duration("benchtime", 5*time.Second, "Benchmarking time for a single benchmark run")
- benchMem = flag.Int("benchmem", 64, "Approx RSS value to aim at in benchmarks, in MB")
- fileLock = flag.String("filelock", "", "File to lock around benchmaring (synchronizes several builders)")
- verbose = flag.Bool("v", false, "verbose")
- report = flag.Bool("report", true, "whether to report results to the dashboard")
-)
-
-var (
- binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`)
- releaseRe = regexp.MustCompile(`^release\.r[0-9\-.]+`)
- allCmd = "all" + suffix
- makeCmd = "make" + suffix
- raceCmd = "race" + suffix
- cleanCmd = "clean" + suffix
- suffix = defaultSuffix()
- exeExt = defaultExeExt()
-
- benchCPU = CpuList([]int{1})
- benchAffinity = CpuList([]int{})
- benchMutex *FileMutex // Isolates benchmarks from other activities
-)
-
-// CpuList is used as flag.Value for -benchcpu flag.
-type CpuList []int
-
-func (cl *CpuList) String() string {
- str := ""
- for _, cpu := range *cl {
- if str == "" {
- str = strconv.Itoa(cpu)
- } else {
- str += fmt.Sprintf(",%v", cpu)
- }
- }
- return str
-}
-
-func (cl *CpuList) Set(str string) error {
- *cl = []int{}
- for _, val := range strings.Split(str, ",") {
- val = strings.TrimSpace(val)
- if val == "" {
- continue
- }
- cpu, err := strconv.Atoi(val)
- if err != nil || cpu <= 0 {
- return fmt.Errorf("%v is a bad value for GOMAXPROCS", val)
- }
- *cl = append(*cl, cpu)
- }
- if len(*cl) == 0 {
- *cl = append(*cl, 1)
- }
- return nil
-}
-
-func main() {
- flag.Var(&benchCPU, "benchcpu", "Comma-delimited list of GOMAXPROCS values for benchmarking")
- flag.Var(&benchAffinity, "benchaffinity", "Comma-delimited list of affinity values for benchmarking")
- flag.Usage = func() {
- fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0])
- flag.PrintDefaults()
- os.Exit(2)
- }
- flag.Parse()
- if len(flag.Args()) == 0 {
- flag.Usage()
- }
-
- vcs.ShowCmd = *verbose
- vcs.Verbose = *verbose
-
- benchMutex = MakeFileMutex(*fileLock)
-
- rr, err := repoForTool()
- if err != nil {
- log.Fatal("Error finding repository:", err)
- }
- rootPath := filepath.Join(*buildroot, "goroot")
- goroot := &Repo{
- Path: rootPath,
- Master: rr,
- }
-
- // set up work environment, use existing environment if possible
- if goroot.Exists() || *failAll {
- log.Print("Found old workspace, will use it")
- } else {
- if err := os.RemoveAll(*buildroot); err != nil {
- log.Fatalf("Error removing build root (%s): %s", *buildroot, err)
- }
- if err := os.Mkdir(*buildroot, mkdirPerm); err != nil {
- log.Fatalf("Error making build root (%s): %s", *buildroot, err)
- }
- var err error
- goroot, err = RemoteRepo(goroot.Master.Root, rootPath)
- if err != nil {
- log.Fatalf("Error creating repository with url (%s): %s", goroot.Master.Root, err)
- }
-
- goroot, err = goroot.Clone(goroot.Path, "")
- if err != nil {
- log.Fatal("Error cloning repository:", err)
- }
- }
-
- // set up builders
- builders := make([]*Builder, len(flag.Args()))
- for i, name := range flag.Args() {
- b, err := NewBuilder(goroot, name)
- if err != nil {
- log.Fatal(err)
- }
- builders[i] = b
- }
-
- if *failAll {
- failMode(builders)
- return
- }
-
- // if specified, build revision and return
- if *buildRevision != "" {
- hash, err := goroot.FullHash(*buildRevision)
- if err != nil {
- log.Fatal("Error finding revision: ", err)
- }
- var exitErr error
- for _, b := range builders {
- if err := b.buildHash(hash); err != nil {
- log.Println(err)
- exitErr = err
- }
- }
- if exitErr != nil && !*report {
- // This mode (-report=false) is used for
- // testing Docker images, making sure the
- // environment is correctly configured. For
- // testing, we want a non-zero exit status, as
- // returned by log.Fatal:
- log.Fatal("Build error.")
- }
- return
- }
-
- if !*doBuild && !*doBench {
- fmt.Fprintf(os.Stderr, "Nothing to do, exiting (specify either -build or -bench or both)\n")
- os.Exit(2)
- }
-
- // go continuous build mode
- // check for new commits and build them
- benchMutex.RLock()
- for {
- built := false
- t := time.Now()
- if *parallel {
- done := make(chan bool)
- for _, b := range builders {
- go func(b *Builder) {
- done <- b.buildOrBench()
- }(b)
- }
- for _ = range builders {
- built = <-done || built
- }
- } else {
- for _, b := range builders {
- built = b.buildOrBench() || built
- }
- }
- // sleep if there was nothing to build
- benchMutex.RUnlock()
- if !built {
- time.Sleep(waitInterval)
- }
- benchMutex.RLock()
- // sleep if we're looping too fast.
- dt := time.Now().Sub(t)
- if dt < waitInterval {
- time.Sleep(waitInterval - dt)
- }
- }
-}
-
-// go continuous fail mode
-// check for new commits and FAIL them
-func failMode(builders []*Builder) {
- for {
- built := false
- for _, b := range builders {
- built = b.failBuild() || built
- }
- // stop if there was nothing to fail
- if !built {
- break
- }
- }
-}
-
-func NewBuilder(goroot *Repo, name string) (*Builder, error) {
- b := &Builder{
- goroot: goroot,
- name: name,
- }
-
- // get builderEnv for this tool
- var err error
- if b.env, err = b.builderEnv(name); err != nil {
- return nil, err
- }
- if *report {
- err = b.setKey()
- }
- return b, err
-}
-
-func (b *Builder) setKey() error {
- // read keys from keyfile
- fn := ""
- switch runtime.GOOS {
- case "plan9":
- fn = os.Getenv("home")
- case "windows":
- fn = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
- default:
- fn = os.Getenv("HOME")
- }
- fn = filepath.Join(fn, ".gobuildkey")
- if s := fn + "-" + b.name; isFile(s) { // builder-specific file
- fn = s
- }
- c, err := ioutil.ReadFile(fn)
- if err != nil {
- // If the on-disk file doesn't exist, also try the
- // Google Compute Engine metadata.
- if v := gceProjectMetadata("buildkey-" + b.name); v != "" {
- b.key = v
- return nil
- }
- return fmt.Errorf("readKeys %s (%s): %s", b.name, fn, err)
- }
- b.key = string(bytes.TrimSpace(bytes.SplitN(c, []byte("\n"), 2)[0]))
- return nil
-}
-
-func gceProjectMetadata(attr string) string {
- client := &http.Client{
- Transport: &http.Transport{
- Dial: (&net.Dialer{
- Timeout: 750 * time.Millisecond,
- KeepAlive: 30 * time.Second,
- }).Dial,
- ResponseHeaderTimeout: 750 * time.Millisecond,
- },
- }
- req, _ := http.NewRequest("GET", "http://metadata.google.internal/computeMetadata/v1/project/attributes/"+attr, nil)
- req.Header.Set("Metadata-Flavor", "Google")
- res, err := client.Do(req)
- if err != nil {
- return ""
- }
- defer res.Body.Close()
- if res.StatusCode != 200 {
- return ""
- }
- slurp, err := ioutil.ReadAll(res.Body)
- if err != nil {
- return ""
- }
- return string(bytes.TrimSpace(slurp))
-}
-
-// builderEnv returns the builderEnv for this buildTool.
-func (b *Builder) builderEnv(name string) (builderEnv, error) {
- // get goos/goarch from builder string
- s := strings.SplitN(b.name, "-", 3)
- if len(s) < 2 {
- return nil, fmt.Errorf("unsupported builder form: %s", name)
- }
- b.goos = s[0]
- b.goarch = s[1]
-
- switch *buildTool {
- case "go":
- return &goEnv{
- goos: s[0],
- goarch: s[1],
- }, nil
- case "gccgo":
- return &gccgoEnv{}, nil
- default:
- return nil, fmt.Errorf("unsupported build tool: %s", *buildTool)
- }
-}
-
-// buildCmd returns the build command to invoke.
-// Builders which contain the string '-race' in their
-// name will override *buildCmd and return raceCmd.
-func (b *Builder) buildCmd() string {
- if strings.Contains(b.name, "-race") {
- return raceCmd
- }
- return *buildCmd
-}
-
-// buildOrBench checks for a new commit for this builder
-// and builds or benchmarks it if one is found.
-// It returns true if a build/benchmark was attempted.
-func (b *Builder) buildOrBench() bool {
- var kinds []string
- if *doBuild {
- kinds = append(kinds, "build-go-commit")
- }
- if *doBench {
- kinds = append(kinds, "benchmark-go-commit")
- }
- kind, hash, benchs, err := b.todo(kinds, "", "")
- if err != nil {
- log.Println(err)
- return false
- }
- if hash == "" {
- return false
- }
- switch kind {
- case "build-go-commit":
- if err := b.buildHash(hash); err != nil {
- log.Println(err)
- }
- return true
- case "benchmark-go-commit":
- if err := b.benchHash(hash, benchs); err != nil {
- log.Println(err)
- }
- return true
- default:
- log.Printf("Unknown todo kind %v", kind)
- return false
- }
-}
-
-func (b *Builder) buildHash(hash string) error {
- log.Println(b.name, "building", hash)
-
- // create place in which to do work
- workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
- if err := os.Mkdir(workpath, mkdirPerm); err != nil {
- if err2 := removePath(workpath); err2 != nil {
- return err
- }
- if err := os.Mkdir(workpath, mkdirPerm); err != nil {
- return err
- }
- }
- defer removePath(workpath)
-
- buildLog, runTime, err := b.buildRepoOnHash(workpath, hash, b.buildCmd())
- if err != nil {
- log.Printf("%s failed at %v: %v", b.name, hash, err)
- // record failure
- return b.recordResult(false, "", hash, "", buildLog, runTime)
- }
-
- // record success
- if err = b.recordResult(true, "", hash, "", "", runTime); err != nil {
- return fmt.Errorf("recordResult: %s", err)
- }
-
- if *buildTool == "go" {
- // build sub-repositories
- goRoot := filepath.Join(workpath, *buildTool)
- goPath := workpath
- b.buildSubrepos(goRoot, goPath, hash)
- }
-
- return nil
-}
-
-// buildRepoOnHash clones repo into workpath and builds it.
-func (b *Builder) buildRepoOnHash(workpath, hash, cmd string) (buildLog string, runTime time.Duration, err error) {
- // Delete the previous workdir, if necessary
- // (benchmarking code can execute several benchmarks in the same workpath).
- if b.lastWorkpath != "" {
- if b.lastWorkpath == workpath {
- panic("workpath already exists: " + workpath)
- }
- removePath(b.lastWorkpath)
- b.lastWorkpath = ""
- }
-
- // pull before cloning to ensure we have the revision
- if err = b.goroot.Pull(); err != nil {
- buildLog = err.Error()
- return
- }
-
- // set up builder's environment.
- srcDir, err := b.env.setup(b.goroot, workpath, hash, b.envv())
- if err != nil {
- buildLog = err.Error()
- return
- }
-
- // build
- var buildbuf bytes.Buffer
- logfile := filepath.Join(workpath, "build.log")
- f, err := os.Create(logfile)
- if err != nil {
- return err.Error(), 0, err
- }
- defer f.Close()
- w := io.MultiWriter(f, &buildbuf)
-
- // go's build command is a script relative to the srcDir, whereas
- // gccgo's build command is usually "make check-go" in the srcDir.
- if *buildTool == "go" {
- if !filepath.IsAbs(cmd) {
- cmd = filepath.Join(srcDir, cmd)
- }
- }
-
- // naive splitting of command from its arguments:
- args := strings.Split(cmd, " ")
- c := exec.Command(args[0], args[1:]...)
- c.Dir = srcDir
- c.Env = b.envv()
- if *verbose {
- c.Stdout = io.MultiWriter(os.Stdout, w)
- c.Stderr = io.MultiWriter(os.Stderr, w)
- } else {
- c.Stdout = w
- c.Stderr = w
- }
-
- startTime := time.Now()
- err = run(c, runTimeout(*buildTimeout))
- runTime = time.Since(startTime)
- if err != nil {
- fmt.Fprintf(w, "Build complete, duration %v. Result: error: %v\n", runTime, err)
- } else {
- fmt.Fprintf(w, "Build complete, duration %v. Result: success\n", runTime)
- }
- return buildbuf.String(), runTime, err
-}
-
-// failBuild checks for a new commit for this builder
-// and fails it if one is found.
-// It returns true if a build was "attempted".
-func (b *Builder) failBuild() bool {
- _, hash, _, err := b.todo([]string{"build-go-commit"}, "", "")
- if err != nil {
- log.Println(err)
- return false
- }
- if hash == "" {
- return false
- }
-
- log.Printf("fail %s %s\n", b.name, hash)
-
- if err := b.recordResult(false, "", hash, "", "auto-fail mode run by "+os.Getenv("USER"), 0); err != nil {
- log.Print(err)
- }
- return true
-}
-
-func (b *Builder) buildSubrepos(goRoot, goPath, goHash string) {
- for _, pkg := range dashboardPackages("subrepo") {
- // get the latest todo for this package
- _, hash, _, err := b.todo([]string{"build-package"}, pkg, goHash)
- if err != nil {
- log.Printf("buildSubrepos %s: %v", pkg, err)
- continue
- }
- if hash == "" {
- continue
- }
-
- // build the package
- if *verbose {
- log.Printf("buildSubrepos %s: building %q", pkg, hash)
- }
- buildLog, err := b.buildSubrepo(goRoot, goPath, pkg, hash)
- if err != nil {
- if buildLog == "" {
- buildLog = err.Error()
- }
- log.Printf("buildSubrepos %s: %v", pkg, err)
- }
-
- // record the result
- err = b.recordResult(err == nil, pkg, hash, goHash, buildLog, 0)
- if err != nil {
- log.Printf("buildSubrepos %s: %v", pkg, err)
- }
- }
-}
-
-// buildSubrepo fetches the given package, updates it to the specified hash,
-// and runs 'go test -short pkg/...'. It returns the build log and any error.
-func (b *Builder) buildSubrepo(goRoot, goPath, pkg, hash string) (string, error) {
- goTool := filepath.Join(goRoot, "bin", "go") + exeExt
- env := append(b.envv(), "GOROOT="+goRoot, "GOPATH="+goPath)
-
- // add $GOROOT/bin and $GOPATH/bin to PATH
- for i, e := range env {
- const p = "PATH="
- if !strings.HasPrefix(e, p) {
- continue
- }
- sep := string(os.PathListSeparator)
- env[i] = p + filepath.Join(goRoot, "bin") + sep + filepath.Join(goPath, "bin") + sep + e[len(p):]
- }
-
- // HACK: check out to new sub-repo location instead of old location.
- pkg = strings.Replace(pkg, "code.google.com/p/go.", "golang.org/x/", 1)
-
- // fetch package and dependencies
- var outbuf bytes.Buffer
- err := run(exec.Command(goTool, "get", "-d", pkg+"/..."), runEnv(env), allOutput(&outbuf), runDir(goPath))
- if err != nil {
- return outbuf.String(), err
- }
- outbuf.Reset()
-
- // hg update to the specified hash
- pkgmaster, err := vcs.RepoRootForImportPath(pkg, *verbose)
- if err != nil {
- return "", fmt.Errorf("Error finding subrepo (%s): %s", pkg, err)
- }
- repo := &Repo{
- Path: filepath.Join(goPath, "src", pkg),
- Master: pkgmaster,
- }
- if err := repo.UpdateTo(hash); err != nil {
- return "", err
- }
-
- // test the package
- err = run(exec.Command(goTool, "test", "-short", pkg+"/..."),
- runTimeout(*buildTimeout), runEnv(env), allOutput(&outbuf), runDir(goPath))
- return outbuf.String(), err
-}
-
-// repoForTool returns the correct RepoRoot for the buildTool, or an error if
-// the tool is unknown.
-func repoForTool() (*vcs.RepoRoot, error) {
- switch *buildTool {
- case "go":
- return vcs.RepoRootForImportPath(*gcPath, *verbose)
- case "gccgo":
- return vcs.RepoRootForImportPath(gofrontendImportPath, *verbose)
- default:
- return nil, fmt.Errorf("unknown build tool: %s", *buildTool)
- }
-}
-
-func isDirectory(name string) bool {
- s, err := os.Stat(name)
- return err == nil && s.IsDir()
-}
-
-func isFile(name string) bool {
- s, err := os.Stat(name)
- return err == nil && !s.IsDir()
-}
-
-// defaultSuffix returns file extension used for command files in
-// current os environment.
-func defaultSuffix() string {
- switch runtime.GOOS {
- case "windows":
- return ".bat"
- case "plan9":
- return ".rc"
- default:
- return ".bash"
- }
-}
-
-func defaultExeExt() string {
- switch runtime.GOOS {
- case "windows":
- return ".exe"
- default:
- return ""
- }
-}
-
-// defaultBuildRoot returns default buildroot directory.
-func defaultBuildRoot() string {
- var d string
- if runtime.GOOS == "windows" {
- // will use c:\, otherwise absolute paths become too long
- // during builder run, see http://golang.org/issue/3358.
- d = `c:\`
- } else {
- d = os.TempDir()
- }
- return filepath.Join(d, "gobuilder")
-}
-
-// removePath is a more robust version of os.RemoveAll.
-// On windows, if remove fails (which can happen if test/benchmark timeouts
-// and keeps some files open) it tries to rename the dir.
-func removePath(path string) error {
- if err := os.RemoveAll(path); err != nil {
- if runtime.GOOS == "windows" {
- err = os.Rename(path, filepath.Clean(path)+"_remove_me")
- }
- return err
- }
- return nil
-}
diff --git a/dashboard/cmd/builder/vcs.go b/dashboard/cmd/builder/vcs.go
deleted file mode 100644
index 2139a90..0000000
--- a/dashboard/cmd/builder/vcs.go
+++ /dev/null
@@ -1,225 +0,0 @@
-// Copyright 2013 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
- "bytes"
- "encoding/xml"
- "fmt"
- "os"
- "os/exec"
- "path/filepath"
- "strings"
- "sync"
-
- "golang.org/x/tools/go/vcs"
-)
-
-// Repo represents a mercurial repository.
-type Repo struct {
- Path string
- Master *vcs.RepoRoot
- sync.Mutex
-}
-
-// RemoteRepo constructs a *Repo representing a remote repository.
-func RemoteRepo(url, path string) (*Repo, error) {
- rr, err := vcs.RepoRootForImportPath(url, *verbose)
- if err != nil {
- return nil, err
- }
- return &Repo{
- Path: path,
- Master: rr,
- }, nil
-}
-
-// Clone clones the current Repo to a new destination
-// returning a new *Repo if successful.
-func (r *Repo) Clone(path, rev string) (*Repo, error) {
- r.Lock()
- defer r.Unlock()
-
- err := timeout(*cmdTimeout, func() error {
- downloadPath := r.Path
- if !r.Exists() {
- downloadPath = r.Master.Repo
- }
- if rev == "" {
- return r.Master.VCS.Create(path, downloadPath)
- }
- return r.Master.VCS.CreateAtRev(path, downloadPath, rev)
- })
- if err != nil {
- return nil, err
- }
- return &Repo{
- Path: path,
- Master: r.Master,
- }, nil
-}
-
-// Export exports the current Repo at revision rev to a new destination.
-func (r *Repo) Export(path, rev string) error {
- // TODO(adg,cmang): implement Export in go/vcs
- _, err := r.Clone(path, rev)
- return err
-}
-
-// UpdateTo updates the working copy of this Repo to the
-// supplied revision.
-func (r *Repo) UpdateTo(hash string) error {
- r.Lock()
- defer r.Unlock()
-
- if r.Master.VCS.Cmd == "git" {
- cmd := exec.Command("git", "reset", "--hard", hash)
- var log bytes.Buffer
- err := run(cmd, runTimeout(*cmdTimeout), runDir(r.Path), allOutput(&log))
- if err != nil {
- return fmt.Errorf("Error running git update -C %v: %v ; output=%s", hash, err, log.Bytes())
- }
- return nil
- }
-
- // Else go down three more levels of abstractions, at
- // least two of which are broken for git.
- return timeout(*cmdTimeout, func() error {
- return r.Master.VCS.TagSync(r.Path, hash)
- })
-}
-
-// Exists reports whether this Repo represents a valid Mecurial repository.
-func (r *Repo) Exists() bool {
- fi, err := os.Stat(filepath.Join(r.Path, "."+r.Master.VCS.Cmd))
- if err != nil {
- return false
- }
- return fi.IsDir()
-}
-
-// Pull pulls changes from the default path, that is, the path
-// this Repo was cloned from.
-func (r *Repo) Pull() error {
- r.Lock()
- defer r.Unlock()
-
- return timeout(*cmdTimeout, func() error {
- return r.Master.VCS.Download(r.Path)
- })
-}
-
-// Log returns the changelog for this repository.
-func (r *Repo) Log() ([]HgLog, error) {
- if err := r.Pull(); err != nil {
- return nil, err
- }
- r.Lock()
- defer r.Unlock()
-
- var logStruct struct {
- Log []HgLog
- }
- err := timeout(*cmdTimeout, func() error {
- data, err := r.Master.VCS.Log(r.Path, xmlLogTemplate)
- if err != nil {
- return err
- }
-
- // We have a commit with description that contains 0x1b byte.
- // Mercurial does not escape it, but xml.Unmarshal does not accept it.
- data = bytes.Replace(data, []byte{0x1b}, []byte{'?'}, -1)
-
- err = xml.Unmarshal([]byte("<Top>"+string(data)+"</Top>"), &logStruct)
- if err != nil {
- return fmt.Errorf("unmarshal %s log: %v", r.Master.VCS, err)
- }
- return nil
- })
- if err != nil {
- return nil, err
- }
- for i, log := range logStruct.Log {
- // Let's pretend there can be only one parent.
- if log.Parent != "" && strings.Contains(log.Parent, " ") {
- logStruct.Log[i].Parent = strings.Split(log.Parent, " ")[0]
- }
- }
- return logStruct.Log, nil
-}
-
-// FullHash returns the full hash for the given Git or Mercurial revision.
-func (r *Repo) FullHash(rev string) (string, error) {
- r.Lock()
- defer r.Unlock()
-
- var hash string
- err := timeout(*cmdTimeout, func() error {
- var data []byte
- // Avoid the vcs package for git, since it's broken
- // for git, and and we're trying to remove levels of
- // abstraction which are increasingly getting
- // difficult to navigate.
- if r.Master.VCS.Cmd == "git" {
- cmd := exec.Command("git", "rev-parse", rev)
- var out bytes.Buffer
- err := run(cmd, runTimeout(*cmdTimeout), runDir(r.Path), allOutput(&out))
- data = out.Bytes()
- if err != nil {
- return fmt.Errorf("Failed to find FullHash of %q; git rev-parse: %v, %s", rev, err, data)
- }
- } else {
- var err error
- data, err = r.Master.VCS.LogAtRev(r.Path, rev, "{node}")
- if err != nil {
- return err
- }
- }
- s := strings.TrimSpace(string(data))
- if s == "" {
- return fmt.Errorf("cannot find revision")
- }
- if len(s) != 40 { // correct for both hg and git
- return fmt.Errorf("%s returned invalid hash: %s", r.Master.VCS, s)
- }
- hash = s
- return nil
- })
- if err != nil {
- return "", err
- }
- return hash, nil
-}
-
-// HgLog represents a single Mercurial revision.
-type HgLog struct {
- Hash string
- Author string
- Date string
- Desc string
- Parent string
- Branch string
- Files string
-
- // Internal metadata
- added bool
- bench bool // needs to be benchmarked?
-}
-
-// xmlLogTemplate is a template to pass to Mercurial to make
-// hg log print the log in valid XML for parsing with xml.Unmarshal.
-// Can not escape branches and files, because it crashes python with:
-// AttributeError: 'NoneType' object has no attribute 'replace'
-const xmlLogTemplate = `
- <Log>
- <Hash>{node|escape}</Hash>
- <Parent>{p1node}</Parent>
- <Author>{author|escape}</Author>
- <Date>{date|rfc3339date}</Date>
- <Desc>{desc|escape}</Desc>
- <Branch>{branches}</Branch>
- <Files>{files}</Files>
- </Log>
-`
diff --git a/dashboard/cmd/buildlet/.gitignore b/dashboard/cmd/buildlet/.gitignore
deleted file mode 100644
index bbd21a2..0000000
--- a/dashboard/cmd/buildlet/.gitignore
+++ /dev/null
@@ -1,5 +0,0 @@
-buildlet
-buildlet.*-*
-stage0/buildlet-stage0.*
-cert.pem
-key.pem
diff --git a/dashboard/cmd/buildlet/Makefile b/dashboard/cmd/buildlet/Makefile
deleted file mode 100644
index 3edd8f8..0000000
--- a/dashboard/cmd/buildlet/Makefile
+++ /dev/null
@@ -1,31 +0,0 @@
-buildlet: buildlet.go
- go build --tags=extdep
-
-buildlet.darwin-amd64: buildlet.go
- GOOS=darwin GOARCH=amd64 go build -o $@ --tags=extdep
- cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
-
-buildlet.freebsd-amd64: buildlet.go
- GOOS=freebsd GOARCH=amd64 go build -o $@ --tags=extdep
- cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
-
-buildlet.linux-amd64: buildlet.go
- GOOS=linux GOARCH=amd64 go build -o $@ --tags=extdep
- cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
-
-buildlet.netbsd-amd64: buildlet.go
- GOOS=netbsd GOARCH=amd64 go build -o $@ --tags=extdep
- cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
-
-buildlet.openbsd-amd64: buildlet.go
- GOOS=openbsd GOARCH=amd64 go build -o $@ --tags=extdep
- cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
-
-buildlet.plan9-386: buildlet.go
- GOOS=plan9 GOARCH=386 go build -o $@ --tags=extdep
- cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
-
-buildlet.windows-amd64: buildlet.go
- GOOS=windows GOARCH=amd64 go build -o $@ --tags=extdep
- cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
-
diff --git a/dashboard/cmd/buildlet/README b/dashboard/cmd/buildlet/README
deleted file mode 100644
index 0dd68cf..0000000
--- a/dashboard/cmd/buildlet/README
+++ /dev/null
@@ -1,12 +0,0 @@
-Local development notes:
-
-Server: (TLS stuff is optional)
-$ go run $GOROOT/src/crypto/tls/generate_cert.go --host=example.com
-$ GCEMETA_password=foo GCEMETA_tls_cert=@cert.pem GCEMETA_tls_key='@key.pem' ./buildlet
-
-Client:
-$ curl -O https://go.googlesource.com/go/+archive/3b76b017cabb.tar.gz
-$ curl -k --user :foo -X PUT --data-binary "@go-3b76b017cabb.tar.gz" https://localhost:5936/writetgz
-$ curl -k --user :foo -d "cmd=src/make.bash" http://127.0.0.1:5937/exec
-etc
-
diff --git a/dashboard/cmd/buildlet/buildlet.go b/dashboard/cmd/buildlet/buildlet.go
deleted file mode 100644
index 95cc2ba..0000000
--- a/dashboard/cmd/buildlet/buildlet.go
+++ /dev/null
@@ -1,654 +0,0 @@
-// Copyright 2014 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-// The buildlet is an HTTP server that untars content to disk and runs
-// commands it has untarred, streaming their output back over HTTP.
-// It is part of Go's continuous build system.
-//
-// This program intentionally allows remote code execution, and
-// provides no security of its own. It is assumed that any user uses
-// it with an appropriately-configured firewall between their VM
-// instances.
-package main // import "golang.org/x/tools/dashboard/cmd/buildlet"
-
-import (
- "archive/tar"
- "compress/gzip"
- "crypto/tls"
- "errors"
- "flag"
- "fmt"
- "io"
- "io/ioutil"
- "log"
- "net"
- "net/http"
- "net/url"
- "os"
- "os/exec"
- "path"
- "path/filepath"
- "runtime"
- "strconv"
- "strings"
- "time"
-
- "google.golang.org/cloud/compute/metadata"
-)
-
-var (
- haltEntireOS = flag.Bool("halt", true, "halt OS in /halt handler. If false, the buildlet process just ends.")
- scratchDir = flag.String("scratchdir", "", "Temporary directory to use. The contents of this directory may be deleted at any time. If empty, TempDir is used to create one.")
- listenAddr = flag.String("listen", defaultListenAddr(), "address to listen on. Warning: this service is inherently insecure and offers no protection of its own. Do not expose this port to the world.")
-)
-
-func defaultListenAddr() string {
- if runtime.GOOS == "darwin" {
- // Darwin will never run on GCE, so let's always
- // listen on a high port (so we don't need to be
- // root).
- return ":5936"
- }
- if !metadata.OnGCE() {
- return "localhost:5936"
- }
- // In production, default to port 80 or 443, depending on
- // whether TLS is configured.
- if metadataValue("tls-cert") != "" {
- return ":443"
- }
- return ":80"
-}
-
-var osHalt func() // set by some machines
-
-func main() {
- flag.Parse()
- onGCE := metadata.OnGCE()
- if !onGCE && !strings.HasPrefix(*listenAddr, "localhost:") {
- log.Printf("** WARNING *** This server is unsafe and offers no security. Be careful.")
- }
- if onGCE {
- fixMTU()
- }
- if runtime.GOOS == "plan9" {
- // Plan 9 is too slow on GCE, so stop running run.rc after the basics.
- // See https://golang.org/cl/2522 and https://golang.org/issue/9491
- // TODO(bradfitz): once the buildlet has environment variable support,
- // the coordinator can send this in, and this variable can be part of
- // the build configuration struct instead of hard-coded here.
- // But no need for environment variables quite yet.
- os.Setenv("GOTESTONLY", "std")
- }
- if *scratchDir == "" {
- dir, err := ioutil.TempDir("", "buildlet-scatch")
- if err != nil {
- log.Fatalf("error creating scratchdir with ioutil.TempDir: %v", err)
- }
- *scratchDir = dir
- }
- // TODO(bradfitz): if this becomes more of a general tool,
- // perhaps we want to remove this hard-coded here. Also,
- // if/once the exec handler ever gets generic environment
- // variable support, it would make sense to remove this too
- // and push it to the client. This hard-codes policy. But
- // that's okay for now.
- os.Setenv("GOROOT_BOOTSTRAP", filepath.Join(*scratchDir, "go1.4"))
- os.Setenv("WORKDIR", *scratchDir) // mostly for demos
-
- if _, err := os.Lstat(*scratchDir); err != nil {
- log.Fatalf("invalid --scratchdir %q: %v", *scratchDir, err)
- }
- http.HandleFunc("/", handleRoot)
- http.HandleFunc("/debug/goroutines", handleGoroutines)
- http.HandleFunc("/debug/x", handleX)
-
- password := metadataValue("password")
- requireAuth := func(handler func(w http.ResponseWriter, r *http.Request)) http.Handler {
- return requirePasswordHandler{http.HandlerFunc(handler), password}
- }
- http.Handle("/writetgz", requireAuth(handleWriteTGZ))
- http.Handle("/exec", requireAuth(handleExec))
- http.Handle("/halt", requireAuth(handleHalt))
- http.Handle("/tgz", requireAuth(handleGetTGZ))
- // TODO: removeall
-
- tlsCert, tlsKey := metadataValue("tls-cert"), metadataValue("tls-key")
- if (tlsCert == "") != (tlsKey == "") {
- log.Fatalf("tls-cert and tls-key must both be supplied, or neither.")
- }
-
- log.Printf("Listening on %s ...", *listenAddr)
- ln, err := net.Listen("tcp", *listenAddr)
- if err != nil {
- log.Fatalf("Failed to listen on %s: %v", *listenAddr, err)
- }
- ln = tcpKeepAliveListener{ln.(*net.TCPListener)}
-
- var srv http.Server
- if tlsCert != "" {
- cert, err := tls.X509KeyPair([]byte(tlsCert), []byte(tlsKey))
- if err != nil {
- log.Fatalf("TLS cert error: %v", err)
- }
- tlsConf := &tls.Config{
- Certificates: []tls.Certificate{cert},
- }
- ln = tls.NewListener(ln, tlsConf)
- }
-
- log.Fatalf("Serve: %v", srv.Serve(ln))
-}
-
-// metadataValue returns the GCE metadata instance value for the given key.
-// If the metadata is not defined, the returned string is empty.
-//
-// If not running on GCE, it falls back to using environment variables
-// for local development.
-func metadataValue(key string) string {
- // The common case:
- if metadata.OnGCE() {
- v, err := metadata.InstanceAttributeValue(key)
- if _, notDefined := err.(metadata.NotDefinedError); notDefined {
- return ""
- }
- if err != nil {
- log.Fatalf("metadata.InstanceAttributeValue(%q): %v", key, err)
- }
- return v
- }
-
- // Else let developers use environment variables to fake
- // metadata keys, for local testing.
- envKey := "GCEMETA_" + strings.Replace(key, "-", "_", -1)
- v := os.Getenv(envKey)
- // Respect curl-style '@' prefix to mean the rest is a filename.
- if strings.HasPrefix(v, "@") {
- slurp, err := ioutil.ReadFile(v[1:])
- if err != nil {
- log.Fatalf("Error reading file for GCEMETA_%v: %v", key, err)
- }
- return string(slurp)
- }
- if v == "" {
- log.Printf("Warning: not running on GCE, and no %v environment variable defined", envKey)
- }
- return v
-}
-
-// tcpKeepAliveListener is a net.Listener that sets TCP keep-alive
-// timeouts on accepted connections.
-type tcpKeepAliveListener struct {
- *net.TCPListener
-}
-
-func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
- tc, err := ln.AcceptTCP()
- if err != nil {
- return
- }
- tc.SetKeepAlive(true)
- tc.SetKeepAlivePeriod(3 * time.Minute)
- return tc, nil
-}
-
-func fixMTU_freebsd() error { return fixMTU_ifconfig("vtnet0") }
-func fixMTU_openbsd() error { return fixMTU_ifconfig("vio0") }
-func fixMTU_ifconfig(iface string) error {
- out, err := exec.Command("/sbin/ifconfig", iface, "mtu", "1460").CombinedOutput()
- if err != nil {
- return fmt.Errorf("/sbin/ifconfig %s mtu 1460: %v, %s", iface, err, out)
- }
- return nil
-}
-
-func fixMTU_plan9() error {
- f, err := os.OpenFile("/net/ipifc/0/ctl", os.O_WRONLY, 0)
- if err != nil {
- return err
- }
- if _, err := io.WriteString(f, "mtu 1400\n"); err != nil { // not 1460
- f.Close()
- return err
- }
- return f.Close()
-}
-
-func fixMTU() {
- fn, ok := map[string]func() error{
- "openbsd": fixMTU_openbsd,
- "freebsd": fixMTU_freebsd,
- "plan9": fixMTU_plan9,
- }[runtime.GOOS]
- if ok {
- if err := fn(); err != nil {
- log.Printf("Failed to set MTU: %v", err)
- } else {
- log.Printf("Adjusted MTU.")
- }
- }
-}
-
-// mtuWriter is a hack for environments where we can't (or can't yet)
-// fix the machine's MTU.
-// Instead of telling the operating system the MTU, we just cut up our
-// writes into small pieces to make sure we don't get too near the
-// MTU, and we hope the kernel doesn't coalesce different flushed
-// writes back together into the same TCP IP packets.
-type mtuWriter struct {
- rw http.ResponseWriter
-}
-
-func (mw mtuWriter) Write(p []byte) (n int, err error) {
- const mtu = 1000 // way less than 1460; since HTTP response headers might be in there too
- for len(p) > 0 {
- chunk := p
- if len(chunk) > mtu {
- chunk = p[:mtu]
- }
- n0, err := mw.rw.Write(chunk)
- n += n0
- if n0 != len(chunk) && err == nil {
- err = io.ErrShortWrite
- }
- if err != nil {
- return n, err
- }
- p = p[n0:]
- mw.rw.(http.Flusher).Flush()
- if len(p) > 0 {
- // Whitelisted operating systems:
- if runtime.GOOS == "openbsd" || runtime.GOOS == "linux" {
- // Nothing
- } else {
- // Try to prevent the kernel from Nagel-ing the IP packets
- // together into one that's too large.
- time.Sleep(250 * time.Millisecond)
- }
- }
- }
- return n, nil
-}
-
-func handleRoot(w http.ResponseWriter, r *http.Request) {
- if r.URL.Path != "/" {
- http.NotFound(w, r)
- return
- }
- fmt.Fprintf(w, "buildlet running on %s-%s\n", runtime.GOOS, runtime.GOARCH)
-}
-
-// unauthenticated /debug/goroutines handler
-func handleGoroutines(rw http.ResponseWriter, r *http.Request) {
- w := mtuWriter{rw}
- log.Printf("Dumping goroutines.")
- rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
- buf := make([]byte, 2<<20)
- buf = buf[:runtime.Stack(buf, true)]
- w.Write(buf)
- log.Printf("Dumped goroutines.")
-}
-
-// unauthenticated /debug/x handler, to test MTU settings.
-func handleX(w http.ResponseWriter, r *http.Request) {
- n, _ := strconv.Atoi(r.FormValue("n"))
- if n > 1<<20 {
- n = 1 << 20
- }
- log.Printf("Dumping %d X.", n)
- w.Header().Set("Content-Type", "text/plain; charset=utf-8")
- buf := make([]byte, n)
- for i := range buf {
- buf[i] = 'X'
- }
- w.Write(buf)
- log.Printf("Dumped X.")
-}
-
-// This is a remote code execution daemon, so security is kinda pointless, but:
-func validRelativeDir(dir string) bool {
- if strings.Contains(dir, `\`) || path.IsAbs(dir) {
- return false
- }
- dir = path.Clean(dir)
- if strings.HasPrefix(dir, "../") || strings.HasSuffix(dir, "/..") || dir == ".." {
- return false
- }
- return true
-}
-
-func handleGetTGZ(rw http.ResponseWriter, r *http.Request) {
- if r.Method != "GET" {
- http.Error(rw, "requires GET method", http.StatusBadRequest)
- return
- }
- dir := r.FormValue("dir")
- if !validRelativeDir(dir) {
- http.Error(rw, "bogus dir", http.StatusBadRequest)
- return
- }
- zw := gzip.NewWriter(mtuWriter{rw})
- tw := tar.NewWriter(zw)
- base := filepath.Join(*scratchDir, filepath.FromSlash(dir))
- err := filepath.Walk(base, func(path string, fi os.FileInfo, err error) error {
- if err != nil {
- return err
- }
- rel := strings.TrimPrefix(strings.TrimPrefix(path, base), "/")
- var linkName string
- if fi.Mode()&os.ModeSymlink != 0 {
- linkName, err = os.Readlink(path)
- if err != nil {
- return err
- }
- }
- th, err := tar.FileInfoHeader(fi, linkName)
- if err != nil {
- return err
- }
- th.Name = rel
- if fi.IsDir() && !strings.HasSuffix(th.Name, "/") {
- th.Name += "/"
- }
- if th.Name == "/" {
- return nil
- }
- if err := tw.WriteHeader(th); err != nil {
- return err
- }
- if fi.Mode().IsRegular() {
- f, err := os.Open(path)
- if err != nil {
- return err
- }
- defer f.Close()
- if _, err := io.Copy(tw, f); err != nil {
- return err
- }
- }
- return nil
- })
- if err != nil {
- log.Printf("Walk error: %v", err)
- // Decent way to signal failure to the caller, since it'll break
- // the chunked response, rather than have a valid EOF.
- conn, _, _ := rw.(http.Hijacker).Hijack()
- conn.Close()
- }
- tw.Close()
- zw.Close()
-}
-
-func handleWriteTGZ(w http.ResponseWriter, r *http.Request) {
- var tgz io.Reader
- switch r.Method {
- case "PUT":
- tgz = r.Body
- case "POST":
- urlStr := r.FormValue("url")
- if urlStr == "" {
- http.Error(w, "missing url POST param", http.StatusBadRequest)
- return
- }
- res, err := http.Get(urlStr)
- if err != nil {
- http.Error(w, fmt.Sprintf("fetching URL %s: %v", urlStr, err), http.StatusInternalServerError)
- return
- }
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- http.Error(w, fmt.Sprintf("fetching provided url: %s", res.Status), http.StatusInternalServerError)
- return
- }
- tgz = res.Body
- default:
- http.Error(w, "requires PUT or POST method", http.StatusBadRequest)
- return
- }
-
- urlParam, _ := url.ParseQuery(r.URL.RawQuery)
- baseDir := *scratchDir
- if dir := urlParam.Get("dir"); dir != "" {
- if !validRelativeDir(dir) {
- http.Error(w, "bogus dir", http.StatusBadRequest)
- return
- }
- dir = filepath.FromSlash(dir)
- baseDir = filepath.Join(baseDir, dir)
- if err := os.MkdirAll(baseDir, 0755); err != nil {
- http.Error(w, "mkdir of base: "+err.Error(), http.StatusInternalServerError)
- return
- }
- }
-
- err := untar(tgz, baseDir)
- if err != nil {
- status := http.StatusInternalServerError
- if he, ok := err.(httpStatuser); ok {
- status = he.httpStatus()
- }
- http.Error(w, err.Error(), status)
- return
- }
- io.WriteString(w, "OK")
-}
-
-// untar reads the gzip-compressed tar file from r and writes it into dir.
-func untar(r io.Reader, dir string) error {
- zr, err := gzip.NewReader(r)
- if err != nil {
- return badRequest("requires gzip-compressed body: " + err.Error())
- }
- tr := tar.NewReader(zr)
- for {
- f, err := tr.Next()
- if err == io.EOF {
- break
- }
- if err != nil {
- log.Printf("tar reading error: %v", err)
- return badRequest("tar error: " + err.Error())
- }
- if !validRelPath(f.Name) {
- return badRequest(fmt.Sprintf("tar file contained invalid name %q", f.Name))
- }
- rel := filepath.FromSlash(f.Name)
- abs := filepath.Join(dir, rel)
-
- fi := f.FileInfo()
- mode := fi.Mode()
- switch {
- case mode.IsRegular():
- // Make the directory. This is redundant because it should
- // already be made by a directory entry in the tar
- // beforehand. Thus, don't check for errors; the next
- // write will fail with the same error.
- os.MkdirAll(filepath.Dir(abs), 0755)
- wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm())
- if err != nil {
- return err
- }
- n, err := io.Copy(wf, tr)
- if closeErr := wf.Close(); closeErr != nil && err == nil {
- err = closeErr
- }
- if err != nil {
- return fmt.Errorf("error writing to %s: %v", abs, err)
- }
- if n != f.Size {
- return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size)
- }
- log.Printf("wrote %s", abs)
- case mode.IsDir():
- if err := os.MkdirAll(abs, 0755); err != nil {
- return err
- }
- default:
- return badRequest(fmt.Sprintf("tar file entry %s contained unsupported file type %v", f.Name, mode))
- }
- }
- return nil
-}
-
-// Process-State is an HTTP Trailer set in the /exec handler to "ok"
-// on success, or os.ProcessState.String() on failure.
-const hdrProcessState = "Process-State"
-
-func handleExec(w http.ResponseWriter, r *http.Request) {
- cn := w.(http.CloseNotifier)
- clientGone := cn.CloseNotify()
- handlerDone := make(chan bool)
- defer close(handlerDone)
-
- if r.Method != "POST" {
- http.Error(w, "requires POST method", http.StatusBadRequest)
- return
- }
- if r.ProtoMajor*10+r.ProtoMinor < 11 {
- // We need trailers, only available in HTTP/1.1 or HTTP/2.
- http.Error(w, "HTTP/1.1 or higher required", http.StatusBadRequest)
- return
- }
-
- w.Header().Set("Trailer", hdrProcessState) // declare it so we can set it
-
- cmdPath := r.FormValue("cmd") // required
- absCmd := cmdPath
- sysMode := r.FormValue("mode") == "sys"
- if sysMode {
- if cmdPath == "" {
- http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest)
- return
- }
- } else {
- if !validRelPath(cmdPath) {
- http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest)
- return
- }
- absCmd = filepath.Join(*scratchDir, filepath.FromSlash(cmdPath))
- }
-
- if f, ok := w.(http.Flusher); ok {
- f.Flush()
- }
-
- cmd := exec.Command(absCmd, r.PostForm["cmdArg"]...)
- if sysMode {
- cmd.Dir = *scratchDir
- } else {
- cmd.Dir = filepath.Dir(absCmd)
- }
- cmdOutput := mtuWriter{w}
- cmd.Stdout = cmdOutput
- cmd.Stderr = cmdOutput
- err := cmd.Start()
- if err == nil {
- go func() {
- select {
- case <-clientGone:
- cmd.Process.Kill()
- case <-handlerDone:
- return
- }
- }()
- err = cmd.Wait()
- }
- state := "ok"
- if err != nil {
- if ps := cmd.ProcessState; ps != nil {
- state = ps.String()
- } else {
- state = err.Error()
- }
- }
- w.Header().Set(hdrProcessState, state)
- log.Printf("Run = %s", state)
-}
-
-func handleHalt(w http.ResponseWriter, r *http.Request) {
- if r.Method != "POST" {
- http.Error(w, "requires POST method", http.StatusBadRequest)
- return
- }
- log.Printf("Halting in 1 second.")
- // do the halt in 1 second, to give the HTTP response time to complete:
- time.AfterFunc(1*time.Second, haltMachine)
-}
-
-func haltMachine() {
- if !*haltEntireOS {
- log.Printf("Ending buildlet process due to halt.")
- os.Exit(0)
- return
- }
- log.Printf("Halting machine.")
- time.AfterFunc(5*time.Second, func() { os.Exit(0) })
- if osHalt != nil {
- // TODO: Windows: http://msdn.microsoft.com/en-us/library/windows/desktop/aa376868%28v=vs.85%29.aspx
- osHalt()
- os.Exit(0)
- }
- // Backup mechanism, if exec hangs for any reason:
- var err error
- switch runtime.GOOS {
- case "openbsd":
- // Quick, no fs flush, and power down:
- err = exec.Command("halt", "-q", "-n", "-p").Run()
- case "freebsd":
- // Power off (-p), via halt (-o), now.
- err = exec.Command("shutdown", "-p", "-o", "now").Run()
- case "linux":
- // Don't sync (-n), force without shutdown (-f), and power off (-p).
- err = exec.Command("/bin/halt", "-n", "-f", "-p").Run()
- case "plan9":
- err = exec.Command("fshalt").Run()
- default:
- err = errors.New("No system-specific halt command run; will just end buildlet process.")
- }
- log.Printf("Shutdown: %v", err)
- log.Printf("Ending buildlet process post-halt")
- os.Exit(0)
-}
-
-func validRelPath(p string) bool {
- if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
- return false
- }
- return true
-}
-
-type httpStatuser interface {
- error
- httpStatus() int
-}
-
-type httpError struct {
- statusCode int
- msg string
-}
-
-func (he httpError) Error() string { return he.msg }
-func (he httpError) httpStatus() int { return he.statusCode }
-
-func badRequest(msg string) error {
- return httpError{http.StatusBadRequest, msg}
-}
-
-// requirePassword is an http.Handler auth wrapper that enforces a
-// HTTP Basic password. The username is ignored.
-type requirePasswordHandler struct {
- h http.Handler
- password string // empty means no password
-}
-
-func (h requirePasswordHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
- _, gotPass, _ := r.BasicAuth()
- if h.password != "" && h.password != gotPass {
- http.Error(w, "invalid password", http.StatusForbidden)
- return
- }
- h.h.ServeHTTP(w, r)
-}
diff --git a/dashboard/cmd/buildlet/stage0/Makefile b/dashboard/cmd/buildlet/stage0/Makefile
deleted file mode 100644
index bf66be3..0000000
--- a/dashboard/cmd/buildlet/stage0/Makefile
+++ /dev/null
@@ -1,3 +0,0 @@
-buildlet-stage0.windows-amd64: stage0.go
- GOOS=windows GOARCH=amd64 go build -o $@ --tags=extdep
- cat $@ | (cd ../../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
diff --git a/dashboard/cmd/buildlet/stage0/stage0.go b/dashboard/cmd/buildlet/stage0/stage0.go
deleted file mode 100644
index f9c6e6c..0000000
--- a/dashboard/cmd/buildlet/stage0/stage0.go
+++ /dev/null
@@ -1,78 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-// The stage0 command looks up the buildlet's URL from the GCE metadata
-// service, downloads it, and runs it. It's used primarily by Windows,
-// since it can be written in a couple lines of shell elsewhere.
-package main
-
-import (
- "fmt"
- "io"
- "log"
- "net/http"
- "os"
- "os/exec"
- "path/filepath"
- "time"
-
- "google.golang.org/cloud/compute/metadata"
-)
-
-const attr = "buildlet-binary-url"
-
-func main() {
- buildletURL, err := metadata.InstanceAttributeValue(attr)
- if err != nil {
- sleepFatalf("Failed to look up %q attribute value: %v", attr, err)
- }
- target := filepath.FromSlash("./buildlet.exe")
- if err := download(target, buildletURL); err != nil {
- sleepFatalf("Downloading %s: %v", buildletURL, err)
- }
- cmd := exec.Command(target)
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
- if err := cmd.Run(); err != nil {
- sleepFatalf("Error running buildlet: %v", err)
- }
-}
-
-func sleepFatalf(format string, args ...interface{}) {
- log.Printf(format, args...)
- time.Sleep(time.Minute) // so user has time to see it in cmd.exe, maybe
- os.Exit(1)
-}
-
-func download(file, url string) error {
- log.Printf("Downloading %s to %s ...\n", url, file)
- res, err := http.Get(url)
- if err != nil {
- return fmt.Errorf("Error fetching %v: %v", url, err)
- }
- if res.StatusCode != 200 {
- return fmt.Errorf("HTTP status code of %s was %v", url, res.Status)
- }
- tmp := file + ".tmp"
- os.Remove(tmp)
- os.Remove(file)
- f, err := os.Create(tmp)
- if err != nil {
- return err
- }
- n, err := io.Copy(f, res.Body)
- res.Body.Close()
- if err != nil {
- return fmt.Errorf("Error reading %v: %v", url, err)
- }
- f.Close()
- err = os.Rename(tmp, file)
- if err != nil {
- return err
- }
- log.Printf("Downloaded %s (%d bytes)", file, n)
- return nil
-}
diff --git a/dashboard/cmd/coordinator/.gitignore b/dashboard/cmd/coordinator/.gitignore
deleted file mode 100644
index 91d0fb0..0000000
--- a/dashboard/cmd/coordinator/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-buildongce/client-*.dat
-buildongce/token.dat
-coordinator
diff --git a/dashboard/cmd/coordinator/Makefile b/dashboard/cmd/coordinator/Makefile
deleted file mode 100644
index 56887c0..0000000
--- a/dashboard/cmd/coordinator/Makefile
+++ /dev/null
@@ -1,9 +0,0 @@
-coordinator: coordinator.go
- GOOS=linux go build --tags=extdep -o coordinator .
-
-# After "make upload", either reboot the machine, or ssh to it and:
-# sudo systemctl restart gobuild.service
-# And watch its logs with:
-# sudo journalctl -f -u gobuild.service
-upload: coordinator
- cat coordinator | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/coordinator)
diff --git a/dashboard/cmd/coordinator/buildongce/create.go b/dashboard/cmd/coordinator/buildongce/create.go
deleted file mode 100644
index a6755bd..0000000
--- a/dashboard/cmd/coordinator/buildongce/create.go
+++ /dev/null
@@ -1,299 +0,0 @@
-// Copyright 2014 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-package main // import "golang.org/x/tools/dashboard/cmd/coordinator/buildongce"
-
-import (
- "bufio"
- "encoding/json"
- "flag"
- "fmt"
- "io/ioutil"
- "log"
- "os"
- "strings"
- "time"
-
- "golang.org/x/oauth2"
- "golang.org/x/oauth2/google"
- compute "google.golang.org/api/compute/v1"
-)
-
-var (
- proj = flag.String("project", "symbolic-datum-552", "name of Project")
- zone = flag.String("zone", "us-central1-a", "GCE zone")
- mach = flag.String("machinetype", "n1-standard-16", "Machine type")
- instName = flag.String("instance_name", "go-builder-1", "Name of VM instance.")
- sshPub = flag.String("ssh_public_key", "", "ssh public key file to authorize. Can modify later in Google's web UI anyway.")
- staticIP = flag.String("static_ip", "", "Static IP to use. If empty, automatic.")
- reuseDisk = flag.Bool("reuse_disk", true, "Whether disk images should be reused between shutdowns/restarts.")
- ssd = flag.Bool("ssd", false, "use a solid state disk (faster, more expensive)")
-)
-
-func readFile(v string) string {
- slurp, err := ioutil.ReadFile(v)
- if err != nil {
- log.Fatalf("Error reading %s: %v", v, err)
- }
- return strings.TrimSpace(string(slurp))
-}
-
-var config = &oauth2.Config{
- // The client-id and secret should be for an "Installed Application" when using
- // the CLI. Later we'll use a web application with a callback.
- ClientID: readFile("client-id.dat"),
- ClientSecret: readFile("client-secret.dat"),
- Endpoint: google.Endpoint,
- Scopes: []string{
- compute.DevstorageFull_controlScope,
- compute.ComputeScope,
- "https://www.googleapis.com/auth/sqlservice",
- "https://www.googleapis.com/auth/sqlservice.admin",
- },
- RedirectURL: "urn:ietf:wg:oauth:2.0:oob",
-}
-
-const baseConfig = `#cloud-config
-coreos:
- update:
- group: alpha
- reboot-strategy: off
- units:
- - name: gobuild.service
- command: start
- content: |
- [Unit]
- Description=Go Builders
- After=docker.service
- Requires=docker.service
-
- [Service]
- ExecStartPre=/bin/bash -c 'mkdir -p /opt/bin && curl -s -o /opt/bin/coordinator http://storage.googleapis.com/go-builder-data/coordinator && chmod +x /opt/bin/coordinator'
- ExecStart=/opt/bin/coordinator
- RestartSec=10s
- Restart=always
- Type=simple
-
- [Install]
- WantedBy=multi-user.target
-`
-
-func main() {
- flag.Parse()
- if *proj == "" {
- log.Fatalf("Missing --project flag")
- }
- prefix := "https://www.googleapis.com/compute/v1/projects/" + *proj
- machType := prefix + "/zones/" + *zone + "/machineTypes/" + *mach
-
- const tokenFileName = "token.dat"
- tokenFile := tokenCacheFile(tokenFileName)
- tokenSource := oauth2.ReuseTokenSource(nil, tokenFile)
- token, err := tokenSource.Token()
- if err != nil {
- log.Printf("Error getting token from %s: %v", tokenFileName, err)
- log.Printf("Get auth code from %v", config.AuthCodeURL("my-state"))
- fmt.Print("\nEnter auth code: ")
- sc := bufio.NewScanner(os.Stdin)
- sc.Scan()
- authCode := strings.TrimSpace(sc.Text())
- token, err = config.Exchange(oauth2.NoContext, authCode)
- if err != nil {
- log.Fatalf("Error exchanging auth code for a token: %v", err)
- }
- if err := tokenFile.WriteToken(token); err != nil {
- log.Fatalf("Error writing to %s: %v", tokenFileName, err)
- }
- tokenSource = oauth2.ReuseTokenSource(token, nil)
- }
-
- oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
-
- computeService, _ := compute.New(oauthClient)
-
- natIP := *staticIP
- if natIP == "" {
- // Try to find it by name.
- aggAddrList, err := computeService.Addresses.AggregatedList(*proj).Do()
- if err != nil {
- log.Fatal(err)
- }
- // https://godoc.org/google.golang.org/api/compute/v1#AddressAggregatedList
- IPLoop:
- for _, asl := range aggAddrList.Items {
- for _, addr := range asl.Addresses {
- if addr.Name == *instName+"-ip" && addr.Status == "RESERVED" {
- natIP = addr.Address
- break IPLoop
- }
- }
- }
- }
-
- cloudConfig := baseConfig
- if *sshPub != "" {
- key := strings.TrimSpace(readFile(*sshPub))
- cloudConfig += fmt.Sprintf("\nssh_authorized_keys:\n - %s\n", key)
- }
- if os.Getenv("USER") == "bradfitz" {
- cloudConfig += fmt.Sprintf("\nssh_authorized_keys:\n - %s\n", "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAwks9dwWKlRC+73gRbvYtVg0vdCwDSuIlyt4z6xa/YU/jTDynM4R4W10hm2tPjy8iR1k8XhDv4/qdxe6m07NjG/By1tkmGpm1mGwho4Pr5kbAAy/Qg+NLCSdAYnnE00FQEcFOC15GFVMOW2AzDGKisReohwH9eIzHPzdYQNPRWXE= bradfitz@papag.bradfitz.com")
- }
- const maxCloudConfig = 32 << 10 // per compute API docs
- if len(cloudConfig) > maxCloudConfig {
- log.Fatalf("cloud config length of %d bytes is over %d byte limit", len(cloudConfig), maxCloudConfig)
- }
-
- instance := &compute.Instance{
- Name: *instName,
- Description: "Go Builder",
- MachineType: machType,
- Disks: []*compute.AttachedDisk{instanceDisk(computeService)},
- Tags: &compute.Tags{
- Items: []string{"http-server", "https-server"},
- },
- Metadata: &compute.Metadata{
- Items: []*compute.MetadataItems{
- {
- Key: "user-data",
- Value: cloudConfig,
- },
- },
- },
- NetworkInterfaces: []*compute.NetworkInterface{
- &compute.NetworkInterface{
- AccessConfigs: []*compute.AccessConfig{
- &compute.AccessConfig{
- Type: "ONE_TO_ONE_NAT",
- Name: "External NAT",
- NatIP: natIP,
- },
- },
- Network: prefix + "/global/networks/default",
- },
- },
- ServiceAccounts: []*compute.ServiceAccount{
- {
- Email: "default",
- Scopes: []string{
- compute.DevstorageFull_controlScope,
- compute.ComputeScope,
- },
- },
- },
- }
-
- log.Printf("Creating instance...")
- op, err := computeService.Instances.Insert(*proj, *zone, instance).Do()
- if err != nil {
- log.Fatalf("Failed to create instance: %v", err)
- }
- opName := op.Name
- log.Printf("Created. Waiting on operation %v", opName)
-OpLoop:
- for {
- time.Sleep(2 * time.Second)
- op, err := computeService.ZoneOperations.Get(*proj, *zone, opName).Do()
- if err != nil {
- log.Fatalf("Failed to get op %s: %v", opName, err)
- }
- switch op.Status {
- case "PENDING", "RUNNING":
- log.Printf("Waiting on operation %v", opName)
- continue
- case "DONE":
- if op.Error != nil {
- for _, operr := range op.Error.Errors {
- log.Printf("Error: %+v", operr)
- }
- log.Fatalf("Failed to start.")
- }
- log.Printf("Success. %+v", op)
- break OpLoop
- default:
- log.Fatalf("Unknown status %q: %+v", op.Status, op)
- }
- }
-
- inst, err := computeService.Instances.Get(*proj, *zone, *instName).Do()
- if err != nil {
- log.Fatalf("Error getting instance after creation: %v", err)
- }
- ij, _ := json.MarshalIndent(inst, "", " ")
- log.Printf("Instance: %s", ij)
-}
-
-func instanceDisk(svc *compute.Service) *compute.AttachedDisk {
- const imageURL = "https://www.googleapis.com/compute/v1/projects/coreos-cloud/global/images/coreos-alpha-402-2-0-v20140807"
- diskName := *instName + "-coreos-stateless-pd"
-
- if *reuseDisk {
- dl, err := svc.Disks.List(*proj, *zone).Do()
- if err != nil {
- log.Fatalf("Error listing disks: %v", err)
- }
- for _, disk := range dl.Items {
- if disk.Name != diskName {
- continue
- }
- return &compute.AttachedDisk{
- AutoDelete: false,
- Boot: true,
- DeviceName: diskName,
- Type: "PERSISTENT",
- Source: disk.SelfLink,
- Mode: "READ_WRITE",
-
- // The GCP web UI's "Show REST API" link includes a
- // "zone" parameter, but it's not in the API
- // description. But it wants this form (disk.Zone, a
- // full zone URL, not *zone):
- // Zone: disk.Zone,
- // ... but it seems to work without it. Keep this
- // comment here until I file a bug with the GCP
- // people.
- }
- }
- }
-
- diskType := ""
- if *ssd {
- diskType = "https://www.googleapis.com/compute/v1/projects/" + *proj + "/zones/" + *zone + "/diskTypes/pd-ssd"
- }
-
- return &compute.AttachedDisk{
- AutoDelete: !*reuseDisk,
- Boot: true,
- Type: "PERSISTENT",
- InitializeParams: &compute.AttachedDiskInitializeParams{
- DiskName: diskName,
- SourceImage: imageURL,
- DiskSizeGb: 50,
- DiskType: diskType,
- },
- }
-}
-
-type tokenCacheFile string
-
-func (f tokenCacheFile) Token() (*oauth2.Token, error) {
- slurp, err := ioutil.ReadFile(string(f))
- if err != nil {
- return nil, err
- }
- t := new(oauth2.Token)
- if err := json.Unmarshal(slurp, t); err != nil {
- return nil, err
- }
- return t, nil
-}
-
-func (f tokenCacheFile) WriteToken(t *oauth2.Token) error {
- jt, err := json.Marshal(t)
- if err != nil {
- return err
- }
- return ioutil.WriteFile(string(f), jt, 0600)
-}
diff --git a/dashboard/cmd/coordinator/coordinator.go b/dashboard/cmd/coordinator/coordinator.go
deleted file mode 100644
index 900b848..0000000
--- a/dashboard/cmd/coordinator/coordinator.go
+++ /dev/null
@@ -1,1220 +0,0 @@
-// Copyright 2014 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-// The coordinator runs on GCE and coordinates builds in Docker containers.
-package main // import "golang.org/x/tools/dashboard/cmd/coordinator"
-
-import (
- "archive/tar"
- "bytes"
- "compress/gzip"
- "crypto/hmac"
- "crypto/md5"
- "crypto/rand"
- "encoding/json"
- "errors"
- "flag"
- "fmt"
- "html"
- "io"
- "io/ioutil"
- "log"
- "net/http"
- "net/url"
- "os"
- "os/exec"
- "path"
- "sort"
- "strconv"
- "strings"
- "sync"
- "time"
-
- "golang.org/x/oauth2"
- "golang.org/x/oauth2/google"
- "golang.org/x/tools/dashboard"
- "golang.org/x/tools/dashboard/buildlet"
- "golang.org/x/tools/dashboard/types"
- "google.golang.org/api/compute/v1"
- "google.golang.org/cloud/compute/metadata"
-)
-
-var (
- masterKeyFile = flag.String("masterkey", "", "Path to builder master key. Else fetched using GCE project attribute 'builder-master-key'.")
- maxLocalBuilds = flag.Int("maxbuilds", 6, "Max concurrent Docker builds (VM builds don't count)")
-
- cleanZones = flag.String("zones", "us-central1-a,us-central1-b,us-central1-f", "Comma-separated list of zones to periodically clean of stale build VMs (ones that failed to shut themselves down)")
-
- // Debug flags:
- just = flag.String("just", "", "If non-empty, run single build in the foreground. Requires rev.")
- rev = flag.String("rev", "", "Revision to build.")
-)
-
-var (
- startTime = time.Now()
- watchers = map[string]watchConfig{} // populated at startup, keyed by repo, e.g. "https://go.googlesource.com/go"
- donec = make(chan builderRev) // reports of finished builders
-
- statusMu sync.Mutex // guards both status (ongoing ones) and statusDone (just finished)
- status = map[builderRev]*buildStatus{}
- statusDone []*buildStatus // finished recently, capped to maxStatusDone
-)
-
-const (
- maxStatusDone = 30
-
- // vmDeleteTimeout is how long before we delete a VM.
- // In practice this need only be as long as the slowest
- // builder (plan9 currently), because on startup this program
- // already deletes all buildlets it doesn't know about
- // (i.e. ones from a previous instance of the coordinator).
- vmDeleteTimeout = 45 * time.Minute
-)
-
-// Initialized by initGCE:
-var (
- projectID string
- projectZone string
- computeService *compute.Service
- externalIP string
- tokenSource oauth2.TokenSource
-)
-
-func initGCE() error {
- if !metadata.OnGCE() {
- return errors.New("not running on GCE; VM support disabled")
- }
- var err error
- projectID, err = metadata.ProjectID()
- if err != nil {
- return fmt.Errorf("failed to get current GCE ProjectID: %v", err)
- }
- projectZone, err = metadata.Get("instance/zone")
- if err != nil || projectZone == "" {
- return fmt.Errorf("failed to get current GCE zone: %v", err)
- }
- // Convert the zone from "projects/1234/zones/us-central1-a" to "us-central1-a".
- projectZone = path.Base(projectZone)
- if !hasComputeScope() {
- return errors.New("The coordinator is not running with access to read and write Compute resources. VM support disabled.")
-
- }
- externalIP, err = metadata.ExternalIP()
- if err != nil {
- return fmt.Errorf("ExternalIP: %v", err)
- }
- tokenSource = google.ComputeTokenSource("default")
- computeService, _ = compute.New(oauth2.NewClient(oauth2.NoContext, tokenSource))
- return nil
-}
-
-type imageInfo struct {
- url string // of tar file
-
- mu sync.Mutex
- lastMod string
-}
-
-var images = map[string]*imageInfo{
- "go-commit-watcher": {url: "https://storage.googleapis.com/go-builder-data/docker-commit-watcher.tar.gz"},
- "gobuilders/linux-x86-base": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.base.tar.gz"},
- "gobuilders/linux-x86-clang": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.clang.tar.gz"},
- "gobuilders/linux-x86-gccgo": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.gccgo.tar.gz"},
- "gobuilders/linux-x86-nacl": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.nacl.tar.gz"},
- "gobuilders/linux-x86-sid": {url: "https://storage.googleapis.com/go-builder-data/docker-linux.sid.tar.gz"},
-}
-
-// recordResult sends build results to the dashboard
-func recordResult(builderName string, ok bool, hash, buildLog string, runTime time.Duration) error {
- req := map[string]interface{}{
- "Builder": builderName,
- "PackagePath": "",
- "Hash": hash,
- "GoHash": "",
- "OK": ok,
- "Log": buildLog,
- "RunTime": runTime,
- }
- args := url.Values{"key": {builderKey(builderName)}, "builder": {builderName}}
- return dash("POST", "result", args, req, nil)
-}
-
-// pingDashboard is a goroutine that periodically POSTS to build.golang.org/building
-// to let it know that we're still working on a build.
-func pingDashboard(st *buildStatus) {
- u := "https://build.golang.org/building?" + url.Values{
- "builder": []string{st.name},
- "key": []string{builderKey(st.name)},
- "hash": []string{st.rev},
- "url": []string{fmt.Sprintf("http://%v/logs?name=%s&rev=%s&st=%p", externalIP, st.name, st.rev, st)},
- }.Encode()
- for {
- st.mu.Lock()
- done := st.done
- st.mu.Unlock()
- if !done.IsZero() {
- return
- }
- if res, _ := http.PostForm(u, nil); res != nil {
- res.Body.Close()
- }
- time.Sleep(60 * time.Second)
- }
-}
-
-type watchConfig struct {
- repo string // "https://go.googlesource.com/go"
- dash string // "https://build.golang.org/" (must end in /)
- interval time.Duration // Polling interval
-}
-
-func main() {
- flag.Parse()
-
- if err := initGCE(); err != nil {
- log.Printf("VM support disabled due to error initializing GCE: %v", err)
- }
-
- addWatcher(watchConfig{repo: "https://go.googlesource.com/go", dash: "https://build.golang.org/"})
- // TODO(adg,cmang): fix gccgo watcher
- // addWatcher(watchConfig{repo: "https://code.google.com/p/gofrontend", dash: "https://build.golang.org/gccgo/"})
-
- if (*just != "") != (*rev != "") {
- log.Fatalf("--just and --rev must be used together")
- }
- if *just != "" {
- conf, ok := dashboard.Builders[*just]
- if !ok {
- log.Fatalf("unknown builder %q", *just)
- }
- args, err := conf.DockerRunArgs(*rev, builderKey(*just))
- if err != nil {
- log.Fatal(err)
- }
- cmd := exec.Command("docker", append([]string{"run"}, args...)...)
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
- if err := cmd.Run(); err != nil {
- log.Fatalf("Build failed: %v", err)
- }
- return
- }
-
- http.HandleFunc("/", handleStatus)
- http.HandleFunc("/logs", handleLogs)
- go http.ListenAndServe(":80", nil)
-
- go cleanUpOldContainers()
- go cleanUpOldVMs()
-
- stopWatchers() // clean up before we start new ones
- for _, watcher := range watchers {
- if err := startWatching(watchers[watcher.repo]); err != nil {
- log.Printf("Error starting watcher for %s: %v", watcher.repo, err)
- }
- }
-
- workc := make(chan builderRev)
- go findWorkLoop(workc)
- // TODO(cmang): gccgo will need its own findWorkLoop
-
- ticker := time.NewTicker(1 * time.Minute)
- for {
- select {
- case work := <-workc:
- log.Printf("workc received %+v; len(status) = %v, maxLocalBuilds = %v; cur = %p", work, len(status), *maxLocalBuilds, status[work])
- if mayBuildRev(work) {
- conf, _ := dashboard.Builders[work.name]
- if st, err := startBuilding(conf, work.rev); err == nil {
- setStatus(work, st)
- go pingDashboard(st)
- } else {
- log.Printf("Error starting to build %v: %v", work, err)
- }
- }
- case done := <-donec:
- log.Printf("%v done", done)
- markDone(done)
- case <-ticker.C:
- if numCurrentBuilds() == 0 && time.Now().After(startTime.Add(10*time.Minute)) {
- // TODO: halt the whole machine to kill the VM or something
- }
- }
- }
-}
-
-func numCurrentBuilds() int {
- statusMu.Lock()
- defer statusMu.Unlock()
- return len(status)
-}
-
-func isBuilding(work builderRev) bool {
- statusMu.Lock()
- defer statusMu.Unlock()
- _, building := status[work]
- return building
-}
-
-// mayBuildRev reports whether the build type & revision should be started.
-// It returns true if it's not already building, and there is capacity.
-func mayBuildRev(work builderRev) bool {
- conf, ok := dashboard.Builders[work.name]
- if !ok {
- return false
- }
-
- statusMu.Lock()
- _, building := status[work]
- statusMu.Unlock()
-
- if building {
- return false
- }
- if conf.UsesVM() {
- // These don't count towards *maxLocalBuilds.
- return true
- }
- numDocker, err := numDockerBuilds()
- if err != nil {
- log.Printf("not starting %v due to docker ps failure: %v", work, err)
- return false
- }
- return numDocker < *maxLocalBuilds
-}
-
-func setStatus(work builderRev, st *buildStatus) {
- statusMu.Lock()
- defer statusMu.Unlock()
- status[work] = st
-}
-
-func markDone(work builderRev) {
- statusMu.Lock()
- defer statusMu.Unlock()
- st, ok := status[work]
- if !ok {
- return
- }
- delete(status, work)
- if len(statusDone) == maxStatusDone {
- copy(statusDone, statusDone[1:])
- statusDone = statusDone[:len(statusDone)-1]
- }
- statusDone = append(statusDone, st)
-}
-
-func vmIsBuilding(instName string) bool {
- if instName == "" {
- log.Printf("bogus empty instance name passed to vmIsBuilding")
- return false
- }
- statusMu.Lock()
- defer statusMu.Unlock()
- for _, st := range status {
- if st.instName == instName {
- return true
- }
- }
- return false
-}
-
-// statusPtrStr disambiguates which status to return if there are
-// multiple in the history (e.g. recent failures where the build
-// didn't finish for reasons outside of all.bash failing)
-func getStatus(work builderRev, statusPtrStr string) *buildStatus {
- statusMu.Lock()
- defer statusMu.Unlock()
- match := func(st *buildStatus) bool {
- return statusPtrStr == "" || fmt.Sprintf("%p", st) == statusPtrStr
- }
- if st, ok := status[work]; ok && match(st) {
- return st
- }
- for _, st := range statusDone {
- if st.builderRev == work && match(st) {
- return st
- }
- }
- return nil
-}
-
-type byAge []*buildStatus
-
-func (s byAge) Len() int { return len(s) }
-func (s byAge) Less(i, j int) bool { return s[i].start.Before(s[j].start) }
-func (s byAge) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
-
-func handleStatus(w http.ResponseWriter, r *http.Request) {
- var active []*buildStatus
- var recent []*buildStatus
- statusMu.Lock()
- for _, st := range status {
- active = append(active, st)
- }
- recent = append(recent, statusDone...)
- numTotal := len(status)
- numDocker, err := numDockerBuilds()
- statusMu.Unlock()
-
- sort.Sort(byAge(active))
- sort.Sort(sort.Reverse(byAge(recent)))
-
- io.WriteString(w, "<html><body><h1>Go build coordinator</h1>")
-
- if err != nil {
- fmt.Fprintf(w, "<h2>Error</h2>Error fetching Docker build count: <i>%s</i>\n", html.EscapeString(err.Error()))
- }
-
- fmt.Fprintf(w, "<h2>running</h2><p>%d total builds active (Docker: %d/%d; VMs: %d/∞):",
- numTotal, numDocker, *maxLocalBuilds, numTotal-numDocker)
-
- io.WriteString(w, "<pre>")
- for _, st := range active {
- io.WriteString(w, st.htmlStatusLine())
- }
- io.WriteString(w, "</pre>")
-
- io.WriteString(w, "<h2>recently completed</h2><pre>")
- for _, st := range recent {
- io.WriteString(w, st.htmlStatusLine())
- }
- io.WriteString(w, "</pre>")
-
- fmt.Fprintf(w, "<h2>disk space</h2><pre>%s</pre></body></html>", html.EscapeString(diskFree()))
-}
-
-func diskFree() string {
- out, _ := exec.Command("df", "-h").Output()
- return string(out)
-}
-
-func handleLogs(w http.ResponseWriter, r *http.Request) {
- st := getStatus(builderRev{r.FormValue("name"), r.FormValue("rev")}, r.FormValue("st"))
- if st == nil {
- http.NotFound(w, r)
- return
- }
- w.Header().Set("Content-Type", "text/plain; charset=utf-8")
- writeStatusHeader(w, st)
-
- io.WriteString(w, st.logs())
- // TODO: if st is still building, stream them to the user with
- // http.Flusher.Flush and CloseNotifier and registering interest
- // of new writes with the buildStatus. Will require moving the
- // BUILDERKEY scrubbing into the Write method.
-}
-
-func writeStatusHeader(w http.ResponseWriter, st *buildStatus) {
- st.mu.Lock()
- defer st.mu.Unlock()
- fmt.Fprintf(w, " builder: %s\n", st.name)
- fmt.Fprintf(w, " rev: %s\n", st.rev)
- if st.container != "" {
- fmt.Fprintf(w, "container: %s\n", st.container)
- }
- if st.instName != "" {
- fmt.Fprintf(w, " vm name: %s\n", st.instName)
- }
- fmt.Fprintf(w, " started: %v\n", st.start)
- done := !st.done.IsZero()
- if done {
- fmt.Fprintf(w, " started: %v\n", st.done)
- fmt.Fprintf(w, " success: %v\n", st.succeeded)
- } else {
- fmt.Fprintf(w, " status: still running\n")
- }
- if len(st.events) > 0 {
- io.WriteString(w, "\nEvents:\n")
- st.writeEventsLocked(w, false)
- }
- io.WriteString(w, "\nBuild log:\n")
-}
-
-// findWorkLoop polls http://build.golang.org/?mode=json looking for new work
-// for the main dashboard. It does not support gccgo.
-// TODO(bradfitz): it also currently does not support subrepos.
-func findWorkLoop(work chan<- builderRev) {
- ticker := time.NewTicker(15 * time.Second)
- for {
- if err := findWork(work); err != nil {
- log.Printf("failed to find new work: %v", err)
- }
- <-ticker.C
- }
-}
-
-func findWork(work chan<- builderRev) error {
- var bs types.BuildStatus
- res, err := http.Get("https://build.golang.org/?mode=json")
- if err != nil {
- return err
- }
- defer res.Body.Close()
- if err := json.NewDecoder(res.Body).Decode(&bs); err != nil {
- return err
- }
- if res.StatusCode != 200 {
- return fmt.Errorf("unexpected http status %v", res.Status)
- }
-
- knownToDashboard := map[string]bool{} // keys are builder
- for _, b := range bs.Builders {
- knownToDashboard[b] = true
- }
-
- var goRevisions []string
- for _, br := range bs.Revisions {
- if br.Repo == "go" {
- goRevisions = append(goRevisions, br.Revision)
- } else {
- // TODO(bradfitz): support these: golang.org/issue/9506
- continue
- }
- if len(br.Results) != len(bs.Builders) {
- return errors.New("bogus JSON response from dashboard: results is too long.")
- }
- for i, res := range br.Results {
- if res != "" {
- // It's either "ok" or a failure URL.
- continue
- }
- builder := bs.Builders[i]
- if _, ok := dashboard.Builders[builder]; !ok {
- // Not managed by the coordinator.
- continue
- }
- br := builderRev{bs.Builders[i], br.Revision}
- if !isBuilding(br) {
- work <- br
- }
- }
- }
-
- // And to bootstrap new builders, see if we have any builders
- // that the dashboard doesn't know about.
- for b := range dashboard.Builders {
- if knownToDashboard[b] {
- continue
- }
- for _, rev := range goRevisions {
- br := builderRev{b, rev}
- if !isBuilding(br) {
- work <- br
- }
- }
- }
- return nil
-}
-
-// builderRev is a build configuration type and a revision.
-type builderRev struct {
- name string // e.g. "linux-amd64-race"
- rev string // lowercase hex git hash
-}
-
-// returns the part after "docker run"
-func (conf watchConfig) dockerRunArgs() (args []string) {
- log.Printf("Running watcher with master key %q", masterKey())
- if key := masterKey(); len(key) > 0 {
- tmpKey := "/tmp/watcher.buildkey"
- if _, err := os.Stat(tmpKey); err != nil {
- if err := ioutil.WriteFile(tmpKey, key, 0600); err != nil {
- log.Fatal(err)
- }
- }
- // Images may look for .gobuildkey in / or /root, so provide both.
- // TODO(adg): fix images that look in the wrong place.
- args = append(args, "-v", tmpKey+":/.gobuildkey")
- args = append(args, "-v", tmpKey+":/root/.gobuildkey")
- }
- args = append(args,
- "go-commit-watcher",
- "/usr/local/bin/watcher",
- "-repo="+conf.repo,
- "-dash="+conf.dash,
- "-poll="+conf.interval.String(),
- )
- return
-}
-
-func addWatcher(c watchConfig) {
- if c.repo == "" {
- c.repo = "https://go.googlesource.com/go"
- }
- if c.dash == "" {
- c.dash = "https://build.golang.org/"
- }
- if c.interval == 0 {
- c.interval = 10 * time.Second
- }
- watchers[c.repo] = c
-}
-
-func condUpdateImage(img string) error {
- ii := images[img]
- if ii == nil {
- return fmt.Errorf("image %q doesn't exist", img)
- }
- ii.mu.Lock()
- defer ii.mu.Unlock()
- res, err := http.Head(ii.url)
- if err != nil {
- return fmt.Errorf("Error checking %s: %v", ii.url, err)
- }
- if res.StatusCode != 200 {
- return fmt.Errorf("Error checking %s: %v", ii.url, res.Status)
- }
- if res.Header.Get("Last-Modified") == ii.lastMod {
- return nil
- }
-
- res, err = http.Get(ii.url)
- if err != nil || res.StatusCode != 200 {
- return fmt.Errorf("Get after Head failed for %s: %v, %v", ii.url, err, res)
- }
- defer res.Body.Close()
-
- log.Printf("Running: docker load of %s\n", ii.url)
- cmd := exec.Command("docker", "load")
- cmd.Stdin = res.Body
-
- var out bytes.Buffer
- cmd.Stdout = &out
- cmd.Stderr = &out
-
- if cmd.Run(); err != nil {
- log.Printf("Failed to pull latest %s from %s and pipe into docker load: %v, %s", img, ii.url, err, out.Bytes())
- return err
- }
- ii.lastMod = res.Header.Get("Last-Modified")
- return nil
-}
-
-// numDockerBuilds finds the number of go builder instances currently running.
-func numDockerBuilds() (n int, err error) {
- out, err := exec.Command("docker", "ps").Output()
- if err != nil {
- return 0, err
- }
- for _, line := range strings.Split(string(out), "\n") {
- if strings.Contains(line, "gobuilders/") {
- n++
- }
- }
- return n, nil
-}
-
-func startBuilding(conf dashboard.BuildConfig, rev string) (*buildStatus, error) {
- if conf.UsesVM() {
- return startBuildingInVM(conf, rev)
- } else {
- return startBuildingInDocker(conf, rev)
- }
-}
-
-func startBuildingInDocker(conf dashboard.BuildConfig, rev string) (*buildStatus, error) {
- if err := condUpdateImage(conf.Image); err != nil {
- log.Printf("Failed to setup container for %v %v: %v", conf.Name, rev, err)
- return nil, err
- }
-
- runArgs, err := conf.DockerRunArgs(rev, builderKey(conf.Name))
- if err != nil {
- return nil, err
- }
- cmd := exec.Command("docker", append([]string{"run", "-d"}, runArgs...)...)
- all, err := cmd.CombinedOutput()
- log.Printf("Docker run for %v %v = err:%v, output:%s", conf.Name, rev, err, all)
- if err != nil {
- return nil, err
- }
- container := strings.TrimSpace(string(all))
- brev := builderRev{
- name: conf.Name,
- rev: rev,
- }
- st := &buildStatus{
- builderRev: brev,
- container: container,
- start: time.Now(),
- }
- log.Printf("%v now building in Docker container %v", brev, st.container)
- go func() {
- all, err := exec.Command("docker", "wait", container).CombinedOutput()
- output := strings.TrimSpace(string(all))
- var ok bool
- if err == nil {
- exit, err := strconv.Atoi(output)
- ok = (err == nil && exit == 0)
- }
- st.setDone(ok)
- log.Printf("docker wait %s/%s: %v, %s", container, rev, err, output)
- donec <- builderRev{conf.Name, rev}
- exec.Command("docker", "rm", container).Run()
- }()
- go func() {
- cmd := exec.Command("docker", "logs", "-f", container)
- cmd.Stdout = st
- cmd.Stderr = st
- if err := cmd.Run(); err != nil {
- // The docker logs subcommand always returns
- // success, even if the underlying process
- // fails.
- log.Printf("failed to follow docker logs of %s: %v", container, err)
- }
- }()
- return st, nil
-}
-
-func randHex(n int) string {
- buf := make([]byte, n/2)
- _, err := rand.Read(buf)
- if err != nil {
- panic("Failed to get randomness: " + err.Error())
- }
- return fmt.Sprintf("%x", buf)
-}
-
-// startBuildingInVM starts a VM on GCE running the buildlet binary to build rev.
-// TODO(bradfitz): move this into a buildlet client package.
-func startBuildingInVM(conf dashboard.BuildConfig, rev string) (*buildStatus, error) {
- brev := builderRev{
- name: conf.Name,
- rev: rev,
- }
- // name is the project-wide unique name of the GCE instance. It can't be longer
- // than 61 bytes, so we only use the first 8 bytes of the rev.
- name := "buildlet-" + conf.Name + "-" + rev[:8] + "-rn" + randHex(6)
-
- st := &buildStatus{
- builderRev: brev,
- start: time.Now(),
- instName: name,
- }
-
- go func() {
- err := buildInVM(conf, st)
- if err != nil {
- if st.hasEvent("instance_created") {
- go deleteVM(projectZone, st.instName)
- }
- }
- st.setDone(err == nil)
- if err != nil {
- fmt.Fprintf(st, "\n\nError: %v\n", err)
- }
- donec <- builderRev{conf.Name, rev}
- }()
- return st, nil
-}
-
-func buildInVM(conf dashboard.BuildConfig, st *buildStatus) (retErr error) {
- bc, err := buildlet.StartNewVM(tokenSource, st.instName, conf.Name, buildlet.VMOpts{
- ProjectID: projectID,
- Zone: projectZone,
- Description: fmt.Sprintf("Go Builder building %s %s", conf.Name, st.rev),
- DeleteIn: vmDeleteTimeout,
- OnInstanceRequested: func() {
- st.logEventTime("instance_create_requested")
- log.Printf("%v now booting VM %v for build", st.builderRev, st.instName)
- },
- OnInstanceCreated: func() {
- st.logEventTime("instance_created")
- },
- OnGotInstanceInfo: func() {
- st.logEventTime("waiting_for_buildlet")
- },
- })
- if err != nil {
- return err
- }
- st.logEventTime("buildlet_up")
- goodRes := func(res *http.Response, err error, what string) bool {
- if err != nil {
- retErr = fmt.Errorf("%s: %v", what, err)
- return false
- }
- if res.StatusCode/100 != 2 {
- slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
- retErr = fmt.Errorf("%s: %v; body: %s", what, res.Status, slurp)
- res.Body.Close()
- return false
-
- }
- return true
- }
-
- // Write the VERSION file.
- st.logEventTime("start_write_version_tar")
- if err := bc.PutTarball(versionTgz(st.rev)); err != nil {
- return fmt.Errorf("writing VERSION tgz: %v", err)
- }
-
- // Feed the buildlet a tar file for it to extract.
- // TODO: cache these.
- st.logEventTime("start_fetch_gerrit_tgz")
- tarRes, err := http.Get("https://go.googlesource.com/go/+archive/" + st.rev + ".tar.gz")
- if !goodRes(tarRes, err, "fetching tarball from Gerrit") {
- return
- }
-
- st.logEventTime("start_write_tar")
- if err := bc.PutTarball(tarRes.Body); err != nil {
- tarRes.Body.Close()
- return fmt.Errorf("writing tarball from Gerrit: %v", err)
- }
- st.logEventTime("end_write_tar")
-
- execStartTime := time.Now()
- st.logEventTime("pre_exec")
-
- remoteErr, err := bc.Exec(conf.AllScript(), buildlet.ExecOpts{
- Output: st,
- OnStartExec: func() { st.logEventTime("running_exec") },
- })
- if err != nil {
- return err
- }
- st.logEventTime("done")
- var log string
- if remoteErr != nil {
- log = st.logs()
- }
- if err := recordResult(st.name, remoteErr == nil, st.rev, log, time.Since(execStartTime)); err != nil {
- if remoteErr != nil {
- return fmt.Errorf("Remote error was %q but failed to report it to the dashboard: %v", remoteErr, err)
- }
- return fmt.Errorf("Build succeeded but failed to report it to the dashboard: %v", err)
- }
- if remoteErr != nil {
- return fmt.Errorf("%s failed: %v", conf.AllScript(), remoteErr)
- }
- return nil
-}
-
-type eventAndTime struct {
- evt string
- t time.Time
-}
-
-// buildStatus is the status of a build.
-type buildStatus struct {
- // Immutable:
- builderRev
- start time.Time
- container string // container ID for docker, else it's a VM
-
- // Immutable, used by VM only:
- instName string
-
- mu sync.Mutex // guards following
- done time.Time // finished running
- succeeded bool // set when done
- output bytes.Buffer // stdout and stderr
- events []eventAndTime
-}
-
-func (st *buildStatus) setDone(succeeded bool) {
- st.mu.Lock()
- defer st.mu.Unlock()
- st.succeeded = succeeded
- st.done = time.Now()
-}
-
-func (st *buildStatus) logEventTime(event string) {
- st.mu.Lock()
- defer st.mu.Unlock()
- st.events = append(st.events, eventAndTime{event, time.Now()})
-}
-
-func (st *buildStatus) hasEvent(event string) bool {
- st.mu.Lock()
- defer st.mu.Unlock()
- for _, e := range st.events {
- if e.evt == event {
- return true
- }
- }
- return false
-}
-
-// htmlStatusLine returns the HTML to show within the <pre> block on
-// the main page's list of active builds.
-func (st *buildStatus) htmlStatusLine() string {
- st.mu.Lock()
- defer st.mu.Unlock()
-
- urlPrefix := "https://go-review.googlesource.com/#/q/"
- if strings.Contains(st.name, "gccgo") {
- urlPrefix = "https://code.google.com/p/gofrontend/source/detail?r="
- }
-
- var buf bytes.Buffer
- fmt.Fprintf(&buf, "<a href='https://github.com/golang/go/wiki/DashboardBuilders'>%s</a> rev <a href='%s%s'>%s</a>",
- st.name, urlPrefix, st.rev, st.rev)
-
- if st.done.IsZero() {
- buf.WriteString(", running")
- } else if st.succeeded {
- buf.WriteString(", succeeded")
- } else {
- buf.WriteString(", failed")
- }
-
- if st.container != "" {
- fmt.Fprintf(&buf, " in container <a href='%s'>%s</a>", st.logsURL(), st.container)
- } else {
- fmt.Fprintf(&buf, " in VM <a href='%s'>%s</a>", st.logsURL(), st.instName)
- }
-
- t := st.done
- if t.IsZero() {
- t = st.start
- }
- fmt.Fprintf(&buf, ", %v ago\n", time.Since(t))
- st.writeEventsLocked(&buf, true)
- return buf.String()
-}
-
-func (st *buildStatus) logsURL() string {
- return fmt.Sprintf("/logs?name=%s&rev=%s&st=%p", st.name, st.rev, st)
-}
-
-// st.mu must be held.
-func (st *buildStatus) writeEventsLocked(w io.Writer, html bool) {
- for i, evt := range st.events {
- var elapsed string
- if i != 0 {
- elapsed = fmt.Sprintf("+%0.1fs", evt.t.Sub(st.events[i-1].t).Seconds())
- }
- msg := evt.evt
- if msg == "running_exec" && html {
- msg = fmt.Sprintf("<a href='%s'>%s</a>", st.logsURL(), msg)
- }
- fmt.Fprintf(w, " %7s %v %s\n", elapsed, evt.t.Format(time.RFC3339), msg)
- }
-}
-
-func (st *buildStatus) logs() string {
- st.mu.Lock()
- logs := st.output.String()
- st.mu.Unlock()
- key := builderKey(st.name)
- return strings.Replace(string(logs), key, "BUILDERKEY", -1)
-}
-
-func (st *buildStatus) Write(p []byte) (n int, err error) {
- st.mu.Lock()
- defer st.mu.Unlock()
- const maxBufferSize = 2 << 20 // 2MB of output is way more than we expect.
- plen := len(p)
- if st.output.Len()+len(p) > maxBufferSize {
- p = p[:maxBufferSize-st.output.Len()]
- }
- st.output.Write(p) // bytes.Buffer can't fail
- return plen, nil
-}
-
-// Stop any previous go-commit-watcher Docker tasks, so they don't
-// pile up upon restarts of the coordinator.
-func stopWatchers() {
- out, err := exec.Command("docker", "ps").Output()
- if err != nil {
- return
- }
- for _, line := range strings.Split(string(out), "\n") {
- if !strings.Contains(line, "go-commit-watcher:") {
- continue
- }
- f := strings.Fields(line)
- exec.Command("docker", "rm", "-f", "-v", f[0]).Run()
- }
-}
-
-func startWatching(conf watchConfig) (err error) {
- defer func() {
- if err != nil {
- restartWatcherSoon(conf)
- }
- }()
- log.Printf("Starting watcher for %v", conf.repo)
- if err := condUpdateImage("go-commit-watcher"); err != nil {
- log.Printf("Failed to setup container for commit watcher: %v", err)
- return err
- }
-
- cmd := exec.Command("docker", append([]string{"run", "-d"}, conf.dockerRunArgs()...)...)
- all, err := cmd.CombinedOutput()
- if err != nil {
- log.Printf("Docker run for commit watcher = err:%v, output: %s", err, all)
- return err
- }
- container := strings.TrimSpace(string(all))
- // Start a goroutine to wait for the watcher to die.
- go func() {
- exec.Command("docker", "wait", container).Run()
- exec.Command("docker", "rm", "-v", container).Run()
- log.Printf("Watcher crashed. Restarting soon.")
- restartWatcherSoon(conf)
- }()
- return nil
-}
-
-func restartWatcherSoon(conf watchConfig) {
- time.AfterFunc(30*time.Second, func() {
- startWatching(conf)
- })
-}
-
-func builderKey(builder string) string {
- master := masterKey()
- if len(master) == 0 {
- return ""
- }
- h := hmac.New(md5.New, master)
- io.WriteString(h, builder)
- return fmt.Sprintf("%x", h.Sum(nil))
-}
-
-func masterKey() []byte {
- keyOnce.Do(loadKey)
- return masterKeyCache
-}
-
-var (
- keyOnce sync.Once
- masterKeyCache []byte
-)
-
-func loadKey() {
- if *masterKeyFile != "" {
- b, err := ioutil.ReadFile(*masterKeyFile)
- if err != nil {
- log.Fatal(err)
- }
- masterKeyCache = bytes.TrimSpace(b)
- return
- }
- masterKey, err := metadata.ProjectAttributeValue("builder-master-key")
- if err != nil {
- log.Fatalf("No builder master key available: %v", err)
- }
- masterKeyCache = []byte(strings.TrimSpace(masterKey))
-}
-
-func cleanUpOldContainers() {
- for {
- for _, cid := range oldContainers() {
- log.Printf("Cleaning old container %v", cid)
- exec.Command("docker", "rm", "-v", cid).Run()
- }
- time.Sleep(30 * time.Second)
- }
-}
-
-func oldContainers() []string {
- out, _ := exec.Command("docker", "ps", "-a", "--filter=status=exited", "--no-trunc", "-q").Output()
- return strings.Fields(string(out))
-}
-
-// cleanUpOldVMs loops forever and periodically enumerates virtual
-// machines and deletes those which have expired.
-//
-// A VM is considered expired if it has a "delete-at" metadata
-// attribute having a unix timestamp before the current time.
-//
-// This is the safety mechanism to delete VMs which stray from the
-// normal deleting process. VMs are created to run a single build and
-// should be shut down by a controlling process. Due to various types
-// of failures, they might get stranded. To prevent them from getting
-// stranded and wasting resources forever, we instead set the
-// "delete-at" metadata attribute on them when created to some time
-// that's well beyond their expected lifetime.
-func cleanUpOldVMs() {
- if computeService == nil {
- return
- }
- for {
- for _, zone := range strings.Split(*cleanZones, ",") {
- zone = strings.TrimSpace(zone)
- if err := cleanZoneVMs(zone); err != nil {
- log.Printf("Error cleaning VMs in zone %q: %v", zone, err)
- }
- }
- time.Sleep(time.Minute)
- }
-}
-
-// cleanZoneVMs is part of cleanUpOldVMs, operating on a single zone.
-func cleanZoneVMs(zone string) error {
- // Fetch the first 500 (default) running instances and clean
- // thoes. We expect that we'll be running many fewer than
- // that. Even if we have more, eventually the first 500 will
- // either end or be cleaned, and then the next call will get a
- // partially-different 500.
- // TODO(bradfitz): revist this code if we ever start running
- // thousands of VMs.
- list, err := computeService.Instances.List(projectID, zone).Do()
- if err != nil {
- return fmt.Errorf("listing instances: %v", err)
- }
- for _, inst := range list.Items {
- if inst.Metadata == nil {
- // Defensive. Not seen in practice.
- continue
- }
- sawDeleteAt := false
- for _, it := range inst.Metadata.Items {
- if it.Key == "delete-at" {
- sawDeleteAt = true
- unixDeadline, err := strconv.ParseInt(it.Value, 10, 64)
- if err != nil {
- log.Printf("invalid delete-at value %q seen; ignoring", it.Value)
- }
- if err == nil && time.Now().Unix() > unixDeadline {
- log.Printf("Deleting expired VM %q in zone %q ...", inst.Name, zone)
- deleteVM(zone, inst.Name)
- }
- }
- }
- // Delete buildlets (things we made) from previous
- // generations. Thenaming restriction (buildlet-*)
- // prevents us from deleting buildlet VMs used by
- // Gophers for interactive development & debugging
- // (non-builder users); those are named "mote-*".
- if sawDeleteAt && strings.HasPrefix(inst.Name, "buildlet-") && !vmIsBuilding(inst.Name) {
- log.Printf("Deleting VM %q in zone %q from an earlier coordinator generation ...", inst.Name, zone)
- deleteVM(zone, inst.Name)
- }
- }
- return nil
-}
-
-func deleteVM(zone, instName string) {
- op, err := computeService.Instances.Delete(projectID, zone, instName).Do()
- if err != nil {
- log.Printf("Failed to delete instance %q in zone %q: %v", instName, zone, err)
- return
- }
- log.Printf("Sent request to delete instance %q in zone %q. Operation ID == %v", instName, zone, op.Id)
-}
-
-func hasComputeScope() bool {
- if !metadata.OnGCE() {
- return false
- }
- scopes, err := metadata.Scopes("default")
- if err != nil {
- log.Printf("failed to query metadata default scopes: %v", err)
- return false
- }
- for _, v := range scopes {
- if v == compute.ComputeScope {
- return true
- }
- }
- return false
-}
-
-// dash is copied from the builder binary. It runs the given method and command on the dashboard.
-//
-// TODO(bradfitz,adg): unify this somewhere?
-//
-// If args is non-nil it is encoded as the URL query string.
-// If req is non-nil it is JSON-encoded and passed as the body of the HTTP POST.
-// If resp is non-nil the server's response is decoded into the value pointed
-// to by resp (resp must be a pointer).
-func dash(meth, cmd string, args url.Values, req, resp interface{}) error {
- const builderVersion = 1 // keep in sync with dashboard/app/build/handler.go
- argsCopy := url.Values{"version": {fmt.Sprint(builderVersion)}}
- for k, v := range args {
- if k == "version" {
- panic(`dash: reserved args key: "version"`)
- }
- argsCopy[k] = v
- }
- var r *http.Response
- var err error
- cmd = "https://build.golang.org/" + cmd + "?" + argsCopy.Encode()
- switch meth {
- case "GET":
- if req != nil {
- log.Panicf("%s to %s with req", meth, cmd)
- }
- r, err = http.Get(cmd)
- case "POST":
- var body io.Reader
- if req != nil {
- b, err := json.Marshal(req)
- if err != nil {
- return err
- }
- body = bytes.NewBuffer(b)
- }
- r, err = http.Post(cmd, "text/json", body)
- default:
- log.Panicf("%s: invalid method %q", cmd, meth)
- panic("invalid method: " + meth)
- }
- if err != nil {
- return err
- }
- defer r.Body.Close()
- if r.StatusCode != http.StatusOK {
- return fmt.Errorf("bad http response: %v", r.Status)
- }
- body := new(bytes.Buffer)
- if _, err := body.ReadFrom(r.Body); err != nil {
- return err
- }
-
- // Read JSON-encoded Response into provided resp
- // and return an error if present.
- var result = struct {
- Response interface{}
- Error string
- }{
- // Put the provided resp in here as it can be a pointer to
- // some value we should unmarshal into.
- Response: resp,
- }
- if err = json.Unmarshal(body.Bytes(), &result); err != nil {
- log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err)
- return err
- }
- if result.Error != "" {
- return errors.New(result.Error)
- }
-
- return nil
-}
-
-func versionTgz(rev string) io.Reader {
- var buf bytes.Buffer
- zw := gzip.NewWriter(&buf)
- tw := tar.NewWriter(zw)
-
- contents := fmt.Sprintf("devel " + rev)
- check(tw.WriteHeader(&tar.Header{
- Name: "VERSION",
- Mode: 0644,
- Size: int64(len(contents)),
- }))
- _, err := io.WriteString(tw, contents)
- check(err)
- check(tw.Close())
- check(zw.Close())
- return bytes.NewReader(buf.Bytes())
-}
-
-// check is only for things which should be impossible (not even rare)
-// to fail.
-func check(err error) {
- if err != nil {
- panic("previously assumed to never fail: " + err.Error())
- }
-}
diff --git a/dashboard/cmd/gomote/auth.go b/dashboard/cmd/gomote/auth.go
deleted file mode 100644
index 56ca0d4..0000000
--- a/dashboard/cmd/gomote/auth.go
+++ /dev/null
@@ -1,87 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-package main
-
-import (
- "io/ioutil"
- "log"
- "os"
- "path/filepath"
- "runtime"
-
- "golang.org/x/oauth2"
- "golang.org/x/tools/dashboard/auth"
- "golang.org/x/tools/dashboard/buildlet"
- "google.golang.org/api/compute/v1"
-)
-
-func username() string {
- if runtime.GOOS == "windows" {
- return os.Getenv("USERNAME")
- }
- return os.Getenv("USER")
-}
-
-func homeDir() string {
- if runtime.GOOS == "windows" {
- return os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
- }
- return os.Getenv("HOME")
-}
-
-func configDir() string {
- if runtime.GOOS == "windows" {
- return filepath.Join(os.Getenv("APPDATA"), "Gomote")
- }
- if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" {
- return filepath.Join(xdg, "gomote")
- }
- return filepath.Join(homeDir(), ".config", "gomote")
-}
-
-func projTokenSource() oauth2.TokenSource {
- ts, err := auth.ProjectTokenSource(*proj, compute.ComputeScope)
- if err != nil {
- log.Fatalf("Failed to get OAuth2 token source for project %s: %v", *proj, err)
- }
- return ts
-}
-
-func userKeyPair() buildlet.KeyPair {
- keyDir := configDir()
- crtFile := filepath.Join(keyDir, "gomote.crt")
- keyFile := filepath.Join(keyDir, "gomote.key")
- _, crtErr := os.Stat(crtFile)
- _, keyErr := os.Stat(keyFile)
- if crtErr == nil && keyErr == nil {
- return buildlet.KeyPair{
- CertPEM: slurpString(crtFile),
- KeyPEM: slurpString(keyFile),
- }
- }
- check := func(what string, err error) {
- if err != nil {
- log.Printf("%s: %v", what, err)
- }
- }
- check("making key dir", os.MkdirAll(keyDir, 0700))
- kp, err := buildlet.NewKeyPair()
- if err != nil {
- log.Fatalf("Error generating new key pair: %v", err)
- }
- check("writing cert file: ", ioutil.WriteFile(crtFile, []byte(kp.CertPEM), 0600))
- check("writing key file: ", ioutil.WriteFile(keyFile, []byte(kp.KeyPEM), 0600))
- return kp
-}
-
-func slurpString(f string) string {
- slurp, err := ioutil.ReadFile(f)
- if err != nil {
- log.Fatal(err)
- }
- return string(slurp)
-}
diff --git a/dashboard/cmd/gomote/create.go b/dashboard/cmd/gomote/create.go
deleted file mode 100644
index 31806af..0000000
--- a/dashboard/cmd/gomote/create.go
+++ /dev/null
@@ -1,77 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-package main
-
-import (
- "flag"
- "fmt"
- "log"
- "os"
- "sort"
- "strings"
- "time"
-
- "golang.org/x/tools/dashboard"
- "golang.org/x/tools/dashboard/buildlet"
-)
-
-func create(args []string) error {
- fs := flag.NewFlagSet("create", flag.ContinueOnError)
- fs.Usage = func() {
- fmt.Fprintln(os.Stderr, "create usage: gomote create [create-opts] <type>\n\n")
- fs.PrintDefaults()
- os.Exit(1)
- }
- var timeout time.Duration
- fs.DurationVar(&timeout, "timeout", 60*time.Minute, "how long the VM will live before being deleted.")
-
- fs.Parse(args)
- if fs.NArg() != 1 {
- fs.Usage()
- }
- builderType := fs.Arg(0)
- conf, ok := dashboard.Builders[builderType]
- if !ok || !conf.UsesVM() {
- var valid []string
- var prefixMatch []string
- for k, conf := range dashboard.Builders {
- if conf.UsesVM() {
- valid = append(valid, k)
- if strings.HasPrefix(k, builderType) {
- prefixMatch = append(prefixMatch, k)
- }
- }
- }
- if len(prefixMatch) == 1 {
- builderType = prefixMatch[0]
- conf, _ = dashboard.Builders[builderType]
- } else {
- sort.Strings(valid)
- return fmt.Errorf("Invalid builder type %q. Valid options include: %q", builderType, valid)
- }
- }
-
- instName := fmt.Sprintf("mote-%s-%s", username(), builderType)
- client, err := buildlet.StartNewVM(projTokenSource(), instName, builderType, buildlet.VMOpts{
- Zone: *zone,
- ProjectID: *proj,
- TLS: userKeyPair(),
- DeleteIn: timeout,
- Description: fmt.Sprintf("gomote buildlet for %s", username()),
- OnInstanceRequested: func() {
- log.Printf("Sent create request. Waiting for operation.")
- },
- OnInstanceCreated: func() {
- log.Printf("Instance created.")
- },
- })
- if err != nil {
- return fmt.Errorf("failed to create VM: %v", err)
- }
- fmt.Printf("%s\t%s\n", builderType, client.URL())
- return nil
-}
diff --git a/dashboard/cmd/gomote/destroy.go b/dashboard/cmd/gomote/destroy.go
deleted file mode 100644
index 1f758ef..0000000
--- a/dashboard/cmd/gomote/destroy.go
+++ /dev/null
@@ -1,81 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-package main
-
-import (
- "errors"
- "flag"
- "fmt"
- "log"
- "os"
- "time"
-
- "golang.org/x/tools/dashboard/buildlet"
-)
-
-func destroy(args []string) error {
- fs := flag.NewFlagSet("destroy", flag.ContinueOnError)
- fs.Usage = func() {
- fmt.Fprintln(os.Stderr, "create usage: gomote destroy <instance>\n\n")
- fs.PrintDefaults()
- os.Exit(1)
- }
-
- fs.Parse(args)
- if fs.NArg() != 1 {
- fs.Usage()
- }
- name := fs.Arg(0)
- bc, err := namedClient(name)
- if err != nil {
- return err
- }
-
- // Ask it to kill itself, and tell GCE to kill it too:
- gceErrc := make(chan error, 1)
- buildletErrc := make(chan error, 1)
- go func() {
- gceErrc <- buildlet.DestroyVM(projTokenSource(), *proj, *zone, fmt.Sprintf("mote-%s-%s", username(), name))
- }()
- go func() {
- buildletErrc <- bc.Destroy()
- }()
- timeout := time.NewTimer(5 * time.Second)
- defer timeout.Stop()
-
- var retErr error
- var gceDone, buildletDone bool
- for !gceDone || !buildletDone {
- select {
- case err := <-gceErrc:
- if err != nil {
- log.Printf("GCE: %v", err)
- retErr = err
- } else {
- log.Printf("Requested GCE delete.")
- }
- gceDone = true
- case err := <-buildletErrc:
- if err != nil {
- log.Printf("Buildlet: %v", err)
- retErr = err
- } else {
- log.Printf("Requested buildlet to shut down.")
- }
- buildletDone = true
- case <-timeout.C:
- if !buildletDone {
- log.Printf("timeout asking buildlet to shut down")
- }
- if !gceDone {
- log.Printf("timeout asking GCE to delete builder VM")
- }
- return errors.New("timeout")
- }
- }
- return retErr
-}
diff --git a/dashboard/cmd/gomote/get.go b/dashboard/cmd/gomote/get.go
deleted file mode 100644
index f78ebd3..0000000
--- a/dashboard/cmd/gomote/get.go
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-package main
-
-import (
- "flag"
- "fmt"
- "io"
- "os"
-)
-
-// get a .tar.gz
-func getTar(args []string) error {
- fs := flag.NewFlagSet("get", flag.ContinueOnError)
- fs.Usage = func() {
- fmt.Fprintln(os.Stderr, "create usage: gomote gettar [get-opts] <buildlet-name>\n")
- fs.PrintDefaults()
- os.Exit(1)
- }
- var dir string
- fs.StringVar(&dir, "dir", "", "relative directory from buildlet's work dir to tar up")
-
- fs.Parse(args)
- if fs.NArg() != 1 {
- fs.Usage()
- }
-
- name := fs.Arg(0)
- bc, err := namedClient(name)
- if err != nil {
- return err
- }
- tgz, err := bc.GetTar(dir)
- if err != nil {
- return err
- }
- defer tgz.Close()
- _, err = io.Copy(os.Stdout, tgz)
- return err
-}
diff --git a/dashboard/cmd/gomote/gomote.go b/dashboard/cmd/gomote/gomote.go
deleted file mode 100644
index 79f4a33..0000000
--- a/dashboard/cmd/gomote/gomote.go
+++ /dev/null
@@ -1,106 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-/*
-The gomote command is a client for the Go builder infrastructure.
-It's a remote control for remote Go builder machines.
-
-Usage:
-
- gomote [global-flags] cmd [cmd-flags]
-
- For example,
- $ gomote create openbsd-amd64-gce56
- $ gomote push
- $ gomote run openbsd-amd64-gce56 src/make.bash
-
-TODO: document more, and figure out the CLI interface more.
-*/
-package main
-
-import (
- "flag"
- "fmt"
- "os"
- "sort"
-)
-
-var (
- proj = flag.String("project", "symbolic-datum-552", "GCE project owning builders")
- zone = flag.String("zone", "us-central1-a", "GCE zone")
-)
-
-type command struct {
- name string
- des string
- run func([]string) error
-}
-
-var commands = map[string]command{}
-
-func sortedCommands() []string {
- s := make([]string, 0, len(commands))
- for name := range commands {
- s = append(s, name)
- }
- sort.Strings(s)
- return s
-}
-
-func usage() {
- fmt.Fprintf(os.Stderr, `Usage of gomote: gomote [global-flags] <cmd> [cmd-flags]
-
-Global flags:
-`)
- flag.PrintDefaults()
- fmt.Fprintf(os.Stderr, "Commands:\n\n")
- for _, name := range sortedCommands() {
- fmt.Fprintf(os.Stderr, " %-10s %s\n", name, commands[name].des)
- }
- os.Exit(1)
-}
-
-func registerCommand(name, des string, run func([]string) error) {
- if _, dup := commands[name]; dup {
- panic("duplicate registration of " + name)
- }
- commands[name] = command{
- name: name,
- des: des,
- run: run,
- }
-}
-
-func registerCommands() {
- registerCommand("create", "create a buildlet", create)
- registerCommand("destroy", "destroy a buildlet", destroy)
- registerCommand("list", "list buildlets", list)
- registerCommand("run", "run a command on a buildlet", run)
- registerCommand("put", "put files on a buildlet", put)
- registerCommand("puttar", "extract a tar.gz to a buildlet", putTar)
- registerCommand("gettar", "extract a tar.gz from a buildlet", getTar)
-}
-
-func main() {
- registerCommands()
- flag.Usage = usage
- flag.Parse()
- args := flag.Args()
- if len(args) == 0 {
- usage()
- }
- cmdName := args[0]
- cmd, ok := commands[cmdName]
- if !ok {
- fmt.Fprintf(os.Stderr, "Unknown command %q\n", cmdName)
- usage()
- }
- err := cmd.run(args[1:])
- if err != nil {
- fmt.Fprintf(os.Stderr, "Error running %s: %v\n", cmdName, err)
- os.Exit(1)
- }
-}
diff --git a/dashboard/cmd/gomote/list.go b/dashboard/cmd/gomote/list.go
deleted file mode 100644
index c410f12..0000000
--- a/dashboard/cmd/gomote/list.go
+++ /dev/null
@@ -1,68 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-package main
-
-import (
- "flag"
- "fmt"
- "os"
- "strings"
-
- "golang.org/x/tools/dashboard/buildlet"
-)
-
-func list(args []string) error {
- fs := flag.NewFlagSet("list", flag.ContinueOnError)
- fs.Usage = func() {
- fmt.Fprintln(os.Stderr, "list usage: gomote list\n\n")
- fs.PrintDefaults()
- os.Exit(1)
- }
- fs.Parse(args)
- if fs.NArg() != 0 {
- fs.Usage()
- }
-
- prefix := fmt.Sprintf("mote-%s-", username())
- vms, err := buildlet.ListVMs(projTokenSource(), *proj, *zone)
- if err != nil {
- return fmt.Errorf("failed to list VMs: %v", err)
- }
- for _, vm := range vms {
- if !strings.HasPrefix(vm.Name, prefix) {
- continue
- }
- fmt.Printf("%s\thttps://%s\n", vm.Type, strings.TrimSuffix(vm.IPPort, ":443"))
- }
- return nil
-}
-
-func namedClient(name string) (*buildlet.Client, error) {
- // TODO(bradfitz): cache the list on disk and avoid the API call?
- vms, err := buildlet.ListVMs(projTokenSource(), *proj, *zone)
- if err != nil {
- return nil, fmt.Errorf("error listing VMs while looking up %q: %v", name, err)
- }
- wantName := fmt.Sprintf("mote-%s-%s", username(), name)
- var matches []buildlet.VM
- for _, vm := range vms {
- if vm.Name == wantName {
- return buildlet.NewClient(vm.IPPort, vm.TLS), nil
- }
- if strings.HasPrefix(vm.Name, wantName) {
- matches = append(matches, vm)
- }
- }
- if len(matches) == 1 {
- vm := matches[0]
- return buildlet.NewClient(vm.IPPort, vm.TLS), nil
- }
- if len(matches) > 1 {
- return nil, fmt.Errorf("prefix %q is ambiguous")
- }
- return nil, fmt.Errorf("buildlet %q not running", name)
-}
diff --git a/dashboard/cmd/gomote/put.go b/dashboard/cmd/gomote/put.go
deleted file mode 100644
index 3ccc3ff..0000000
--- a/dashboard/cmd/gomote/put.go
+++ /dev/null
@@ -1,84 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-package main
-
-import (
- "flag"
- "fmt"
- "io"
- "os"
-)
-
-// put a .tar.gz
-func putTar(args []string) error {
- fs := flag.NewFlagSet("put", flag.ContinueOnError)
- fs.Usage = func() {
- fmt.Fprintln(os.Stderr, "create usage: gomote puttar [put-opts] <buildlet-name> [tar.gz file or '-' for stdin]\n")
- fs.PrintDefaults()
- os.Exit(1)
- }
- var rev string
- fs.StringVar(&rev, "gorev", "", "If non-empty, git hash to download from gerrit and put to the buildlet. e.g. 886b02d705ff for Go 1.4.1. This just maps to the --URL flag, so the two options are mutually exclusive.")
- var dir string
- fs.StringVar(&dir, "dir", "", "relative directory from buildlet's work dir to extra tarball into")
- var tarURL string
- fs.StringVar(&tarURL, "url", "", "URL of tarball, instead of provided file.")
-
- fs.Parse(args)
- if fs.NArg() < 1 || fs.NArg() > 2 {
- fs.Usage()
- }
- if rev != "" {
- if tarURL != "" {
- fmt.Fprintln(os.Stderr, "--gorev and --url are mutually exclusive")
- fs.Usage()
- }
- tarURL = "https://go.googlesource.com/go/+archive/" + rev + ".tar.gz"
- }
-
- name := fs.Arg(0)
- bc, err := namedClient(name)
- if err != nil {
- return err
- }
-
- if tarURL != "" {
- if fs.NArg() != 1 {
- fs.Usage()
- }
- return bc.PutTarFromURL(tarURL, dir)
- }
-
- var tgz io.Reader = os.Stdin
- if fs.NArg() == 2 && fs.Arg(1) != "-" {
- f, err := os.Open(fs.Arg(1))
- if err != nil {
- return err
- }
- defer f.Close()
- tgz = f
- }
- return bc.PutTar(tgz, dir)
-}
-
-// put single files
-func put(args []string) error {
- fs := flag.NewFlagSet("put", flag.ContinueOnError)
- fs.Usage = func() {
- fmt.Fprintln(os.Stderr, "create usage: gomote put [put-opts] <type>\n\n")
- fs.PrintDefaults()
- os.Exit(1)
- }
- fs.Parse(args)
- if fs.NArg() != 1 {
- fs.Usage()
- }
- return fmt.Errorf("TODO")
- builderType := fs.Arg(0)
- _ = builderType
- return nil
-}
diff --git a/dashboard/cmd/gomote/run.go b/dashboard/cmd/gomote/run.go
deleted file mode 100644
index ff2d47c..0000000
--- a/dashboard/cmd/gomote/run.go
+++ /dev/null
@@ -1,47 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-package main
-
-import (
- "flag"
- "fmt"
- "os"
- "strings"
-
- "golang.org/x/tools/dashboard/buildlet"
-)
-
-func run(args []string) error {
- fs := flag.NewFlagSet("run", flag.ContinueOnError)
- fs.Usage = func() {
- fmt.Fprintln(os.Stderr, "create usage: gomote run [run-opts] <instance> <cmd> [args...]")
- fs.PrintDefaults()
- os.Exit(1)
- }
- var sys bool
- fs.BoolVar(&sys, "system", false, "run inside the system, and not inside the workdir; this is implicit if cmd starts with '/'")
-
- fs.Parse(args)
- if fs.NArg() < 2 {
- fs.Usage()
- }
- name, cmd := fs.Arg(0), fs.Arg(1)
- bc, err := namedClient(name)
- if err != nil {
- return err
- }
-
- remoteErr, execErr := bc.Exec(cmd, buildlet.ExecOpts{
- SystemLevel: sys || strings.HasPrefix(cmd, "/"),
- Output: os.Stdout,
- Args: fs.Args()[2:],
- })
- if execErr != nil {
- return fmt.Errorf("Error trying to execute %s: %v", cmd, execErr)
- }
- return remoteErr
-}
diff --git a/dashboard/cmd/retrybuilds/retrybuilds.go b/dashboard/cmd/retrybuilds/retrybuilds.go
deleted file mode 100644
index bead61e..0000000
--- a/dashboard/cmd/retrybuilds/retrybuilds.go
+++ /dev/null
@@ -1,238 +0,0 @@
-// Copyright 2014 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// The retrybuilds command clears build failures from the build.golang.org dashboard
-// to force them to be rebuilt.
-//
-// Valid usage modes:
-//
-// retrybuilds -loghash=f45f0eb8
-// retrybuilds -builder=openbsd-amd64
-// retrybuilds -builder=openbsd-amd64 -hash=6fecb7
-// retrybuilds -redo-flaky
-// retrybuilds -redo-flaky -builder=linux-amd64-clang
-package main
-
-import (
- "bytes"
- "crypto/hmac"
- "crypto/md5"
- "flag"
- "fmt"
- "io/ioutil"
- "log"
- "net/http"
- "net/url"
- "os"
- "path/filepath"
- "strings"
- "sync"
-)
-
-var (
- masterKeyFile = flag.String("masterkey", filepath.Join(os.Getenv("HOME"), "keys", "gobuilder-master.key"), "path to Go builder master key. If present, the key argument is not necessary")
- keyFile = flag.String("key", "", "path to key file")
- builder = flag.String("builder", "", "builder to wipe a result for.")
- hash = flag.String("hash", "", "Hash to wipe. If empty, all will be wiped.")
- redoFlaky = flag.Bool("redo-flaky", false, "Reset all flaky builds. If builder is empty, the master key is required.")
- builderPrefix = flag.String("builder-prefix", "https://build.golang.org", "builder URL prefix")
- logHash = flag.String("loghash", "", "If non-empty, clear the build that failed with this loghash prefix")
-)
-
-type Failure struct {
- Builder string
- Hash string
- LogURL string
-}
-
-func main() {
- flag.Parse()
- *builderPrefix = strings.TrimSuffix(*builderPrefix, "/")
- if *logHash != "" {
- substr := "/log/" + *logHash
- for _, f := range failures() {
- if strings.Contains(f.LogURL, substr) {
- wipe(f.Builder, f.Hash)
- }
- }
- return
- }
- if *redoFlaky {
- fixTheFlakes()
- return
- }
- if *builder == "" {
- log.Fatalf("Missing -builder, -redo-flaky, or -loghash flag.")
- }
- wipe(*builder, fullHash(*hash))
-}
-
-func fixTheFlakes() {
- gate := make(chan bool, 50)
- var wg sync.WaitGroup
- for _, f := range failures() {
- f := f
- if *builder != "" && f.Builder != *builder {
- continue
- }
- gate <- true
- wg.Add(1)
- go func() {
- defer wg.Done()
- defer func() { <-gate }()
- res, err := http.Get(f.LogURL)
- if err != nil {
- log.Fatalf("Error fetching %s: %v", f.LogURL, err)
- }
- defer res.Body.Close()
- failLog, err := ioutil.ReadAll(res.Body)
- if err != nil {
- log.Fatalf("Error reading %s: %v", f.LogURL, err)
- }
- if isFlaky(string(failLog)) {
- log.Printf("Restarting flaky %+v", f)
- wipe(f.Builder, f.Hash)
- }
- }()
- }
- wg.Wait()
-}
-
-var flakePhrases = []string{
- "No space left on device",
- "fatal error: error in backend: IO failure on output stream",
- "Boffset: unknown state 0",
- "Bseek: unknown state 0",
- "error exporting repository: exit status",
- "remote error: User Is Over Quota",
- "fatal: remote did not send all necessary objects",
-}
-
-func isFlaky(failLog string) bool {
- if strings.HasPrefix(failLog, "exit status ") {
- return true
- }
- if strings.HasPrefix(failLog, "timed out after ") {
- return true
- }
- for _, phrase := range flakePhrases {
- if strings.Contains(failLog, phrase) {
- return true
- }
- }
- numLines := strings.Count(failLog, "\n")
- if numLines < 20 && strings.Contains(failLog, "error: exit status") {
- return true
- }
- // e.g. fatal: destination path 'go.tools.TMP' already exists and is not an empty directory.
- // To be fixed in golang.org/issue/9407
- if strings.Contains(failLog, "fatal: destination path '") &&
- strings.Contains(failLog, "' already exists and is not an empty directory.") {
- return true
- }
- return false
-}
-
-func fullHash(h string) string {
- if h == "" || len(h) == 40 {
- return h
- }
- for _, f := range failures() {
- if strings.HasPrefix(f.Hash, h) {
- return f.Hash
- }
- }
- log.Fatalf("invalid hash %q; failed to finds its full hash. Not a recent failure?", h)
- panic("unreachable")
-}
-
-// hash may be empty
-func wipe(builder, hash string) {
- if hash != "" {
- log.Printf("Clearing %s, hash %s", builder, hash)
- } else {
- log.Printf("Clearing all builds for %s", builder)
- }
- vals := url.Values{
- "builder": {builder},
- "hash": {hash},
- "key": {builderKey(builder)},
- }
- res, err := http.PostForm(*builderPrefix+"/clear-results?"+vals.Encode(), nil)
- if err != nil {
- log.Fatal(err)
- }
- defer res.Body.Close()
- if res.StatusCode != 200 {
- log.Fatalf("Error clearing %v hash %q: %v", builder, hash, res.Status)
- }
-}
-
-func builderKey(builder string) string {
- if v, ok := builderKeyFromMaster(builder); ok {
- return v
- }
- if *keyFile == "" {
- log.Fatalf("No --key specified for builder %s", builder)
- }
- slurp, err := ioutil.ReadFile(*keyFile)
- if err != nil {
- log.Fatalf("Error reading builder key %s: %v", builder, err)
- }
- return strings.TrimSpace(string(slurp))
-}
-
-func builderKeyFromMaster(builder string) (key string, ok bool) {
- if *masterKeyFile == "" {
- return
- }
- slurp, err := ioutil.ReadFile(*masterKeyFile)
- if err != nil {
- return
- }
- h := hmac.New(md5.New, bytes.TrimSpace(slurp))
- h.Write([]byte(builder))
- return fmt.Sprintf("%x", h.Sum(nil)), true
-}
-
-var (
- failMu sync.Mutex
- failCache []Failure
-)
-
-func failures() (ret []Failure) {
- failMu.Lock()
- ret = failCache
- failMu.Unlock()
- if ret != nil {
- return
- }
- ret = []Failure{} // non-nil
-
- res, err := http.Get(*builderPrefix + "/?mode=failures")
- if err != nil {
- log.Fatal(err)
- }
- defer res.Body.Close()
- slurp, err := ioutil.ReadAll(res.Body)
- if err != nil {
- log.Fatal(err)
- }
- body := string(slurp)
- for _, line := range strings.Split(body, "\n") {
- f := strings.Fields(line)
- if len(f) == 3 {
- ret = append(ret, Failure{
- Hash: f[0],
- Builder: f[1],
- LogURL: f[2],
- })
- }
- }
-
- failMu.Lock()
- failCache = ret
- failMu.Unlock()
- return ret
-}
diff --git a/dashboard/cmd/upload/upload.go b/dashboard/cmd/upload/upload.go
deleted file mode 100644
index 087be49..0000000
--- a/dashboard/cmd/upload/upload.go
+++ /dev/null
@@ -1,117 +0,0 @@
-// Copyright 2015 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build extdep
-
-// The upload command writes a file to Google Cloud Storage. It's used
-// exclusively by the Makefiles in the Go project repos. Think of it
-// as a very light version of gsutil or gcloud, but with some
-// Go-specific configuration knowledge baked in.
-package main
-
-import (
- "bytes"
- "flag"
- "fmt"
- "io"
- "log"
- "net/http"
- "os"
- "strings"
-
- "golang.org/x/oauth2"
- "golang.org/x/tools/dashboard/auth"
- "google.golang.org/cloud"
- "google.golang.org/cloud/storage"
-)
-
-var (
- public = flag.Bool("public", false, "object should be world-readable")
- file = flag.String("file", "-", "Filename to read object from, or '-' for stdin.")
- verbose = flag.Bool("verbose", false, "verbose logging")
-)
-
-func main() {
- flag.Usage = func() {
- fmt.Fprintf(os.Stderr, "Usage: upload [--public] [--file=...] <bucket/object>\n")
- flag.PrintDefaults()
- }
- flag.Parse()
- if flag.NArg() != 1 {
- flag.Usage()
- os.Exit(1)
- }
- args := strings.SplitN(flag.Arg(0), "/", 2)
- if len(args) != 2 {
- flag.Usage()
- os.Exit(1)
- }
- bucket, object := args[0], args[1]
-
- proj, ok := bucketProject[bucket]
- if !ok {
- log.Fatalf("bucket %q doesn't have an associated project in upload.go")
- }
-
- ts, err := tokenSource(bucket)
- if err != nil {
- log.Fatalf("Failed to get an OAuth2 token source: %v", err)
- }
- httpClient := oauth2.NewClient(oauth2.NoContext, ts)
-
- ctx := cloud.NewContext(proj, httpClient)
- w := storage.NewWriter(ctx, bucket, object)
- // If you don't give the owners access, the web UI seems to
- // have a bug and doesn't have access to see that it's public, so
- // won't render the "Shared Publicly" link. So we do that, even
- // though it's dumb and unnecessary otherwise:
- w.ACL = append(w.ACL, storage.ACLRule{Entity: storage.ACLEntity("project-owners-" + proj), Role: storage.RoleOwner})
- if *public {
- w.ACL = append(w.ACL, storage.ACLRule{Entity: storage.AllUsers, Role: storage.RoleReader})
- }
- var content io.Reader
- if *file == "-" {
- content = os.Stdin
- } else {
- content, err = os.Open(*file)
- if err != nil {
- log.Fatal(err)
- }
- }
-
- const maxSlurp = 1 << 20
- var buf bytes.Buffer
- n, err := io.CopyN(&buf, content, maxSlurp)
- if err != nil && err != io.EOF {
- log.Fatalf("Error reading from stdin: %v, %v", n, err)
- }
- w.ContentType = http.DetectContentType(buf.Bytes())
-
- _, err = io.Copy(w, io.MultiReader(&buf, content))
- if cerr := w.Close(); cerr != nil && err == nil {
- err = cerr
- }
- if err != nil {
- log.Fatalf("Write error: %v", err)
- }
- if *verbose {
- log.Printf("Wrote %v", object)
- }
- os.Exit(0)
-}
-
-var bucketProject = map[string]string{
- "go-builder-data": "symbolic-datum-552",
- "http2-demo-server-tls": "symbolic-datum-552",
- "winstrap": "999119582588",
- "gobuilder": "999119582588", // deprecated
-}
-
-func tokenSource(bucket string) (oauth2.TokenSource, error) {
- proj, ok := bucketProject[bucket]
- if !ok {
- return nil, fmt.Errorf("unknown project for bucket %q", bucket)
- }
- return auth.ProjectTokenSource(proj, storage.ScopeReadWrite)
-}
diff --git a/dashboard/env/commit-watcher/Dockerfile b/dashboard/env/commit-watcher/Dockerfile
deleted file mode 100644
index dce074a..0000000
--- a/dashboard/env/commit-watcher/Dockerfile
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-# Commit watcher for Go repos.
-
-FROM debian:wheezy
-MAINTAINER golang-dev <golang-dev@googlegroups.com>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-ADD /scripts/install-apt-deps.sh /scripts/
-RUN /scripts/install-apt-deps.sh
-
-ADD /scripts/build-commit-watcher.sh /scripts/
-RUN GO_REV=go1.4 WATCHER_REV=a54d0066172 /scripts/build-commit-watcher.sh && test -f /usr/local/bin/watcher
diff --git a/dashboard/env/commit-watcher/Makefile b/dashboard/env/commit-watcher/Makefile
deleted file mode 100644
index f844ecc..0000000
--- a/dashboard/env/commit-watcher/Makefile
+++ /dev/null
@@ -1,9 +0,0 @@
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-docker: Dockerfile
- docker build -t go-commit-watcher .
-
-docker-commit-watcher.tar.gz: docker
- docker save go-commit-watcher | gzip | (cd ../../cmd/upload && go run --tags=extdep upload.go --public go-builder-data/docker-commit-watcher.tar.gz)
diff --git a/dashboard/env/commit-watcher/scripts/build-commit-watcher.sh b/dashboard/env/commit-watcher/scripts/build-commit-watcher.sh
deleted file mode 100755
index ca03b41..0000000
--- a/dashboard/env/commit-watcher/scripts/build-commit-watcher.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-set -ex
-
-export GOPATH=/gopath
-export GOROOT=/goroot
-PREFIX=/usr/local
-: ${GO_REV:?"need to be set to the golang repo revision used to build the commit watcher."}
-: ${WATCHER_REV:?"need to be set to the go.tools repo revision for the commit watcher."}
-
-mkdir -p $GOROOT
-git clone https://go.googlesource.com/go $GOROOT
-(cd $GOROOT/src && git reset --hard $GO_REV && find && ./make.bash)
-
-GO_TOOLS=$GOPATH/src/golang.org/x/tools
-mkdir -p $GO_TOOLS
-git clone https://go.googlesource.com/tools $GO_TOOLS
-
-mkdir -p $PREFIX/bin
-(cd $GO_TOOLS && git reset --hard $WATCHER_REV && GOBIN=$PREFIX/bin /goroot/bin/go install golang.org/x/tools/dashboard/watcher)
-
-rm -fR $GOROOT/bin $GOROOT/pkg $GOPATH
diff --git a/dashboard/env/commit-watcher/scripts/install-apt-deps.sh b/dashboard/env/commit-watcher/scripts/install-apt-deps.sh
deleted file mode 100755
index 02e6b07..0000000
--- a/dashboard/env/commit-watcher/scripts/install-apt-deps.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-set -ex
-
-apt-get update
-apt-get install -y --no-install-recommends ca-certificates
-# For building Go's bootstrap 'dist' prog
-apt-get install -y --no-install-recommends gcc libc6-dev
-# For interacting with the Go source & subrepos:
-apt-get install -y --no-install-recommends git-core
-
-apt-get clean
-rm -fr /var/lib/apt/lists
diff --git a/dashboard/env/linux-x86-base/Dockerfile b/dashboard/env/linux-x86-base/Dockerfile
deleted file mode 100644
index 866b8a0..0000000
--- a/dashboard/env/linux-x86-base/Dockerfile
+++ /dev/null
@@ -1,19 +0,0 @@
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-# Base builder image: gobuilders/linux-x86-base
-
-FROM debian:wheezy
-MAINTAINER golang-dev <golang-dev@googlegroups.com>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-ADD /scripts/install-apt-deps.sh /scripts/
-RUN /scripts/install-apt-deps.sh
-
-ADD /scripts/build-go-builder.sh /scripts/
-RUN GO_REV=go1.4 BUILDER_REV=d79e0375a /scripts/build-go-builder.sh && test -f /usr/local/bin/builder
-
-RUN mkdir -p /go1.4-386 && (curl --silent https://storage.googleapis.com/golang/go1.4.linux-386.tar.gz | tar -C /go1.4-386 -zxv)
-RUN mkdir -p /go1.4-amd64 && (curl --silent https://storage.googleapis.com/golang/go1.4.linux-amd64.tar.gz | tar -C /go1.4-amd64 -zxv)
diff --git a/dashboard/env/linux-x86-base/Makefile b/dashboard/env/linux-x86-base/Makefile
deleted file mode 100644
index ecfee99..0000000
--- a/dashboard/env/linux-x86-base/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-docker: Dockerfile
- docker build -t gobuilders/linux-x86-base .
-
-docker-linux.base.tar.gz: docker
- docker save gobuilders/linux-x86-base | gzip | (cd ../../cmd/upload && go run --tags=extdep upload.go --public go-builder-data/docker-linux.base.tar.gz)
-
-check: docker
- docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-base /usr/local/bin/builder -rev=20a10e7ddd1 -buildroot=/ -v -report=false linux-amd64-temp
-
-check32: docker
- docker run -e GOROOT_BOOTSTRAP=/go1.4-386/go gobuilders/linux-x86-base /usr/local/bin/builder -rev=20a10e7ddd1 -buildroot=/ -v -report=false linux-386-temp
diff --git a/dashboard/env/linux-x86-base/README b/dashboard/env/linux-x86-base/README
deleted file mode 100644
index a511909..0000000
--- a/dashboard/env/linux-x86-base/README
+++ /dev/null
@@ -1,11 +0,0 @@
-For now, you can at least do a single build of a single revision:
-
-$ export BUILD=linux-amd64-temp
-$ docker run \
- -v $HOME/keys/$BUILD.buildkey:/.gobuildkey \
- gobuilders/linux-x86-base \
- /usr/local/bin/builder -rev=50ac9eded6ad -buildroot=/ -v $BUILD
-
-TODO(bradfitz): automate with CoreOS + GCE, ala:
- https://github.com/bradfitz/camlistore/blob/master/misc/gce/create.go
-
diff --git a/dashboard/env/linux-x86-base/scripts/build-go-builder.sh b/dashboard/env/linux-x86-base/scripts/build-go-builder.sh
deleted file mode 100755
index 13b7d73..0000000
--- a/dashboard/env/linux-x86-base/scripts/build-go-builder.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-set -ex
-
-export GOPATH=/gopath
-export GOROOT=/goroot
-PREFIX=/usr/local
-: ${GO_REV:?"need to be set to the golang repo revision used to build the builder."}
-: ${BUILDER_REV:?"need to be set to the go.tools repo revision for the builder."}
-
-mkdir -p $GOROOT
-git clone https://go.googlesource.com/go $GOROOT
-(cd $GOROOT/src && git checkout $GO_REV && find && ./make.bash)
-
-GO_TOOLS=$GOPATH/src/golang.org/x/tools
-mkdir -p $GO_TOOLS
-git clone https://go.googlesource.com/tools $GO_TOOLS
-
-mkdir -p $PREFIX/bin
-(cd $GO_TOOLS && git reset --hard $BUILDER_REV && GOBIN=$PREFIX/bin /goroot/bin/go install golang.org/x/tools/dashboard/builder)
-
-rm -fR $GOROOT/bin $GOROOT/pkg $GOPATH
-
-cd $GOROOT
-git clean -f -d -x
-git checkout master
diff --git a/dashboard/env/linux-x86-base/scripts/install-apt-deps.sh b/dashboard/env/linux-x86-base/scripts/install-apt-deps.sh
deleted file mode 100755
index db1b635..0000000
--- a/dashboard/env/linux-x86-base/scripts/install-apt-deps.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-set -ex
-
-apt-get update
-apt-get install -y --no-install-recommends ca-certificates
-# Optionally used by some net/http tests:
-apt-get install -y --no-install-recommends strace
-# For building Go's bootstrap 'dist' prog
-apt-get install -y --no-install-recommends gcc libc6-dev
-# For 32-bit builds:
-# TODO(bradfitz): move these into a 386 image that derives from this one.
-apt-get install -y --no-install-recommends libc6-dev-i386 gcc-multilib
-# For interacting with the Go source & subrepos:
-apt-get install -y --no-install-recommends git-core
-# For downloading Go 1.4:
-apt-get install -y --no-install-recommends curl
-
-apt-get clean
-rm -fr /var/lib/apt/lists
diff --git a/dashboard/env/linux-x86-clang/Dockerfile b/dashboard/env/linux-x86-clang/Dockerfile
deleted file mode 100644
index 939606c..0000000
--- a/dashboard/env/linux-x86-clang/Dockerfile
+++ /dev/null
@@ -1,23 +0,0 @@
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-# gobuilders/linux-x86-clang for building with clang instead of gcc.
-
-FROM debian:wheezy
-MAINTAINER golang-dev <golang-dev@googlegroups.com>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-ADD /sources/clang-deps.list /etc/apt/sources.list.d/
-
-ADD /scripts/install-apt-deps.sh /scripts/
-RUN /scripts/install-apt-deps.sh
-
-ADD /scripts/build-go-builder.sh /scripts/
-RUN GO_REV=go1.4 BUILDER_REV=d79e0375a /scripts/build-go-builder.sh && test -f /usr/local/bin/builder
-
-ENV CC /usr/bin/clang
-
-RUN mkdir -p /go1.4-386 && (curl --silent https://storage.googleapis.com/golang/go1.4.linux-386.tar.gz | tar -C /go1.4-386 -zxv)
-RUN mkdir -p /go1.4-amd64 && (curl --silent https://storage.googleapis.com/golang/go1.4.linux-amd64.tar.gz | tar -C /go1.4-amd64 -zxv)
diff --git a/dashboard/env/linux-x86-clang/Makefile b/dashboard/env/linux-x86-clang/Makefile
deleted file mode 100644
index 0ae0fd0..0000000
--- a/dashboard/env/linux-x86-clang/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-docker: Dockerfile
- docker build -t gobuilders/linux-x86-clang .
-
-docker-linux.clang.tar.gz: docker
- docker save gobuilders/linux-x86-clang | gzip | (cd ../../cmd/upload && go run --tags=extdep upload.go --public go-builder-data/docker-linux.clang.tar.gz)
-
-check: docker
- docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-clang /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-temp
-
-check32: docker
- docker run -e GOROOT_BOOTSTRAP=/go1.4-386/go gobuilders/linux-x86-clang /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-386-temp
diff --git a/dashboard/env/linux-x86-clang/scripts/build-go-builder.sh b/dashboard/env/linux-x86-clang/scripts/build-go-builder.sh
deleted file mode 100755
index 13b7d73..0000000
--- a/dashboard/env/linux-x86-clang/scripts/build-go-builder.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-set -ex
-
-export GOPATH=/gopath
-export GOROOT=/goroot
-PREFIX=/usr/local
-: ${GO_REV:?"need to be set to the golang repo revision used to build the builder."}
-: ${BUILDER_REV:?"need to be set to the go.tools repo revision for the builder."}
-
-mkdir -p $GOROOT
-git clone https://go.googlesource.com/go $GOROOT
-(cd $GOROOT/src && git checkout $GO_REV && find && ./make.bash)
-
-GO_TOOLS=$GOPATH/src/golang.org/x/tools
-mkdir -p $GO_TOOLS
-git clone https://go.googlesource.com/tools $GO_TOOLS
-
-mkdir -p $PREFIX/bin
-(cd $GO_TOOLS && git reset --hard $BUILDER_REV && GOBIN=$PREFIX/bin /goroot/bin/go install golang.org/x/tools/dashboard/builder)
-
-rm -fR $GOROOT/bin $GOROOT/pkg $GOPATH
-
-cd $GOROOT
-git clean -f -d -x
-git checkout master
diff --git a/dashboard/env/linux-x86-clang/scripts/install-apt-deps.sh b/dashboard/env/linux-x86-clang/scripts/install-apt-deps.sh
deleted file mode 100755
index f5d3fda..0000000
--- a/dashboard/env/linux-x86-clang/scripts/install-apt-deps.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-set -ex
-
-apt-get update
-apt-get install -y --no-install-recommends ca-certificates
-# Optionally used by some net/http tests:
-apt-get install -y --no-install-recommends strace
-# For building Go's bootstrap 'dist' prog
-apt-get install -y --no-install-recommends wget
-wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key | apt-key add -
-apt-get update
-apt-get install -y --no-install-recommends clang-3.5
-# TODO(cmang): move these into a 386 image that derives from this one.
-apt-get install -y --no-install-recommends libc6-dev-i386 gcc-multilib
-# Remove gcc binary so it doesn't interfere with clang
-rm -f /usr/bin/gcc
-# For interacting with the Go source & subrepos:
-apt-get install -y --no-install-recommends git-core
-# For installing Go 1.4:
-apt-get install -y --no-install-recommends curl
-
-apt-get clean
-rm -fr /var/lib/apt/lists
diff --git a/dashboard/env/linux-x86-clang/sources/clang-deps.list b/dashboard/env/linux-x86-clang/sources/clang-deps.list
deleted file mode 100644
index eb3e244..0000000
--- a/dashboard/env/linux-x86-clang/sources/clang-deps.list
+++ /dev/null
@@ -1,3 +0,0 @@
-# The debian sources for stable clang builds, taken from http://llvm.org/apt/
-deb http://llvm.org/apt/wheezy/ llvm-toolchain-wheezy main
-deb-src http://llvm.org/apt/wheezy/ llvm-toolchain-wheezy main \ No newline at end of file
diff --git a/dashboard/env/linux-x86-gccgo/Dockerfile b/dashboard/env/linux-x86-gccgo/Dockerfile
deleted file mode 100644
index 2ccd0d9..0000000
--- a/dashboard/env/linux-x86-gccgo/Dockerfile
+++ /dev/null
@@ -1,19 +0,0 @@
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-# gobuilders/linux-x86-gccgo for 32- and 64-bit gccgo.
-
-FROM debian:wheezy
-MAINTAINER golang-dev <golang-dev@googlegroups.com>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-ADD /scripts/install-apt-deps.sh /scripts/
-RUN /scripts/install-apt-deps.sh
-
-ADD /scripts/install-gold.sh /scripts/
-RUN /scripts/install-gold.sh
-
-ADD /scripts/install-gccgo-builder.sh /scripts/
-RUN /scripts/install-gccgo-builder.sh && test -f /usr/local/bin/builder \ No newline at end of file
diff --git a/dashboard/env/linux-x86-gccgo/Makefile b/dashboard/env/linux-x86-gccgo/Makefile
deleted file mode 100644
index a84ce89..0000000
--- a/dashboard/env/linux-x86-gccgo/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-docker: Dockerfile
- docker build -t gobuilders/linux-x86-gccgo .
-
-docker-linux.gccgo.tar.gz: docker
- docker save gobuilders/linux-x86-gccgo | gzip | (cd ../../cmd/upload && go run --tags=extdep upload.go --public go-builder-data/docker-linux.gccgo.tar.gz)
-
-check: docker
- docker run gobuilders/linux-x86-gccgo /usr/local/bin/builder -tool="gccgo" -rev=b9151e911a54 -v -cmd='make RUNTESTFLAGS="--target_board=unix/-m64" check-go' -report=false linux-amd64-gccgo-temp
-
-check32: docker
- docker run gobuilders/linux-x86-gccgo /usr/local/bin/builder -tool="gccgo" -rev=b9151e911a54 -v -cmd='make RUNTESTFLAGS="--target_board=unix/-m32" check-go' -report=false linux-386-gccgo-temp
diff --git a/dashboard/env/linux-x86-gccgo/README b/dashboard/env/linux-x86-gccgo/README
deleted file mode 100644
index 65180bc..0000000
--- a/dashboard/env/linux-x86-gccgo/README
+++ /dev/null
@@ -1,6 +0,0 @@
-$ export BUILD=linux-amd64-gccgo
-$ export BUILDREV=b9151e911a54
-$ docker run \
- -v $HOME/keys/$BUILD.buildkey:/.gobuildkey \
- gobuilders/linux-x86-gccgo \
- /usr/local/bin/builder -tool=gccgo -dashboard='https://build.golang.org/gccgo' -rev=$BUILDREV -buildroot=/gccgo -v -cmd='make check-go -kj' $BUILD
diff --git a/dashboard/env/linux-x86-gccgo/scripts/install-apt-deps.sh b/dashboard/env/linux-x86-gccgo/scripts/install-apt-deps.sh
deleted file mode 100755
index 90dbac1..0000000
--- a/dashboard/env/linux-x86-gccgo/scripts/install-apt-deps.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-set -ex
-
-apt-get update
-# For running curl to get the gccgo builder binary.
-apt-get install -y --no-install-recommends curl ca-certificates
-# Optionally used by some net/http tests:
-apt-get install -y --no-install-recommends strace
-# For using numeric libraries within GCC.
-apt-get install -y --no-install-recommends libgmp10-dev libmpc-dev libmpfr-dev
-# For building binutils and gcc from source.
-apt-get install -y --no-install-recommends make g++ flex bison
-# Same as above, but for 32-bit builds as well.
-apt-get install -y --no-install-recommends libc6-dev-i386 g++-multilib
-# For running the extended gccgo testsuite
-apt-get install -y --no-install-recommends dejagnu
-# For interacting with the gccgo source and git mirror:
-apt-get install -y --no-install-recommends mercurial git-core
-
-apt-get clean
-rm -rf /var/lib/apt/lists
diff --git a/dashboard/env/linux-x86-gccgo/scripts/install-gccgo-builder.sh b/dashboard/env/linux-x86-gccgo/scripts/install-gccgo-builder.sh
deleted file mode 100755
index fd3785d..0000000
--- a/dashboard/env/linux-x86-gccgo/scripts/install-gccgo-builder.sh
+++ /dev/null
@@ -1,7 +0,0 @@
-set -ex
-
-# Installs a version of the go.tools dashboard builder that runs the gccgo build
-# command assuming there are 16 cores available to speed up build times.
-# TODO(cmang): There should be an option in the builder to specify this.
-
-curl -o /usr/local/bin/builder http://storage.googleapis.com/go-builder-data/gccgo_builder && chmod +x /usr/local/bin/builder
diff --git a/dashboard/env/linux-x86-gccgo/scripts/install-gold.sh b/dashboard/env/linux-x86-gccgo/scripts/install-gold.sh
deleted file mode 100755
index 77f96ac..0000000
--- a/dashboard/env/linux-x86-gccgo/scripts/install-gold.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-set -ex
-
-# gccgo uses the Gold linker from binutils.
-export BINUTILS_VERSION=binutils-2.24
-mkdir -p binutils-objdir
-curl -s http://ftp.gnu.org/gnu/binutils/$BINUTILS_VERSION.tar.gz | tar x --no-same-owner -zv
-(cd binutils-objdir && ../$BINUTILS_VERSION/configure --enable-gold --enable-plugins --prefix=/opt/gold && make -sj && make install -sj)
-
-rm -rf binutils* \ No newline at end of file
diff --git a/dashboard/env/linux-x86-nacl/Dockerfile b/dashboard/env/linux-x86-nacl/Dockerfile
deleted file mode 100644
index 6c491d2..0000000
--- a/dashboard/env/linux-x86-nacl/Dockerfile
+++ /dev/null
@@ -1,27 +0,0 @@
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-# gobuilders/linux-x86-nacl for 32- and 64-bit nacl.
-#
-# We need more modern libc than Debian stable as used in base, so we're
-# using Ubuntu LTS here.
-#
-# TODO(bradfitz): make both be Ubuntu? But we also want Debian, Fedora,
-# etc coverage., so deal with unifying these later, once there's a plan
-# or a generator for them and the other builders are turned down.
-
-FROM ubuntu:trusty
-MAINTAINER golang-dev <golang-dev@googlegroups.com>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-ADD /scripts/install-apt-deps.sh /scripts/
-RUN /scripts/install-apt-deps.sh
-
-ADD /scripts/build-go-builder.sh /scripts/
-RUN GO_REV=go1.4 BUILDER_REV=6735829f /scripts/build-go-builder.sh && test -f /usr/local/bin/builder
-
-ADD build-command.pl /usr/local/bin/
-
-ENV PATH /usr/local/bin:$GOROOT/bin:$PATH
diff --git a/dashboard/env/linux-x86-nacl/Makefile b/dashboard/env/linux-x86-nacl/Makefile
deleted file mode 100644
index 398221b..0000000
--- a/dashboard/env/linux-x86-nacl/Makefile
+++ /dev/null
@@ -1,12 +0,0 @@
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-docker: Dockerfile
- docker build -t gobuilders/linux-x86-nacl .
-
-upload: docker
- docker save gobuilders/linux-x86-nacl | gzip | (cd ../../cmd/upload && go run --tags=extdep upload.go --public go-builder-data/docker-linux.nacl.tar.gz)
-
-check: docker
- docker run gobuilders/linux-x86-nacl /usr/local/bin/builder -rev=77e96c9208d0 -buildroot=/ -v -cmd=/usr/local/bin/build-command.pl -report=false nacl-amd64p32
diff --git a/dashboard/env/linux-x86-nacl/README b/dashboard/env/linux-x86-nacl/README
deleted file mode 100644
index 5862ee1..0000000
--- a/dashboard/env/linux-x86-nacl/README
+++ /dev/null
@@ -1,6 +0,0 @@
-$ export BUILD=nacl-amd64p32-temp
-$ export BUILDREV=59b1bb4bf045
-$ docker run \
- -v $HOME/keys/$BUILD.buildkey:/.gobuildkey \
- gobuilders/linux-x86-nacl \
- /usr/local/bin/builder -rev=$BUILDREV -buildroot=/ -v -cmd=/usr/local/bin/build-command.pl $BUILD
diff --git a/dashboard/env/linux-x86-nacl/build-command.pl b/dashboard/env/linux-x86-nacl/build-command.pl
deleted file mode 100755
index 0eb9edb..0000000
--- a/dashboard/env/linux-x86-nacl/build-command.pl
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/usr/bin/perl
-
-use strict;
-
-if ($ENV{GOOS} eq "nacl") {
- delete $ENV{GOROOT_FINAL};
- exec("./nacltest.bash", @ARGV);
- die "Failed to run nacltest.bash: $!\n";
-}
-
-exec("./all.bash", @ARGV);
-die "Failed to run all.bash: $!\n";
-
diff --git a/dashboard/env/linux-x86-nacl/scripts/build-go-builder.sh b/dashboard/env/linux-x86-nacl/scripts/build-go-builder.sh
deleted file mode 100755
index 8cf9c27..0000000
--- a/dashboard/env/linux-x86-nacl/scripts/build-go-builder.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-set -ex
-
-export GOPATH=/gopath
-export GOROOT=/goroot
-PREFIX=/usr/local
-: ${GO_REV:?"need to be set to the golang repo revision used to build the builder."}
-: ${BUILDER_REV:?"need to be set to the go.tools repo revision for the builder."}
-
-mkdir -p $GOROOT
-git clone https://go.googlesource.com/go $GOROOT
-(cd $GOROOT/src && git checkout $GO_REV && find && ./make.bash)
-
-GO_TOOLS=$GOPATH/src/golang.org/x/tools
-mkdir -p $GO_TOOLS
-git clone https://go.googlesource.com/tools $GO_TOOLS
-
-mkdir -p $PREFIX/bin
-(cd $GO_TOOLS && git reset --hard $BUILDER_REV && GOBIN=$PREFIX/bin /goroot/bin/go install golang.org/x/tools/dashboard/builder)
-
-rm -fR $GOROOT/bin $GOROOT/pkg $GOPATH
-
-(cd /usr/local/bin && curl -s -O https://storage.googleapis.com/gobuilder/sel_ldr_x86_32 && chmod +x sel_ldr_x86_32)
-(cd /usr/local/bin && curl -s -O https://storage.googleapis.com/gobuilder/sel_ldr_x86_64 && chmod +x sel_ldr_x86_64)
-
-ln -s $GOROOT/misc/nacl/go_nacl_386_exec /usr/local/bin/
-ln -s $GOROOT/misc/nacl/go_nacl_amd64p32_exec /usr/local/bin/
-
-cd $GOROOT
-git clean -f -d -x
-git checkout master
diff --git a/dashboard/env/linux-x86-nacl/scripts/install-apt-deps.sh b/dashboard/env/linux-x86-nacl/scripts/install-apt-deps.sh
deleted file mode 100755
index 0034fa6..0000000
--- a/dashboard/env/linux-x86-nacl/scripts/install-apt-deps.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-set -ex
-
-apt-get update
-# curl is needed to fetch the sel_ldr nacl binaries:
-apt-get install -y --no-install-recommends curl ca-certificates
-# For building Go's bootstrap 'dist' prog
-apt-get install -y --no-install-recommends gcc libc6-dev
-# For interacting with the Go source & subrepos:
-apt-get install -y --no-install-recommends git-core
-# For 32-bit nacl:
-apt-get install -y --no-install-recommends libc6-i386 libc6-dev-i386 lib32stdc++6 gcc-multilib
-
-apt-get clean
-rm -fr /var/lib/apt/lists
diff --git a/dashboard/env/linux-x86-sid/Dockerfile b/dashboard/env/linux-x86-sid/Dockerfile
deleted file mode 100644
index f591339..0000000
--- a/dashboard/env/linux-x86-sid/Dockerfile
+++ /dev/null
@@ -1,17 +0,0 @@
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-FROM debian:sid
-MAINTAINER golang-dev <golang-dev@googlegroups.com>
-
-ENV DEBIAN_FRONTEND noninteractive
-
-ADD /scripts/install-apt-deps.sh /scripts/
-RUN /scripts/install-apt-deps.sh
-
-ADD /scripts/build-go-builder.sh /scripts/
-RUN GO_REV=go1.4 BUILDER_REV=d79e0375a /scripts/build-go-builder.sh && test -f /usr/local/bin/builder
-
-RUN mkdir -p /go1.4-386 && (curl --silent https://storage.googleapis.com/golang/go1.4.linux-386.tar.gz | tar -C /go1.4-386 -zxv)
-RUN mkdir -p /go1.4-amd64 && (curl --silent https://storage.googleapis.com/golang/go1.4.linux-amd64.tar.gz | tar -C /go1.4-amd64 -zxv)
diff --git a/dashboard/env/linux-x86-sid/Makefile b/dashboard/env/linux-x86-sid/Makefile
deleted file mode 100644
index d40634a..0000000
--- a/dashboard/env/linux-x86-sid/Makefile
+++ /dev/null
@@ -1,15 +0,0 @@
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-docker: Dockerfile
- docker build -t gobuilders/linux-x86-sid .
-
-docker-linux.sid.tar.gz: docker
- docker save gobuilders/linux-x86-sid | gzip | (cd ../../cmd/upload && go run --tags=extdep upload.go --public go-builder-data/docker-linux.sid.tar.gz)
-
-check: docker
- docker run -e GOROOT_BOOTSTRAP=/go1.4-amd64/go gobuilders/linux-x86-sid /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-amd64-sid
-
-check32: docker
- docker run -e GOROOT_BOOTSTRAP=/go1.4-386/go gobuilders/linux-x86-sid /usr/local/bin/builder -rev=20a10e7ddd1b -buildroot=/ -v -report=false linux-386-sid
diff --git a/dashboard/env/linux-x86-sid/scripts/build-go-builder.sh b/dashboard/env/linux-x86-sid/scripts/build-go-builder.sh
deleted file mode 100755
index 13b7d73..0000000
--- a/dashboard/env/linux-x86-sid/scripts/build-go-builder.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-set -ex
-
-export GOPATH=/gopath
-export GOROOT=/goroot
-PREFIX=/usr/local
-: ${GO_REV:?"need to be set to the golang repo revision used to build the builder."}
-: ${BUILDER_REV:?"need to be set to the go.tools repo revision for the builder."}
-
-mkdir -p $GOROOT
-git clone https://go.googlesource.com/go $GOROOT
-(cd $GOROOT/src && git checkout $GO_REV && find && ./make.bash)
-
-GO_TOOLS=$GOPATH/src/golang.org/x/tools
-mkdir -p $GO_TOOLS
-git clone https://go.googlesource.com/tools $GO_TOOLS
-
-mkdir -p $PREFIX/bin
-(cd $GO_TOOLS && git reset --hard $BUILDER_REV && GOBIN=$PREFIX/bin /goroot/bin/go install golang.org/x/tools/dashboard/builder)
-
-rm -fR $GOROOT/bin $GOROOT/pkg $GOPATH
-
-cd $GOROOT
-git clean -f -d -x
-git checkout master
diff --git a/dashboard/env/linux-x86-sid/scripts/install-apt-deps.sh b/dashboard/env/linux-x86-sid/scripts/install-apt-deps.sh
deleted file mode 100755
index 173d03a..0000000
--- a/dashboard/env/linux-x86-sid/scripts/install-apt-deps.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-set -ex
-
-apt-get update
-apt-get install -y --no-install-recommends ca-certificates
-# Optionally used by some net/http tests:
-apt-get install -y --no-install-recommends strace
-# For building Go's bootstrap 'dist' prog
-apt-get install -y --no-install-recommends gcc libc6-dev
-# For 32-bit builds:
-# TODO(bradfitz): move these into a 386 image that derives from this one.
-apt-get install -y --no-install-recommends libc6-dev-i386 gcc-multilib
-# For interacting with the Go source & subrepos:
-apt-get install -y --no-install-recommends git-core
-# For installing Go 1.4:
-apt-get install -y --no-install-recommends curl
-
-apt-get clean
-rm -fr /var/lib/apt/lists
diff --git a/dashboard/env/openbsd-amd64/.gitignore b/dashboard/env/openbsd-amd64/.gitignore
deleted file mode 100644
index f1bb5ed..0000000
--- a/dashboard/env/openbsd-amd64/.gitignore
+++ /dev/null
@@ -1,8 +0,0 @@
-boot.conf
-disk.raw
-etc
-install.site
-install56-patched.iso
-install56.iso
-random.seed
-site56.tgz
diff --git a/dashboard/env/openbsd-amd64/README b/dashboard/env/openbsd-amd64/README
deleted file mode 100644
index f54a076..0000000
--- a/dashboard/env/openbsd-amd64/README
+++ /dev/null
@@ -1,20 +0,0 @@
-make.bash creates a Google Compute Engine VM image to run the Go
-OpenBSD builder, booting up to run the buildlet.
-
-make.bash should be run on a Linux box with qemu.
-
-After it completes, it creates a file openbsd-amd64-gce.tar.gz
-
-Then:
- gsutil cp -a public-read openbsd-amd64-gce.tar.gz gs://go-builder-data/openbsd-amd64-gce.tar.gz
-Or just use the web UI at:
- https://console.developers.google.com/project/symbolic-datum-552/storage/browser/go-builder-data/
-
-Then:
- gcloud compute --project symbolic-datum-552 images delete openbsd-amd64-56
- gcloud compute --project symbolic-datum-552 images create openbsd-amd64-56 --source-uri gs://go-builder-data/openbsd-amd64-gce.tar.gz
-
-The VM needs to be run with the GCE metadata attribute "buildlet-binary-url" set to a URL
-of the OpenBSD buildlet (cross-compiled, typically).
-
- buildlet-binary-url == http://storage.googleapis.com/go-builder-data/buildlet.openbsd-amd64
diff --git a/dashboard/env/openbsd-amd64/make.bash b/dashboard/env/openbsd-amd64/make.bash
deleted file mode 100755
index 9ed91cd..0000000
--- a/dashboard/env/openbsd-amd64/make.bash
+++ /dev/null
@@ -1,227 +0,0 @@
-#!/bin/bash
-# Copyright 2014 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-set -e
-
-# Download kernel, sets, etc. from ftp.usa.openbsd.org
-if ! [ -e install56.iso ]; then
- curl -O ftp://ftp.usa.openbsd.org/pub/OpenBSD/5.6/amd64/install56.iso
-fi
-
-# XXX: Download and save bash, curl, and their dependencies too?
-# Currently we download them from the network during the install process.
-
-# Create custom site56.tgz set.
-mkdir -p etc
-cat >install.site <<EOF
-#!/bin/sh
-env PKG_PATH=ftp://ftp.usa.openbsd.org/pub/OpenBSD/5.6/packages/amd64 pkg_add -iv bash curl git
-
-# See https://code.google.com/p/google-compute-engine/issues/detail?id=77
-echo "ignore classless-static-routes;" >> /etc/dhclient.conf
-EOF
-cat >etc/rc.local <<EOF
-(
- set -x
- echo "starting buildlet script"
- netstat -rn
- cat /etc/resolv.conf
- dig metadata.google.internal
- (
- set -e
- export PATH="\$PATH:/usr/local/bin"
- /usr/local/bin/curl -o /buildlet \$(/usr/local/bin/curl -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/attributes/buildlet-binary-url)
- chmod +x /buildlet
- exec /buildlet
- )
- echo "giving up"
- sleep 10
- halt -p
-)
-EOF
-chmod +x install.site
-tar -zcvf site56.tgz install.site etc/rc.local
-
-# Hack install CD a bit.
-echo 'set tty com0' > boot.conf
-dd if=/dev/urandom of=random.seed bs=4096 count=1
-cp install56.iso install56-patched.iso
-growisofs -M install56-patched.iso -l -R -graft-points \
- /5.6/amd64/site56.tgz=site56.tgz \
- /etc/boot.conf=boot.conf \
- /etc/random.seed=random.seed
-
-# Initialize disk image.
-rm -f disk.raw
-qemu-img create -f raw disk.raw 10G
-
-# Run the installer to create the disk image.
-expect <<EOF
-spawn qemu-system-x86_64 -nographic -smp 2 -drive if=virtio,file=disk.raw -cdrom install56-patched.iso -net nic,model=virtio -net user -boot once=d
-
-expect "boot>"
-send "\n"
-
-# Need to wait for the kernel to boot.
-expect -timeout 600 "\(I\)nstall, \(U\)pgrade, \(A\)utoinstall or \(S\)hell\?"
-send "i\n"
-
-expect "Terminal type\?"
-send "vt220\n"
-
-expect "System hostname\?"
-send "buildlet\n"
-
-expect "Which network interface do you wish to configure\?"
-send "vio0\n"
-
-expect "IPv4 address for vio0\?"
-send "dhcp\n"
-
-expect "IPv6 address for vio0\?"
-send "none\n"
-
-expect "Which network interface do you wish to configure\?"
-send "done\n"
-
-expect "Password for root account\?"
-send "root\n"
-
-expect "Password for root account\?"
-send "root\n"
-
-expect "Start sshd\(8\) by default\?"
-send "yes\n"
-
-expect "Start ntpd\(8\) by default\?"
-send "no\n"
-
-expect "Do you expect to run the X Window System\?"
-send "no\n"
-
-expect "Do you want the X Window System to be started by xdm\(1\)\?"
-send "no\n"
-
-expect "Do you want to suspend on lid close\?"
-send "no\n"
-
-expect "Change the default console to com0\?"
-send "yes\n"
-
-expect "Which speed should com0 use\?"
-send "115200\n"
-
-expect "Setup a user\?"
-send "gopher\n"
-
-expect "Full name for user gopher\?"
-send "Gopher Gopherson\n"
-
-expect "Password for user gopher\?"
-send "gopher\n"
-
-expect "Password for user gopher\?"
-send "gopher\n"
-
-expect "Since you set up a user, disable sshd\(8\) logins to root\?"
-send "yes\n"
-
-expect "What timezone are you in\?"
-send "US/Pacific\n"
-
-expect "Which disk is the root disk\?"
-send "sd0\n"
-
-expect "Use DUIDs rather than device names in fstab\?"
-send "yes\n"
-
-expect "Use \(W\)hole disk or \(E\)dit the MBR\?"
-send "whole\n"
-
-expect "Use \(A\)uto layout, \(E\)dit auto layout, or create \(C\)ustom layout\?"
-send "custom\n"
-
-expect "> "
-send "z\n"
-
-expect "> "
-send "a b\n"
-expect "offset: "
-send "\n"
-expect "size: "
-send "1G\n"
-expect "FS type: "
-send "swap\n"
-
-expect "> "
-send "a a\n"
-expect "offset: "
-send "\n"
-expect "size: "
-send "\n"
-expect "FS type: "
-send "4.2BSD\n"
-expect "mount point: "
-send "/\n"
-
-expect "> "
-send "w\n"
-expect "> "
-send "q\n"
-
-expect "Location of sets\?"
-send "cd\n"
-
-expect "Which CD-ROM contains the install media\?"
-send "cd0\n"
-
-expect "Pathname to the sets\?"
-send "5.6/amd64\n"
-
-expect "Set name\(s\)\?"
-send "+*\n"
-
-expect "Set name\(s\)\?"
-send " -x*\n"
-
-expect "Set name\(s\)\?"
-send " -game*\n"
-
-expect "Set name\(s\)\?"
-send " -man*\n"
-
-expect "Set name\(s\)\?"
-send "done\n"
-
-expect "Directory does not contain SHA256\.sig\. Continue without verification\?"
-send "yes\n"
-
-# Need to wait for previous sets to unpack.
-expect -timeout 600 "Location of sets\?"
-send "done\n"
-
-expect "Ambiguous: choose dependency for git"
-send "0\n"
-
-# Need to wait for install.site to install curl, git, et
-expect -timeout 600 "CONGRATULATIONS!"
-
-expect "# "
-send "halt\n"
-
-expect "Please press any key to reboot.\n"
-send "\n"
-
-expect "boot>"
-send "\n"
-
-expect -timeout 600 eof
-EOF
-
-# Create Compute Engine disk image.
-echo "Archiving disk.raw... (this may take a while)"
-tar -Szcf openbsd-amd64-gce.tar.gz disk.raw
-
-echo "Done. GCE image is openbsd-amd64-gce.tar.gz."
diff --git a/dashboard/env/plan9-386/.gitignore b/dashboard/env/plan9-386/.gitignore
deleted file mode 100644
index b5a531d..0000000
--- a/dashboard/env/plan9-386/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-disk.raw
-plan9-gce.iso
diff --git a/dashboard/env/plan9-386/README b/dashboard/env/plan9-386/README
deleted file mode 100644
index f1e99d6..0000000
--- a/dashboard/env/plan9-386/README
+++ /dev/null
@@ -1,21 +0,0 @@
-make.bash creates a Google Compute Engine VM image to run the Go
-Plan 9 builder, booting up to run the buildlet.
-
-make.bash should be run on a Linux box with qemu.
-
-After it completes, it creates a file plan9-386-gce.tar.gz
-
-The make.bash script depends on the following packages:
-
-$ sudo apt-get install bzip2 curl expect qemu
-or
-$ sudo yum install bzip2 curl expect qemu
-
-It has been tested with QEMU 1.4.2 to 2.2.0, as provided with:
-
- - Ubuntu 14.04 (Trusty Tahr) and above
- - Debian 8.0 (Jessie) and above
- - Fedora 19 and above
-
-Also, due to an ATA bug affecting QEMU 1.6 and 1.7, the
-Plan 9 CD can't be booted with these versions.
diff --git a/dashboard/env/plan9-386/make.bash b/dashboard/env/plan9-386/make.bash
deleted file mode 100755
index 6c113ba..0000000
--- a/dashboard/env/plan9-386/make.bash
+++ /dev/null
@@ -1,236 +0,0 @@
-# Copyright 2015 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-set -e
-
-# Download Plan 9
-if ! sha1sum -c plan9-gce.iso.sha1; then
- curl --fail -O http://9legacy.org/download/go/plan9-gce.iso.bz2
- bunzip2 plan9-gce.iso.bz2
- sha1sum -c plan9-gce.iso.sha1
-fi
-
-# Initialize disk image.
-rm -f disk.raw
-qemu-img create -f raw disk.raw 10G
-
-# Run the installer to create the disk image.
-expect <<EOF
-spawn qemu-system-i386 -machine accel=kvm -nographic -net user -net nic,model=virtio -m 1024 -vga none -drive if=none,id=hd,file=disk.raw -device virtio-scsi-pci,id=scsi -device scsi-hd,drive=hd -cdrom plan9-gce.iso -boot d
-
-expect -exact "Selection:"
-send "1\n"
-
-expect -exact "Plan 9"
-sleep 5
-
-# Need to wait for the kernel to boot.
-expect -timeout 600 -exact "use DMA for ide drives\[yes\]:"
-send "\n"
-
-expect -exact "mouseport is (ps2, ps2intellimouse, 0, 1, 2)\[ps2\]:"
-send "ps2intellimouse\n"
-
-expect -exact "vgasize \[640x480x8\]:"
-send "1280x1024x32\n"
-
-expect -exact "monitor is \[xga\]:"
-send "vesa\n"
-
-expect -exact "% "
-send "inst/textonly\n"
-
-expect -exact "Task to do \[configfs\]:"
-send "\n"
-
-expect -exact "File system (fossil, fossil+venti)\[fossil\]:"
-send "\n"
-
-expect -exact "Task to do \[partdisk\]:"
-send "\n"
-
-expect -exact "Disk to partition (sd00, sdD0)\[no default\]:"
-send "sd00\n"
-
-expect -exact "Install mbr \(y, n\)\[no default\]:"
-send "y\n"
-
-expect -exact ">>> "
-send "w\n"
-
-expect -exact ">>> "
-send "q\n"
-
-expect -exact "Task to do \[prepdisk\]:"
-send "\n"
-
-expect -exact "Plan 9 partition to subdivide \(/dev/sd00/plan9\)\[/dev/sd00/plan9\]:"
-send "\n"
-
-expect -exact ">>> "
-send "w\n"
-
-expect -exact ">>> "
-send "q\n"
-
-expect -exact "Task to do \[fmtfossil\]:"
-send "\n"
-
-expect -exact "Fossil partition to format \(/dev/sd00/fossil\)\[/dev/sd00/fossil\]:"
-send "\n"
-
-expect -exact "Task to do \[mountfs\]:"
-send "\n"
-
-expect -exact "Fossil partition \(/dev/sd00/fossil\)\[/dev/sd00/fossil\]:"
-send "\n"
-
-expect -exact "Task to do \[configdist\]:"
-send "\n"
-
-expect -exact "Distribution is from \(local, net\)\[local\]:"
-send "\n"
-
-expect -exact "Task to do \[mountdist\]:"
-send "\n"
-
-expect -exact "Distribution disk \[no default\]:"
-send "/dev/sdD0/data\n"
-
-expect -exact "Location of archives \[browse\]:"
-send "/\n"
-
-expect -exact "Task to do \[copydist\]:"
-send "\n"
-
-# Need to wait for the copy to finish.
-expect -timeout 600 -exact "Task to do \[bootsetup\]:"
-send "\n"
-
-expect -exact "Enable boot method (floppy, plan9, win9x, winnt)\[no default\]:"
-send "plan9\n"
-
-expect -exact "Install the Plan 9 master boot record (y, n)\[no default\]:"
-send "y\n"
-
-expect -exact "Task to do \[finish\]:"
-send "\n"
-
-expect -exact "Feel free to turn off your computer."
-exit
-EOF
-
-# Configuration.
-expect <<EOF
-spawn qemu-system-i386 -machine accel=kvm -nographic -net user -net nic,model=virtio -m 1024 -vga none -drive if=none,id=hd,file=disk.raw -device virtio-scsi-pci,id=scsi -device scsi-hd,drive=hd -cdrom plan9-gce.iso -boot c
-
-expect -exact "Plan 9"
-sleep 5
-
-# Need to wait for the kernel to boot.
-expect -timeout 600 -exact "term% "
-send "\n"
-
-expect -exact "term% "
-send "9fat:\n"
-
-expect -exact "term% "
-send "sed s/9pcf/9pccpuf/ /n/9fat/plan9.ini >/tmp/plan9.ini\n"
-
-expect -exact "term% "
-send "mv /tmp/plan9.ini /n/9fat/plan9.ini\n"
-
-expect -exact "term% "
-send "cp /386/9pccpuf /n/9fat\n"
-
-expect -exact "term% "
-send "sed s/sd00/sd01/ /n/9fat/plan9.ini >/tmp/plan9.ini\n"
-
-expect -exact "term% "
-send "mv /tmp/plan9.ini /n/9fat/plan9.ini\n"
-
-expect -exact "term% "
-send "unmount /n/9fat\n"
-
-expect -exact "term% "
-send "fossil/conf /dev/sd00/fossil | sed s/sd00/sd01/ >/tmp/fossil.conf\n"
-
-expect -exact "term% "
-send "fossil/conf -w /dev/sd00/fossil /tmp/fossil.conf\n"
-
-expect -exact "term% "
-send "rm /tmp/fossil.conf\n"
-
-expect -exact "term% "
-send "mkdir /cfg/helix\n"
-
-expect -exact "term% "
-send "dircp /cfg/example /cfg/helix\n"
-
-expect -exact "term% "
-send "echo ip/ipconfig >>/cfg/helix/cpurc\n"
-
-expect -exact "term% "
-send "echo ndb/dns -r >>/cfg/helix/cpurc\n"
-
-expect -exact "term% "
-send "echo echo remove 10.0.0.0 /104 10.0.0.0 '>'/net/iproute >>/cfg/helix/cpurc\n"
-
-expect -exact "term% "
-send "echo ramfs -u >>/cfg/helix/cpustart\n"
-
-expect -exact "term% "
-send "echo echo downloading git >>/cfg/helix/cpustart\n"
-
-expect -exact "term% "
-send "echo hget http://9legacy.org/9legacy/tools/git '>'/usr/glenda/bin/rc/git >>/cfg/helix/cpustart\n"
-
-expect -exact "term% "
-send "echo chmod +x /usr/glenda/bin/rc/git >>/cfg/helix/cpustart\n"
-
-expect -exact "term% "
-send "echo >>/cfg/helix/cpustart\n"
-
-expect -exact "term% "
-send "echo echo starting buildlet script >>/cfg/helix/cpustart\n"
-
-expect -exact "term% "
-send "echo 'hget \`{hget -r '''Metadata-Flavor: Google''' http://metadata.google.internal/computeMetadata/v1/instance/attributes/buildlet-binary-url} >/tmp/buildlet' >>/cfg/helix/cpustart\n"
-
-expect -exact "term% "
-send "echo chmod +x /tmp/buildlet >>/cfg/helix/cpustart\n"
-
-expect -exact "term% "
-send "echo exec /tmp/buildlet >>/cfg/helix/cpustart\n"
-
-expect -exact "term% "
-send "echo fshalt >>/cfg/helix/cpustart\n"
-
-expect -exact "term% "
-send "auth/wrkey\n"
-
-expect -exact "authid: "
-send "glenda\n"
-
-expect -exact "authdom: "
-send "go\n"
-
-expect -exact "auth password: "
-send "glenda123\n"
-
-expect -exact "secstore password: "
-send "glenda123\n"
-
-expect -exact "term% "
-send "fshalt\n"
-
-expect -exact "done halting"
-exit
-EOF
-
-# Create Compute Engine disk image.
-echo "Archiving disk.raw... (this may take a while)"
-tar -Szcf plan9-386-gce.tar.gz disk.raw
-
-echo "Done. GCE image is plan9-386-gce.tar.gz."
diff --git a/dashboard/env/plan9-386/plan9-gce.iso.sha1 b/dashboard/env/plan9-386/plan9-gce.iso.sha1
deleted file mode 100644
index 3bc91fa..0000000
--- a/dashboard/env/plan9-386/plan9-gce.iso.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6c3ca633d65d6d00dff668b2b2dd3746c9883145 plan9-gce.iso
diff --git a/dashboard/env/windows/README b/dashboard/env/windows/README
deleted file mode 100644
index c8a2e39..0000000
--- a/dashboard/env/windows/README
+++ /dev/null
@@ -1,70 +0,0 @@
-Unlike the other operating systems which have a bash script that runs
-on Linux and scripts qemu to prepare a GCE image, the Windows image
-builder image is prepared by hand, following this list of
-instructions:
-
--- in GCE, create a new Windows 2008 server instance. Set its initial username
- to "wingopher" and the password to something.
-
--- boot it. first boot is slow.
-
--- Connect with RDP to it. (I used the OS X Microsoft Remote Desktop
- in the Mac App Store)
-
--- In initial “Initial Configuration Tasks” window:
- * disable automatic installation of updates. They’ll just interrupt
- the builds later. And we don't care about security since this isn't
- going to be Internet-facing. No ports will be accessible
- * disable Windows Error Report (“Not participating”)
-
-TODO: disable the Windows firewall too. might break some outgoing net
- tests? Figure out where to do this.
-
--- Enable auto-login:
- * Start > Run > Open: "control userpasswords2" [OK]
- * Uncheck the "Users must enter a user name..." box (it already was
- unchecked for me)
- * Press "OK"
- * Enter password twice.
-
--- Download winstrap.
- * Bring up cmd.exe (Start > Run > "cmd" [OK])
- * Copy/paste (Right click: paste) this into cmd.exe:
-
- bitsadmin /transfer mydownloadjob /download /priority normal https://storage.googleapis.com/winstrap/winstrap-2015-01-03-440bb77.exe c:\users\wingopher\Desktop\winstrap.exe
-
- * It should appear on the desktop.
-
--- double click winstrap. type "go" and press <enter>. abort it once
- it's downloaded everything. (close the window)
-
--- make the Buildlet start on boot:
- * Start > All Programs > right click "Startup"
- * Right-mouse-drag "Start Buildlet" from the desktop (from winstrap) to the
- Startup folder and "Create shortcuts here"
-
--- Install git with middle option: "Use Git from Windows Command Prompt",
- otherwise all the defaults.
-
--- click yes yes yes on the tdm64-gcc installer question defaults.
- Then do the tdm64-gcc installer again but do 32-bit this time. You
- should then have two TDM installs at c:\TDM-GCC-64 and
- c:\TDM-GCC-32. Don’t pick other locations. The gobuilder adds
- those to your path.
-
-TODO(bradfitz): this document is incomplete. Cover gcesysprep,
-regedit(?), windows-startup-script-cmd, etc. And I think winstrap
-will need to write things to something outside of C:\Users, since
-sysprep on reboot nukes all users, at least on GCE. We probably need
-to run the buildlet's stage0 from the windows-startup-script-cmd. Find
-& ask Windows experts.
-
---
-
-Docs:
-http://www.win2008workstation.com/configure-auto-logon/
-https://cloud.google.com/compute/docs/operating-systems/windows
- -- notably "windows-startup-script-cmd" ala:
- gcloud compute instances add-metadata <INSTANCE> --zone <ZONE> --metadata windows-startup-script-cmd="net user <USERNAME> <PASSWORD>"
-https://github.com/golang/winstrap
-http://golang.org/s/windowsbuilderhowto
diff --git a/dashboard/types/types.go b/dashboard/types/types.go
deleted file mode 100644
index b8dfa48..0000000
--- a/dashboard/types/types.go
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package types contains common types used by the Go continuous build
-// system.
-package types
-
-// BuildStatus is the data structure that's marshalled as JSON
-// for the http://build.golang.org/?mode=json page.
-type BuildStatus struct {
- // Builders is a list of all known builders.
- // The order that builders appear is the same order as the build results for a revision.
- Builders []string `json:"builders"`
-
- // Revisions are the revisions shown on the front page of build.golang.org,
- // in the same order. It starts with the "go" repo, from recent to old, and then
- // it has 1 each of the subrepos, with only their most recent commit.
- Revisions []BuildRevision `json:"revisions"`
-}
-
-// BuildRevision is the status of a commit across all builders.
-// It corresponds to a single row of http://build.golang.org/
-type BuildRevision struct {
- // Repo is "go" for the main repo, else "tools", "crypto", "net", etc.
- // These are repos as listed at https://go.googlesource.com/
- Repo string `json:"repo"`
-
- // Revision is the full git hash of the repo.
- Revision string `json:"revision"`
-
- // GoRevision is the full git hash of the "go" repo, if Repo is not "go" itself.
- // Otherwise this is empty.
- GoRevision string `json:"goRevision,omitempty"`
-
- // Results are the build results for each of the builders in
- // the same length slice BuildStatus.Builders.
- // Each string is either "" (if no data), "ok", or the URL to failure logs.
- Results []string `json:"results"`
-}
diff --git a/dashboard/watcher/watcher.go b/dashboard/watcher/watcher.go
deleted file mode 100644
index f29d5a5..0000000
--- a/dashboard/watcher/watcher.go
+++ /dev/null
@@ -1,775 +0,0 @@
-// Copyright 2014 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Command watcher watches the specified repository for new commits
-// and reports them to the build dashboard.
-package main // import "golang.org/x/tools/dashboard/watcher"
-
-import (
- "bufio"
- "bytes"
- "encoding/json"
- "errors"
- "flag"
- "fmt"
- "io"
- "io/ioutil"
- "log"
- "net/http"
- "net/url"
- "os"
- "os/exec"
- "path"
- "path/filepath"
- "runtime"
- "sort"
- "strings"
- "sync"
- "time"
-)
-
-const (
- goBase = "https://go.googlesource.com/"
- watcherVersion = 3 // must match dashboard/app/build/handler.go's watcherVersion
- origin = "origin/"
- master = origin + "master" // name of the master branch
- metaURL = goBase + "?b=master&format=JSON"
-)
-
-var (
- repoURL = flag.String("repo", goBase+"go", "Repository URL")
- dashboard = flag.String("dash", "https://build.golang.org/", "Dashboard URL (must end in /)")
- keyFile = flag.String("key", defaultKeyFile, "Build dashboard key file")
- pollInterval = flag.Duration("poll", 10*time.Second, "Remote repo poll interval")
- network = flag.Bool("network", true, "Enable network calls (disable for testing)")
-)
-
-var (
- defaultKeyFile = filepath.Join(homeDir(), ".gobuildkey")
- dashboardKey = ""
- networkSeen = make(map[string]bool) // track known hashes for testing
-)
-
-func main() {
- flag.Parse()
- go pollGerritAndTickle()
-
- err := run()
- fmt.Fprintln(os.Stderr, err)
- os.Exit(1)
-}
-
-// run is a little wrapper so we can use defer and return to signal
-// errors. It should only return a non-nil error.
-func run() error {
- if !strings.HasSuffix(*dashboard, "/") {
- return errors.New("dashboard URL (-dashboard) must end in /")
- }
-
- if k, err := readKey(); err != nil {
- return err
- } else {
- dashboardKey = k
- }
-
- dir, err := ioutil.TempDir("", "watcher")
- if err != nil {
- return err
- }
- defer os.RemoveAll(dir)
-
- errc := make(chan error)
-
- go func() {
- r, err := NewRepo(dir, *repoURL, "")
- if err != nil {
- errc <- err
- return
- }
- errc <- r.Watch()
- }()
-
- subrepos, err := subrepoList()
- if err != nil {
- return err
- }
- for _, path := range subrepos {
- go func(path string) {
- url := goBase + strings.TrimPrefix(path, "golang.org/x/")
- r, err := NewRepo(dir, url, path)
- if err != nil {
- errc <- err
- return
- }
- errc <- r.Watch()
- }(path)
- }
-
- // Must be non-nil.
- return <-errc
-}
-
-// Repo represents a repository to be watched.
-type Repo struct {
- root string // on-disk location of the git repo
- path string // base import path for repo (blank for main repo)
- commits map[string]*Commit // keyed by full commit hash (40 lowercase hex digits)
- branches map[string]*Branch // keyed by branch name, eg "release-branch.go1.3" (or empty for default)
-}
-
-// NewRepo checks out a new instance of the Mercurial repository
-// specified by url to a new directory inside dir.
-// The path argument is the base import path of the repository,
-// and should be empty for the main Go repo.
-func NewRepo(dir, url, path string) (*Repo, error) {
- r := &Repo{
- path: path,
- root: filepath.Join(dir, filepath.Base(path)),
- commits: make(map[string]*Commit),
- branches: make(map[string]*Branch),
- }
-
- r.logf("cloning %v", url)
- cmd := exec.Command("git", "clone", url, r.root)
- if out, err := cmd.CombinedOutput(); err != nil {
- return nil, fmt.Errorf("%v\n\n%s", err, out)
- }
-
- r.logf("loading commit log")
- if err := r.update(false); err != nil {
- return nil, err
- }
-
- r.logf("found %v branches among %v commits\n", len(r.branches), len(r.commits))
- return r, nil
-}
-
-// Watch continuously runs "git fetch" in the repo, checks for
-// new commits, and posts any new commits to the dashboard.
-// It only returns a non-nil error.
-func (r *Repo) Watch() error {
- tickler := repoTickler(r.name())
- for {
- if err := r.fetch(); err != nil {
- return err
- }
- if err := r.update(true); err != nil {
- return err
- }
- remotes, err := r.remotes()
- if err != nil {
- return err
- }
- for _, name := range remotes {
- b, ok := r.branches[name]
- if !ok {
- // skip branch; must be already merged
- continue
- }
- if err := r.postNewCommits(b); err != nil {
- return err
- }
- }
- // We still run a timer but a very slow one, just
- // in case the mechanism updating the repo tickler
- // breaks for some reason.
- timer := time.NewTimer(5 * time.Minute)
- select {
- case <-tickler:
- timer.Stop()
- case <-timer.C:
- }
- }
-}
-
-func (r *Repo) name() string {
- if r.path == "" {
- return "go"
- }
- return path.Base(r.path)
-}
-
-func (r *Repo) logf(format string, args ...interface{}) {
- log.Printf(r.name()+": "+format, args...)
-}
-
-// postNewCommits looks for unseen commits on the specified branch and
-// posts them to the dashboard.
-func (r *Repo) postNewCommits(b *Branch) error {
- if b.Head == b.LastSeen {
- return nil
- }
- c := b.LastSeen
- if c == nil {
- // Haven't seen anything on this branch yet:
- if b.Name == master {
- // For the master branch, bootstrap by creating a dummy
- // commit with a lone child that is the initial commit.
- c = &Commit{}
- for _, c2 := range r.commits {
- if c2.Parent == "" {
- c.children = []*Commit{c2}
- break
- }
- }
- if c.children == nil {
- return fmt.Errorf("couldn't find initial commit")
- }
- } else {
- // Find the commit that this branch forked from.
- base, err := r.mergeBase(b.Name, master)
- if err != nil {
- return err
- }
- var ok bool
- c, ok = r.commits[base]
- if !ok {
- return fmt.Errorf("couldn't find base commit: %v", base)
- }
- }
- }
- if err := r.postChildren(b, c); err != nil {
- return err
- }
- b.LastSeen = b.Head
- return nil
-}
-
-// postChildren posts to the dashboard all descendants of the given parent.
-// It ignores descendants that are not on the given branch.
-func (r *Repo) postChildren(b *Branch, parent *Commit) error {
- for _, c := range parent.children {
- if c.Branch != b.Name {
- continue
- }
- if err := r.postCommit(c); err != nil {
- return err
- }
- }
- for _, c := range parent.children {
- if err := r.postChildren(b, c); err != nil {
- return err
- }
- }
- return nil
-}
-
-// postCommit sends a commit to the build dashboard.
-func (r *Repo) postCommit(c *Commit) error {
- r.logf("sending commit to dashboard: %v", c)
-
- t, err := time.Parse("Mon, 2 Jan 2006 15:04:05 -0700", c.Date)
- if err != nil {
- return fmt.Errorf("postCommit: parsing date %q for commit %v: %v", c.Date, c, err)
- }
- dc := struct {
- PackagePath string // (empty for main repo commits)
- Hash string
- ParentHash string
-
- User string
- Desc string
- Time time.Time
- Branch string
-
- NeedsBenchmarking bool
- }{
- PackagePath: r.path,
- Hash: c.Hash,
- ParentHash: c.Parent,
-
- User: c.Author,
- Desc: c.Desc,
- Time: t,
- Branch: strings.TrimPrefix(c.Branch, origin),
-
- NeedsBenchmarking: c.NeedsBenchmarking(),
- }
- b, err := json.Marshal(dc)
- if err != nil {
- return fmt.Errorf("postCommit: marshaling request body: %v", err)
- }
-
- if !*network {
- if c.Parent != "" {
- if !networkSeen[c.Parent] {
- r.logf("%v: %v", c.Parent, r.commits[c.Parent])
- return fmt.Errorf("postCommit: no parent %v found on dashboard for %v", c.Parent, c)
- }
- }
- if networkSeen[c.Hash] {
- return fmt.Errorf("postCommit: already seen %v", c)
- }
- networkSeen[c.Hash] = true
- return nil
- }
-
- u := fmt.Sprintf("%vcommit?version=%v&key=%v", *dashboard, watcherVersion, dashboardKey)
- resp, err := http.Post(u, "text/json", bytes.NewReader(b))
- if err != nil {
- return err
- }
- defer resp.Body.Close()
- if resp.StatusCode != 200 {
- return fmt.Errorf("postCommit: status: %v", resp.Status)
- }
-
- var s struct {
- Error string
- }
- err = json.NewDecoder(resp.Body).Decode(&s)
- if err != nil {
- return fmt.Errorf("postCommit: decoding response: %v", err)
- }
- if s.Error != "" {
- return fmt.Errorf("postCommit: error: %v", s.Error)
- }
- return nil
-}
-
-// update looks for new commits and branches,
-// and updates the commits and branches maps.
-func (r *Repo) update(noisy bool) error {
- remotes, err := r.remotes()
- if err != nil {
- return err
- }
- for _, name := range remotes {
- b := r.branches[name]
-
- // Find all unseen commits on this branch.
- revspec := name
- if b != nil {
- // If we know about this branch,
- // only log commits down to the known head.
- revspec = b.Head.Hash + ".." + name
- } else if revspec != master {
- // If this is an unknown non-master branch,
- // log up to where it forked from master.
- base, err := r.mergeBase(name, master)
- if err != nil {
- return err
- }
- revspec = base + ".." + name
- }
- log, err := r.log("--topo-order", revspec)
- if err != nil {
- return err
- }
- if len(log) == 0 {
- // No commits to handle; carry on.
- continue
- }
-
- // Add unknown commits to r.commits.
- var added []*Commit
- for _, c := range log {
- // Sanity check: we shouldn't see the same commit twice.
- if _, ok := r.commits[c.Hash]; ok {
- return fmt.Errorf("found commit we already knew about: %v", c.Hash)
- }
- if noisy {
- r.logf("found new commit %v", c)
- }
- c.Branch = name
- r.commits[c.Hash] = c
- added = append(added, c)
- }
-
- // Link added commits.
- for _, c := range added {
- if c.Parent == "" {
- // This is the initial commit; no parent.
- r.logf("no parents for initial commit %v", c)
- continue
- }
- // Find parent commit.
- p, ok := r.commits[c.Parent]
- if !ok {
- return fmt.Errorf("can't find parent %q for %v", c.Parent, c)
- }
- // Link parent Commit.
- c.parent = p
- // Link child Commits.
- p.children = append(p.children, c)
- }
-
- // Update branch head, or add newly discovered branch.
- head := log[0]
- if b != nil {
- // Known branch; update head.
- b.Head = head
- r.logf("updated branch head: %v", b)
- } else {
- // It's a new branch; add it.
- seen, err := r.lastSeen(head.Hash)
- if err != nil {
- return err
- }
- b = &Branch{Name: name, Head: head, LastSeen: seen}
- r.branches[name] = b
- r.logf("found branch: %v", b)
- }
- }
-
- return nil
-}
-
-// lastSeen finds the most recent commit the dashboard has seen,
-// starting at the specified head. If the dashboard hasn't seen
-// any of the commits from head to the beginning, it returns nil.
-func (r *Repo) lastSeen(head string) (*Commit, error) {
- h, ok := r.commits[head]
- if !ok {
- return nil, fmt.Errorf("lastSeen: can't find %q in commits", head)
- }
-
- var s []*Commit
- for c := h; c != nil; c = c.parent {
- s = append(s, c)
- }
-
- var err error
- i := sort.Search(len(s), func(i int) bool {
- if err != nil {
- return false
- }
- ok, err = r.dashSeen(s[i].Hash)
- return ok
- })
- switch {
- case err != nil:
- return nil, fmt.Errorf("lastSeen: %v", err)
- case i < len(s):
- return s[i], nil
- default:
- // Dashboard saw no commits.
- return nil, nil
- }
-}
-
-// dashSeen reports whether the build dashboard knows the specified commit.
-func (r *Repo) dashSeen(hash string) (bool, error) {
- if !*network {
- return networkSeen[hash], nil
- }
- v := url.Values{"hash": {hash}, "packagePath": {r.path}}
- u := *dashboard + "commit?" + v.Encode()
- resp, err := http.Get(u)
- if err != nil {
- return false, err
- }
- defer resp.Body.Close()
- if resp.StatusCode != 200 {
- return false, fmt.Errorf("status: %v", resp.Status)
- }
- var s struct {
- Error string
- }
- err = json.NewDecoder(resp.Body).Decode(&s)
- if err != nil {
- return false, err
- }
- switch s.Error {
- case "":
- // Found one.
- return true, nil
- case "Commit not found":
- // Commit not found, keep looking for earlier commits.
- return false, nil
- default:
- return false, fmt.Errorf("dashboard: %v", s.Error)
- }
-}
-
-// mergeBase returns the hash of the merge base for revspecs a and b.
-func (r *Repo) mergeBase(a, b string) (string, error) {
- cmd := exec.Command("git", "merge-base", a, b)
- cmd.Dir = r.root
- out, err := cmd.CombinedOutput()
- if err != nil {
- return "", fmt.Errorf("git merge-base: %v", err)
- }
- return string(bytes.TrimSpace(out)), nil
-}
-
-// remotes returns a slice of remote branches known to the git repo.
-// It always puts "origin/master" first.
-func (r *Repo) remotes() ([]string, error) {
- cmd := exec.Command("git", "branch", "-r")
- cmd.Dir = r.root
- out, err := cmd.CombinedOutput()
- if err != nil {
- return nil, fmt.Errorf("git branch: %v", err)
- }
- bs := []string{master}
- for _, b := range strings.Split(string(out), "\n") {
- b = strings.TrimSpace(b)
- // Ignore aliases, blank lines, and master (it's already in bs).
- if b == "" || strings.Contains(b, "->") || b == master {
- continue
- }
- // Ignore pre-go1 release branches; they are just noise.
- if strings.HasPrefix(b, origin+"release-branch.r") {
- continue
- }
- bs = append(bs, b)
- }
- return bs, nil
-}
-
-const logFormat = `--format=format:%H
-%P
-%an <%ae>
-%cD
-%B
-` + logBoundary
-
-const logBoundary = `_-_- magic boundary -_-_`
-
-// log runs "git log" with the supplied arguments
-// and parses the output into Commit values.
-func (r *Repo) log(dir string, args ...string) ([]*Commit, error) {
- args = append([]string{"log", "--date=rfc", logFormat}, args...)
- cmd := exec.Command("git", args...)
- cmd.Dir = r.root
- out, err := cmd.CombinedOutput()
- if err != nil {
- return nil, fmt.Errorf("git log %v: %v", strings.Join(args, " "), err)
- }
-
- // We have a commit with description that contains 0x1b byte.
- // Mercurial does not escape it, but xml.Unmarshal does not accept it.
- // TODO(adg): do we still need to scrub this? Probably.
- out = bytes.Replace(out, []byte{0x1b}, []byte{'?'}, -1)
-
- var cs []*Commit
- for _, text := range strings.Split(string(out), logBoundary) {
- text = strings.TrimSpace(text)
- if text == "" {
- continue
- }
- p := strings.SplitN(text, "\n", 5)
- if len(p) != 5 {
- return nil, fmt.Errorf("git log %v: malformed commit: %q", strings.Join(args, " "), text)
- }
- cs = append(cs, &Commit{
- Hash: p[0],
- // TODO(adg): This may break with branch merges.
- Parent: strings.Split(p[1], " ")[0],
- Author: p[2],
- Date: p[3],
- Desc: strings.TrimSpace(p[4]),
- // TODO(adg): populate Files
- })
- }
- return cs, nil
-}
-
-// fetch runs "git fetch" in the repository root.
-// It tries three times, just in case it failed because of a transient error.
-func (r *Repo) fetch() error {
- var err error
- for tries := 0; tries < 3; tries++ {
- time.Sleep(time.Duration(tries) * 5 * time.Second) // Linear back-off.
- cmd := exec.Command("git", "fetch", "--all")
- cmd.Dir = r.root
- if out, e := cmd.CombinedOutput(); err != nil {
- e = fmt.Errorf("%v\n\n%s", e, out)
- log.Printf("git fetch error %v: %v", r.root, e)
- if err == nil {
- err = e
- }
- continue
- }
- return nil
- }
- return err
-}
-
-// Branch represents a Mercurial branch.
-type Branch struct {
- Name string
- Head *Commit
- LastSeen *Commit // the last commit posted to the dashboard
-}
-
-func (b *Branch) String() string {
- return fmt.Sprintf("%q(Head: %v LastSeen: %v)", b.Name, b.Head, b.LastSeen)
-}
-
-// Commit represents a single Git commit.
-type Commit struct {
- Hash string
- Author string
- Date string // Format: "Mon, 2 Jan 2006 15:04:05 -0700"
- Desc string // Plain text, first line is a short description.
- Parent string
- Branch string
- Files string
-
- // For walking the graph.
- parent *Commit
- children []*Commit
-}
-
-func (c *Commit) String() string {
- s := c.Hash
- if c.Branch != "" {
- s += fmt.Sprintf("[%v]", strings.TrimPrefix(c.Branch, origin))
- }
- s += fmt.Sprintf("(%q)", strings.SplitN(c.Desc, "\n", 2)[0])
- return s
-}
-
-// NeedsBenchmarking reports whether the Commit needs benchmarking.
-func (c *Commit) NeedsBenchmarking() bool {
- // Do not benchmark branch commits, they are usually not interesting
- // and fall out of the trunk succession.
- if c.Branch != master {
- return false
- }
- // Do not benchmark commits that do not touch source files (e.g. CONTRIBUTORS).
- for _, f := range strings.Split(c.Files, " ") {
- if (strings.HasPrefix(f, "include") || strings.HasPrefix(f, "src")) &&
- !strings.HasSuffix(f, "_test.go") && !strings.Contains(f, "testdata") {
- return true
- }
- }
- return false
-}
-
-func homeDir() string {
- switch runtime.GOOS {
- case "plan9":
- return os.Getenv("home")
- case "windows":
- return os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
- }
- return os.Getenv("HOME")
-}
-
-func readKey() (string, error) {
- c, err := ioutil.ReadFile(*keyFile)
- if err != nil {
- return "", err
- }
- return string(bytes.TrimSpace(bytes.SplitN(c, []byte("\n"), 2)[0])), nil
-}
-
-// subrepoList fetches a list of sub-repositories from the dashboard
-// and returns them as a slice of base import paths.
-// Eg, []string{"golang.org/x/tools", "golang.org/x/net"}.
-func subrepoList() ([]string, error) {
- if !*network {
- return nil, nil
- }
-
- r, err := http.Get(*dashboard + "packages?kind=subrepo")
- if err != nil {
- return nil, fmt.Errorf("subrepo list: %v", err)
- }
- defer r.Body.Close()
- if r.StatusCode != 200 {
- return nil, fmt.Errorf("subrepo list: got status %v", r.Status)
- }
- var resp struct {
- Response []struct {
- Path string
- }
- Error string
- }
- err = json.NewDecoder(r.Body).Decode(&resp)
- if err != nil {
- return nil, fmt.Errorf("subrepo list: %v", err)
- }
- if resp.Error != "" {
- return nil, fmt.Errorf("subrepo list: %v", resp.Error)
- }
- var pkgs []string
- for _, r := range resp.Response {
- pkgs = append(pkgs, r.Path)
- }
- return pkgs, nil
-}
-
-var (
- ticklerMu sync.Mutex
- ticklers = make(map[string]chan bool)
-)
-
-// repo is the gerrit repo: e.g. "go", "net", "crypto", ...
-func repoTickler(repo string) chan bool {
- ticklerMu.Lock()
- defer ticklerMu.Unlock()
- if c, ok := ticklers[repo]; ok {
- return c
- }
- c := make(chan bool, 1)
- ticklers[repo] = c
- return c
-}
-
-// pollGerritAndTickle polls Gerrit's JSON meta URL of all its URLs
-// and their current branch heads. When this sees that one has
-// changed, it tickles the channel for that repo and wakes up its
-// poller, if its poller is in a sleep.
-func pollGerritAndTickle() {
- last := map[string]string{} // repo -> last seen hash
- for {
- for repo, hash := range gerritMetaMap() {
- if hash != last[repo] {
- last[repo] = hash
- select {
- case repoTickler(repo) <- true:
- log.Printf("tickled the %s repo poller", repo)
- default:
- }
- }
- }
- time.Sleep(*pollInterval)
- }
-}
-
-// gerritMetaMap returns the map from repo name (e.g. "go") to its
-// latest master hash.
-// The returned map is nil on any transient error.
-func gerritMetaMap() map[string]string {
- res, err := http.Get(metaURL)
- if err != nil {
- return nil
- }
- defer res.Body.Close()
- defer io.Copy(ioutil.Discard, res.Body) // ensure EOF for keep-alive
- if res.StatusCode != 200 {
- return nil
- }
- var meta map[string]struct {
- Branches map[string]string
- }
- br := bufio.NewReader(res.Body)
- // For security reasons or something, this URL starts with ")]}'\n" before
- // the JSON object. So ignore that.
- // Shawn Pearce says it's guaranteed to always be just one line, ending in '\n'.
- for {
- b, err := br.ReadByte()
- if err != nil {
- return nil
- }
- if b == '\n' {
- break
- }
- }
- if err := json.NewDecoder(br).Decode(&meta); err != nil {
- log.Printf("JSON decoding error from %v: %s", metaURL, err)
- return nil
- }
- m := map[string]string{}
- for repo, v := range meta {
- if master, ok := v.Branches["master"]; ok {
- m[repo] = master
- }
- }
- return m
-}