dashboard: buildlet client, server, and gomote enhancements

- get tarballs out of buildlets
- gomote can pass arguments to buildlet's exec handler
- buildlet's exec handler can run system-level commands
- hard-code GOROOT_BOOTSTRAP to be "go1.4" under the workdir
- adjust MTU size on GCE

etc

Change-Id: I73e18b7a5e395a889f5ee93ba9850d331ffb7812
Reviewed-on: https://go-review.googlesource.com/3052
Reviewed-by: Andrew Gerrand <adg@golang.org>

9 files changed, 406 insertions, 48 deletions

diff --git a/dashboard/buildlet/buildletclient.go b/dashboard/buildlet/buildletclient.go
index ef9c0a2..53b2c6a 100644
--- a/dashboard/buildlet/buildletclient.go
+++ b/dashboard/buildlet/buildletclient.go
@@ -102,12 +102,38 @@ func (c *Client) PutTarFromURL(tarURL, dir string) error {
 	return c.doOK(req)
 }
 
+// GetTar returns a .tar.gz stream of the given directory, relative to the buildlet's work dir.
+// The provided dir may be empty to get everything.
+func (c *Client) GetTar(dir string) (tgz io.ReadCloser, err error) {
+	req, err := http.NewRequest("GET", c.URL()+"/tgz?dir="+url.QueryEscape(dir), nil)
+	if err != nil {
+		return nil, err
+	}
+	res, err := c.do(req)
+	if err != nil {
+		return nil, err
+	}
+	if res.StatusCode != http.StatusOK {
+		slurp, _ := ioutil.ReadAll(io.LimitReader(res.Body, 4<<10))
+		res.Body.Close()
+		return nil, fmt.Errorf("%v; body: %s", res.Status, slurp)
+	}
+	return res.Body, nil
+}
+
 // ExecOpts are options for a remote command invocation.
 type ExecOpts struct {
 	// Output is the output of stdout and stderr.
 	// If nil, the output is discarded.
 	Output io.Writer
 
+	// Args are the arguments to pass to the cmd given to Client.Exec.
+	Args []string
+
+	// SystemLevel controls whether the command is run outside of
+	// the buildlet's environment.
+	SystemLevel bool
+
 	// OnStartExec is an optional hook that runs after the 200 OK
 	// response from the buildlet, but before the output begins
 	// writing to Output.
@@ -122,8 +148,14 @@ type ExecOpts struct {
 // seen to completition. If execErr is non-nil, the remoteErr is
 // meaningless.
 func (c *Client) Exec(cmd string, opts ExecOpts) (remoteErr, execErr error) {
+	var mode string
+	if opts.SystemLevel {
+		mode = "sys"
+	}
 	form := url.Values{
-		"cmd": {cmd},
+		"cmd":    {cmd},
+		"mode":   {mode},
+		"cmdArg": opts.Args,
 	}
 	req, err := http.NewRequest("POST", c.URL()+"/exec", strings.NewReader(form.Encode()))
 	if err != nil {
diff --git a/dashboard/cmd/buildlet/Makefile b/dashboard/cmd/buildlet/Makefile
index 078a4e9..3edd8f8 100644
--- a/dashboard/cmd/buildlet/Makefile
+++ b/dashboard/cmd/buildlet/Makefile
@@ -1,10 +1,22 @@
 buildlet: buildlet.go
 	go build --tags=extdep
 
+buildlet.darwin-amd64: buildlet.go
+	GOOS=darwin GOARCH=amd64 go build -o $@ --tags=extdep
+	cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
+
+buildlet.freebsd-amd64: buildlet.go
+	GOOS=freebsd GOARCH=amd64 go build -o $@ --tags=extdep
+	cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
+
 buildlet.linux-amd64: buildlet.go
 	GOOS=linux GOARCH=amd64 go build -o $@ --tags=extdep
 	cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
 
+buildlet.netbsd-amd64: buildlet.go
+	GOOS=netbsd GOARCH=amd64 go build -o $@ --tags=extdep
+	cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
+
 buildlet.openbsd-amd64: buildlet.go
 	GOOS=openbsd GOARCH=amd64 go build -o $@ --tags=extdep
 	cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
@@ -17,10 +29,3 @@ buildlet.windows-amd64: buildlet.go
 	GOOS=windows GOARCH=amd64 go build -o $@ --tags=extdep
 	cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
 
-buildlet.darwin-amd64: buildlet.go
-	GOOS=darwin GOARCH=amd64 go build -o $@ --tags=extdep
-	cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
-
-buildlet.netbsd-amd64: buildlet.go
-	GOOS=netbsd GOARCH=amd64 go build -o $@ --tags=extdep
-	cat $@ | (cd ../upload && go run --tags=extdep upload.go --public go-builder-data/$@)
diff --git a/dashboard/cmd/buildlet/buildlet.go b/dashboard/cmd/buildlet/buildlet.go
index b4d3f7e..95cc2ba 100644
--- a/dashboard/cmd/buildlet/buildlet.go
+++ b/dashboard/cmd/buildlet/buildlet.go
@@ -29,10 +29,11 @@ import (
 	"net/url"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"google.golang.org/cloud/compute/metadata"
@@ -66,9 +67,13 @@ var osHalt func() // set by some machines
 
 func main() {
 	flag.Parse()
-	if !metadata.OnGCE() && !strings.HasPrefix(*listenAddr, "localhost:") {
+	onGCE := metadata.OnGCE()
+	if !onGCE && !strings.HasPrefix(*listenAddr, "localhost:") {
 		log.Printf("** WARNING ***  This server is unsafe and offers no security. Be careful.")
 	}
+	if onGCE {
+		fixMTU()
+	}
 	if runtime.GOOS == "plan9" {
 		// Plan 9 is too slow on GCE, so stop running run.rc after the basics.
 		// See https://golang.org/cl/2522 and https://golang.org/issue/9491
@@ -78,7 +83,6 @@ func main() {
 		// But no need for environment variables quite yet.
 		os.Setenv("GOTESTONLY", "std")
 	}
-
 	if *scratchDir == "" {
 		dir, err := ioutil.TempDir("", "buildlet-scatch")
 		if err != nil {
@@ -86,10 +90,21 @@ func main() {
 		}
 		*scratchDir = dir
 	}
+	// TODO(bradfitz): if this becomes more of a general tool,
+	// perhaps we want to remove this hard-coded here. Also,
+	// if/once the exec handler ever gets generic environment
+	// variable support, it would make sense to remove this too
+	// and push it to the client.  This hard-codes policy. But
+	// that's okay for now.
+	os.Setenv("GOROOT_BOOTSTRAP", filepath.Join(*scratchDir, "go1.4"))
+	os.Setenv("WORKDIR", *scratchDir) // mostly for demos
+
 	if _, err := os.Lstat(*scratchDir); err != nil {
 		log.Fatalf("invalid --scratchdir %q: %v", *scratchDir, err)
 	}
 	http.HandleFunc("/", handleRoot)
+	http.HandleFunc("/debug/goroutines", handleGoroutines)
+	http.HandleFunc("/debug/x", handleX)
 
 	password := metadataValue("password")
 	requireAuth := func(handler func(w http.ResponseWriter, r *http.Request)) http.Handler {
@@ -98,6 +113,7 @@ func main() {
 	http.Handle("/writetgz", requireAuth(handleWriteTGZ))
 	http.Handle("/exec", requireAuth(handleExec))
 	http.Handle("/halt", requireAuth(handleHalt))
+	http.Handle("/tgz", requireAuth(handleGetTGZ))
 	// TODO: removeall
 
 	tlsCert, tlsKey := metadataValue("tls-cert"), metadataValue("tls-key")
@@ -179,10 +195,193 @@ func (ln tcpKeepAliveListener) Accept() (c net.Conn, err error) {
 	return tc, nil
 }
 
+func fixMTU_freebsd() error { return fixMTU_ifconfig("vtnet0") }
+func fixMTU_openbsd() error { return fixMTU_ifconfig("vio0") }
+func fixMTU_ifconfig(iface string) error {
+	out, err := exec.Command("/sbin/ifconfig", iface, "mtu", "1460").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("/sbin/ifconfig %s mtu 1460: %v, %s", iface, err, out)
+	}
+	return nil
+}
+
+func fixMTU_plan9() error {
+	f, err := os.OpenFile("/net/ipifc/0/ctl", os.O_WRONLY, 0)
+	if err != nil {
+		return err
+	}
+	if _, err := io.WriteString(f, "mtu 1400\n"); err != nil { // not 1460
+		f.Close()
+		return err
+	}
+	return f.Close()
+}
+
+func fixMTU() {
+	fn, ok := map[string]func() error{
+		"openbsd": fixMTU_openbsd,
+		"freebsd": fixMTU_freebsd,
+		"plan9":   fixMTU_plan9,
+	}[runtime.GOOS]
+	if ok {
+		if err := fn(); err != nil {
+			log.Printf("Failed to set MTU: %v", err)
+		} else {
+			log.Printf("Adjusted MTU.")
+		}
+	}
+}
+
+// mtuWriter is a hack for environments where we can't (or can't yet)
+// fix the machine's MTU.
+// Instead of telling the operating system the MTU, we just cut up our
+// writes into small pieces to make sure we don't get too near the
+// MTU, and we hope the kernel doesn't coalesce different flushed
+// writes back together into the same TCP IP packets.
+type mtuWriter struct {
+	rw http.ResponseWriter
+}
+
+func (mw mtuWriter) Write(p []byte) (n int, err error) {
+	const mtu = 1000 // way less than 1460; since HTTP response headers might be in there too
+	for len(p) > 0 {
+		chunk := p
+		if len(chunk) > mtu {
+			chunk = p[:mtu]
+		}
+		n0, err := mw.rw.Write(chunk)
+		n += n0
+		if n0 != len(chunk) && err == nil {
+			err = io.ErrShortWrite
+		}
+		if err != nil {
+			return n, err
+		}
+		p = p[n0:]
+		mw.rw.(http.Flusher).Flush()
+		if len(p) > 0 {
+			// Whitelisted operating systems:
+			if runtime.GOOS == "openbsd" || runtime.GOOS == "linux" {
+				// Nothing
+			} else {
+				// Try to prevent the kernel from Nagel-ing the IP packets
+				// together into one that's too large.
+				time.Sleep(250 * time.Millisecond)
+			}
+		}
+	}
+	return n, nil
+}
+
 func handleRoot(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path != "/" {
+		http.NotFound(w, r)
+		return
+	}
 	fmt.Fprintf(w, "buildlet running on %s-%s\n", runtime.GOOS, runtime.GOARCH)
 }
 
+// unauthenticated /debug/goroutines handler
+func handleGoroutines(rw http.ResponseWriter, r *http.Request) {
+	w := mtuWriter{rw}
+	log.Printf("Dumping goroutines.")
+	rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	buf := make([]byte, 2<<20)
+	buf = buf[:runtime.Stack(buf, true)]
+	w.Write(buf)
+	log.Printf("Dumped goroutines.")
+}
+
+// unauthenticated /debug/x handler, to test MTU settings.
+func handleX(w http.ResponseWriter, r *http.Request) {
+	n, _ := strconv.Atoi(r.FormValue("n"))
+	if n > 1<<20 {
+		n = 1 << 20
+	}
+	log.Printf("Dumping %d X.", n)
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	buf := make([]byte, n)
+	for i := range buf {
+		buf[i] = 'X'
+	}
+	w.Write(buf)
+	log.Printf("Dumped X.")
+}
+
+// This is a remote code execution daemon, so security is kinda pointless, but:
+func validRelativeDir(dir string) bool {
+	if strings.Contains(dir, `\`) || path.IsAbs(dir) {
+		return false
+	}
+	dir = path.Clean(dir)
+	if strings.HasPrefix(dir, "../") || strings.HasSuffix(dir, "/..") || dir == ".." {
+		return false
+	}
+	return true
+}
+
+func handleGetTGZ(rw http.ResponseWriter, r *http.Request) {
+	if r.Method != "GET" {
+		http.Error(rw, "requires GET method", http.StatusBadRequest)
+		return
+	}
+	dir := r.FormValue("dir")
+	if !validRelativeDir(dir) {
+		http.Error(rw, "bogus dir", http.StatusBadRequest)
+		return
+	}
+	zw := gzip.NewWriter(mtuWriter{rw})
+	tw := tar.NewWriter(zw)
+	base := filepath.Join(*scratchDir, filepath.FromSlash(dir))
+	err := filepath.Walk(base, func(path string, fi os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		rel := strings.TrimPrefix(strings.TrimPrefix(path, base), "/")
+		var linkName string
+		if fi.Mode()&os.ModeSymlink != 0 {
+			linkName, err = os.Readlink(path)
+			if err != nil {
+				return err
+			}
+		}
+		th, err := tar.FileInfoHeader(fi, linkName)
+		if err != nil {
+			return err
+		}
+		th.Name = rel
+		if fi.IsDir() && !strings.HasSuffix(th.Name, "/") {
+			th.Name += "/"
+		}
+		if th.Name == "/" {
+			return nil
+		}
+		if err := tw.WriteHeader(th); err != nil {
+			return err
+		}
+		if fi.Mode().IsRegular() {
+			f, err := os.Open(path)
+			if err != nil {
+				return err
+			}
+			defer f.Close()
+			if _, err := io.Copy(tw, f); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		log.Printf("Walk error: %v", err)
+		// Decent way to signal failure to the caller, since it'll break
+		// the chunked response, rather than have a valid EOF.
+		conn, _, _ := rw.(http.Hijacker).Hijack()
+		conn.Close()
+	}
+	tw.Close()
+	zw.Close()
+}
+
 func handleWriteTGZ(w http.ResponseWriter, r *http.Request) {
 	var tgz io.Reader
 	switch r.Method {
@@ -213,12 +412,11 @@ func handleWriteTGZ(w http.ResponseWriter, r *http.Request) {
 	urlParam, _ := url.ParseQuery(r.URL.RawQuery)
 	baseDir := *scratchDir
 	if dir := urlParam.Get("dir"); dir != "" {
-		dir = filepath.FromSlash(dir)
-		if strings.Contains(dir, "../") {
-			// This is a remote code execution daemon, so security is kinda pointless, but:
+		if !validRelativeDir(dir) {
 			http.Error(w, "bogus dir", http.StatusBadRequest)
 			return
 		}
+		dir = filepath.FromSlash(dir)
 		baseDir = filepath.Join(baseDir, dir)
 		if err := os.MkdirAll(baseDir, 0755); err != nil {
 			http.Error(w, "mkdir of base: "+err.Error(), http.StatusInternalServerError)
@@ -300,6 +498,11 @@ func untar(r io.Reader, dir string) error {
 const hdrProcessState = "Process-State"
 
 func handleExec(w http.ResponseWriter, r *http.Request) {
+	cn := w.(http.CloseNotifier)
+	clientGone := cn.CloseNotify()
+	handlerDone := make(chan bool)
+	defer close(handlerDone)
+
 	if r.Method != "POST" {
 		http.Error(w, "requires POST method", http.StatusBadRequest)
 		return
@@ -313,21 +516,46 @@ func handleExec(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Trailer", hdrProcessState) // declare it so we can set it
 
 	cmdPath := r.FormValue("cmd") // required
-	if !validRelPath(cmdPath) {
-		http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest)
-		return
+	absCmd := cmdPath
+	sysMode := r.FormValue("mode") == "sys"
+	if sysMode {
+		if cmdPath == "" {
+			http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest)
+			return
+		}
+	} else {
+		if !validRelPath(cmdPath) {
+			http.Error(w, "requires 'cmd' parameter", http.StatusBadRequest)
+			return
+		}
+		absCmd = filepath.Join(*scratchDir, filepath.FromSlash(cmdPath))
 	}
+
 	if f, ok := w.(http.Flusher); ok {
 		f.Flush()
 	}
 
-	absCmd := filepath.Join(*scratchDir, filepath.FromSlash(cmdPath))
 	cmd := exec.Command(absCmd, r.PostForm["cmdArg"]...)
-	cmd.Dir = filepath.Dir(absCmd)
-	cmdOutput := &flushWriter{w: w}
+	if sysMode {
+		cmd.Dir = *scratchDir
+	} else {
+		cmd.Dir = filepath.Dir(absCmd)
+	}
+	cmdOutput := mtuWriter{w}
 	cmd.Stdout = cmdOutput
 	cmd.Stderr = cmdOutput
-	err := cmd.Run()
+	err := cmd.Start()
+	if err == nil {
+		go func() {
+			select {
+			case <-clientGone:
+				cmd.Process.Kill()
+			case <-handlerDone:
+				return
+			}
+		}()
+		err = cmd.Wait()
+	}
 	state := "ok"
 	if err != nil {
 		if ps := cmd.ProcessState; ps != nil {
@@ -385,23 +613,6 @@ func haltMachine() {
 	os.Exit(0)
 }
 
-// flushWriter is an io.Writer wrapper that writes to w and
-// Flushes the output immediately, if w is an http.Flusher.
-type flushWriter struct {
-	mu sync.Mutex
-	w  http.ResponseWriter
-}
-
-func (hw *flushWriter) Write(p []byte) (n int, err error) {
-	hw.mu.Lock()
-	defer hw.mu.Unlock()
-	n, err = hw.w.Write(p)
-	if f, ok := hw.w.(http.Flusher); ok {
-		f.Flush()
-	}
-	return
-}
-
 func validRelPath(p string) bool {
 	if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
 		return false
diff --git a/dashboard/cmd/gomote/create.go b/dashboard/cmd/gomote/create.go
index bb25a8e..31806af 100644
--- a/dashboard/cmd/gomote/create.go
+++ b/dashboard/cmd/gomote/create.go
@@ -12,6 +12,7 @@ import (
 	"log"
 	"os"
 	"sort"
+	"strings"
 	"time"
 
 	"golang.org/x/tools/dashboard"
@@ -36,13 +37,22 @@ func create(args []string) error {
 	conf, ok := dashboard.Builders[builderType]
 	if !ok || !conf.UsesVM() {
 		var valid []string
+		var prefixMatch []string
 		for k, conf := range dashboard.Builders {
 			if conf.UsesVM() {
 				valid = append(valid, k)
+				if strings.HasPrefix(k, builderType) {
+					prefixMatch = append(prefixMatch, k)
+				}
 			}
 		}
-		sort.Strings(valid)
-		return fmt.Errorf("Invalid builder type %q. Valid options include: %q", builderType, valid)
+		if len(prefixMatch) == 1 {
+			builderType = prefixMatch[0]
+			conf, _ = dashboard.Builders[builderType]
+		} else {
+			sort.Strings(valid)
+			return fmt.Errorf("Invalid builder type %q. Valid options include: %q", builderType, valid)
+		}
 	}
 
 	instName := fmt.Sprintf("mote-%s-%s", username(), builderType)
diff --git a/dashboard/cmd/gomote/destroy.go b/dashboard/cmd/gomote/destroy.go
index ca9b4de..1f758ef 100644
--- a/dashboard/cmd/gomote/destroy.go
+++ b/dashboard/cmd/gomote/destroy.go
@@ -7,9 +7,12 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
+	"log"
 	"os"
+	"time"
 
 	"golang.org/x/tools/dashboard/buildlet"
 )
@@ -32,11 +35,47 @@ func destroy(args []string) error {
 		return err
 	}
 
-	// First ask it to kill itself, and then tell GCE to kill it too:
-	shutErr := bc.Destroy()
-	gceErr := buildlet.DestroyVM(projTokenSource(), *proj, *zone, fmt.Sprintf("mote-%s-%s", username(), name))
-	if shutErr != nil {
-		return shutErr
+	// Ask it to kill itself, and tell GCE to kill it too:
+	gceErrc := make(chan error, 1)
+	buildletErrc := make(chan error, 1)
+	go func() {
+		gceErrc <- buildlet.DestroyVM(projTokenSource(), *proj, *zone, fmt.Sprintf("mote-%s-%s", username(), name))
+	}()
+	go func() {
+		buildletErrc <- bc.Destroy()
+	}()
+	timeout := time.NewTimer(5 * time.Second)
+	defer timeout.Stop()
+
+	var retErr error
+	var gceDone, buildletDone bool
+	for !gceDone || !buildletDone {
+		select {
+		case err := <-gceErrc:
+			if err != nil {
+				log.Printf("GCE: %v", err)
+				retErr = err
+			} else {
+				log.Printf("Requested GCE delete.")
+			}
+			gceDone = true
+		case err := <-buildletErrc:
+			if err != nil {
+				log.Printf("Buildlet: %v", err)
+				retErr = err
+			} else {
+				log.Printf("Requested buildlet to shut down.")
+			}
+			buildletDone = true
+		case <-timeout.C:
+			if !buildletDone {
+				log.Printf("timeout asking buildlet to shut down")
+			}
+			if !gceDone {
+				log.Printf("timeout asking GCE to delete builder VM")
+			}
+			return errors.New("timeout")
+		}
 	}
-	return gceErr
+	return retErr
 }
diff --git a/dashboard/cmd/gomote/get.go b/dashboard/cmd/gomote/get.go
new file mode 100644
index 0000000..f78ebd3
--- /dev/null
+++ b/dashboard/cmd/gomote/get.go
@@ -0,0 +1,44 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build extdep
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"os"
+)
+
+// get a .tar.gz
+func getTar(args []string) error {
+	fs := flag.NewFlagSet("get", flag.ContinueOnError)
+	fs.Usage = func() {
+		fmt.Fprintln(os.Stderr, "create usage: gomote gettar [get-opts] <buildlet-name>\n")
+		fs.PrintDefaults()
+		os.Exit(1)
+	}
+	var dir string
+	fs.StringVar(&dir, "dir", "", "relative directory from buildlet's work dir to tar up")
+
+	fs.Parse(args)
+	if fs.NArg() != 1 {
+		fs.Usage()
+	}
+
+	name := fs.Arg(0)
+	bc, err := namedClient(name)
+	if err != nil {
+		return err
+	}
+	tgz, err := bc.GetTar(dir)
+	if err != nil {
+		return err
+	}
+	defer tgz.Close()
+	_, err = io.Copy(os.Stdout, tgz)
+	return err
+}
diff --git a/dashboard/cmd/gomote/gomote.go b/dashboard/cmd/gomote/gomote.go
index 7ede494..79f4a33 100644
--- a/dashboard/cmd/gomote/gomote.go
+++ b/dashboard/cmd/gomote/gomote.go
@@ -81,6 +81,7 @@ func registerCommands() {
 	registerCommand("run", "run a command on a buildlet", run)
 	registerCommand("put", "put files on a buildlet", put)
 	registerCommand("puttar", "extract a tar.gz to a buildlet", putTar)
+	registerCommand("gettar", "extract a tar.gz from a buildlet", getTar)
 }
 
 func main() {
diff --git a/dashboard/cmd/gomote/list.go b/dashboard/cmd/gomote/list.go
index cd3d1f8..c410f12 100644
--- a/dashboard/cmd/gomote/list.go
+++ b/dashboard/cmd/gomote/list.go
@@ -48,10 +48,21 @@ func namedClient(name string) (*buildlet.Client, error) {
 		return nil, fmt.Errorf("error listing VMs while looking up %q: %v", name, err)
 	}
 	wantName := fmt.Sprintf("mote-%s-%s", username(), name)
+	var matches []buildlet.VM
 	for _, vm := range vms {
 		if vm.Name == wantName {
 			return buildlet.NewClient(vm.IPPort, vm.TLS), nil
 		}
+		if strings.HasPrefix(vm.Name, wantName) {
+			matches = append(matches, vm)
+		}
+	}
+	if len(matches) == 1 {
+		vm := matches[0]
+		return buildlet.NewClient(vm.IPPort, vm.TLS), nil
+	}
+	if len(matches) > 1 {
+		return nil, fmt.Errorf("prefix %q is ambiguous")
 	}
 	return nil, fmt.Errorf("buildlet %q not running", name)
 }
diff --git a/dashboard/cmd/gomote/run.go b/dashboard/cmd/gomote/run.go
index 6be9346..ff2d47c 100644
--- a/dashboard/cmd/gomote/run.go
+++ b/dashboard/cmd/gomote/run.go
@@ -10,6 +10,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"strings"
 
 	"golang.org/x/tools/dashboard/buildlet"
 )
@@ -17,10 +18,12 @@ import (
 func run(args []string) error {
 	fs := flag.NewFlagSet("run", flag.ContinueOnError)
 	fs.Usage = func() {
-		fmt.Fprintln(os.Stderr, "create usage: gomote run [run-opts] <buildlet-name> <cmd> [args...]")
+		fmt.Fprintln(os.Stderr, "create usage: gomote run [run-opts] <instance> <cmd> [args...]")
 		fs.PrintDefaults()
 		os.Exit(1)
 	}
+	var sys bool
+	fs.BoolVar(&sys, "system", false, "run inside the system, and not inside the workdir; this is implicit if cmd starts with '/'")
 
 	fs.Parse(args)
 	if fs.NArg() < 2 {
@@ -33,7 +36,9 @@ func run(args []string) error {
 	}
 
 	remoteErr, execErr := bc.Exec(cmd, buildlet.ExecOpts{
-		Output: os.Stdout,
+		SystemLevel: sys || strings.HasPrefix(cmd, "/"),
+		Output:      os.Stdout,
+		Args:        fs.Args()[2:],
 	})
 	if execErr != nil {
 		return fmt.Errorf("Error trying to execute %s: %v", cmd, execErr)