Merge "Add upload-only mode and manual build-started-time flag."

3 files changed, 133 insertions, 44 deletions

diff --git a/cmd/soong_ui/main.go b/cmd/soong_ui/main.go
index 713ccbe72..8c99988f2 100644
--- a/cmd/soong_ui/main.go
+++ b/cmd/soong_ui/main.go
@@ -58,7 +58,7 @@ type command struct {
 	stdio func() terminal.StdioInterface
 
 	// run the command
-	run func(ctx build.Context, config build.Config, args []string, logsDir string)
+	run func(ctx build.Context, config build.Config, args []string)
 }
 
 // list of supported commands (flags) supported by soong ui
@@ -91,6 +91,15 @@ var commands = []command{
 		config:      buildActionConfig,
 		stdio:       stdio,
 		run:         runMake,
+	}, {
+		flag:        "--upload-metrics-only",
+		description: "upload metrics without building anything",
+		config:      uploadOnlyConfig,
+		stdio:       stdio,
+		// Upload-only mode mostly skips to the metrics-uploading phase of soong_ui.
+		// However, this invocation marks the true "end of the build", and thus we
+		// need to update the total runtime of the build to include this upload step.
+		run: updateTotalRealTime,
 	},
 }
 
@@ -182,18 +191,49 @@ func main() {
 	}}
 
 	config := c.config(buildCtx, args...)
-
-	build.SetupOutDir(buildCtx, config)
-
-	// Set up files to be outputted in the log directory.
+	config.SetLogsPrefix(c.logsPrefix)
 	logsDir := config.LogsDir()
+	buildStarted = config.BuildStartedTimeOrDefault(buildStarted)
 
-	// Common list of metric file definition.
 	buildErrorFile := filepath.Join(logsDir, c.logsPrefix+"build_error")
-	rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
 	soongMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_metrics")
+	rbeMetricsFile := filepath.Join(logsDir, c.logsPrefix+"rbe_metrics.pb")
 	bp2buildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"bp2build_metrics.pb")
-	soongBuildMetricsFile := filepath.Join(logsDir, c.logsPrefix+"soong_build_metrics.pb")
+	metricsFiles := []string{
+		buildErrorFile,           // build error strings
+		rbeMetricsFile,           // high level metrics related to remote build execution.
+		bp2buildMetricsFile,      // high level metrics related to bp2build.
+		soongMetricsFile,         // high level metrics related to this build system.
+		config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
+	}
+
+	os.MkdirAll(logsDir, 0777)
+
+	log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log"))
+
+	trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace"))
+
+	c.run(buildCtx, config, args)
+
+	defer met.Dump(soongMetricsFile)
+	if !config.SkipMetricsUpload() {
+		defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, metricsFiles...)
+	}
+
+}
+
+func logAndSymlinkSetup(buildCtx build.Context, config build.Config) {
+	log := buildCtx.ContextImpl.Logger
+	logsPrefix := config.GetLogsPrefix()
+	build.SetupOutDir(buildCtx, config)
+	logsDir := config.LogsDir()
+
+	// Common list of metric file definition.
+	buildErrorFile := filepath.Join(logsDir, logsPrefix+"build_error")
+	rbeMetricsFile := filepath.Join(logsDir, logsPrefix+"rbe_metrics.pb")
+	soongMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_metrics")
+	bp2buildMetricsFile := filepath.Join(logsDir, logsPrefix+"bp2build_metrics.pb")
+	soongBuildMetricsFile := filepath.Join(logsDir, logsPrefix+"soong_build_metrics.pb")
 
 	//Delete the stale metrics files
 	staleFileSlice := []string{buildErrorFile, rbeMetricsFile, soongMetricsFile, bp2buildMetricsFile, soongBuildMetricsFile}
@@ -203,14 +243,12 @@ func main() {
 
 	build.PrintOutDirWarning(buildCtx, config)
 
-	os.MkdirAll(logsDir, 0777)
-	log.SetOutput(filepath.Join(logsDir, c.logsPrefix+"soong.log"))
-	trace.SetOutput(filepath.Join(logsDir, c.logsPrefix+"build.trace"))
-	stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, c.logsPrefix+"verbose.log")))
-	stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, c.logsPrefix+"error.log")))
+	stat := buildCtx.Status
+	stat.AddOutput(status.NewVerboseLog(log, filepath.Join(logsDir, logsPrefix+"verbose.log")))
+	stat.AddOutput(status.NewErrorLog(log, filepath.Join(logsDir, logsPrefix+"error.log")))
 	stat.AddOutput(status.NewProtoErrorLog(log, buildErrorFile))
 	stat.AddOutput(status.NewCriticalPath(log))
-	stat.AddOutput(status.NewBuildProgressLog(log, filepath.Join(logsDir, c.logsPrefix+"build_progress.pb")))
+	stat.AddOutput(status.NewBuildProgressLog(log, filepath.Join(logsDir, logsPrefix+"build_progress.pb")))
 
 	buildCtx.Verbosef("Detected %.3v GB total RAM", float32(config.TotalRAM())/(1024*1024*1024))
 	buildCtx.Verbosef("Parallelism (local/remote/highmem): %v/%v/%v",
@@ -218,27 +256,7 @@ func main() {
 
 	setMaxFiles(buildCtx)
 
-	{
-		// The order of the function calls is important. The last defer function call
-		// is the first one that is executed to save the rbe metrics to a protobuf
-		// file. The soong metrics file is then next. Bazel profiles are written
-		// before the uploadMetrics is invoked. The written files are then uploaded
-		// if the uploading of the metrics is enabled.
-		files := []string{
-			buildErrorFile,           // build error strings
-			rbeMetricsFile,           // high level metrics related to remote build execution.
-			soongBuildMetricsFile,    // high level metrics related to soong build(except bp2build).
-			bp2buildMetricsFile,      // high level metrics related to bp2build.
-			soongMetricsFile,         // high level metrics related to this build system.
-			config.BazelMetricsDir(), // directory that contains a set of bazel metrics.
-		}
-
-		if !config.SkipMetricsUpload() {
-			defer build.UploadMetrics(buildCtx, config, c.simpleOutput, buildStarted, files...)
-		}
-		defer met.Dump(soongMetricsFile)
-		defer build.CheckProdCreds(buildCtx, config)
-	}
+	defer build.CheckProdCreds(buildCtx, config)
 
 	// Read the time at the starting point.
 	if start, ok := os.LookupEnv("TRACE_BEGIN_SOONG"); ok {
@@ -253,7 +271,7 @@ func main() {
 		}
 
 		if executable, err := os.Executable(); err == nil {
-			trace.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace"))
+			buildCtx.ContextImpl.Tracer.ImportMicrofactoryLog(filepath.Join(filepath.Dir(executable), "."+filepath.Base(executable)+".trace"))
 		}
 	}
 
@@ -266,8 +284,6 @@ func main() {
 	f := build.NewSourceFinder(buildCtx, config)
 	defer f.Shutdown()
 	build.FindSources(buildCtx, config, f)
-
-	c.run(buildCtx, config, args, logsDir)
 }
 
 func fixBadDanglingLink(ctx build.Context, name string) {
@@ -284,7 +300,8 @@ func fixBadDanglingLink(ctx build.Context, name string) {
 	}
 }
 
-func dumpVar(ctx build.Context, config build.Config, args []string, _ string) {
+func dumpVar(ctx build.Context, config build.Config, args []string) {
+	logAndSymlinkSetup(ctx, config)
 	flags := flag.NewFlagSet("dumpvar", flag.ExitOnError)
 	flags.SetOutput(ctx.Writer)
 
@@ -336,7 +353,9 @@ func dumpVar(ctx build.Context, config build.Config, args []string, _ string) {
 	}
 }
 
-func dumpVars(ctx build.Context, config build.Config, args []string, _ string) {
+func dumpVars(ctx build.Context, config build.Config, args []string) {
+	logAndSymlinkSetup(ctx, config)
+
 	flags := flag.NewFlagSet("dumpvars", flag.ExitOnError)
 	flags.SetOutput(ctx.Writer)
 
@@ -421,6 +440,14 @@ func dumpVarConfig(ctx build.Context, args ...string) build.Config {
 	return build.NewConfig(ctx)
 }
 
+// uploadOnlyConfig explicitly requires no arguments.
+func uploadOnlyConfig(ctx build.Context, args ...string) build.Config {
+	if len(args) > 0 {
+		fmt.Printf("--upload-only does not require arguments.")
+	}
+	return build.UploadOnlyConfig(ctx)
+}
+
 func buildActionConfig(ctx build.Context, args ...string) build.Config {
 	flags := flag.NewFlagSet("build-mode", flag.ContinueOnError)
 	flags.SetOutput(ctx.Writer)
@@ -514,7 +541,9 @@ func buildActionConfig(ctx build.Context, args ...string) build.Config {
 	return build.NewBuildActionConfig(buildAction, *dir, ctx, args...)
 }
 
-func runMake(ctx build.Context, config build.Config, _ []string, logsDir string) {
+func runMake(ctx build.Context, config build.Config, _ []string) {
+	logAndSymlinkSetup(ctx, config)
+	logsDir := config.LogsDir()
 	if config.IsVerbose() {
 		writer := ctx.Writer
 		fmt.Fprintln(writer, "! The argument `showcommands` is no longer supported.")
@@ -659,3 +688,19 @@ func setMaxFiles(ctx build.Context) {
 		ctx.Println("Failed to increase file limit:", err)
 	}
 }
+
+func updateTotalRealTime(ctx build.Context, config build.Config, args []string) {
+	soongMetricsFile := filepath.Join(config.LogsDir(), "soong_metrics")
+
+	//read file into proto
+	data, err := os.ReadFile(soongMetricsFile)
+	if err != nil {
+		ctx.Fatal(err)
+	}
+	met := ctx.ContextImpl.Metrics
+
+	err = met.UpdateTotalRealTime(data)
+	if err != nil {
+		ctx.Fatal(err)
+	}
+}
diff --git a/ui/build/config.go b/ui/build/config.go
index a6bba159e..61f6b1c9a 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -63,6 +63,7 @@ type configImpl struct {
 	environ       *Environment
 	distDir       string
 	buildDateTime string
+	logsPrefix    string
 
 	// From the arguments
 	parallel          int
@@ -84,6 +85,7 @@ type configImpl struct {
 	skipSoongTests    bool
 	searchApiDir      bool // Scan the Android.bp files generated in out/api_surfaces
 	skipMetricsUpload bool
+	buildStartedTime  int64 // For metrics-upload-only - manually specify a build-started time
 
 	// From the product config
 	katiArgs        []string
@@ -255,6 +257,14 @@ func defaultBazelProdMode(cfg *configImpl) bool {
 	return true
 }
 
+func UploadOnlyConfig(ctx Context, _ ...string) Config {
+	ret := &configImpl{
+		environ:       OsEnvironment(),
+		sandboxConfig: &SandboxConfig{},
+	}
+	return Config{ret}
+}
+
 func NewConfig(ctx Context, args ...string) Config {
 	ret := &configImpl{
 		environ:       OsEnvironment(),
@@ -266,9 +276,7 @@ func NewConfig(ctx Context, args ...string) Config {
 	ret.keepGoing = 1
 
 	ret.totalRAM = detectTotalRAM(ctx)
-
 	ret.parseArgs(ctx, args)
-
 	// Make sure OUT_DIR is set appropriately
 	if outDir, ok := ret.environ.Get("OUT_DIR"); ok {
 		ret.environ.Set("OUT_DIR", filepath.Clean(outDir))
@@ -756,6 +764,14 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
 			ctx.Metrics.SetBuildCommand([]string{buildCmd})
 		} else if strings.HasPrefix(arg, "--bazel-force-enabled-modules=") {
 			c.bazelForceEnabledModules = strings.TrimPrefix(arg, "--bazel-force-enabled-modules=")
+		} else if strings.HasPrefix(arg, "--build-started-time-unix-millis=") {
+			buildTimeStr := strings.TrimPrefix(arg, "--build-started-time-unix-millis=")
+			val, err := strconv.ParseInt(buildTimeStr, 10, 64)
+			if err == nil {
+				c.buildStartedTime = val
+			} else {
+				ctx.Fatalf("Error parsing build-time-started-unix-millis", err)
+			}
 		} else if len(arg) > 0 && arg[0] == '-' {
 			parseArgNum := func(def int) int {
 				if len(arg) > 2 {
@@ -1092,6 +1108,14 @@ func (c *configImpl) SetIncludeTags(i []string) {
 	c.includeTags = i
 }
 
+func (c *configImpl) GetLogsPrefix() string {
+	return c.logsPrefix
+}
+
+func (c *configImpl) SetLogsPrefix(prefix string) {
+	c.logsPrefix = prefix
+}
+
 func (c *configImpl) HighmemParallel() int {
 	if i, ok := c.environ.GetInt("NINJA_HIGHMEM_NUM_JOBS"); ok {
 		return i
@@ -1519,6 +1543,15 @@ func (c *configImpl) SkipMetricsUpload() bool {
 	return c.skipMetricsUpload
 }
 
+// Returns a Time object if one was passed via a command-line flag.
+// Otherwise returns the passed default.
+func (c *configImpl) BuildStartedTimeOrDefault(defaultTime time.Time) time.Time {
+	if c.buildStartedTime == 0 {
+		return defaultTime
+	}
+	return time.UnixMilli(c.buildStartedTime)
+}
+
 func GetMetricsUploader(topDir string, env *Environment) string {
 	if p, ok := env.Get("METRICS_UPLOADER"); ok {
 		metricsUploader := filepath.Join(topDir, p)
diff --git a/ui/metrics/metrics.go b/ui/metrics/metrics.go
index ce2d94667..717530c47 100644
--- a/ui/metrics/metrics.go
+++ b/ui/metrics/metrics.go
@@ -32,13 +32,13 @@ package metrics
 // of what an event is and how the metrics system is a stack based system.
 
 import (
+	"fmt"
 	"os"
 	"runtime"
 	"strings"
 	"time"
 
 	"android/soong/shared"
-
 	"google.golang.org/protobuf/proto"
 
 	soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
@@ -223,6 +223,17 @@ func (m *Metrics) SetBuildDateTime(buildTimestamp time.Time) {
 	m.metrics.BuildDateTimestamp = proto.Int64(buildTimestamp.UnixNano() / int64(time.Second))
 }
 
+func (m *Metrics) UpdateTotalRealTime(data []byte) error {
+	if err := proto.Unmarshal(data, &m.metrics); err != nil {
+		return fmt.Errorf("Failed to unmarshal proto", err)
+	}
+	startTime := *m.metrics.Total.StartTime
+	endTime := uint64(time.Now().UnixNano())
+
+	*m.metrics.Total.RealTime = *proto.Uint64(endTime - startTime)
+	return nil
+}
+
 // SetBuildCommand adds the build command specified by the user to the
 // list of collected metrics.
 func (m *Metrics) SetBuildCommand(cmd []string) {