From 7c59ee050a4e4935f9ac505f2776d8eea1c4d557 Mon Sep 17 00:00:00 2001 From: Ryan Beltran Date: Mon, 22 Feb 2021 23:58:40 +0000 Subject: compiler_wrapper: disable goma, ccache w/ rusage Disables Goma and CCahce when rusage monitoring is requested. This allows for usable metrics to be produced for more packages. BUG=chromium:1156314 TEST=Modified and ran unit tests Change-Id: I36415b4df6ce033b23ff8055e900233886e1889d Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2713228 Tested-by: Ryan Beltran Commit-Queue: Ryan Beltran Reviewed-by: George Burgess Reviewed-by: Denis Nikitin --- compiler_wrapper/ccache_flag_test.go | 12 +++++++++ compiler_wrapper/compiler_wrapper.go | 43 ++++++++++++++++++--------------- compiler_wrapper/disable_werror_flag.go | 6 ++--- compiler_wrapper/gomacc_flag.go | 4 +-- compiler_wrapper/gomacc_flag_test.go | 17 +++++++++++++ compiler_wrapper/rusage_flag.go | 9 +++++-- 6 files changed, 64 insertions(+), 27 deletions(-) (limited to 'compiler_wrapper') diff --git a/compiler_wrapper/ccache_flag_test.go b/compiler_wrapper/ccache_flag_test.go index 03a74de8..2b18c8ca 100644 --- a/compiler_wrapper/ccache_flag_test.go +++ b/compiler_wrapper/ccache_flag_test.go @@ -5,6 +5,7 @@ package main import ( + "path/filepath" "testing" ) @@ -160,3 +161,14 @@ func withCCacheEnabledTestContext(t *testing.T, work func(ctx *testContext)) { work(ctx) }) } + +func TestRusagePreventsCCache(t *testing.T) { + withCCacheEnabledTestContext(t, func(ctx *testContext) { + ctx.env = append(ctx.env, "TOOLCHAIN_RUSAGE_OUTPUT="+filepath.Join(ctx.tempDir, "rusage.log")) + cmd := ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyPath(cmd, gccX86_64+".real"); err != nil { + t.Error(err) + } + }) +} diff --git a/compiler_wrapper/compiler_wrapper.go b/compiler_wrapper/compiler_wrapper.go index 009cc871..2de2f6a4 100644 --- a/compiler_wrapper/compiler_wrapper.go +++ b/compiler_wrapper/compiler_wrapper.go @@ -76,6 +76,12 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int var compilerCmd *command clangSyntax := processClangSyntaxFlag(mainBuilder) + rusageEnabled := isRusageEnabled(env) + + // Disable goma for rusage logs + allowGoma := !rusageEnabled + allowCCache := !rusageEnabled + workAroundKernelBugWithRetries := false if cfg.isAndroidWrapper { mainBuilder.path = calculateAndroidWrapperPath(mainBuilder.path, mainBuilder.absWrapperPath) @@ -84,7 +90,7 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int mainBuilder.addPreUserArgs(mainBuilder.cfg.clangFlags...) mainBuilder.addPreUserArgs(mainBuilder.cfg.commonFlags...) mainBuilder.addPostUserArgs(mainBuilder.cfg.clangPostFlags...) - if _, err := processGomaCccFlags(mainBuilder); err != nil { + if _, err := processGomaCccFlags(allowGoma, mainBuilder); err != nil { return 0, err } compilerCmd = mainBuilder.build() @@ -100,7 +106,6 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int if err != nil { return 0, err } - allowCCache := true if tidyMode != tidyModeNone { allowCCache = false clangCmdWithoutGomaAndCCache := mainBuilder.build() @@ -121,24 +126,24 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int return 0, err } } - if err := processGomaCCacheFlags(allowCCache, mainBuilder); err != nil { + if err := processGomaCCacheFlags(allowGoma, allowCCache, mainBuilder); err != nil { return 0, err } compilerCmd = mainBuilder.build() } else { if clangSyntax { - allowCCache := false - clangCmd, err := calcClangCommand(allowCCache, mainBuilder.clone()) + allowCCache = false + clangCmd, err := calcClangCommand(allowGoma, allowCCache, mainBuilder.clone()) if err != nil { return 0, err } - gccCmd, err := calcGccCommand(mainBuilder) + gccCmd, err := calcGccCommand(rusageEnabled, mainBuilder) if err != nil { return 0, err } return checkClangSyntax(env, clangCmd, gccCmd) } - compilerCmd, err = calcGccCommand(mainBuilder) + compilerCmd, err = calcGccCommand(rusageEnabled, mainBuilder) if err != nil { return 0, err } @@ -146,10 +151,9 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int } } - rusageLogfileName := getRusageLogFilename(env) bisectStage := getBisectStage(env) - if rusageLogfileName != "" { + if rusageEnabled { compilerCmd = removeRusageFromCommand(compilerCmd) } @@ -157,10 +161,10 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int if bisectStage != "" { return 0, newUserErrorf("BISECT_STAGE is meaningless with FORCE_DISABLE_WERROR") } - return doubleBuildWithWNoError(env, cfg, compilerCmd, rusageLogfileName) + return doubleBuildWithWNoError(env, cfg, compilerCmd) } if shouldCompileWithFallback(env) { - if rusageLogfileName != "" { + if rusageEnabled { return 0, newUserErrorf("TOOLCHAIN_RUSAGE_OUTPUT is meaningless with ANDROID_LLVM_PREBUILT_COMPILER_PATH") } if bisectStage != "" { @@ -169,7 +173,7 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int return compileWithFallback(env, cfg, compilerCmd, mainBuilder.absWrapperPath) } if bisectStage != "" { - if rusageLogfileName != "" { + if rusageEnabled { return 0, newUserErrorf("TOOLCHAIN_RUSAGE_OUTPUT is meaningless with BISECT_STAGE") } compilerCmd, err = calcBisectCommand(env, cfg, bisectStage, compilerCmd) @@ -228,7 +232,7 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int for { var exitCode int - commitRusage, err := maybeCaptureRusage(env, rusageLogfileName, compilerCmd, func(willLogRusage bool) error { + commitRusage, err := maybeCaptureRusage(env, compilerCmd, func(willLogRusage bool) error { var err error exitCode, err = runCompiler(willLogRusage) return err @@ -262,18 +266,18 @@ func prepareClangCommand(builder *commandBuilder) (err error) { return processClangFlags(builder) } -func calcClangCommand(allowCCache bool, builder *commandBuilder) (*command, error) { +func calcClangCommand(allowGoma bool, allowCCache bool, builder *commandBuilder) (*command, error) { err := prepareClangCommand(builder) if err != nil { return nil, err } - if err := processGomaCCacheFlags(allowCCache, builder); err != nil { + if err := processGomaCCacheFlags(allowGoma, allowCCache, builder); err != nil { return nil, err } return builder.build(), nil } -func calcGccCommand(builder *commandBuilder) (*command, error) { +func calcGccCommand(enableRusage bool, builder *commandBuilder) (*command, error) { if !builder.cfg.isHostWrapper { processSysrootFlag(builder) } @@ -281,8 +285,7 @@ func calcGccCommand(builder *commandBuilder) (*command, error) { calcCommonPreUserArgs(builder) processGccFlags(builder) if !builder.cfg.isHostWrapper { - allowCCache := true - if err := processGomaCCacheFlags(allowCCache, builder); err != nil { + if err := processGomaCCacheFlags(!enableRusage, !enableRusage, builder); err != nil { return nil, err } } @@ -300,10 +303,10 @@ func calcCommonPreUserArgs(builder *commandBuilder) { processSanitizerFlags(builder) } -func processGomaCCacheFlags(allowCCache bool, builder *commandBuilder) (err error) { +func processGomaCCacheFlags(allowGoma bool, allowCCache bool, builder *commandBuilder) (err error) { gomaccUsed := false if !builder.cfg.isHostWrapper { - gomaccUsed, err = processGomaCccFlags(builder) + gomaccUsed, err = processGomaCccFlags(allowGoma, builder) if err != nil { return err } diff --git a/compiler_wrapper/disable_werror_flag.go b/compiler_wrapper/disable_werror_flag.go index 9bbf2e25..5c21b1ad 100644 --- a/compiler_wrapper/disable_werror_flag.go +++ b/compiler_wrapper/disable_werror_flag.go @@ -61,7 +61,7 @@ func isLikelyAConfTest(cfg *config, cmd *command) bool { return false } -func doubleBuildWithWNoError(env env, cfg *config, originalCmd *command, rusageLogfileName string) (exitCode int, err error) { +func doubleBuildWithWNoError(env env, cfg *config, originalCmd *command) (exitCode int, err error) { originalStdoutBuffer := &bytes.Buffer{} originalStderrBuffer := &bytes.Buffer{} // TODO: This is a bug in the old wrapper that it drops the ccache path @@ -76,7 +76,7 @@ func doubleBuildWithWNoError(env env, cfg *config, originalCmd *command, rusageL } var originalExitCode int - commitOriginalRusage, err := maybeCaptureRusage(env, rusageLogfileName, originalCmd, func(willLogRusage bool) error { + commitOriginalRusage, err := maybeCaptureRusage(env, originalCmd, func(willLogRusage bool) error { originalExitCode, err = wrapSubprocessErrorWithSourceLoc(originalCmd, env.run(originalCmd, getStdin(), originalStdoutBuffer, originalStderrBuffer)) return err @@ -111,7 +111,7 @@ func doubleBuildWithWNoError(env env, cfg *config, originalCmd *command, rusageL } var retryExitCode int - commitRetryRusage, err := maybeCaptureRusage(env, rusageLogfileName, retryCommand, func(willLogRusage bool) error { + commitRetryRusage, err := maybeCaptureRusage(env, retryCommand, func(willLogRusage bool) error { retryExitCode, err = wrapSubprocessErrorWithSourceLoc(retryCommand, env.run(retryCommand, getStdin(), retryStdoutBuffer, retryStderrBuffer)) return err diff --git a/compiler_wrapper/gomacc_flag.go b/compiler_wrapper/gomacc_flag.go index ac298b12..1305c60b 100644 --- a/compiler_wrapper/gomacc_flag.go +++ b/compiler_wrapper/gomacc_flag.go @@ -8,7 +8,7 @@ import ( "os" ) -func processGomaCccFlags(builder *commandBuilder) (gomaUsed bool, err error) { +func processGomaCccFlags(allowGoma bool, builder *commandBuilder) (gomaUsed bool, err error) { gomaPath := "" nextArgIsGomaPath := false builder.transformArgs(func(arg builderArg) string { @@ -31,7 +31,7 @@ func processGomaCccFlags(builder *commandBuilder) (gomaUsed bool, err error) { if gomaPath == "" { gomaPath, _ = builder.env.getenv("GOMACC_PATH") } - if gomaPath != "" { + if allowGoma && gomaPath != "" { if _, err := os.Lstat(gomaPath); err == nil { builder.wrapPath(gomaPath) return true, nil diff --git a/compiler_wrapper/gomacc_flag_test.go b/compiler_wrapper/gomacc_flag_test.go index d7b2b0b7..e2aa4420 100644 --- a/compiler_wrapper/gomacc_flag_test.go +++ b/compiler_wrapper/gomacc_flag_test.go @@ -6,6 +6,7 @@ package main import ( "path" + "path/filepath" "testing" ) @@ -92,3 +93,19 @@ func TestOmitGomaccByDefault(t *testing.T) { } }) } + +func TestRusagePreventsGoma(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + gomaPath := path.Join(ctx.tempDir, "gomacc") + ctx.writeFile(gomaPath, "") + ctx.env = []string{ + "GOMACC_PATH=" + gomaPath, + "TOOLCHAIN_RUSAGE_OUTPUT=" + filepath.Join(ctx.tempDir, "rusage.log"), + } + cmd := ctx.must(callCompiler(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyPath(cmd, gccX86_64+".real"); err != nil { + t.Error(err) + } + }) +} diff --git a/compiler_wrapper/rusage_flag.go b/compiler_wrapper/rusage_flag.go index 87a994b4..4aa40b4d 100644 --- a/compiler_wrapper/rusage_flag.go +++ b/compiler_wrapper/rusage_flag.go @@ -18,6 +18,10 @@ func getRusageLogFilename(env env) string { return value } +func isRusageEnabled(env env) bool { + return getRusageLogFilename(env) != "" +} + func lockFileExclusive(fd uintptr) error { maxTries := 100 for i := 0; i < maxTries; i++ { @@ -61,8 +65,9 @@ func removeRusageFromCommand(compilerCmd *command) *command { // unless action returns an error or logFileName is "" // a function is returned which saves the rusage log data at logFileName unless logFileName is "" // an error is returned if action returns an error, or rusage commands in syscall fails -func maybeCaptureRusage(env env, logFileName string, compilerCmd *command, action func(willLogRusage bool) error) (maybeCommitToFile func(exitCode int) error, err error) { - willLogRusage := logFileName != "" +func maybeCaptureRusage(env env, compilerCmd *command, action func(willLogRusage bool) error) (maybeCommitToFile func(exitCode int) error, err error) { + logFileName := getRusageLogFilename(env) + willLogRusage := isRusageEnabled(env) if !willLogRusage { if err := action(willLogRusage); err != nil { return nil, err -- cgit v1.2.3