diff options
author | Tobias Bosch <tbosch@google.com> | 2019-10-01 15:00:52 -0700 |
---|---|---|
committer | Tobias Bosch <tbosch@google.com> | 2019-10-02 16:53:48 +0000 |
commit | c88ee8a1d2c3d925a90f7a880ae085237b117454 (patch) | |
tree | e1c35815b9b354ffebe485c077e17c22099b59e2 /compiler_wrapper | |
parent | 5322d4af4264a1e63fa33c732cdfb32454347a5c (diff) | |
download | toolchain-utils-c88ee8a1d2c3d925a90f7a880ae085237b117454.tar.gz |
Android wrapper: Support compile with fallback.
The old android wrapper supports to use a fallback
compiler in case of errors. This replicates this
logic.
See
https://cs.corp.google.com/android/toolchain/llvm_android/compiler_wrapper.py.
BUG=chromium:773875
TEST=unit test, golden tests comparing to old wrapper
Change-Id: Iff7281e6e21c4834f9a4493884ed7b3d66b87967
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1833898
Tested-by: Tobias Bosch <tbosch@google.com>
Reviewed-by: George Burgess <gbiv@chromium.org>
Diffstat (limited to 'compiler_wrapper')
-rw-r--r-- | compiler_wrapper/android_config_test.go | 42 | ||||
-rw-r--r-- | compiler_wrapper/compile_with_fallback.go | 105 | ||||
-rw-r--r-- | compiler_wrapper/compile_with_fallback_test.go | 292 | ||||
-rw-r--r-- | compiler_wrapper/compiler_wrapper.go | 9 | ||||
-rw-r--r-- | compiler_wrapper/testdata/android_golden/compile_with_fallback.json | 115 | ||||
-rw-r--r-- | compiler_wrapper/testutil_test.go | 1 |
6 files changed, 564 insertions, 0 deletions
diff --git a/compiler_wrapper/android_config_test.go b/compiler_wrapper/android_config_test.go index 74881a02..6d51bfe9 100644 --- a/compiler_wrapper/android_config_test.go +++ b/compiler_wrapper/android_config_test.go @@ -26,6 +26,7 @@ func TestAndroidConfig(t *testing.T) { runGoldenRecords(ctx, androidGoldenDir, []goldenFile{ createAndroidClangPathGoldenInputs(ctx), createBisectGoldenInputs(filepath.Join(ctx.tempDir, "clang")), + createAndroidCompileWithFallbackGoldenInputs(ctx), }) }) } @@ -74,3 +75,44 @@ func createAndroidClangPathGoldenInputs(ctx *testContext) goldenFile { }, } } + +func createAndroidCompileWithFallbackGoldenInputs(ctx *testContext) goldenFile { + env := []string{ + "ANDROID_LLVM_PREBUILT_COMPILER_PATH=fallback_compiler", + "ANDROID_LLVM_STDERR_REDIRECT=" + filepath.Join(ctx.tempDir, "fallback_stderr"), + "ANDROID_LLVM_FALLBACK_DISABLED_WARNINGS=-a -b", + } + defaultPath := filepath.Join(ctx.tempDir, "clang") + return goldenFile{ + Name: "compile_with_fallback.json", + Records: []goldenRecord{ + { + WrapperCmd: newGoldenCmd(defaultPath, mainCc), + Env: env, + Cmds: okResults, + }, + { + WrapperCmd: newGoldenCmd(defaultPath, mainCc), + Env: env, + Cmds: []commandResult{ + { + ExitCode: 1, + }, + okResult, + }, + }, + { + WrapperCmd: newGoldenCmd(defaultPath, mainCc), + Env: env, + Cmds: []commandResult{ + { + ExitCode: 1, + }, + { + ExitCode: 1, + }, + }, + }, + }, + } +} diff --git a/compiler_wrapper/compile_with_fallback.go b/compiler_wrapper/compile_with_fallback.go new file mode 100644 index 00000000..a3b00bf1 --- /dev/null +++ b/compiler_wrapper/compile_with_fallback.go @@ -0,0 +1,105 @@ +// Copyright 2019 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package main + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "syscall" + "time" +) + +const prebuiltCompilerPathKey = "ANDROID_LLVM_PREBUILT_COMPILER_PATH" + +func shouldCompileWithFallback(env env) bool { + value, _ := env.getenv(prebuiltCompilerPathKey) + return value != "" +} + +// FIXME: Deduplicate this logic with the logic for FORCE_DISABLE_WERROR +// (the logic here is from Android, the logic for FORCE_DISABLE_WERROR is from ChromeOS) +func compileWithFallback(env env, cfg *config, originalCmd *command, absWrapperPath string) (exitCode int, err error) { + firstCmd := &command{ + Path: originalCmd.Path, + Args: originalCmd.Args, + EnvUpdates: originalCmd.EnvUpdates, + } + // We only want to pass extra flags to clang and clang++. + if base := filepath.Base(originalCmd.Path); base == "clang.real" || base == "clang++.real" { + // We may introduce some new warnings after rebasing and we need to + // disable them before we fix those warnings. + extraArgs, _ := env.getenv("ANDROID_LLVM_FALLBACK_DISABLED_WARNINGS") + firstCmd.Args = append( + append(firstCmd.Args, "-fno-color-diagnostics"), + strings.Split(extraArgs, " ")..., + ) + } + + firstCmdStdinBuffer := &bytes.Buffer{} + firstCmdStderrBuffer := &bytes.Buffer{} + firstCmdExitCode, err := wrapSubprocessErrorWithSourceLoc(firstCmd, + env.run(firstCmd, teeStdinIfNeeded(env, firstCmd, firstCmdStdinBuffer), env.stdout(), io.MultiWriter(env.stderr(), firstCmdStderrBuffer))) + if err != nil { + return 0, err + } + + if firstCmdExitCode == 0 { + return 0, nil + } + stderrRedirectPath, _ := env.getenv("ANDROID_LLVM_STDERR_REDIRECT") + f, err := os.OpenFile(stderrRedirectPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return 0, wrapErrorwithSourceLocf(err, "error opening stderr file %s", stderrRedirectPath) + } + lockSuccess := false + for i := 0; i < 30; i++ { + err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) + if err == nil { + lockSuccess = true + break + } + if errno, ok := err.(syscall.Errno); ok { + if errno == syscall.EAGAIN || errno == syscall.EACCES { + time.Sleep(500 * time.Millisecond) + err = nil + } + } + if err != nil { + return 0, wrapErrorwithSourceLocf(err, "error waiting to lock file %s", stderrRedirectPath) + } + } + if !lockSuccess { + return 0, wrapErrorwithSourceLocf(err, "timeout waiting to lock file %s", stderrRedirectPath) + } + w := bufio.NewWriter(f) + w.WriteString("==================COMMAND:====================\n") + fmt.Fprintf(w, "%s %s\n\n", firstCmd.Path, strings.Join(firstCmd.Args, " ")) + firstCmdStderrBuffer.WriteTo(w) + w.WriteString("==============================================\n\n") + if err := w.Flush(); err != nil { + return 0, wrapErrorwithSourceLocf(err, "unable to write to file %s", stderrRedirectPath) + } + if err := f.Close(); err != nil { + return 0, wrapErrorwithSourceLocf(err, "error closing file %s", stderrRedirectPath) + } + + prebuiltCompilerPath, _ := env.getenv(prebuiltCompilerPathKey) + fallbackCmd := &command{ + Path: filepath.Join(prebuiltCompilerPath, filepath.Base(absWrapperPath)), + // Don't use extra args added (from ANDROID_LLVM_FALLBACK_DISABLED_WARNINGS) for clang and + // clang++ above. They may not be recognized by the fallback clang. + Args: originalCmd.Args, + // Delete prebuiltCompilerPathKey so the fallback doesn't keep + // calling itself in case of an error. + EnvUpdates: append(originalCmd.EnvUpdates, prebuiltCompilerPathKey+"="), + } + return wrapSubprocessErrorWithSourceLoc(fallbackCmd, + env.run(fallbackCmd, bytes.NewReader(firstCmdStdinBuffer.Bytes()), env.stdout(), env.stderr())) +} diff --git a/compiler_wrapper/compile_with_fallback_test.go b/compiler_wrapper/compile_with_fallback_test.go new file mode 100644 index 00000000..32d1915b --- /dev/null +++ b/compiler_wrapper/compile_with_fallback_test.go @@ -0,0 +1,292 @@ +// Copyright 2019 The Chromium OS Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package main + +import ( + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestOmitFallbackCompileForSuccessfulCall(t *testing.T) { + withCompileWithFallbackTestContext(t, func(ctx *testContext) { + ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangAndroid, mainCc))) + if ctx.cmdCount != 1 { + t.Errorf("expected 1 call. Got: %d", ctx.cmdCount) + } + }) +} + +func TestOmitFallbackCompileForGeneralError(t *testing.T) { + withCompileWithFallbackTestContext(t, func(ctx *testContext) { + ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + return errors.New("someerror") + } + stderr := ctx.mustFail(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangAndroid, mainCc))) + if err := verifyInternalError(stderr); err != nil { + t.Fatal(err) + } + if !strings.Contains(stderr, "someerror") { + t.Errorf("unexpected error. Got: %s", stderr) + } + if ctx.cmdCount != 1 { + t.Errorf("expected 1 call. Got: %d", ctx.cmdCount) + } + }) +} + +func TestCompileWithFallbackForNonZeroExitCode(t *testing.T) { + withCompileWithFallbackTestContext(t, func(ctx *testContext) { + ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + switch ctx.cmdCount { + case 1: + return newExitCodeError(1) + case 2: + if err := verifyPath(cmd, "fallback_compiler/clang"); err != nil { + return err + } + if err := verifyEnvUpdate(cmd, "ANDROID_LLVM_PREBUILT_COMPILER_PATH="); err != nil { + return err + } + return nil + default: + t.Fatalf("unexpected command: %#v", cmd) + return nil + } + } + ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangAndroid, mainCc))) + if ctx.cmdCount != 2 { + t.Errorf("expected 2 calls. Got: %d", ctx.cmdCount) + } + }) +} + +func TestCompileWithFallbackForwardStdoutAndStderr(t *testing.T) { + withCompileWithFallbackTestContext(t, func(ctx *testContext) { + ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + switch ctx.cmdCount { + case 1: + fmt.Fprint(stdout, "originalmessage") + fmt.Fprint(stderr, "originalerror") + return newExitCodeError(1) + case 2: + fmt.Fprint(stdout, "fallbackmessage") + fmt.Fprint(stderr, "fallbackerror") + return nil + default: + t.Fatalf("unexpected command: %#v", cmd) + return nil + } + } + ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangAndroid, mainCc))) + if err := verifyNonInternalError(ctx.stderrString(), "originalerrorfallbackerror"); err != nil { + t.Error(err) + } + if !strings.Contains(ctx.stdoutString(), "originalmessagefallbackmessage") { + t.Errorf("unexpected stdout. Got: %s", ctx.stdoutString()) + } + }) +} + +func TestForwardGeneralErrorWhenFallbackCompileFails(t *testing.T) { + withCompileWithFallbackTestContext(t, func(ctx *testContext) { + ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + switch ctx.cmdCount { + case 1: + return newExitCodeError(1) + case 2: + return errors.New("someerror") + default: + t.Fatalf("unexpected command: %#v", cmd) + return nil + } + } + stderr := ctx.mustFail(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangAndroid, mainCc))) + if err := verifyInternalError(stderr); err != nil { + t.Error(err) + } + if !strings.Contains(stderr, "someerror") { + t.Errorf("unexpected stderr. Got: %s", stderr) + } + }) +} + +func TestForwardExitCodeWhenFallbackCompileFails(t *testing.T) { + withCompileWithFallbackTestContext(t, func(ctx *testContext) { + ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + switch ctx.cmdCount { + case 1: + return newExitCodeError(1) + case 2: + return newExitCodeError(2) + default: + t.Fatalf("unexpected command: %#v", cmd) + return nil + } + } + exitCode := callCompiler(ctx, ctx.cfg, ctx.newCommand(clangAndroid, mainCc)) + if exitCode != 2 { + t.Errorf("unexpected exit code. Got: %d", exitCode) + } + }) +} + +func TestForwardStdinToFallbackCompile(t *testing.T) { + withCompileWithFallbackTestContext(t, func(ctx *testContext) { + ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + stdinStr := ctx.readAllString(stdin) + if stdinStr != "someinput" { + return fmt.Errorf("unexpected stdin. Got: %s", stdinStr) + } + + switch ctx.cmdCount { + case 1: + return newExitCodeError(1) + case 2: + return nil + default: + t.Fatalf("unexpected command: %#v", cmd) + return nil + } + } + io.WriteString(&ctx.stdinBuffer, "someinput") + ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangAndroid, "-", mainCc))) + }) +} + +func TestCompileWithFallbackExtraArgs(t *testing.T) { + withCompileWithFallbackTestContext(t, func(ctx *testContext) { + testData := []struct { + compiler string + expectExtraArgs bool + }{ + {"./clang", true}, + {"./clang++", true}, + {"./some_clang", false}, + } + ctx.env = append(ctx.env, "ANDROID_LLVM_FALLBACK_DISABLED_WARNINGS=-a -b") + extraArgs := []string{"-fno-color-diagnostics", "-a", "-b"} + for _, tt := range testData { + ctx.cmdCount = 0 + ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + switch ctx.cmdCount { + case 1: + if tt.expectExtraArgs { + if err := verifyArgOrder(cmd, extraArgs...); err != nil { + return err + } + } else { + for _, arg := range extraArgs { + if err := verifyArgCount(cmd, 0, arg); err != nil { + return err + } + } + } + return newExitCodeError(1) + case 2: + for _, arg := range extraArgs { + if err := verifyArgCount(cmd, 0, arg); err != nil { + return err + } + } + return nil + default: + t.Fatalf("unexpected command: %#v", cmd) + return nil + } + } + ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(tt.compiler, mainCc))) + if ctx.cmdCount != 2 { + t.Errorf("expected 2 calls. Got: %d", ctx.cmdCount) + } + } + }) +} + +func TestCompileWithFallbackLogCommandAndErrors(t *testing.T) { + withCompileWithFallbackTestContext(t, func(ctx *testContext) { + ctx.env = append(ctx.env, "ANDROID_LLVM_FALLBACK_DISABLED_WARNINGS=-a -b") + ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + switch ctx.cmdCount { + case 1: + fmt.Fprint(stderr, "someerror\n") + return newExitCodeError(1) + case 2: + return nil + default: + t.Fatalf("unexpected command: %#v", cmd) + return nil + } + } + ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangAndroid, mainCc))) + + log := readCompileWithFallbackErrorLog(ctx) + if log != `==================COMMAND:==================== +clang.real main.cc -fno-color-diagnostics -a -b + +someerror +============================================== + +` { + t.Errorf("unexpected log. Got: %s", log) + } + + entry, _ := os.Lstat(filepath.Join(ctx.tempDir, "fallback_stderr")) + if entry.Mode()&0777 != 0644 { + t.Errorf("unexpected mode for logfile. Got: %#o", entry.Mode()) + } + }) +} + +func TestCompileWithFallbackAppendToLog(t *testing.T) { + withCompileWithFallbackTestContext(t, func(ctx *testContext) { + ctx.writeFile(filepath.Join(ctx.tempDir, "fallback_stderr"), "oldContent\n") + ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { + switch ctx.cmdCount { + case 1: + return newExitCodeError(1) + case 2: + return nil + default: + t.Fatalf("unexpected command: %#v", cmd) + return nil + } + } + ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangAndroid, mainCc))) + + log := readCompileWithFallbackErrorLog(ctx) + if !strings.Contains(log, "oldContent") { + t.Errorf("old content not present: %s", log) + } + if !strings.Contains(log, "clang.real") { + t.Errorf("new content not present: %s", log) + } + }) +} + +func withCompileWithFallbackTestContext(t *testing.T, work func(ctx *testContext)) { + withTestContext(t, func(ctx *testContext) { + ctx.cfg.isAndroidWrapper = true + ctx.env = []string{ + "ANDROID_LLVM_PREBUILT_COMPILER_PATH=fallback_compiler", + "ANDROID_LLVM_STDERR_REDIRECT=" + filepath.Join(ctx.tempDir, "fallback_stderr"), + } + work(ctx) + }) +} + +func readCompileWithFallbackErrorLog(ctx *testContext) string { + logFile := filepath.Join(ctx.tempDir, "fallback_stderr") + data, err := ioutil.ReadFile(logFile) + if err != nil { + ctx.t.Fatalf("error reading log file %s: %s", logFile, err) + } + return string(data) +} diff --git a/compiler_wrapper/compiler_wrapper.go b/compiler_wrapper/compiler_wrapper.go index 3dec7e66..b157687f 100644 --- a/compiler_wrapper/compiler_wrapper.go +++ b/compiler_wrapper/compiler_wrapper.go @@ -118,6 +118,15 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int } return doubleBuildWithWNoError(env, cfg, compilerCmd) } + if shouldCompileWithFallback(env) { + if rusageLogfileName != "" { + return 0, newUserErrorf("GETRUSAGE is meaningless with FORCE_DISABLE_WERROR") + } + if bisectStage != "" { + return 0, newUserErrorf("BISECT_STAGE is meaningless with FORCE_DISABLE_WERROR") + } + return compileWithFallback(env, cfg, compilerCmd, mainBuilder.absWrapperPath) + } if rusageLogfileName != "" { if bisectStage != "" { return 0, newUserErrorf("BISECT_STAGE is meaningless with GETRUSAGE") diff --git a/compiler_wrapper/testdata/android_golden/compile_with_fallback.json b/compiler_wrapper/testdata/android_golden/compile_with_fallback.json new file mode 100644 index 00000000..509583ae --- /dev/null +++ b/compiler_wrapper/testdata/android_golden/compile_with_fallback.json @@ -0,0 +1,115 @@ +[ + { + "wd": "/tmp/stable", + "env": [ + "ANDROID_LLVM_PREBUILT_COMPILER_PATH=fallback_compiler", + "ANDROID_LLVM_STDERR_REDIRECT=/tmp/stable/fallback_stderr", + "ANDROID_LLVM_FALLBACK_DISABLED_WARNINGS=-a -b" + ], + "wrapper": { + "cmd": { + "path": "/tmp/stable/clang", + "args": [ + "main.cc" + ] + } + }, + "cmds": [ + { + "cmd": { + "path": "/tmp/stable/clang.real", + "args": [ + "main.cc", + "-fno-color-diagnostics", + "-a", + "-b" + ] + } + } + ] + }, + { + "wd": "/tmp/stable", + "env": [ + "ANDROID_LLVM_PREBUILT_COMPILER_PATH=fallback_compiler", + "ANDROID_LLVM_STDERR_REDIRECT=/tmp/stable/fallback_stderr", + "ANDROID_LLVM_FALLBACK_DISABLED_WARNINGS=-a -b" + ], + "wrapper": { + "cmd": { + "path": "/tmp/stable/clang", + "args": [ + "main.cc" + ] + } + }, + "cmds": [ + { + "cmd": { + "path": "/tmp/stable/clang.real", + "args": [ + "main.cc", + "-fno-color-diagnostics", + "-a", + "-b" + ] + }, + "exitcode": 1 + }, + { + "cmd": { + "path": "fallback_compiler/clang", + "args": [ + "main.cc" + ], + "env_updates": [ + "ANDROID_LLVM_PREBUILT_COMPILER_PATH=" + ] + } + } + ] + }, + { + "wd": "/tmp/stable", + "env": [ + "ANDROID_LLVM_PREBUILT_COMPILER_PATH=fallback_compiler", + "ANDROID_LLVM_STDERR_REDIRECT=/tmp/stable/fallback_stderr", + "ANDROID_LLVM_FALLBACK_DISABLED_WARNINGS=-a -b" + ], + "wrapper": { + "cmd": { + "path": "/tmp/stable/clang", + "args": [ + "main.cc" + ] + }, + "exitcode": 1 + }, + "cmds": [ + { + "cmd": { + "path": "/tmp/stable/clang.real", + "args": [ + "main.cc", + "-fno-color-diagnostics", + "-a", + "-b" + ] + }, + "exitcode": 1 + }, + { + "cmd": { + "path": "fallback_compiler/clang", + "args": [ + "main.cc" + ], + "env_updates": [ + "ANDROID_LLVM_PREBUILT_COMPILER_PATH=" + ] + }, + "exitcode": 1 + } + ] + } +] diff --git a/compiler_wrapper/testutil_test.go b/compiler_wrapper/testutil_test.go index e63cc5c1..c0e49fdd 100644 --- a/compiler_wrapper/testutil_test.go +++ b/compiler_wrapper/testutil_test.go @@ -22,6 +22,7 @@ var crosRootDirFlag = flag.String("crosroot", "", "root dir of the chrome os too var androidPrebuiltsDirFlag = flag.String("androidprebuilts", "", "prebuilts dir of android") const mainCc = "main.cc" +const clangAndroid = "./clang" const clangX86_64 = "./x86_64-cros-linux-gnu-clang" const gccX86_64 = "./x86_64-cros-linux-gnu-gcc" const gccX86_64Eabi = "./x86_64-cros-eabi-gcc" |