diff options
Diffstat (limited to 'compiler_wrapper')
36 files changed, 2271 insertions, 0 deletions
diff --git a/compiler_wrapper/README.md b/compiler_wrapper/README.md new file mode 100644 index 00000000..fd38d894 --- /dev/null +++ b/compiler_wrapper/README.md @@ -0,0 +1,3 @@ +# Compiler wrapper + +See the comments on the top of main.go diff --git a/compiler_wrapper/build_cros_hardened_wrapper.sh b/compiler_wrapper/build_cros_hardened_wrapper.sh new file mode 100755 index 00000000..82d9e14c --- /dev/null +++ b/compiler_wrapper/build_cros_hardened_wrapper.sh @@ -0,0 +1,5 @@ +#!/bin/bash -eu +# Copyright 2019 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +go build -tags "cros hardened" "$@" diff --git a/compiler_wrapper/build_cros_nonhardened_wrapper.sh b/compiler_wrapper/build_cros_nonhardened_wrapper.sh new file mode 100755 index 00000000..94e591fc --- /dev/null +++ b/compiler_wrapper/build_cros_nonhardened_wrapper.sh @@ -0,0 +1,5 @@ +#!/bin/bash -eu +# Copyright 2019 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +go build -tags "cros nonhardened" "$@" diff --git a/compiler_wrapper/ccache_flag.go b/compiler_wrapper/ccache_flag.go new file mode 100644 index 00000000..13ee234d --- /dev/null +++ b/compiler_wrapper/ccache_flag.go @@ -0,0 +1,60 @@ +package main + +func processCCacheFlag(sysroot string, builder *commandBuilder) { + // We should be able to share the objects across compilers as + // the pre-processed output will differ. This allows boards + // that share compiler flags (like x86 boards) to share caches. + const ccacheDir = "/var/cache/distfiles/ccache" + + useCCache := true + builder.transformArgs(func(arg builderArg) string { + if arg.Value == "-noccache" { + useCCache = false + return "" + } + return arg.Value + }) + + if useCCache { + // We need to get ccache to make relative paths from within the + // sysroot. This lets us share cached files across boards (if + // all other things are equal of course like CFLAGS) as well as + // across versions. A quick test is something like: + // $ export CFLAGS='-O2 -g -pipe' CXXFLAGS='-O2 -g -pipe' + // $ BOARD=x86-alex + // $ cros_workon-$BOARD stop cros-disks + // $ emerge-$BOARD cros-disks + // $ cros_workon-$BOARD start cros-disks + // $ emerge-$BOARD cros-disks + // $ BOARD=amd64-generic + // $ cros_workon-$BOARD stop cros-disks + // $ emerge-$BOARD cros-disks + // $ cros_workon-$BOARD start cros-disks + // $ emerge-$BOARD cros-disks + // All of those will get cache hits (ignoring the first one + // which will seed the cache) due to this setting. + builder.updateEnv("CCACHE_BASEDIR=" + sysroot) + if builder.env.getenv("CCACHE_DISABLE") != "" { + // Portage likes to set this for us when it has FEATURES=-ccache. + // The other vars we need to setup manually because of tools like + // scons that scrubs the env before we get executed. + builder.updateEnv("CCACHE_DISABLE=") + } + // If RESTRICT=sandbox is enabled, then sandbox won't be setup, + // and the env vars won't be available for appending. + if sandboxRewrite := builder.env.getenv("SANDBOX_WRITE"); sandboxRewrite != "" { + builder.updateEnv("SANDBOX_WRITE=" + sandboxRewrite + ":" + ccacheDir) + } + + // Make sure we keep the cached files group writable. + builder.updateEnv("CCACHE_DIR="+ccacheDir, "CCACHE_UMASK=002") + + // ccache may generate false positive warnings. + // Workaround bug https://crbug.com/649740 + if builder.target.compilerType == clangType { + builder.updateEnv("CCACHE_CPP2=yes") + } + + builder.wrapPath("/usr/bin/ccache") + } +} diff --git a/compiler_wrapper/ccache_flag_test.go b/compiler_wrapper/ccache_flag_test.go new file mode 100644 index 00000000..15fc3177 --- /dev/null +++ b/compiler_wrapper/ccache_flag_test.go @@ -0,0 +1,117 @@ +package main + +import ( + "testing" +) + +func TestCallCCache(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + wrapperCmd := ctx.newCommand(gccX86_64, mainCc) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, wrapperCmd)) + if err := verifyPath(cmd, "/usr/bin/ccache"); err != nil { + t.Error(err) + } + if err := verifyArgOrder(cmd, wrapperCmd.path+".real", mainCc); err != nil { + t.Error(err) + } + }) +} + +func TestNotCallCCacheIfNoCCacheArgGiven(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + wrapperCmd := ctx.newCommand(gccX86_64, "-noccache", mainCc) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, wrapperCmd)) + if err := verifyPath(cmd, wrapperCmd.path+".real"); err != nil { + t.Error(err) + } + if err := verifyArgCount(cmd, 0, "-noccache"); err != nil { + t.Error(err) + } + }) +} + +func TestSetCacheDir(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyEnvUpdate(cmd, "CCACHE_DIR=/var/cache/distfiles/ccache"); err != nil { + t.Error(err) + } + }) +} + +func TestSetCacheBaseDirToSysroot(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyEnvUpdate(cmd, + "CCACHE_BASEDIR="+ctx.tempDir+"/usr/x86_64-cros-linux-gnu"); err != nil { + t.Error(err) + } + }) +} + +func TestSetCacheUmask(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyEnvUpdate(cmd, "CCACHE_UMASK=002"); err != nil { + t.Error(err) + } + }) +} + +func TestUpdateSandboxRewrite(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyNoEnvUpdate(cmd, "SANDBOX_WRITE"); err != nil { + t.Error(err) + } + + ctx.env = []string{"SANDBOX_WRITE=xyz"} + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyEnvUpdate(cmd, + "SANDBOX_WRITE=xyz:/var/cache/distfiles/ccache"); err != nil { + t.Error(err) + } + }) +} + +func TestClearCacheDisable(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyNoEnvUpdate(cmd, "SANDBOX_WRITE"); err != nil { + t.Error(err) + } + + ctx.env = []string{"CCACHE_DISABLE=true"} + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyEnvUpdate(cmd, "CCACHE_DISABLE="); err != nil { + t.Error(err) + } + }) +} + +func TestAddCCacheCpp2FlagForClang(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, mainCc))) + if err := verifyEnvUpdate(cmd, "CCACHE_CPP2=yes"); err != nil { + t.Error(err) + } + }) +} + +func TestOmitCCacheCpp2FlagForGcc(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyNoEnvUpdate(cmd, "CCACHE_CPP2"); err != nil { + t.Error(err) + } + }) +} diff --git a/compiler_wrapper/clang_flags.go b/compiler_wrapper/clang_flags.go new file mode 100644 index 00000000..9679ace9 --- /dev/null +++ b/compiler_wrapper/clang_flags.go @@ -0,0 +1,137 @@ +package main + +import ( + "path/filepath" + "strings" +) + +func processClangFlags(rootPath string, builder *commandBuilder) error { + clangDir := builder.env.getenv("CLANG") + + if clangDir == "" { + clangDir = filepath.Join(rootPath, "usr/bin/") + if !filepath.IsAbs(builder.path) { + // If sysroot_wrapper is invoked by relative path, call actual compiler in + // relative form. This is neccesary to remove absolute path from compile + // outputs. + var err error + clangDir, err = filepath.Rel(builder.env.getwd(), clangDir) + if err != nil { + return err + } + } + } else { + clangDir = filepath.Dir(clangDir) + } + builder.path = filepath.Join(clangDir, builder.target.compiler) + + // GCC flags to remove from the clang command line. + // TODO: Once clang supports GCC compatibility mode, remove + // these checks. + // + // Use of -Qunused-arguments allows this set to be small, just those + // that clang still warns about. + unsupported := map[string]bool{ + "-mno-movbe": true, + "-pass-exit-codes": true, + "-Wclobbered": true, + "-Wno-psabi": true, + "-Wlogical-op": true, + "-Wmissing-parameter-type": true, + "-Wold-style-declaration": true, + "-Woverride-init": true, + "-Wunsafe-loop-optimizations": true, + } + + unsupportedPrefixes := []string{"-Wstrict-aliasing=", "-finline-limit="} + + // clang with '-ftrapv' generates 'call __mulodi4', which is only implemented + // in compiler-rt library. However compiler-rt library only has i386/x86_64 + // backends (see '/usr/lib/clang/3.7.0/lib/linux/libclang_rt.*'). GCC, on the + // other hand, generate 'call __mulvdi3', which is implemented in libgcc. See + // bug chromium:503229. + armUnsupported := map[string]bool{"-ftrapv": true} + + // Clang may use different options for the same or similar functionality. + gccToClang := map[string]string{ + "-Wno-error=cpp": "-Wno-#warnings", + "-Wno-error=maybe-uninitialized": "-Wno-error=uninitialized", + "-Wno-error=unused-but-set-variable": "-Wno-error=unused-variable", + "-Wno-unused-but-set-variable": "-Wno-unused-variable", + "-Wunused-but-set-variable": "-Wunused-variable", + } + + builder.transformArgs(func(arg builderArg) string { + if mapped, ok := gccToClang[arg.Value]; ok { + return mapped + } + + if unsupported[arg.Value] { + return "" + } + + for _, prefix := range unsupportedPrefixes { + if strings.HasPrefix(arg.Value, prefix) { + return "" + } + } + + if builder.target.arch == "armv7a" && builder.target.sys == "linux" { + if armUnsupported[arg.Value] { + return "" + } + } + + if clangOnly := "-Xclang-only="; strings.HasPrefix(arg.Value, clangOnly) { + return arg.Value[len(clangOnly):] + } + + return arg.Value + }) + + // Specify the target for clang. + linkerPath, err := getLinkerPath(builder.env, builder.target.target+"-ld", rootPath) + if err != nil { + return err + } + relLinkerPath, err := filepath.Rel(builder.env.getwd(), linkerPath) + if err != nil { + return err + } + builder.addPostUserArgs("-B" + relLinkerPath) + if startswithI86(builder.target.arch) { + // TODO: -target i686-pc-linux-gnu causes clang to search for + // libclang_rt.asan-i686.a which doesn't exist because it's packaged + // as libclang_rt.asan-i386.a. We can't use -target i386-pc-linux-gnu + // because then it would try to run i386-pc-linux-gnu-ld which doesn't + // exist. Consider renaming the runtime library to use i686 in its name. + builder.addPostUserArgs("-m32") + // clang does not support -mno-movbe. This is the alternate way to do it. + builder.addPostUserArgs("-Xclang", "-target-feature", "-Xclang", "-movbe") + } else { + builder.addPostUserArgs("-target", builder.target.target) + } + return nil +} + +// Return the a directory which contains an 'ld' that gcc is using. +func getLinkerPath(env env, linkerCmd string, rootPath string) (string, error) { + // We did not pass the tuple i686-pc-linux-gnu to x86-32 clang. Instead, + // we passed '-m32' to clang. As a result, clang does not want to use the + // i686-pc-linux-gnu-ld, so we need to add this to help clang find the right + // linker. + for _, path := range strings.Split(env.getenv("PATH"), ":") { + if linkerPath, err := filepath.EvalSymlinks(filepath.Join(path, linkerCmd)); err == nil { + return filepath.Dir(linkerPath), nil + } + } + + // When using the sdk outside chroot, we need to provide the cross linker path + // to the compiler via -B ${linker_path}. This is because for gcc, it can + // find the right linker via searching its internal paths. Clang does not have + // such feature, and it falls back to $PATH search only. However, the path of + // ${SDK_LOCATION}/bin is not necessarily in the ${PATH}. To fix this, we + // provide the directory that contains the cross linker wrapper to clang. + // Outside chroot, it is the top bin directory form the sdk tarball. + return filepath.Join(rootPath, "bin"), nil +} diff --git a/compiler_wrapper/clang_flags_test.go b/compiler_wrapper/clang_flags_test.go new file mode 100644 index 00000000..f50757fa --- /dev/null +++ b/compiler_wrapper/clang_flags_test.go @@ -0,0 +1,189 @@ +package main + +import ( + "path/filepath" + "testing" +) + +func TestClangBasename(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + var tests = []struct { + in string + out string + }{ + {"./x86_64-cros-linux-gnu-clang", ".*/clang"}, + {"./x86_64-cros-linux-gnu-clang++", ".*/clang\\+\\+"}, + } + + for _, tt := range tests { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(tt.in, "-noccache", mainCc))) + if err := verifyPath(cmd, tt.out); err != nil { + t.Error(err) + } + } + }) +} + +func TestClangPathGivenClangEnv(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.env = []string{"CLANG=/a/b/clang"} + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, "-noccache", mainCc))) + if err := verifyPath(cmd, "/a/b/clang"); err != nil { + t.Error(err) + } + }) +} + +func TestAbsoluteClangPathBasedOnRootPath(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.cfg.rootRelPath = "somepath" + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(filepath.Join(ctx.tempDir, clangX86_64), "-noccache", mainCc))) + if err := verifyPath(cmd, filepath.Join(ctx.tempDir, "somepath/usr/bin/clang")); err != nil { + t.Error(err) + } + }) +} + +func TestRelativeClangPathBasedOnRootPath(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.cfg.rootRelPath = "somepath" + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, "-noccache", mainCc))) + if err := verifyPath(cmd, "somepath/usr/bin/clang"); err != nil { + t.Error(err) + } + }) +} + +func TestConvertGccToClangFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + var tests = []struct { + in string + out string + }{ + {"-Wno-error=unused-but-set-variable", "-Wno-error=unused-variable"}, + {"-Wno-error=maybe-uninitialized", "-Wno-error=uninitialized"}, + {"-Wno-unused-but-set-variable", "-Wno-unused-variable"}, + {"-Wunused-but-set-variable", "-Wunused-variable"}, + {"-Wno-error=cpp", "-Wno-#warnings"}, + {"-Xclang-only=-abc=xyz", "-abc=xyz"}, + } + + for _, tt := range tests { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, tt.in, mainCc))) + if err := verifyArgCount(cmd, 0, tt.in); err != nil { + t.Error(err) + } + if err := verifyArgOrder(cmd, tt.out, mainCc); err != nil { + t.Error(err) + } + } + }) +} + +func TestFilterUnsupportedClangFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + var tests = []struct { + compiler string + flag string + expectedCount int + }{ + {clangX86_64, "-pass-exit-codes", 0}, + {clangX86_64, "-Wclobbered", 0}, + {clangX86_64, "-Wunsafe-loop-optimizations", 0}, + {clangX86_64, "-Wlogical-op", 0}, + {clangX86_64, "-Wmissing-parameter-type", 0}, + {clangX86_64, "-Woverride-init", 0}, + {clangX86_64, "-Wold-style-declaration", 0}, + {clangX86_64, "-Wno-psabi", 0}, + {clangX86_64, "-mno-movbe", 0}, + {clangX86_64, "-Wstrict-aliasing=xyz", 0}, + {clangX86_64, "-finline-limit=xyz", 0}, + {"./armv7a-cros-linux-gnu-clang", "-ftrapv", 0}, + {"./armv7a-cros-win-gnu-clang", "-ftrapv", 1}, + {"./armv8a-cros-win-gnu-clang", "-ftrapv", 1}, + {clangX86_64, "-ftrapv", 1}, + } + + for _, tt := range tests { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(tt.compiler, tt.flag, mainCc))) + if err := verifyArgCount(cmd, tt.expectedCount, tt.flag); err != nil { + t.Error(err) + } + } + }) +} + +func TestClangArchFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + var tests = []struct { + compiler string + flags []string + }{ + {"./i686_64-cros-linux-gnu-clang", []string{mainCc, "-m32", "-Xclang", "-target-feature", "-Xclang", "-movbe"}}, + {"./x86_64-cros-linux-gnu-clang", []string{mainCc, "-target", "x86_64-cros-linux-gnu"}}, + } + for _, tt := range tests { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(tt.compiler, mainCc))) + if err := verifyArgOrder(cmd, tt.flags...); err != nil { + t.Error(err) + } + } + }) +} + +func TestClangLinkerPathProbesBinariesOnPath(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + linkerPath := filepath.Join(ctx.tempDir, "a/b/c") + ctx.writeFile(filepath.Join(linkerPath, "x86_64-cros-linux-gnu-ld"), "") + ctx.env = []string{"PATH=nonExistantPath:" + linkerPath} + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand("./x86_64-cros-linux-gnu-clang", mainCc))) + if err := verifyArgOrder(cmd, "-Ba/b/c"); err != nil { + t.Error(err) + } + }) +} + +func TestClangLinkerPathEvaluatesSymlinksForBinariesOnPath(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + realLinkerPath := filepath.Join(ctx.tempDir, "a/original/path/somelinker") + ctx.writeFile(realLinkerPath, "") + linkedLinkerPath := filepath.Join(ctx.tempDir, "a/linked/path/x86_64-cros-linux-gnu-ld") + ctx.symlink(realLinkerPath, linkedLinkerPath) + + ctx.env = []string{"PATH=nonExistantPath:" + filepath.Dir(linkedLinkerPath)} + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand("./x86_64-cros-linux-gnu-clang", mainCc))) + if err := verifyArgOrder(cmd, "-Ba/original/path"); err != nil { + t.Error(err) + } + }) +} + +func TestClangFallbackLinkerPathRelativeToRootDir(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, mainCc))) + if err := verifyArgOrder(cmd, "-Bbin"); err != nil { + t.Error(err) + } + }) +} + +func TestClangLinkerPathRelativeToRootDir(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.cfg.rootRelPath = "somepath" + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, mainCc))) + if err := verifyArgOrder(cmd, "-Bsomepath/bin"); err != nil { + t.Error(err) + } + }) +} diff --git a/compiler_wrapper/command.go b/compiler_wrapper/command.go new file mode 100644 index 00000000..c73acb86 --- /dev/null +++ b/compiler_wrapper/command.go @@ -0,0 +1,186 @@ +package main + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "sort" + "strings" +) + +type command struct { + path string + args []string + envUpdates []string +} + +func newProcessCommand() *command { + return &command{ + path: os.Args[0], + args: os.Args[1:], + } +} + +func newExecCmd(env env, cmd *command) *exec.Cmd { + execCmd := exec.Command(cmd.path, cmd.args...) + execCmd.Env = env.environ() + execCmd.Dir = env.getwd() + return execCmd +} + +// TODO: Move to test once we no longer compare the calculated command against +// the command produced by the old wrapper in the released binary. +func (actual *command) verifySimilarTo(expected *command) error { + var differences []string + if actual.path != expected.path { + differences = append(differences, fmt.Sprintf("Paths are different. Expected: %q. Actual: %q", expected.path, actual.path)) + } + + if !reflect.DeepEqual(actual.args, expected.args) { + differences = append(differences, fmt.Sprintf("Args are different. Expected: %q. Actual: %q", expected.args, actual.args)) + } + + // Sort the environment as we don't care in which order + // it was modified. + actualEnvUpdates := actual.envUpdates + sort.Strings(actualEnvUpdates) + expectedEnvUpdates := expected.envUpdates + sort.Strings(expectedEnvUpdates) + + if !reflect.DeepEqual(actualEnvUpdates, expectedEnvUpdates) { + differences = append(differences, fmt.Sprintf("Env updates are different. Expected: %q. Actual: %q", expectedEnvUpdates, actualEnvUpdates)) + } + + if len(differences) > 0 { + return errors.New("commands differ:\n" + strings.Join(differences, "\n")) + } + return nil +} + +func newCommandBuilder(env env, cmd *command) (*commandBuilder, error) { + basename := filepath.Base(cmd.path) + nameParts := strings.Split(basename, "-") + if len(nameParts) != 5 { + return nil, fmt.Errorf("expected 5 parts in the compiler name. Actual: %s", basename) + } + + compiler := nameParts[4] + var compilerType compilerType + switch { + case strings.HasPrefix(compiler, "clang"): + compilerType = clangType + case strings.HasPrefix(compiler, "gcc"): + compilerType = gccType + case strings.HasPrefix(compiler, "g++"): + compilerType = gccType + default: + return nil, fmt.Errorf("expected clang or gcc. Actual: %s", basename) + } + return &commandBuilder{ + path: cmd.path, + args: createBuilderArgs( /*fromUser=*/ true, cmd.args), + env: env, + target: builderTarget{ + target: strings.Join(nameParts[:4], "-"), + arch: nameParts[0], + vendor: nameParts[1], + sys: nameParts[2], + abi: nameParts[3], + compiler: compiler, + compilerType: compilerType, + }, + }, nil +} + +type commandBuilder struct { + path string + target builderTarget + args []builderArg + envUpdates []string + env env +} + +type builderArg struct { + Value string + FromUser bool +} + +type compilerType int32 + +const ( + gccType compilerType = iota + clangType +) + +type builderTarget struct { + target string + arch string + vendor string + sys string + abi string + compiler string + compilerType compilerType +} + +func createBuilderArgs(fromUser bool, args []string) []builderArg { + builderArgs := make([]builderArg, len(args)) + for i, arg := range args { + builderArgs[i] = builderArg{Value: arg, FromUser: fromUser} + } + return builderArgs +} + +func (builder *commandBuilder) wrapPath(path string) { + builder.args = append([]builderArg{{Value: builder.path, FromUser: false}}, builder.args...) + builder.path = path +} + +func (builder *commandBuilder) addPreUserArgs(args ...string) { + index := 0 + for _, arg := range builder.args { + if arg.FromUser { + break + } + index++ + } + builder.args = append(builder.args[:index], append(createBuilderArgs( /*fromUser=*/ false, args), builder.args[index:]...)...) +} + +func (builder *commandBuilder) addPostUserArgs(args ...string) { + builder.args = append(builder.args, createBuilderArgs( /*fromUser=*/ false, args)...) +} + +// Allows to map and filter arguments. Filters when the callback returns an empty string. +func (builder *commandBuilder) transformArgs(transform func(arg builderArg) string) { + // See https://github.com/golang/go/wiki/SliceTricks + newArgs := builder.args[:0] + for _, arg := range builder.args { + newArg := transform(arg) + if newArg != "" { + newArgs = append(newArgs, builderArg{ + Value: newArg, + FromUser: arg.FromUser, + }) + } + } + builder.args = newArgs +} + +func (builder *commandBuilder) updateEnv(updates ...string) { + builder.envUpdates = append(builder.envUpdates, updates...) +} + +func (builder *commandBuilder) build() *command { + cmdArgs := make([]string, len(builder.args)) + for i, builderArg := range builder.args { + cmdArgs[i] = builderArg.Value + } + return &command{ + path: builder.path, + args: cmdArgs, + envUpdates: builder.envUpdates, + } +} diff --git a/compiler_wrapper/compiler_wrapper.go b/compiler_wrapper/compiler_wrapper.go new file mode 100644 index 00000000..72e1749c --- /dev/null +++ b/compiler_wrapper/compiler_wrapper.go @@ -0,0 +1,102 @@ +package main + +import ( + "log" + "path/filepath" + "strings" +) + +func calcCompilerCommand(env env, cfg *config, wrapperCmd *command) (*command, error) { + absWrapperDir, err := getAbsWrapperDir(env, wrapperCmd.path) + if err != nil { + return nil, err + } + rootPath := filepath.Join(absWrapperDir, cfg.rootRelPath) + if err := checkUnsupportedFlags(wrapperCmd); err != nil { + return nil, err + } + builder, err := newCommandBuilder(env, wrapperCmd) + if err != nil { + return nil, err + } + useClang := builder.target.compilerType == clangType + sysroot := processSysrootFlag(rootPath, builder) + if useClang { + builder.addPreUserArgs(cfg.clangFlags...) + } else { + builder.addPreUserArgs(cfg.gccFlags...) + } + builder.addPreUserArgs(cfg.commonFlags...) + processPieFlags(builder) + processStackProtectorFlags(builder) + processThumbCodeFlags(builder) + processX86Flags(builder) + processSanitizerFlags(builder) + if useClang { + if err := processClangFlags(rootPath, builder); err != nil { + return nil, err + } + } else { + processGccFlags(builder) + } + gomaccUsed := processGomaCccFlags(builder) + if !gomaccUsed { + processCCacheFlag(sysroot, builder) + } + + return builder.build(), nil +} + +func calcCompilerCommandAndCompareToOld(env env, cfg *config, wrapperCmd *command) (*command, error) { + compilerCmd, err := calcCompilerCommand(env, cfg, wrapperCmd) + if err != nil { + return nil, err + } + if cfg.oldWrapperPath == "" { + return compilerCmd, nil + } + oldCmds, err := calcOldCompilerCommands(env, cfg, wrapperCmd) + if err != nil { + return nil, err + } + if err := compilerCmd.verifySimilarTo(oldCmds[0]); err != nil { + return nil, err + } + return compilerCmd, nil +} + +func getAbsWrapperDir(env env, wrapperPath string) (string, error) { + if !filepath.IsAbs(wrapperPath) { + wrapperPath = filepath.Join(env.getwd(), wrapperPath) + } + evaledCmdPath, err := filepath.EvalSymlinks(wrapperPath) + if err != nil { + log.Printf("Unable to EvalSymlinks for %s. Error: %s", evaledCmdPath, err) + return "", err + } + return filepath.Dir(evaledCmdPath), nil +} + +// Whether the command should be executed by the old wrapper as we don't +// support it yet. +func shouldForwardToOldWrapper(env env, wrapperCmd *command) bool { + for _, arg := range wrapperCmd.args { + switch { + case strings.HasPrefix(arg, "-Xclang-path="): + fallthrough + case arg == "-clang-syntax": + return true + } + } + switch { + case env.getenv("WITH_TIDY") != "": + fallthrough + case env.getenv("FORCE_DISABLE_WERROR") != "": + fallthrough + case env.getenv("GETRUSAGE") != "": + fallthrough + case env.getenv("BISECT_STAGE") != "": + return true + } + return false +} diff --git a/compiler_wrapper/compiler_wrapper_test.go b/compiler_wrapper/compiler_wrapper_test.go new file mode 100644 index 00000000..6ab25e4e --- /dev/null +++ b/compiler_wrapper/compiler_wrapper_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "testing" +) + +func TestAddCommonFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.cfg.commonFlags = []string{"-someflag"} + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyArgOrder(cmd, "-someflag", mainCc); err != nil { + t.Error(err) + } + }) +} + +func TestAddGccConfigFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.cfg.gccFlags = []string{"-someflag"} + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyArgOrder(cmd, "-someflag", mainCc); err != nil { + t.Error(err) + } + }) +} + +func TestAddClangConfigFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.cfg.clangFlags = []string{"-someflag"} + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(clangX86_64, mainCc))) + if err := verifyArgOrder(cmd, "-someflag", mainCc); err != nil { + t.Error(err) + } + }) +} + +func TestShouldForwardToOldWrapperBecauseOfArgs(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + testdata := []struct { + arg string + shouldForward bool + }{ + {"abc", false}, + {"-Xclang-path=abc", true}, + {"-clang-syntax", true}, + {"-clang-syntaxabc", false}, + } + for _, tt := range testdata { + if actual := shouldForwardToOldWrapper(ctx, ctx.newCommand(clangX86_64, tt.arg)); actual != tt.shouldForward { + t.Fatalf("Forward to old wrapper incorrect for arg %s. Expected %t but was %t.", tt.arg, tt.shouldForward, actual) + } + } + }) +} + +func TestShouldForwardToOldWrapperBecauseOfEnv(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + testdata := []struct { + env string + shouldForward bool + }{ + {"PATH=abc", false}, + {"WITH_TIDY=abc", true}, + {"FORCE_DISABLE_WERROR=abc", true}, + {"GETRUSAGE=abc", true}, + {"BISECT_STAGE=abc", true}, + } + for _, tt := range testdata { + ctx.env = []string{tt.env} + if actual := shouldForwardToOldWrapper(ctx, ctx.newCommand(clangX86_64)); actual != tt.shouldForward { + t.Fatalf("Forward to old wrapper incorrect for env %s. Expected %t but was %t.", tt.env, tt.shouldForward, actual) + } + } + }) +} diff --git a/compiler_wrapper/config.go b/compiler_wrapper/config.go new file mode 100644 index 00000000..909edf0a --- /dev/null +++ b/compiler_wrapper/config.go @@ -0,0 +1,70 @@ +package main + +type config struct { + // Flags to add to gcc and clang. + commonFlags []string + // Flags to add to gcc only. + gccFlags []string + // Flags to add to clang only. + clangFlags []string + // Toolchain root path relative to the wrapper binary. + rootRelPath string + // Path of the old wrapper using the toolchain root. + oldWrapperPath string + overrideOldWrapperConfig bool +} + +// Full hardening. +// Temporarily disable function splitting because of chromium:434751. +var crosHardenedConfig = config{ + rootRelPath: "../../../../..", + oldWrapperPath: "./sysroot_wrapper.hardened.old", + commonFlags: []string{ + "-fPIE", + "-D_FORTIFY_SOURCE=2", + "-fstack-protector-strong", + "-pie", + "-fno-omit-frame-pointer", + }, + gccFlags: []string{ + "-Wno-unused-local-typedefs", + "-Wno-maybe-uninitialized", + "-fno-reorder-blocks-and-partition", + }, + // Temporarily disable tautological-*-compare chromium:778316. + // Temporarily add no-unknown-warning-option to deal with old clang versions. + // Temporarily disable Wsection since kernel gets a bunch of these. chromium:778867 + // Disable "-faddrsig" since it produces object files that strip doesn't understand, chromium:915742. + clangFlags: []string{ + "-Wno-tautological-unsigned-enum-zero-compare", + "-Qunused-arguments", + "-grecord-gcc-switches", + "-Wno-section", + "-Wno-unknown-warning-option", + "-fno-addrsig", + "-Wno-tautological-constant-compare", + }, +} + +// Flags to be added to non-hardened toolchain. +var crosNonHardenedConfig = config{ + rootRelPath: "../../../../..", + oldWrapperPath: "./sysroot_wrapper.old", + commonFlags: []string{}, + gccFlags: []string{ + "-Wno-unused-local-typedefs", + "-Wno-maybe-uninitialized", + "-Wtrampolines", + "-Wno-deprecated-declarations", + }, + // Temporarily disable tautological-*-compare chromium:778316. + // Temporarily add no-unknown-warning-option to deal with old clang versions. + // Temporarily disable Wsection since kernel gets a bunch of these. chromium:778867 + clangFlags: []string{ + "-Wno-unknown-warning-option", + "-Qunused-arguments", + "-Wno-section", + "-Wno-tautological-unsigned-enum-zero-compare", + "-Wno-tautological-constant-compare", + }, +} diff --git a/compiler_wrapper/config_test.go b/compiler_wrapper/config_test.go new file mode 100644 index 00000000..9a75acbd --- /dev/null +++ b/compiler_wrapper/config_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "path/filepath" + "testing" +) + +func TestFullHardeningConfigAndGcc(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initFullHardeningConfig(ctx) + wrapperCmd := ctx.newCommand(gccX86_64, mainCc) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, wrapperCmd)) + if err := verifyPath(cmd, "/usr/bin/ccache"); err != nil { + t.Error(err) + } + if err := verifyArgOrder(cmd, wrapperCmd.path+".real", "--sysroot=/usr/x86_64-cros-linux-gnu", "-Wno-unused-local-typedefs", + "-Wno-maybe-uninitialized", "-fno-reorder-blocks-and-partition", "-fPIE", "-D_FORTIFY_SOURCE=2", "-fstack-protector-strong", + "-pie", "-fno-omit-frame-pointer", "main.cc", "-mno-movbe"); err != nil { + t.Error(err) + } + }) +} + +func TestFullHardeningConfigAndClang(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initFullHardeningConfig(ctx) + wrapperCmd := ctx.newCommand(clangX86_64, mainCc) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, wrapperCmd)) + if err := verifyPath(cmd, "/usr/bin/ccache"); err != nil { + t.Error(err) + } + clangPath, err := filepath.Rel(ctx.tempDir, "/usr/bin/clang") + if err != nil { + t.Error(err) + } + binPath, err := filepath.Rel(ctx.tempDir, "/bin") + if err := verifyArgOrder(cmd, clangPath, "--sysroot=/usr/x86_64-cros-linux-gnu", "-Wno-tautological-unsigned-enum-zero-compare", + "-Qunused-arguments", "-grecord-gcc-switches", "-Wno-section", "-Wno-unknown-warning-option", "-fno-addrsig", + "-Wno-tautological-constant-compare", "-fPIE", "-D_FORTIFY_SOURCE=2", "-fstack-protector-strong", "-pie", + "-fno-omit-frame-pointer", "main.cc", "-B"+binPath, "-target", "x86_64-cros-linux-gnu"); err != nil { + t.Error(err) + } + }) +} + +func TestNonHardeningConfigAndGcc(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initNonHardeningConfig(ctx) + wrapperCmd := ctx.newCommand(gccX86_64, mainCc) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, wrapperCmd)) + if err := verifyPath(cmd, "/usr/bin/ccache"); err != nil { + t.Error(err) + } + if err := verifyArgOrder(cmd, wrapperCmd.path+".real", "--sysroot=/usr/x86_64-cros-linux-gnu", + "-Wno-unused-local-typedefs", "-Wno-maybe-uninitialized", "-Wtrampolines", + "-Wno-deprecated-declarations", "main.cc", "-mno-movbe"); err != nil { + t.Error(err) + } + }) +} + +func TestNonHardeningConfigAndClang(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initNonHardeningConfig(ctx) + wrapperCmd := ctx.newCommand(clangX86_64, mainCc) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, wrapperCmd)) + if err := verifyPath(cmd, "/usr/bin/ccache"); err != nil { + t.Error(err) + } + clangPath, err := filepath.Rel(ctx.tempDir, "/usr/bin/clang") + if err != nil { + t.Error(err) + } + binPath, err := filepath.Rel(ctx.tempDir, "/bin") + if err := verifyArgOrder(cmd, clangPath, "--sysroot=/usr/x86_64-cros-linux-gnu", "-Wno-unknown-warning-option", + "-Qunused-arguments", "-Wno-section", "-Wno-tautological-unsigned-enum-zero-compare", + "-Wno-tautological-constant-compare", "main.cc", "-B"+binPath, "-target", "x86_64-cros-linux-gnu"); err != nil { + t.Error(err) + } + }) +} + +func initFullHardeningConfig(ctx *testContext) { + *ctx.cfg = crosHardenedConfig + ctx.setOldWrapperPath(oldHardenedWrapperPathForTest) +} + +func initNonHardeningConfig(ctx *testContext) { + *ctx.cfg = crosNonHardenedConfig + ctx.setOldWrapperPath(oldNonHardenedWrapperPathForTest) +} diff --git a/compiler_wrapper/cros_hardened_config.go b/compiler_wrapper/cros_hardened_config.go new file mode 100644 index 00000000..0ab8ab40 --- /dev/null +++ b/compiler_wrapper/cros_hardened_config.go @@ -0,0 +1,7 @@ +// +build cros,hardened + +package main + +func getRealConfig() *config { + return &crosHardenedConfig +} diff --git a/compiler_wrapper/cros_nonhardened_config.go b/compiler_wrapper/cros_nonhardened_config.go new file mode 100644 index 00000000..9106a4da --- /dev/null +++ b/compiler_wrapper/cros_nonhardened_config.go @@ -0,0 +1,7 @@ +// +build cros,nonhardened + +package main + +func getRealConfig() *config { + return &crosNonHardenedConfig +} diff --git a/compiler_wrapper/env.go b/compiler_wrapper/env.go new file mode 100644 index 00000000..3c106d4a --- /dev/null +++ b/compiler_wrapper/env.go @@ -0,0 +1,37 @@ +package main + +import ( + "os" +) + +type env interface { + getenv(key string) string + environ() []string + getwd() string +} + +type processEnv struct { + wd string +} + +func newProcessEnv() (env, error) { + wd, err := os.Getwd() + if err != nil { + return nil, err + } + return &processEnv{wd: wd}, nil +} + +var _ env = (*processEnv)(nil) + +func (env *processEnv) getenv(key string) string { + return os.Getenv(key) +} + +func (env *processEnv) environ() []string { + return os.Environ() +} + +func (env *processEnv) getwd() string { + return env.wd +} diff --git a/compiler_wrapper/gcc_flags.go b/compiler_wrapper/gcc_flags.go new file mode 100644 index 00000000..352b38e4 --- /dev/null +++ b/compiler_wrapper/gcc_flags.go @@ -0,0 +1,25 @@ +package main + +func processGccFlags(builder *commandBuilder) { + // Flags not supported by GCC. + unsupported := map[string]bool{"-Xcompiler": true} + + // Conversion for flags supported by clang but not gcc. + clangToGcc := map[string]string{ + "-march=goldmont": "-march=silvermont", + "-march=goldmont-plus": "-march=silvermont", + "-march=skylake": "-march=corei7", + } + + builder.transformArgs(func(arg builderArg) string { + if unsupported[arg.Value] { + return "" + } + if mapped, ok := clangToGcc[arg.Value]; ok { + return mapped + } + return arg.Value + }) + + builder.path += ".real" +} diff --git a/compiler_wrapper/gcc_flags_test.go b/compiler_wrapper/gcc_flags_test.go new file mode 100644 index 00000000..12119f0b --- /dev/null +++ b/compiler_wrapper/gcc_flags_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "testing" +) + +func TestCallRealGcc(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + wrapperCmd := ctx.newCommand(gccX86_64, "-noccache", mainCc) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, wrapperCmd)) + if err := verifyPath(cmd, wrapperCmd.path+".real"); err != nil { + t.Error(err) + } + }) +} + +func TestCallRealGPlusPlus(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + wrapperCmd := ctx.newCommand("./x86_64-cros-linux-gnu-g++", "-noccache", mainCc) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, wrapperCmd)) + if err := verifyPath(cmd, "\\./x86_64-cros-linux-gnu-g\\+\\+\\.real"); err != nil { + t.Error(err) + } + }) +} + +func TestConvertClangToGccFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + var tests = []struct { + in string + out string + }{ + {"-march=goldmont", "-march=silvermont"}, + {"-march=goldmont-plus", "-march=silvermont"}, + {"-march=skylake", "-march=corei7"}, + } + + for _, tt := range tests { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, tt.in, mainCc))) + if err := verifyArgCount(cmd, 0, tt.in); err != nil { + t.Error(err) + } + if err := verifyArgOrder(cmd, tt.out, mainCc); err != nil { + t.Error(err) + } + } + }) +} + +func TestFilterUnsupportedGccFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-Xcompiler", mainCc))) + if err := verifyArgCount(cmd, 0, "-Xcompiler"); err != nil { + t.Error(err) + } + }) +} diff --git a/compiler_wrapper/gomacc_flag.go b/compiler_wrapper/gomacc_flag.go new file mode 100644 index 00000000..b82a94c5 --- /dev/null +++ b/compiler_wrapper/gomacc_flag.go @@ -0,0 +1,15 @@ +package main + +import ( + "os" +) + +func processGomaCccFlags(builder *commandBuilder) (gomaUsed bool) { + if gomaPath := builder.env.getenv("GOMACC_PATH"); gomaPath != "" { + if _, err := os.Lstat(gomaPath); err == nil { + builder.wrapPath(gomaPath) + return true + } + } + return false +} diff --git a/compiler_wrapper/gomacc_flag_test.go b/compiler_wrapper/gomacc_flag_test.go new file mode 100644 index 00000000..1372df22 --- /dev/null +++ b/compiler_wrapper/gomacc_flag_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "path" + "testing" +) + +func TestCallGomaccIfEnvIsGivenAndValid(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + gomaPath := path.Join(ctx.tempDir, "gomacc") + // Create a file so the gomacc path is valid. + ctx.writeFile(gomaPath, "") + ctx.env = []string{"GOMACC_PATH=" + gomaPath} + wrapperCmd := ctx.newCommand(gccX86_64, mainCc) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, wrapperCmd)) + if err := verifyPath(cmd, gomaPath); err != nil { + t.Error(err) + } + if err := verifyArgOrder(cmd, wrapperCmd.path+".real", mainCc); err != nil { + t.Error(err) + } + }) +} + +func TestOmitGomaccIfEnvIsGivenButInvalid(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + // Note: This path does not point to a valid file. + gomaPath := path.Join(ctx.tempDir, "gomacc") + ctx.env = []string{"GOMACC_PATH=" + gomaPath} + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyPath(cmd, "/usr/bin/ccache"); err != nil { + t.Error(err) + } + }) +} + +func TestOmitGomaccByDefault(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyPath(cmd, "/usr/bin/ccache"); err != nil { + t.Error(err) + } + }) +} diff --git a/compiler_wrapper/main.go b/compiler_wrapper/main.go new file mode 100644 index 00000000..5323a301 --- /dev/null +++ b/compiler_wrapper/main.go @@ -0,0 +1,56 @@ +// +build cros + +// This binary uses the following build tags: +// - cros: Whether the wrapper should be built for ChromeOS +// - nonhardened: For a non-hardened set of compiler flags +// - hardened: For a hardened set of compiler flags +// +// There is a bash script for every meaningful combination. +// E.g. ./build_cros_hardened_wrapper.sh will build the ChromeOS +// hardened compiler wrapper. +// +// Test arguments: +// - crosroot: Specifies the ChromeOS toolchain root directory (aka chroot). +// If this is given, tests will compare the produced commands against the +// old compiler wrapper. +// +// Examples: +// - run all tests and compare output against old compiler wrapper: +// go test third_party/toolchain-utils/compiler_wrapper/ -v --crosroot=$HOME/chromiumos/chroot/ +// - run all tests in isolation: +// go test third_party/toolchain-utils/compiler_wrapper/ -v +package main + +import ( + "log" + "os/exec" + "syscall" +) + +func main() { + wrapperCmd := newProcessCommand() + env, err := newProcessEnv() + if err != nil { + log.Fatal(err) + } + cfg := getRealConfig() + if shouldForwardToOldWrapper(env, wrapperCmd) { + err := forwardToOldWrapper(env, cfg, wrapperCmd) + if err != nil { + log.Fatal(err) + } + return + } + + cmd, err := calcCompilerCommandAndCompareToOld(env, cfg, wrapperCmd) + if err != nil { + log.Fatal(err) + } + if err := execCmd(newExecCmd(env, cmd)); err != nil { + log.Fatal(err) + } +} + +func execCmd(cmd *exec.Cmd) error { + return syscall.Exec(cmd.Path, cmd.Args, cmd.Env) +} diff --git a/compiler_wrapper/oldwrapper.go b/compiler_wrapper/oldwrapper.go new file mode 100644 index 00000000..bc9d5eac --- /dev/null +++ b/compiler_wrapper/oldwrapper.go @@ -0,0 +1,178 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" +) + +func calcOldCompilerCommands(env env, cfg *config, wrapperCmd *command) ([]*command, error) { + stdoutBuffer := bytes.Buffer{} + stderrBuffer := bytes.Buffer{} + pipes := exec.Cmd{ + Stdin: strings.NewReader(""), + Stdout: &stdoutBuffer, + Stderr: &stderrBuffer, + } + mockForks := true + if err := callOldWrapper(env, cfg, wrapperCmd, &pipes, mockForks); err != nil { + return nil, fmt.Errorf("error: %s. %s", err, stderrBuffer.String()) + } + + // Parse the nested commands. + allStderrLines := strings.Split(stderrBuffer.String(), "\n") + var commands []*command + for _, line := range allStderrLines { + const commandPrefix = "command:" + const envupdatePrefix = ".EnvUpdate:" + envUpdateIdx := strings.Index(line, ".EnvUpdate:") + if strings.Index(line, commandPrefix) >= 0 { + if envUpdateIdx == -1 { + envUpdateIdx = len(line) - 1 + } + args := strings.Fields(line[len(commandPrefix):envUpdateIdx]) + envUpdateStr := line[envUpdateIdx+len(envupdatePrefix):] + envUpdate := strings.Fields(envUpdateStr) + if len(envUpdate) == 0 { + // normalize empty slice to nil to make comparing empty envUpdates + // simpler. + envUpdate = nil + } + command := &command{ + path: args[0], + args: args[1:], + envUpdates: envUpdate, + } + commands = append(commands, command) + } + } + return commands, nil +} + +func forwardToOldWrapper(env env, cfg *config, wrapperCmd *command) error { + pipes := exec.Cmd{ + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + } + mockForks := false + return callOldWrapper(env, cfg, wrapperCmd, &pipes, mockForks) +} + +func callOldWrapper(env env, cfg *config, wrapperCmd *command, pipes *exec.Cmd, mockForks bool) error { + mockFile, err := ioutil.TempFile("", "compiler_wrapper_mock") + if err != nil { + return err + } + defer os.Remove(mockFile.Name()) + + if err := writeOldWrapperMock(mockFile, env, cfg, wrapperCmd, mockForks); err != nil { + return err + } + if err := mockFile.Close(); err != nil { + return err + } + + // Call the wrapper. + cmd := newExecCmd(env, wrapperCmd) + ensurePathEnv(cmd) + // Note: Using a self executable wrapper does not work due to a race condition + // on unix systems. See https://github.com/golang/go/issues/22315 + cmd.Args = append([]string{"/usr/bin/python2", "-S", mockFile.Name()}, cmd.Args[1:]...) + cmd.Path = cmd.Args[0] + cmd.Stdin = pipes.Stdin + cmd.Stdout = pipes.Stdout + cmd.Stderr = pipes.Stderr + return cmd.Run() +} + +func writeOldWrapperMock(writer io.Writer, env env, cfg *config, wrapperCmd *command, mockForks bool) error { + absOldWrapperPath := cfg.oldWrapperPath + if !filepath.IsAbs(absOldWrapperPath) { + absWrapperDir, err := getAbsWrapperDir(env, wrapperCmd.path) + if err != nil { + return err + } + absOldWrapperPath = filepath.Join(absWrapperDir, cfg.oldWrapperPath) + } + // Note: Fieldnames need to be upper case so that they can be read via reflection. + mockData := struct { + CmdPath string + WrapperPath string + RootRelPath string + MockForks bool + OverwriteConfig bool + CommonFlags []string + GccFlags []string + ClangFlags []string + }{ + wrapperCmd.path, + absOldWrapperPath, + cfg.rootRelPath, + mockForks, + cfg.overrideOldWrapperConfig, + cfg.commonFlags, + cfg.gccFlags, + cfg.clangFlags, + } + + const mockTemplate = `from __future__ import print_function +import imp +import os +import sys + +{{if .MockForks}} +init_env = os.environ.copy() + +def serialize_cmd(args): + current_env = os.environ + envupdate = [k + "=" + current_env.get(k, '') for k in set(list(current_env.keys()) + list(init_env.keys())) if current_env.get(k, '') != init_env.get(k, '')] + print('command:%s.EnvUpdate:%s\n' % (' '.join(args), ' '.join(envupdate)), file=sys.stderr) + +def execv_mock(binary, args): + serialize_cmd([binary] + args[1:]) + sys.exit(0) + +os.execv = execv_mock +{{end}} + +sys.argv[0] = '{{.CmdPath}}' + +# Equivalent to "import wrapper", but does not need a ".py" suffix. +# Also avoids writing '.pyc' files +sys.dont_write_bytecode = True +wrapper = imp.load_source('wrapper', '{{.WrapperPath}}') + +wrapper.ROOT_REL_PATH = '{{.RootRelPath}}' + +{{if .OverwriteConfig}} +wrapper.FLAGS_TO_ADD=set([{{range .CommonFlags}}'{{.}}',{{end}}]) +wrapper.GCC_FLAGS_TO_ADD=set([{{range .GccFlags}}'{{.}}',{{end}}]) +wrapper.CLANG_FLAGS_TO_ADD=set([{{range .ClangFlags}}'{{.}}',{{end}}]) +{{end}} + +wrapper.main() +` + + tmpl, err := template.New("mock").Parse(mockTemplate) + if err != nil { + return err + } + + return tmpl.Execute(writer, mockData) +} + +func ensurePathEnv(cmd *exec.Cmd) { + for _, env := range cmd.Env { + if strings.HasPrefix(env, "PATH=") { + return + } + } + cmd.Env = append(cmd.Env, "PATH=") +} diff --git a/compiler_wrapper/pie_flags.go b/compiler_wrapper/pie_flags.go new file mode 100644 index 00000000..73b3a224 --- /dev/null +++ b/compiler_wrapper/pie_flags.go @@ -0,0 +1,39 @@ +package main + +func processPieFlags(builder *commandBuilder) { + fpieMap := map[string]bool{"-D__KERNEL__": true, "-fPIC": true, "-fPIE": true, "-fno-PIC": true, "-fno-PIE": true, + "-fno-pic": true, "-fno-pie": true, "-fpic": true, "-fpie": true, "-nopie": true, + "-nostartfiles": true, "-nostdlib": true, "-pie": true, "-static": true} + + pieMap := map[string]bool{"-D__KERNEL__": true, "-A": true, "-fno-PIC": true, "-fno-PIE": true, "-fno-pic": true, "-fno-pie": true, + "-nopie": true, "-nostartfiles": true, "-nostdlib": true, "-pie": true, "-r": true, "--shared": true, + "-shared": true, "-static": true} + + pie := false + fpie := false + if builder.target.abi != "eabi" { + for _, arg := range builder.args { + if arg.FromUser { + if fpieMap[arg.Value] { + fpie = true + } + if pieMap[arg.Value] { + pie = true + } + } + } + } + builder.transformArgs(func(arg builderArg) string { + // Remove -nopie as it is a non-standard flag. + if arg.Value == "-nopie" { + return "" + } + if fpie && !arg.FromUser && arg.Value == "-fPIE" { + return "" + } + if pie && !arg.FromUser && arg.Value == "-pie" { + return "" + } + return arg.Value + }) +} diff --git a/compiler_wrapper/pie_flags_test.go b/compiler_wrapper/pie_flags_test.go new file mode 100644 index 00000000..56250fcd --- /dev/null +++ b/compiler_wrapper/pie_flags_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "testing" +) + +func TestAddPieFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initPieConfig(ctx.cfg) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, ctx.newCommand(gccX86_64, mainCc))) + + if err := verifyArgOrder(cmd, "-pie", mainCc); err != nil { + t.Error(err) + } + if err := verifyArgOrder(cmd, "-fPIE", mainCc); err != nil { + t.Error(err) + } + }) +} + +func TestOmitPieFlagsWhenNoPieArgGiven(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initPieConfig(ctx.cfg) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-nopie", mainCc))) + if err := verifyArgCount(cmd, 0, "-nopie"); err != nil { + t.Error(err) + } + if err := verifyArgCount(cmd, 0, "-pie"); err != nil { + t.Error(err) + } + if err := verifyArgCount(cmd, 0, "-fPIE"); err != nil { + t.Error(err) + } + + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-fno-pie", mainCc))) + if err := verifyArgCount(cmd, 0, "-pie"); err != nil { + t.Error(err) + } + if err := verifyArgCount(cmd, 0, "-fPIE"); err != nil { + t.Error(err) + } + }) +} + +func TestOmitPieFlagsWhenKernelDefined(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initPieConfig(ctx.cfg) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-D__KERNEL__", mainCc))) + if err := verifyArgCount(cmd, 0, "-pie"); err != nil { + t.Error(err) + } + if err := verifyArgCount(cmd, 0, "-fPIE"); err != nil { + t.Error(err) + } + }) +} + +func TestAddPieFlagsForEabiEvenIfNoPieGiven(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initPieConfig(ctx.cfg) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64Eabi, "-nopie", mainCc))) + if err := verifyArgCount(cmd, 0, "-nopie"); err != nil { + t.Error(err) + } + if err := verifyArgCount(cmd, 1, "-pie"); err != nil { + t.Error(err) + } + if err := verifyArgCount(cmd, 1, "-fPIE"); err != nil { + t.Error(err) + } + }) +} + +func initPieConfig(cfg *config) { + cfg.commonFlags = []string{"-fPIE", "-pie"} +} diff --git a/compiler_wrapper/sanitizer_flags.go b/compiler_wrapper/sanitizer_flags.go new file mode 100644 index 00000000..de4f76c4 --- /dev/null +++ b/compiler_wrapper/sanitizer_flags.go @@ -0,0 +1,33 @@ +package main + +import ( + "strings" +) + +func processSanitizerFlags(builder *commandBuilder) { + filterSanitizerFlags := false + for _, arg := range builder.args { + // TODO: This should probably be -fsanitize= to not match on + // e.g. -fsanitize-blacklist + if arg.FromUser && strings.HasPrefix(arg.Value, "-fsanitize") { + filterSanitizerFlags = true + break + } + } + if filterSanitizerFlags { + // Flags not supported by sanitizers (ASan etc.) + var unsupportedSanitizerFlags = map[string]bool{ + "-D_FORTIFY_SOURCE=1": true, + "-D_FORTIFY_SOURCE=2": true, + "-Wl,--no-undefined": true, + "-Wl,-z,defs": true, + } + + builder.transformArgs(func(arg builderArg) string { + if unsupportedSanitizerFlags[arg.Value] { + return "" + } + return arg.Value + }) + } +} diff --git a/compiler_wrapper/sanitizer_flags_test.go b/compiler_wrapper/sanitizer_flags_test.go new file mode 100644 index 00000000..b54398a6 --- /dev/null +++ b/compiler_wrapper/sanitizer_flags_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "testing" +) + +func TestFilterUnsupportedSanitizerFlagsIfSanitizeGiven(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-fsanitize=kernel-address", "-Wl,--no-undefined", mainCc))) + if err := verifyArgCount(cmd, 0, "-Wl,--no-undefined"); err != nil { + t.Error(err) + } + + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-fsanitize=kernel-address", + "-Wl,-z,defs", mainCc))) + if err := verifyArgCount(cmd, 0, "-Wl,-z,defs"); err != nil { + t.Error(err) + } + + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-fsanitize=kernel-address", "-D_FORTIFY_SOURCE=1", mainCc))) + if err := verifyArgCount(cmd, 0, "-D_FORTIFY_SOURCE=1"); err != nil { + t.Error(err) + } + + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-fsanitize=kernel-address", "-D_FORTIFY_SOURCE=2", mainCc))) + if err := verifyArgCount(cmd, 0, "-D_FORTIFY_SOURCE=2"); err != nil { + t.Error(err) + } + }) +} + +func TestKeepSanitizerFlagsIfNoSanitizeGiven(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-Wl,--no-undefined", mainCc))) + if err := verifyArgCount(cmd, 1, "-Wl,--no-undefined"); err != nil { + t.Error(err) + } + + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-Wl,-z,defs", mainCc))) + if err := verifyArgCount(cmd, 1, "-Wl,-z,defs"); err != nil { + t.Error(err) + } + + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-D_FORTIFY_SOURCE=1", mainCc))) + if err := verifyArgCount(cmd, 1, "-D_FORTIFY_SOURCE=1"); err != nil { + t.Error(err) + } + + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-D_FORTIFY_SOURCE=2", mainCc))) + if err := verifyArgCount(cmd, 1, "-D_FORTIFY_SOURCE=2"); err != nil { + t.Error(err) + } + }) +} + +func TestKeepSanitizerFlagsIfSanitizeGivenInCommonFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.cfg.commonFlags = []string{"-fsanitize=kernel-address"} + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-Wl,--no-undefined", mainCc))) + if err := verifyArgCount(cmd, 1, "-Wl,--no-undefined"); err != nil { + t.Error(err) + } + }) +} diff --git a/compiler_wrapper/stackprotector_flags.go b/compiler_wrapper/stackprotector_flags.go new file mode 100644 index 00000000..caadb6ed --- /dev/null +++ b/compiler_wrapper/stackprotector_flags.go @@ -0,0 +1,25 @@ +package main + +func processStackProtectorFlags(builder *commandBuilder) { + fstackMap := map[string]bool{"-D__KERNEL__": true, "-fno-stack-protector": true, "-nodefaultlibs": true, + "-nostdlib": true} + + fstack := false + if builder.target.abi != "eabi" { + for _, arg := range builder.args { + if arg.FromUser && fstackMap[arg.Value] { + fstack = true + break + } + } + } + if fstack { + builder.addPreUserArgs("-fno-stack-protector") + builder.transformArgs(func(arg builderArg) string { + if !arg.FromUser && arg.Value == "-fstack-protector-strong" { + return "" + } + return arg.Value + }) + } +} diff --git a/compiler_wrapper/stackprotector_flags_test.go b/compiler_wrapper/stackprotector_flags_test.go new file mode 100644 index 00000000..1a50a9b0 --- /dev/null +++ b/compiler_wrapper/stackprotector_flags_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "testing" +) + +func TestAddStrongStackProtectorFlag(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initStackProtectorStrongConfig(ctx.cfg) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyArgOrder(cmd, "-fstack-protector-strong", mainCc); err != nil { + t.Error(err) + } + }) +} + +func TestAddNoStackProtectorFlagWhenKernelDefined(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initStackProtectorStrongConfig(ctx.cfg) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-D__KERNEL__", mainCc))) + if err := verifyArgOrder(cmd, "-fno-stack-protector", mainCc); err != nil { + t.Error(err) + } + }) +} + +func TestOmitNoStackProtectorFlagWhenAlreadyInCommonFlags(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.cfg.commonFlags = []string{"-fno-stack-protector"} + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyArgCount(cmd, 1, "-fno-stack-protector"); err != nil { + t.Error(err) + } + }) +} + +func TestAddStrongStackProtectorFlagForEabiEvenIfNoStackProtectorGiven(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initStackProtectorStrongConfig(ctx.cfg) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64Eabi, "-fno-stack-protector", mainCc))) + if err := verifyArgCount(cmd, 1, "-fstack-protector-strong"); err != nil { + t.Error(err) + } + }) +} + +func initStackProtectorStrongConfig(cfg *config) { + cfg.commonFlags = []string{"-fstack-protector-strong"} +} diff --git a/compiler_wrapper/sysroot_flag.go b/compiler_wrapper/sysroot_flag.go new file mode 100644 index 00000000..fa8a3aef --- /dev/null +++ b/compiler_wrapper/sysroot_flag.go @@ -0,0 +1,27 @@ +package main + +import ( + "path/filepath" + "strings" +) + +func processSysrootFlag(rootPath string, builder *commandBuilder) string { + fromUser := false + for _, arg := range builder.args { + if arg.FromUser && strings.HasPrefix(arg.Value, "--sysroot=") { + fromUser = true + break + } + } + sysroot := builder.env.getenv("SYSROOT") + if sysroot != "" { + builder.updateEnv("SYSROOT=") + } else { + // Use the bundled sysroot by default. + sysroot = filepath.Join(rootPath, "usr", builder.target.target) + } + if !fromUser { + builder.addPreUserArgs("--sysroot=" + sysroot) + } + return sysroot +} diff --git a/compiler_wrapper/sysroot_flag_test.go b/compiler_wrapper/sysroot_flag_test.go new file mode 100644 index 00000000..dd726291 --- /dev/null +++ b/compiler_wrapper/sysroot_flag_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "path" + "testing" +) + +func TestOmitSysrootGivenUserDefinedSysroot(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + runWithCompiler := func(compiler string) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(compiler, "--sysroot=/somepath", mainCc))) + if err := verifyArgOrder(cmd, "--sysroot=/somepath", mainCc); err != nil { + t.Error(err) + } + if err := verifyArgCount(cmd, 1, "--sysroot.*"); err != nil { + t.Error(err) + } + } + + runWithCompiler(gccX86_64) + runWithCompiler(clangX86_64) + }) +} + +func TestSetSysrootFlagFromEnv(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.env = []string{"SYSROOT=/envpath"} + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyArgOrder(cmd, "--sysroot=/envpath", mainCc); err != nil { + t.Error(err) + } + }) +} + +func TestSetSysrootRelativeToWrapperPath(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.cfg.rootRelPath = "somepath" + wrapperCmd := ctx.newCommand(gccX86_64, mainCc) + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, wrapperCmd)) + if err := verifyArgOrder(cmd, + "--sysroot="+ctx.tempDir+"/somepath/usr/x86_64-cros-linux-gnu", mainCc); err != nil { + t.Error(err) + } + }) +} + +func TestSetSysrootRelativeToSymlinkedWrapperPath(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + ctx.cfg.rootRelPath = "somepath" + linkedWrapperPath := path.Join(ctx.tempDir, "a/linked/path/x86_64-cros-linux-gnu-gcc") + ctx.symlink(path.Join(ctx.tempDir, gccX86_64), linkedWrapperPath) + + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(linkedWrapperPath, mainCc))) + if err := verifyArgOrder(cmd, + "--sysroot="+ctx.tempDir+"/somepath/usr/x86_64-cros-linux-gnu", mainCc); err != nil { + t.Error(err) + } + }) +} diff --git a/compiler_wrapper/testutil_test.go b/compiler_wrapper/testutil_test.go new file mode 100644 index 00000000..5ebe0084 --- /dev/null +++ b/compiler_wrapper/testutil_test.go @@ -0,0 +1,193 @@ +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +var crosRootDirFlag = flag.String("crosroot", "", "root dir of the chrome os toolchain") + +const mainCc = "main.cc" +const clangX86_64 = "./x86_64-cros-linux-gnu-clang" +const gccX86_64 = "./x86_64-cros-linux-gnu-gcc" +const gccX86_64Eabi = "./x86_64-cros-linux-eabi-gcc" +const gccArmV7 = "./armv7m-cros-linux-gnu-gcc" +const gccArmV7Eabi = "./armv7m-cros-linux-eabi-gcc" +const gccArmV8 = "./armv8m-cros-linux-gnu-gcc" +const gccArmV8Eabi = "./armv8m-cros-linux-eabi-gcc" + +const oldHardenedWrapperPathForTest = "/usr/x86_64-pc-linux-gnu/x86_64-cros-linux-gnu/gcc-bin/4.9.x/sysroot_wrapper.hardened" +const oldNonHardenedWrapperPathForTest = "/usr/x86_64-pc-linux-gnu/arm-none-eabi/gcc-bin/4.9.x/sysroot_wrapper" + +type testContext struct { + t *testing.T + tempDir string + env []string + cfg *config +} + +func withTestContext(t *testing.T, work func(ctx *testContext)) { + t.Parallel() + tempDir, err := ioutil.TempDir("", "compiler_wrapper") + if err != nil { + t.Fatalf("Unable to create the temp dir. Error: %s", err) + } + defer os.RemoveAll(tempDir) + + ctx := testContext{ + t: t, + tempDir: tempDir, + env: nil, + cfg: &config{ + oldWrapperPath: "FilledLater", + overrideOldWrapperConfig: true, + }, + } + // Note: It's ok to use the hardened wrapper here, as we replace its config + // on each run. + ctx.setOldWrapperPath(oldHardenedWrapperPathForTest) + + work(&ctx) +} + +var _ env = (*testContext)(nil) + +func (ctx *testContext) getenv(key string) string { + for i := len(ctx.env) - 1; i >= 0; i-- { + entry := ctx.env[i] + if strings.HasPrefix(entry, key+"=") { + return entry[len(key)+1:] + } + } + return "" +} + +func (ctx *testContext) environ() []string { + return ctx.env +} + +func (ctx *testContext) getwd() string { + return ctx.tempDir +} + +func (ctx *testContext) must(cmd *command, err error) *command { + if err != nil { + ctx.t.Fatalf("Expected no error, but got %s", err) + } + return cmd +} + +func (ctx *testContext) setOldWrapperPath(chrootPath string) { + if *crosRootDirFlag != "" { + ctx.cfg.oldWrapperPath = filepath.Join(*crosRootDirFlag, chrootPath) + } else { + ctx.cfg.oldWrapperPath = "" + } +} + +func (ctx *testContext) newCommand(path string, args ...string) *command { + // Create an empty wrapper at the given path. + // Needed as we are resolving symlinks which stats the wrapper file. + ctx.writeFile(path, "") + return &command{ + path: path, + args: args, + } +} + +func (ctx *testContext) writeFile(fullFileName string, fileContent string) { + if !filepath.IsAbs(fullFileName) { + fullFileName = filepath.Join(ctx.tempDir, fullFileName) + } + if err := os.MkdirAll(filepath.Dir(fullFileName), 0777); err != nil { + ctx.t.Fatal(err) + } + if err := ioutil.WriteFile(fullFileName, []byte(fileContent), 0777); err != nil { + ctx.t.Fatal(err) + } +} + +func (ctx *testContext) symlink(oldname string, newname string) { + if err := os.MkdirAll(filepath.Dir(newname), 0777); err != nil { + ctx.t.Fatal(err) + } + if err := os.Symlink(oldname, newname); err != nil { + ctx.t.Fatal(err) + } +} + +func verifyPath(cmd *command, expectedRegex string) error { + compiledRegex := regexp.MustCompile(matchFullString(expectedRegex)) + if !compiledRegex.MatchString(cmd.path) { + return fmt.Errorf("path does not match %s. Actual %s", expectedRegex, cmd.path) + } + return nil +} + +func verifyArgCount(cmd *command, expectedCount int, expectedRegex string) error { + compiledRegex := regexp.MustCompile(matchFullString(expectedRegex)) + count := 0 + for _, arg := range cmd.args { + if compiledRegex.MatchString(arg) { + count++ + } + } + if count != expectedCount { + return fmt.Errorf("expected %d matches for arg %s. All args: %s", + expectedCount, expectedRegex, cmd.args) + } + return nil +} + +func verifyArgOrder(cmd *command, expectedRegexes ...string) error { + compiledRegexes := []*regexp.Regexp{} + for _, regex := range expectedRegexes { + compiledRegexes = append(compiledRegexes, regexp.MustCompile(matchFullString(regex))) + } + expectedArgIndex := 0 + for _, arg := range cmd.args { + if expectedArgIndex == len(compiledRegexes) { + break + } else if compiledRegexes[expectedArgIndex].MatchString(arg) { + expectedArgIndex++ + } + } + if expectedArgIndex != len(expectedRegexes) { + return fmt.Errorf("expected args %s in order. All args: %s", + expectedRegexes, cmd.args) + } + return nil +} + +func verifyEnvUpdate(cmd *command, expectedRegex string) error { + compiledRegex := regexp.MustCompile(matchFullString(expectedRegex)) + for _, update := range cmd.envUpdates { + if compiledRegex.MatchString(update) { + return nil + } + } + return fmt.Errorf("expected at least one match for env update %s. All env updates: %s", + expectedRegex, cmd.envUpdates) +} + +func verifyNoEnvUpdate(cmd *command, expectedRegex string) error { + compiledRegex := regexp.MustCompile(matchFullString(expectedRegex)) + updates := cmd.envUpdates + for _, update := range updates { + if compiledRegex.MatchString(update) { + return fmt.Errorf("expected no match for env update %s. All env updates: %s", + expectedRegex, cmd.envUpdates) + } + } + return nil +} + +func matchFullString(regex string) string { + return "^" + regex + "$" +} diff --git a/compiler_wrapper/thumb_flags.go b/compiler_wrapper/thumb_flags.go new file mode 100644 index 00000000..fa65875e --- /dev/null +++ b/compiler_wrapper/thumb_flags.go @@ -0,0 +1,23 @@ +package main + +import ( + "strings" +) + +func processThumbCodeFlags(builder *commandBuilder) { + arch := builder.target.arch + if builder.target.abi != "eabi" && (strings.HasPrefix(arch, "armv7") || strings.HasPrefix(arch, "armv8")) { + // ARM32 specfic: + // 1. Generate thumb codes by default. GCC is configured with + // --with-mode=thumb and defaults to thumb mode already. This + // changes the default behavior of clang and doesn't affect GCC. + // 2. Do not force frame pointers on ARM32 (https://crbug.com/693137). + builder.addPreUserArgs("-mthumb") + builder.transformArgs(func(arg builderArg) string { + if !arg.FromUser && arg.Value == "-fno-omit-frame-pointer" { + return "" + } + return arg.Value + }) + } +} diff --git a/compiler_wrapper/thumb_flags_test.go b/compiler_wrapper/thumb_flags_test.go new file mode 100644 index 00000000..463ce260 --- /dev/null +++ b/compiler_wrapper/thumb_flags_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "testing" +) + +func TestAddThumbFlagForArm(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccArmV7, mainCc))) + if err := verifyArgOrder(cmd, "-mthumb", mainCc); err != nil { + t.Error(err) + } + + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccArmV8, mainCc))) + if err := verifyArgOrder(cmd, "-mthumb", mainCc); err != nil { + t.Error(err) + } + }) +} + +func TestOmitThumbFlagForNonArm(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyArgCount(cmd, 0, "-mthumb"); err != nil { + t.Error(err) + } + }) +} + +func TestOmitThumbFlagForEabiArm(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccArmV7Eabi, mainCc))) + if err := verifyArgCount(cmd, 0, "-mthumb"); err != nil { + t.Error(err) + } + + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccArmV8Eabi, mainCc))) + if err := verifyArgCount(cmd, 0, "-mthumb"); err != nil { + t.Error(err) + } + }) +} + +func TestRemoveNoOmitFramePointerFlagForArm(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initNoOmitFramePointerConfig(ctx.cfg) + + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccArmV7, mainCc))) + if err := verifyArgCount(cmd, 0, "-fno-omit-frame-pointer"); err != nil { + t.Error(err) + } + + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccArmV8, mainCc))) + if err := verifyArgCount(cmd, 0, "-fno-omit-frame-pointer"); err != nil { + t.Error(err) + } + }) +} + +func TestKeepNoOmitFramePointerFlagForNonArm(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initNoOmitFramePointerConfig(ctx.cfg) + + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyArgCount(cmd, 1, "-fno-omit-frame-pointer"); err != nil { + t.Error(err) + } + }) +} + +func TestKeepNoOmitFramePointerFlagForEabiArm(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + initNoOmitFramePointerConfig(ctx.cfg) + + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccArmV7Eabi, mainCc))) + if err := verifyArgCount(cmd, 1, "-fno-omit-frame-pointer"); err != nil { + t.Error(err) + } + + cmd = ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccArmV8Eabi, mainCc))) + if err := verifyArgCount(cmd, 1, "-fno-omit-frame-pointer"); err != nil { + t.Error(err) + } + }) +} + +func TestKeepNoOmitFramePointIfGivenByUser(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccArmV7, "-fno-omit-frame-pointer", mainCc))) + if err := verifyArgCount(cmd, 1, "-fno-omit-frame-pointer"); err != nil { + t.Error(err) + } + }) +} + +func initNoOmitFramePointerConfig(cfg *config) { + cfg.commonFlags = []string{"-fno-omit-frame-pointer"} +} diff --git a/compiler_wrapper/unsupported_flags.go b/compiler_wrapper/unsupported_flags.go new file mode 100644 index 00000000..604f94ce --- /dev/null +++ b/compiler_wrapper/unsupported_flags.go @@ -0,0 +1,14 @@ +package main + +import ( + "errors" +) + +func checkUnsupportedFlags(cmd *command) error { + for _, arg := range cmd.args { + if arg == "-fstack-check" { + return errors.New(`option "-fstack-check" is not supported; crbug/485492`) + } + } + return nil +} diff --git a/compiler_wrapper/unsupported_flags_test.go b/compiler_wrapper/unsupported_flags_test.go new file mode 100644 index 00000000..0b174a9b --- /dev/null +++ b/compiler_wrapper/unsupported_flags_test.go @@ -0,0 +1,15 @@ +package main + +import ( + "testing" +) + +func TestErrorOnFstatCheckFlag(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + _, err := calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, "-fstack-check", mainCc)) + if err == nil || err.Error() != `option "-fstack-check" is not supported; crbug/485492` { + t.Errorf("Expected error not found. Got: %s", err) + } + }) +} diff --git a/compiler_wrapper/x64_flags.go b/compiler_wrapper/x64_flags.go new file mode 100644 index 00000000..7325cf85 --- /dev/null +++ b/compiler_wrapper/x64_flags.go @@ -0,0 +1,17 @@ +package main + +import ( + "strings" +) + +func processX86Flags(builder *commandBuilder) { + arch := builder.target.arch + if strings.HasPrefix(arch, "x86_64") || startswithI86(arch) { + builder.addPostUserArgs("-mno-movbe") + } +} + +// Returns true if s starts with i.86. +func startswithI86(s string) bool { + return len(s) >= 4 && s[0] == 'i' && s[2:4] == "86" +} diff --git a/compiler_wrapper/x64_flags_test.go b/compiler_wrapper/x64_flags_test.go new file mode 100644 index 00000000..90bad4e7 --- /dev/null +++ b/compiler_wrapper/x64_flags_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "testing" +) + +func TestAddNoMovbeFlagOnX86(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccX86_64, mainCc))) + if err := verifyArgOrder(cmd, mainCc, "-mno-movbe"); err != nil { + t.Error(err) + } + }) +} + +func TestAddNoMovbeFlagOnI686(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand("./i686-cros-linux-gnu-gcc", mainCc))) + if err := verifyArgOrder(cmd, mainCc, "-mno-movbe"); err != nil { + t.Error(err) + } + }) +} + +func TestDoNotAddNoMovbeFlagOnArm(t *testing.T) { + withTestContext(t, func(ctx *testContext) { + cmd := ctx.must(calcCompilerCommandAndCompareToOld(ctx, ctx.cfg, + ctx.newCommand(gccArmV7, mainCc))) + if err := verifyArgCount(cmd, 0, "-mno-movbe"); err != nil { + t.Error(err) + } + }) +} |