Android wrapper: Support @param files for bisection.

BUG=chromium:773875
TEST=unit test

Change-Id: I94f99977f823e098aae39c79b9e01603fd84f5fa
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1832309
Tested-by: Tobias Bosch <tbosch@google.com>
Reviewed-by: George Burgess <gbiv@chromium.org>

13 files changed, 529 insertions, 47 deletions

diff --git a/compiler_wrapper/android_config_test.go b/compiler_wrapper/android_config_test.go
index fddde817..74881a02 100644
--- a/compiler_wrapper/android_config_test.go
+++ b/compiler_wrapper/android_config_test.go
@@ -25,6 +25,7 @@ func TestAndroidConfig(t *testing.T) {
 
 		runGoldenRecords(ctx, androidGoldenDir, []goldenFile{
 			createAndroidClangPathGoldenInputs(ctx),
+			createBisectGoldenInputs(filepath.Join(ctx.tempDir, "clang")),
 		})
 	})
 }
diff --git a/compiler_wrapper/bisect_flag.go b/compiler_wrapper/bisect_flag.go
index 2a805352..62bcbd31 100644
--- a/compiler_wrapper/bisect_flag.go
+++ b/compiler_wrapper/bisect_flag.go
@@ -4,22 +4,57 @@
 
 package main
 
-const bisectPythonCommand = "import bisect_driver; sys.exit(bisect_driver.bisect_driver(sys.argv[1], sys.argv[2], sys.argv[3:]))"
+import (
+	"os"
+	"path/filepath"
+)
+
+// Note: We keep this code in python as golang has no builtin
+// shlex function.
+const bisectPythonCommand = `
+import bisect_driver
+import shlex
+import sys
+
+def ExpandArgs(args, target):
+	for arg in args:
+		if arg[0] == '@':
+			with open(arg[1:], 'rb') as f:
+				ExpandArgs(shlex.split(f.read()), target)
+		else:
+			target.append(arg)
+	return target
+
+stage = sys.argv[1]
+dir = sys.argv[2]
+execargs = ExpandArgs(sys.argv[3:], [])
+
+sys.exit(bisect_driver.bisect_driver(stage, dir, execargs))
+`
 
 func getBisectStage(env env) string {
 	value, _ := env.getenv("BISECT_STAGE")
 	return value
 }
 
-func calcBisectCommand(env env, bisectStage string, compilerCmd *command) *command {
+func calcBisectCommand(env env, cfg *config, bisectStage string, compilerCmd *command) (*command, error) {
 	bisectDir, _ := env.getenv("BISECT_DIR")
 	if bisectDir == "" {
-		bisectDir = "/tmp/sysroot_bisect"
+		if cfg.isAndroidWrapper {
+			homeDir, err := os.UserHomeDir()
+			if err != nil {
+				return nil, err
+			}
+			bisectDir = filepath.Join(homeDir, "ANDROID_BISECT")
+		} else {
+			bisectDir = "/tmp/sysroot_bisect"
+		}
 	}
 	absCompilerPath := getAbsCmdPath(env, compilerCmd)
 	return &command{
-		Path: "/usr/bin/python2",
+		Path: "/usr/bin/env",
 		Args: append([]string{
+			"python",
 			"-c",
 			bisectPythonCommand,
 			bisectStage,
@@ -27,5 +62,5 @@ func calcBisectCommand(env env, bisectStage string, compilerCmd *command) *comma
 			absCompilerPath,
 		}, compilerCmd.Args...),
 		EnvUpdates: compilerCmd.EnvUpdates,
-	}
+	}, nil
 }
diff --git a/compiler_wrapper/bisect_flag_test.go b/compiler_wrapper/bisect_flag_test.go
index 11ab2dcf..43c3095b 100644
--- a/compiler_wrapper/bisect_flag_test.go
+++ b/compiler_wrapper/bisect_flag_test.go
@@ -8,8 +8,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"path/filepath"
-	"regexp"
 	"strings"
 	"testing"
 )
@@ -20,25 +20,44 @@ func TestCallBisectDriver(t *testing.T) {
 			"BISECT_STAGE=someBisectStage",
 			"BISECT_DIR=someBisectDir",
 		}
-		cmd := ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(gccX86_64, mainCc)))
-		if err := verifyPath(cmd, "/usr/bin/python2"); err != nil {
+		cmd := mustCallBisectDriver(ctx, callCompiler(ctx, ctx.cfg, ctx.newCommand(gccX86_64, mainCc)))
+		if err := verifyPath(cmd, "bisect_driver"); err != nil {
 			t.Error(err)
 		}
-		if err := verifyArgOrder(cmd, "-c", regexp.QuoteMeta(bisectPythonCommand),
+		if err := verifyArgOrder(cmd,
 			"someBisectStage", "someBisectDir", filepath.Join(ctx.tempDir, gccX86_64+".real"), "--sysroot=.*", mainCc); err != nil {
 			t.Error(err)
 		}
 	})
 }
 
+func TestCallBisectDriverWithParamsFile(t *testing.T) {
+	withBisectTestContext(t, func(ctx *testContext) {
+		ctx.env = []string{
+			"BISECT_STAGE=someBisectStage",
+			"BISECT_DIR=someBisectDir",
+		}
+		paramsFile1 := filepath.Join(ctx.tempDir, "params1")
+		ctx.writeFile(paramsFile1, "a\n#comment\n@params2")
+		paramsFile2 := filepath.Join(ctx.tempDir, "params2")
+		ctx.writeFile(paramsFile2, "b\n"+mainCc)
+
+		cmd := mustCallBisectDriver(ctx, callCompiler(ctx, ctx.cfg, ctx.newCommand(gccX86_64, "@"+paramsFile1)))
+		if err := verifyArgOrder(cmd,
+			"a", "b", mainCc); err != nil {
+			t.Error(err)
+		}
+	})
+}
+
 func TestCallBisectDriverWithCCache(t *testing.T) {
 	withBisectTestContext(t, func(ctx *testContext) {
 		ctx.cfg.useCCache = true
 		cmd := ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(gccX86_64, mainCc)))
-		if err := verifyPath(cmd, "/usr/bin/python2"); err != nil {
+		if err := verifyPath(cmd, "/usr/bin/env"); err != nil {
 			t.Error(err)
 		}
-		if err := verifyArgCount(cmd, 1, "/usr/bin/ccache"); err != nil {
+		if err := verifyArgOrder(cmd, "python", "/usr/bin/ccache"); err != nil {
 			t.Error(err)
 		}
 		if err := verifyEnvUpdate(cmd, "CCACHE_DIR=.*"); err != nil {
@@ -47,19 +66,34 @@ func TestCallBisectDriverWithCCache(t *testing.T) {
 	})
 }
 
-func TestDefaultBisectDir(t *testing.T) {
+func TestDefaultBisectDirCros(t *testing.T) {
 	withBisectTestContext(t, func(ctx *testContext) {
 		ctx.env = []string{
 			"BISECT_STAGE=someBisectStage",
 		}
-		cmd := ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(gccX86_64, mainCc)))
-		if err := verifyArgOrder(cmd, "-c", regexp.QuoteMeta(bisectPythonCommand),
+		cmd := mustCallBisectDriver(ctx, callCompiler(ctx, ctx.cfg, ctx.newCommand(gccX86_64, mainCc)))
+		if err := verifyArgOrder(cmd,
 			"someBisectStage", "/tmp/sysroot_bisect"); err != nil {
 			t.Error(err)
 		}
 	})
 }
 
+func TestDefaultBisectDirAndroid(t *testing.T) {
+	withBisectTestContext(t, func(ctx *testContext) {
+		ctx.env = []string{
+			"BISECT_STAGE=someBisectStage",
+		}
+		ctx.cfg.isAndroidWrapper = true
+		cmd := mustCallBisectDriver(ctx, callCompiler(ctx, ctx.cfg, ctx.newCommand(gccX86_64, mainCc)))
+		userHome, _ := os.UserHomeDir()
+		if err := verifyArgOrder(cmd,
+			"someBisectStage", filepath.Join(userHome, "ANDROID_BISECT")); err != nil {
+			t.Error(err)
+		}
+	})
+}
+
 func TestForwardStdOutAndStdErrAndExitCodeFromBisect(t *testing.T) {
 	withBisectTestContext(t, func(ctx *testContext) {
 		ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
@@ -103,6 +137,47 @@ func withBisectTestContext(t *testing.T, work func(ctx *testContext)) {
 		// sub command.
 		ctx.cfg.oldWrapperPath = ""
 		ctx.env = []string{"BISECT_STAGE=xyz"}
+		// We execute the python script but replace the call to the bisect_driver with
+		// a mock that logs the data in the same way as the oldwrapper. This way
+		// we can reuse the parseOldWrapperCommands to get the values.
+		ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
+			if err := verifyPath(cmd, "/usr/bin/env"); err != nil {
+				return err
+			}
+			if cmd.Args[0] != "python" {
+				return fmt.Errorf("expected a call to python. Got: %s", cmd.Args[0])
+			}
+			if cmd.Args[1] != "-c" {
+				return fmt.Errorf("expected an inline python script. Got: %s", cmd.Args)
+			}
+			script := cmd.Args[2]
+			mock := `
+class BisectDriver:
+	def __init__(self):
+		self.VALID_MODES = ['POPULATE_GOOD', 'POPULATE_BAD', 'TRIAGE']
+	def bisect_driver(self, bisect_stage, bisect_dir, execargs):
+		print('command bisect_driver')
+		print('arg %s' % bisect_stage)
+		print('arg %s' % bisect_dir)
+		for arg in execargs:
+			print('arg %s' % arg)
+
+bisect_driver = BisectDriver()
+`
+			script = mock + script
+			script = strings.Replace(script, "import bisect_driver", "", -1)
+			cmdCopy := *cmd
+			cmdCopy.Args = append(append(cmd.Args[:2], script), cmd.Args[3:]...)
+			// Evaluate the python script, but replace the call to the bisect_driver
+			// with a log statement so that we can assert it.
+			return runCmd(ctx, &cmdCopy, nil, stdout, stderr)
+		}
 		work(ctx)
 	})
 }
+
+func mustCallBisectDriver(ctx *testContext, exitCode int) *command {
+	ctx.must(exitCode)
+	cmds, _ := parseOldWrapperCommands(ctx.stdoutString())
+	return cmds[0]
+}
diff --git a/compiler_wrapper/compiler_wrapper.go b/compiler_wrapper/compiler_wrapper.go
index 09ef41d0..3dec7e66 100644
--- a/compiler_wrapper/compiler_wrapper.go
+++ b/compiler_wrapper/compiler_wrapper.go
@@ -125,7 +125,10 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int
 		return logRusage(env, rusageLogfileName, compilerCmd)
 	}
 	if bisectStage != "" {
-		compilerCmd = calcBisectCommand(env, bisectStage, compilerCmd)
+		compilerCmd, err = calcBisectCommand(env, cfg, bisectStage, compilerCmd)
+		if err != nil {
+			return 0, err
+		}
 	}
 	// Note: We return an exit code only if the underlying env is not
 	// really doing an exec, e.g. commandRecordingEnv.
diff --git a/compiler_wrapper/cros_hardened_config_test.go b/compiler_wrapper/cros_hardened_config_test.go
index b6d47da8..c3881303 100644
--- a/compiler_wrapper/cros_hardened_config_test.go
+++ b/compiler_wrapper/cros_hardened_config_test.go
@@ -64,7 +64,7 @@ func TestCrosHardenedConfigWithoutCCache(t *testing.T) {
 			createGccPathGoldenInputs(ctx, gomaEnv),
 			createClangPathGoldenInputs(ctx, gomaEnv),
 			createClangSyntaxGoldenInputs(gomaEnv),
-			createBisectGoldenInputs(),
+			createBisectGoldenInputs(clangX86_64),
 			createForceDisableWErrorGoldenInputs(),
 			createClangTidyGoldenInputs(gomaEnv),
 		})
@@ -108,7 +108,7 @@ LLVM_NEXT_FLAGS_TO_ADD = set(['-Wno-reorder-init-list',
 			createGccPathGoldenInputs(ctx, gomaEnv),
 			createClangPathGoldenInputs(ctx, gomaEnv),
 			createClangSyntaxGoldenInputs(gomaEnv),
-			createBisectGoldenInputs(),
+			createBisectGoldenInputs(clangX86_64),
 			createForceDisableWErrorGoldenInputs(),
 			createClangTidyGoldenInputs(gomaEnv),
 		})
@@ -133,7 +133,7 @@ func createSyswrapperGoldenInputs(ctx *testContext) []goldenFile {
 		createSysrootWrapperCommonGoldenInputs("clang", gomaEnv),
 		createSanitizerGoldenInputs("clang"),
 		createClangArgsGoldenInputs(),
-		createBisectGoldenInputs(),
+		createBisectGoldenInputs(clangX86_64),
 		createForceDisableWErrorGoldenInputs(),
 		createClangTidyGoldenInputs(gomaEnv),
 	}
@@ -184,7 +184,7 @@ func createGoldenInputsForAllTargets(compiler string, args ...string) goldenFile
 	}
 }
 
-func createBisectGoldenInputs() goldenFile {
+func createBisectGoldenInputs(compiler string) goldenFile {
 	return goldenFile{
 		Name: "bisect.json",
 		// Disable comparing to the old wrapper as that calls the bisect_driver
@@ -193,7 +193,14 @@ func createBisectGoldenInputs() goldenFile {
 		ignoreOldWrapper: true,
 		Records: []goldenRecord{
 			{
-				WrapperCmd: newGoldenCmd(clangX86_64, mainCc),
+				WrapperCmd: newGoldenCmd(compiler, mainCc),
+				Env: []string{
+					"BISECT_STAGE=someBisectStage",
+				},
+				Cmds: okResults,
+			},
+			{
+				WrapperCmd: newGoldenCmd(compiler, mainCc),
 				Env: []string{
 					"BISECT_STAGE=someBisectStage",
 					"BISECT_DIR=someBisectDir",
@@ -201,7 +208,7 @@ func createBisectGoldenInputs() goldenFile {
 				Cmds: okResults,
 			},
 			{
-				WrapperCmd: newGoldenCmd(clangX86_64, mainCc),
+				WrapperCmd: newGoldenCmd(compiler, mainCc),
 				Env: []string{
 					"BISECT_STAGE=someBisectStage",
 					"BISECT_DIR=someBisectDir",
diff --git a/compiler_wrapper/cros_host_config_test.go b/compiler_wrapper/cros_host_config_test.go
index 5137b561..d2be9ab6 100644
--- a/compiler_wrapper/cros_host_config_test.go
+++ b/compiler_wrapper/cros_host_config_test.go
@@ -34,7 +34,7 @@ func TestCrosClangHostConfig(t *testing.T) {
 			createGoldenInputsForAllTargets("clang", "-ftrapv", mainCc),
 			createSanitizerGoldenInputs("clang"),
 			createClangArgsGoldenInputs(),
-			createBisectGoldenInputs(),
+			createBisectGoldenInputs(clangX86_64),
 			createForceDisableWErrorGoldenInputs(),
 			createClangTidyGoldenInputs(gomaEnv),
 			createClangHostWrapperInputs(),
diff --git a/compiler_wrapper/goldenutil_test.go b/compiler_wrapper/goldenutil_test.go
index 9c164442..0bb2b11c 100644
--- a/compiler_wrapper/goldenutil_test.go
+++ b/compiler_wrapper/goldenutil_test.go
@@ -175,10 +175,11 @@ func fillGoldenResults(ctx *testContext, files []goldenFile) []goldenFile {
 func writeGoldenRecords(ctx *testContext, writer io.Writer, records []goldenRecord) {
 	// Replace the temp dir with a stable path so that the goldens stay stable.
 	stableTempDir := filepath.Join(filepath.Dir(ctx.tempDir), "stable")
+	homeDir, _ := os.UserHomeDir()
 	writer = &replacingWriter{
 		Writer: writer,
-		old:    []byte(ctx.tempDir),
-		new:    []byte(stableTempDir),
+		old:    [][]byte{[]byte(ctx.tempDir), []byte(homeDir)},
+		new:    [][]byte{[]byte(stableTempDir), []byte("$HOME")},
 	}
 	enc := json.NewEncoder(writer)
 	enc.SetIndent("", "  ")
@@ -189,12 +190,14 @@ func writeGoldenRecords(ctx *testContext, writer io.Writer, records []goldenReco
 
 type replacingWriter struct {
 	io.Writer
-	old []byte
-	new []byte
+	old [][]byte
+	new [][]byte
 }
 
 func (writer *replacingWriter) Write(p []byte) (n int, err error) {
 	// TODO: Use bytes.ReplaceAll once cros sdk uses golang >= 1.12
-	p = bytes.Replace(p, writer.old, writer.new, -1)
+	for i, old := range writer.old {
+		p = bytes.Replace(p, old, writer.new[i], -1)
+	}
 	return writer.Writer.Write(p)
 }
diff --git a/compiler_wrapper/testdata/android_golden/bisect.json b/compiler_wrapper/testdata/android_golden/bisect.json
new file mode 100644
index 00000000..41310ca1
--- /dev/null
+++ b/compiler_wrapper/testdata/android_golden/bisect.json
@@ -0,0 +1,100 @@
+[
+  {
+    "wd": "/tmp/stable",
+    "env": [
+      "BISECT_STAGE=someBisectStage"
+    ],
+    "wrapper": {
+      "cmd": {
+        "path": "/tmp/stable/clang",
+        "args": [
+          "main.cc"
+        ]
+      }
+    },
+    "cmds": [
+      {
+        "cmd": {
+          "path": "/usr/bin/env",
+          "args": [
+            "python",
+            "-c",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
+            "someBisectStage",
+            "$HOME/ANDROID_BISECT",
+            "/tmp/stable/clang.real",
+            "main.cc"
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "wd": "/tmp/stable",
+    "env": [
+      "BISECT_STAGE=someBisectStage",
+      "BISECT_DIR=someBisectDir"
+    ],
+    "wrapper": {
+      "cmd": {
+        "path": "/tmp/stable/clang",
+        "args": [
+          "main.cc"
+        ]
+      }
+    },
+    "cmds": [
+      {
+        "cmd": {
+          "path": "/usr/bin/env",
+          "args": [
+            "python",
+            "-c",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
+            "someBisectStage",
+            "someBisectDir",
+            "/tmp/stable/clang.real",
+            "main.cc"
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "wd": "/tmp/stable",
+    "env": [
+      "BISECT_STAGE=someBisectStage",
+      "BISECT_DIR=someBisectDir"
+    ],
+    "wrapper": {
+      "cmd": {
+        "path": "/tmp/stable/clang",
+        "args": [
+          "main.cc"
+        ]
+      },
+      "stdout": "somemessage",
+      "stderr": "someerror",
+      "exitcode": 1
+    },
+    "cmds": [
+      {
+        "cmd": {
+          "path": "/usr/bin/env",
+          "args": [
+            "python",
+            "-c",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
+            "someBisectStage",
+            "someBisectDir",
+            "/tmp/stable/clang.real",
+            "main.cc"
+          ]
+        },
+        "stdout": "somemessage",
+        "stderr": "someerror",
+        "exitcode": 1
+      }
+    ]
+  }
+]
diff --git a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json
index 47ed1949..5a67a635 100644
--- a/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_clang_host_golden/bisect.json
@@ -2,6 +2,45 @@
   {
     "wd": "/tmp/stable",
     "env": [
+      "BISECT_STAGE=someBisectStage"
+    ],
+    "wrapper": {
+      "cmd": {
+        "path": "./x86_64-cros-linux-gnu-clang",
+        "args": [
+          "main.cc"
+        ]
+      }
+    },
+    "cmds": [
+      {
+        "cmd": {
+          "path": "/usr/bin/env",
+          "args": [
+            "python",
+            "-c",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
+            "someBisectStage",
+            "/tmp/sysroot_bisect",
+            "/tmp/stable/clang",
+            "-Qunused-arguments",
+            "-grecord-gcc-switches",
+            "-fno-addrsig",
+            "-fuse-ld=lld",
+            "-Wno-unused-local-typedefs",
+            "-Wno-deprecated-declarations",
+            "-Wno-tautological-constant-compare",
+            "-Wno-tautological-unsigned-enum-zero-compare",
+            "-Wno-unknown-warning-option",
+            "main.cc"
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "wd": "/tmp/stable",
+    "env": [
       "BISECT_STAGE=someBisectStage",
       "BISECT_DIR=someBisectDir"
     ],
@@ -16,10 +55,11 @@
     "cmds": [
       {
         "cmd": {
-          "path": "/usr/bin/python2",
+          "path": "/usr/bin/env",
           "args": [
+            "python",
             "-c",
-            "import bisect_driver; sys.exit(bisect_driver.bisect_driver(sys.argv[1], sys.argv[2], sys.argv[3:]))",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
             "someBisectStage",
             "someBisectDir",
             "/tmp/stable/clang",
@@ -58,10 +98,11 @@
     "cmds": [
       {
         "cmd": {
-          "path": "/usr/bin/python2",
+          "path": "/usr/bin/env",
           "args": [
+            "python",
             "-c",
-            "import bisect_driver; sys.exit(bisect_driver.bisect_driver(sys.argv[1], sys.argv[2], sys.argv[3:]))",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
             "someBisectStage",
             "someBisectDir",
             "/tmp/stable/clang",
diff --git a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json
index 25764de2..e0709e04 100644
--- a/compiler_wrapper/testdata/cros_hardened_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_hardened_golden/bisect.json
@@ -2,6 +2,61 @@
   {
     "wd": "/tmp/stable",
     "env": [
+      "BISECT_STAGE=someBisectStage"
+    ],
+    "wrapper": {
+      "cmd": {
+        "path": "./x86_64-cros-linux-gnu-clang",
+        "args": [
+          "main.cc"
+        ]
+      }
+    },
+    "cmds": [
+      {
+        "cmd": {
+          "path": "/usr/bin/env",
+          "args": [
+            "python",
+            "-c",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
+            "someBisectStage",
+            "/tmp/sysroot_bisect",
+            "/usr/bin/ccache",
+            "../../usr/bin/clang",
+            "--sysroot=/usr/x86_64-cros-linux-gnu",
+            "-Qunused-arguments",
+            "-grecord-gcc-switches",
+            "-fno-addrsig",
+            "-Wno-tautological-constant-compare",
+            "-Wno-tautological-unsigned-enum-zero-compare",
+            "-Wno-unknown-warning-option",
+            "-Wno-section",
+            "-static-libgcc",
+            "-fuse-ld=lld",
+            "-fstack-protector-strong",
+            "-fPIE",
+            "-pie",
+            "-D_FORTIFY_SOURCE=2",
+            "-fno-omit-frame-pointer",
+            "main.cc",
+            "-B../../bin",
+            "-target",
+            "x86_64-cros-linux-gnu"
+          ],
+          "env_updates": [
+            "CCACHE_BASEDIR=/usr/x86_64-cros-linux-gnu",
+            "CCACHE_DIR=/var/cache/distfiles/ccache",
+            "CCACHE_UMASK=002",
+            "CCACHE_CPP2=yes"
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "wd": "/tmp/stable",
+    "env": [
       "BISECT_STAGE=someBisectStage",
       "BISECT_DIR=someBisectDir"
     ],
@@ -16,10 +71,11 @@
     "cmds": [
       {
         "cmd": {
-          "path": "/usr/bin/python2",
+          "path": "/usr/bin/env",
           "args": [
+            "python",
             "-c",
-            "import bisect_driver; sys.exit(bisect_driver.bisect_driver(sys.argv[1], sys.argv[2], sys.argv[3:]))",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
             "someBisectStage",
             "someBisectDir",
             "/usr/bin/ccache",
@@ -74,10 +130,11 @@
     "cmds": [
       {
         "cmd": {
-          "path": "/usr/bin/python2",
+          "path": "/usr/bin/env",
           "args": [
+            "python",
             "-c",
-            "import bisect_driver; sys.exit(bisect_driver.bisect_driver(sys.argv[1], sys.argv[2], sys.argv[3:]))",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
             "someBisectStage",
             "someBisectDir",
             "/usr/bin/ccache",
diff --git a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json
index 2987a557..f158f3da 100644
--- a/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_hardened_llvmnext_golden/bisect.json
@@ -2,6 +2,65 @@
   {
     "wd": "/tmp/stable",
     "env": [
+      "BISECT_STAGE=someBisectStage"
+    ],
+    "wrapper": {
+      "cmd": {
+        "path": "./x86_64-cros-linux-gnu-clang",
+        "args": [
+          "main.cc"
+        ]
+      }
+    },
+    "cmds": [
+      {
+        "cmd": {
+          "path": "/usr/bin/env",
+          "args": [
+            "python",
+            "-c",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
+            "someBisectStage",
+            "/tmp/sysroot_bisect",
+            "/usr/bin/ccache",
+            "../../usr/bin/clang",
+            "--sysroot=/usr/x86_64-cros-linux-gnu",
+            "-Qunused-arguments",
+            "-grecord-gcc-switches",
+            "-fno-addrsig",
+            "-Wno-tautological-constant-compare",
+            "-Wno-tautological-unsigned-enum-zero-compare",
+            "-Wno-unknown-warning-option",
+            "-Wno-section",
+            "-static-libgcc",
+            "-fuse-ld=lld",
+            "-Wno-reorder-init-list",
+            "-Wno-final-dtor-non-final-class",
+            "-Wno-implicit-int-float-conversion",
+            "-Wno-return-stack-address",
+            "-fstack-protector-strong",
+            "-fPIE",
+            "-pie",
+            "-D_FORTIFY_SOURCE=2",
+            "-fno-omit-frame-pointer",
+            "main.cc",
+            "-B../../bin",
+            "-target",
+            "x86_64-cros-linux-gnu"
+          ],
+          "env_updates": [
+            "CCACHE_BASEDIR=/usr/x86_64-cros-linux-gnu",
+            "CCACHE_DIR=/var/cache/distfiles/ccache",
+            "CCACHE_UMASK=002",
+            "CCACHE_CPP2=yes"
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "wd": "/tmp/stable",
+    "env": [
       "BISECT_STAGE=someBisectStage",
       "BISECT_DIR=someBisectDir"
     ],
@@ -16,10 +75,11 @@
     "cmds": [
       {
         "cmd": {
-          "path": "/usr/bin/python2",
+          "path": "/usr/bin/env",
           "args": [
+            "python",
             "-c",
-            "import bisect_driver; sys.exit(bisect_driver.bisect_driver(sys.argv[1], sys.argv[2], sys.argv[3:]))",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
             "someBisectStage",
             "someBisectDir",
             "/usr/bin/ccache",
@@ -78,10 +138,11 @@
     "cmds": [
       {
         "cmd": {
-          "path": "/usr/bin/python2",
+          "path": "/usr/bin/env",
           "args": [
+            "python",
             "-c",
-            "import bisect_driver; sys.exit(bisect_driver.bisect_driver(sys.argv[1], sys.argv[2], sys.argv[3:]))",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
             "someBisectStage",
             "someBisectDir",
             "/usr/bin/ccache",
diff --git a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json
index fd6e5a36..c8255e56 100644
--- a/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_hardened_noccache_golden/bisect.json
@@ -2,6 +2,54 @@
   {
     "wd": "/tmp/stable",
     "env": [
+      "BISECT_STAGE=someBisectStage"
+    ],
+    "wrapper": {
+      "cmd": {
+        "path": "./x86_64-cros-linux-gnu-clang",
+        "args": [
+          "main.cc"
+        ]
+      }
+    },
+    "cmds": [
+      {
+        "cmd": {
+          "path": "/usr/bin/env",
+          "args": [
+            "python",
+            "-c",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
+            "someBisectStage",
+            "/tmp/sysroot_bisect",
+            "/usr/bin/clang",
+            "--sysroot=/usr/x86_64-cros-linux-gnu",
+            "-Qunused-arguments",
+            "-grecord-gcc-switches",
+            "-fno-addrsig",
+            "-Wno-tautological-constant-compare",
+            "-Wno-tautological-unsigned-enum-zero-compare",
+            "-Wno-unknown-warning-option",
+            "-Wno-section",
+            "-static-libgcc",
+            "-fuse-ld=lld",
+            "-fstack-protector-strong",
+            "-fPIE",
+            "-pie",
+            "-D_FORTIFY_SOURCE=2",
+            "-fno-omit-frame-pointer",
+            "main.cc",
+            "-B../../bin",
+            "-target",
+            "x86_64-cros-linux-gnu"
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "wd": "/tmp/stable",
+    "env": [
       "BISECT_STAGE=someBisectStage",
       "BISECT_DIR=someBisectDir"
     ],
@@ -16,10 +64,11 @@
     "cmds": [
       {
         "cmd": {
-          "path": "/usr/bin/python2",
+          "path": "/usr/bin/env",
           "args": [
+            "python",
             "-c",
-            "import bisect_driver; sys.exit(bisect_driver.bisect_driver(sys.argv[1], sys.argv[2], sys.argv[3:]))",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
             "someBisectStage",
             "someBisectDir",
             "/usr/bin/clang",
@@ -67,10 +116,11 @@
     "cmds": [
       {
         "cmd": {
-          "path": "/usr/bin/python2",
+          "path": "/usr/bin/env",
           "args": [
+            "python",
             "-c",
-            "import bisect_driver; sys.exit(bisect_driver.bisect_driver(sys.argv[1], sys.argv[2], sys.argv[3:]))",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
             "someBisectStage",
             "someBisectDir",
             "/usr/bin/clang",
diff --git a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json
index e8a5653c..d81622fd 100644
--- a/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json
+++ b/compiler_wrapper/testdata/cros_nonhardened_golden/bisect.json
@@ -2,6 +2,53 @@
   {
     "wd": "/tmp/stable",
     "env": [
+      "BISECT_STAGE=someBisectStage"
+    ],
+    "wrapper": {
+      "cmd": {
+        "path": "./x86_64-cros-linux-gnu-clang",
+        "args": [
+          "main.cc"
+        ]
+      }
+    },
+    "cmds": [
+      {
+        "cmd": {
+          "path": "/usr/bin/env",
+          "args": [
+            "python",
+            "-c",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
+            "someBisectStage",
+            "/tmp/sysroot_bisect",
+            "/usr/bin/ccache",
+            "../../usr/bin/clang",
+            "--sysroot=/usr/x86_64-cros-linux-gnu",
+            "-Qunused-arguments",
+            "-Wno-tautological-constant-compare",
+            "-Wno-tautological-unsigned-enum-zero-compare",
+            "-Wno-unknown-warning-option",
+            "-Wno-section",
+            "-static-libgcc",
+            "main.cc",
+            "-B../../bin",
+            "-target",
+            "x86_64-cros-linux-gnu"
+          ],
+          "env_updates": [
+            "CCACHE_BASEDIR=/usr/x86_64-cros-linux-gnu",
+            "CCACHE_DIR=/var/cache/distfiles/ccache",
+            "CCACHE_UMASK=002",
+            "CCACHE_CPP2=yes"
+          ]
+        }
+      }
+    ]
+  },
+  {
+    "wd": "/tmp/stable",
+    "env": [
       "BISECT_STAGE=someBisectStage",
       "BISECT_DIR=someBisectDir"
     ],
@@ -16,10 +63,11 @@
     "cmds": [
       {
         "cmd": {
-          "path": "/usr/bin/python2",
+          "path": "/usr/bin/env",
           "args": [
+            "python",
             "-c",
-            "import bisect_driver; sys.exit(bisect_driver.bisect_driver(sys.argv[1], sys.argv[2], sys.argv[3:]))",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
             "someBisectStage",
             "someBisectDir",
             "/usr/bin/ccache",
@@ -66,10 +114,11 @@
     "cmds": [
       {
         "cmd": {
-          "path": "/usr/bin/python2",
+          "path": "/usr/bin/env",
           "args": [
+            "python",
             "-c",
-            "import bisect_driver; sys.exit(bisect_driver.bisect_driver(sys.argv[1], sys.argv[2], sys.argv[3:]))",
+            "\nimport bisect_driver\nimport shlex\nimport sys\n\ndef ExpandArgs(args, target):\n\tfor arg in args:\n\t\tif arg[0] == '@':\n\t\t\twith open(arg[1:], 'rb') as f:\n\t\t\t\tExpandArgs(shlex.split(f.read()), target)\n\t\telse:\n\t\t\ttarget.append(arg)\n\treturn target\n\nstage = sys.argv[1]\ndir = sys.argv[2]\nexecargs = ExpandArgs(sys.argv[3:], [])\n\nsys.exit(bisect_driver.bisect_driver(stage, dir, execargs))\n",
             "someBisectStage",
             "someBisectDir",
             "/usr/bin/ccache",