compiler_wrapper: dump tricium clang-tidy crash information

This CL has us dump extra info when we observe a crash while using
`WITH_TIDY=tricium`. In particular:
- We'll now try to generate a -E'ed source file, and stash that in a
  subdirectory of our general crash_diagnostics directory.
- Our JSON object that represents clang-tidy output has metadata about
  the aforementioned crash file, if it was generated.

BUG=chromium:1113442
TEST=`go test`; ran locally with a crashy clang-tidy

Change-Id: I577e5ec8e990af3b7cf1d8a9fea347582d5b227b
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2343974
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Tested-by: George Burgess <gbiv@chromium.org>

5 files changed, 134 insertions, 29 deletions

diff --git a/compiler_wrapper/clang_tidy_flag.go b/compiler_wrapper/clang_tidy_flag.go
index 2d97ddd3..01387fd6 100644
--- a/compiler_wrapper/clang_tidy_flag.go
+++ b/compiler_wrapper/clang_tidy_flag.go
@@ -9,12 +9,15 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 )
 
 type useTidyMode int
 
+const clangTidyCrashSubstring = "PLEASE submit a bug report"
+
 const (
 	tidyModeNone useTidyMode = iota
 	tidyModeAll
@@ -95,7 +98,7 @@ func calcClangTidyInvocation(env env, clangCmd *command, cSrcFile string, tidyFl
 	}, nil
 }
 
-func runClangTidyForTricium(env env, clangCmd *command, cSrcFile, fixesDir string, extraTidyFlags []string) error {
+func runClangTidyForTricium(env env, clangCmd *command, cSrcFile, fixesDir string, extraTidyFlags []string, crashArtifactsDir string) error {
 	if err := os.MkdirAll(fixesDir, 0777); err != nil {
 		return fmt.Errorf("creating fixes directory at %q: %v", fixesDir, err)
 	}
@@ -128,13 +131,58 @@ func runClangTidyForTricium(env env, clangCmd *command, cSrcFile, fixesDir strin
 		return err
 	}
 
+	type crashOutput struct {
+		CrashReproducerPath string `json:"crash_reproducer_path"`
+		Stdstreams          string `json:"stdstreams"`
+	}
+
 	type metadata struct {
-		Args       []string `json:"args"`
-		Executable string   `json:"executable"`
-		ExitCode   int      `json:"exit_code"`
-		LintTarget string   `json:"lint_target"`
-		Stdstreams string   `json:"stdstreams"`
-		Wd         string   `json:"wd"`
+		Args        []string     `json:"args"`
+		CrashOutput *crashOutput `json:"crash_output"`
+		Executable  string       `json:"executable"`
+		ExitCode    int          `json:"exit_code"`
+		LintTarget  string       `json:"lint_target"`
+		Stdstreams  string       `json:"stdstreams"`
+		Wd          string       `json:"wd"`
+	}
+
+	meta := &metadata{
+		Args:        clangTidyCmd.Args,
+		CrashOutput: nil,
+		Executable:  clangTidyCmd.Path,
+		ExitCode:    exitCode,
+		LintTarget:  cSrcFile,
+		Stdstreams:  stdstreams.String(),
+		Wd:          env.getwd(),
+	}
+
+	// Sometimes, clang-tidy crashes. Unfortunately, these don't get funnelled through the
+	// standard clang crash machinery. :(. Try to work with our own.
+	if crashArtifactsDir != "" && strings.Contains(meta.Stdstreams, clangTidyCrashSubstring) {
+		tidyCrashArtifacts := path.Join(crashArtifactsDir, "clang-tidy")
+		if err := os.MkdirAll(tidyCrashArtifacts, 0777); err != nil {
+			return fmt.Errorf("creating crash artifacts directory at %q: %v", tidyCrashArtifacts, err)
+		}
+
+		f, err := ioutil.TempFile(tidyCrashArtifacts, "crash-")
+		if err != nil {
+			return fmt.Errorf("making tempfile for crash output: %v", err)
+		}
+		f.Close()
+
+		reproCmd := &command{}
+		*reproCmd = *clangCmd
+		reproCmd.Args = append(reproCmd.Args, "-E", "-o", f.Name())
+
+		reproOut := &strings.Builder{}
+		_, err = wrapSubprocessErrorWithSourceLoc(reproCmd, env.run(reproCmd, nil, reproOut, reproOut))
+		if err != nil {
+			return fmt.Errorf("attempting to produce a clang-tidy crash reproducer: %v", err)
+		}
+		meta.CrashOutput = &crashOutput{
+			CrashReproducerPath: f.Name(),
+			Stdstreams:          reproOut.String(),
+		}
 	}
 
 	f, err = os.Create(fixesMetadataPath)
@@ -142,14 +190,6 @@ func runClangTidyForTricium(env env, clangCmd *command, cSrcFile, fixesDir strin
 		return fmt.Errorf("creating fixes metadata: %v", err)
 	}
 
-	meta := &metadata{
-		Args:       clangTidyCmd.Args,
-		Executable: clangTidyCmd.Path,
-		ExitCode:   exitCode,
-		LintTarget: cSrcFile,
-		Stdstreams: stdstreams.String(),
-		Wd:         env.getwd(),
-	}
 	if err := json.NewEncoder(f).Encode(meta); err != nil {
 		return fmt.Errorf("writing fixes metadata: %v", err)
 	}
diff --git a/compiler_wrapper/clang_tidy_flag_test.go b/compiler_wrapper/clang_tidy_flag_test.go
index 54159cdc..4293bb21 100644
--- a/compiler_wrapper/clang_tidy_flag_test.go
+++ b/compiler_wrapper/clang_tidy_flag_test.go
@@ -386,6 +386,63 @@ func TestClangTidyFlagsAreFilteredFromGccInvocations(t *testing.T) {
 	})
 }
 
+func TestTriciumReportsClangTidyCrashesGracefully(t *testing.T) {
+	withClangTidyTestContext(t, func(ctx *testContext) {
+		ctx.env = []string{"WITH_TIDY=tricium"}
+		ctx.cmdMock = func(cmd *command, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
+			switch ctx.cmdCount {
+			case 1:
+				if err := verifyPath(cmd, "usr/bin/clang"); err != nil {
+					t.Error(err)
+				}
+				return nil
+			case 2:
+				if err := verifyPath(cmd, "usr/bin/clang-tidy"); err != nil {
+					return err
+				}
+
+				if _, err := io.WriteString(stdout, clangTidyCrashSubstring); err != nil {
+					return err
+				}
+				return nil
+			case 3:
+				if err := verifyPath(cmd, "usr/bin/clang"); err != nil {
+					t.Error(err)
+				}
+
+				args := cmd.Args
+				if len(args) < 3 {
+					t.Errorf("insufficient number of args provided; got %d; want at least 3", len(args))
+					return nil
+				}
+
+				lastArgs := args[len(args)-3:]
+				eArg, oArg, outFileArg := lastArgs[0], lastArgs[1], lastArgs[2]
+				if eArg != "-E" {
+					t.Errorf("got eArg=%q; wanted -E", eArg)
+				}
+
+				if oArg != "-o" {
+					t.Errorf("got oArg=%q; wanted -o", oArg)
+				}
+
+				wantPrefix := path.Join(ctx.cfg.crashArtifactsDir, "clang-tidy")
+				if !strings.HasPrefix(outFileArg, wantPrefix) {
+					t.Errorf("got out file %q; wanted one starting with %q", outFileArg, wantPrefix)
+				}
+
+				return nil
+			default:
+				return nil
+			}
+		}
+		ctx.must(callCompiler(ctx, ctx.cfg, ctx.newCommand(clangX86_64, mainCc)))
+		if ctx.cmdCount != 4 {
+			t.Errorf("expected 3 calls. Got: %d", ctx.cmdCount)
+		}
+	})
+}
+
 func withClangTidyTestContext(t *testing.T, work func(ctx *testContext)) {
 	withTestContext(t, func(ctx *testContext) {
 		ctx.env = []string{"WITH_TIDY=1"}
diff --git a/compiler_wrapper/compiler_wrapper.go b/compiler_wrapper/compiler_wrapper.go
index a5d35a2c..da712a33 100644
--- a/compiler_wrapper/compiler_wrapper.go
+++ b/compiler_wrapper/compiler_wrapper.go
@@ -106,7 +106,7 @@ func callCompilerInternal(env env, cfg *config, inputCmd *command) (exitCode int
 					if cfg.triciumNitsDir == "" {
 						return 0, newErrorwithSourceLocf("tricium linting was requested, but no nits directory is configured")
 					}
-					err = runClangTidyForTricium(env, clangCmdWithoutGomaAndCCache, cSrcFile, cfg.triciumNitsDir, tidyFlags)
+					err = runClangTidyForTricium(env, clangCmdWithoutGomaAndCCache, cSrcFile, cfg.triciumNitsDir, tidyFlags, cfg.crashArtifactsDir)
 				case tidyModeAll:
 					err = runClangTidy(env, clangCmdWithoutGomaAndCCache, cSrcFile, tidyFlags)
 				default:
diff --git a/compiler_wrapper/config.go b/compiler_wrapper/config.go
index 546a0e11..e87c7baa 100644
--- a/compiler_wrapper/config.go
+++ b/compiler_wrapper/config.go
@@ -31,6 +31,8 @@ type config struct {
 	newWarningsDir string
 	// Directory to store nits in when using `WITH_TIDY=tricium`.
 	triciumNitsDir string
+	// Directory to store crash artifacts in.
+	crashArtifactsDir string
 	// Version. Only used for printing via -print-cmd.
 	version string
 }
@@ -143,6 +145,8 @@ var crosHardenedConfig = &config{
 	},
 	newWarningsDir: "/tmp/fatal_clang_warnings",
 	triciumNitsDir: "/tmp/linting_output/clang-tidy",
+	// FIXME(gbiv): apply -fcrash-diagnostics-dir based on this.
+	crashArtifactsDir: "/tmp/clang_crash_diagnostics",
 }
 
 // Flags to be added to non-hardened toolchain.
@@ -173,8 +177,9 @@ var crosNonHardenedConfig = &config{
 	clangPostFlags: []string{
 		"-Wno-implicit-int-float-conversion",
 	},
-	newWarningsDir: "/tmp/fatal_clang_warnings",
-	triciumNitsDir: "/tmp/linting_output/clang-tidy",
+	newWarningsDir:    "/tmp/fatal_clang_warnings",
+	triciumNitsDir:    "/tmp/linting_output/clang-tidy",
+	crashArtifactsDir: "/tmp/clang_crash_diagnostics",
 }
 
 // Flags to be added to host toolchain.
@@ -211,18 +216,20 @@ var crosHostConfig = &config{
 	clangPostFlags: []string{
 		"-Wno-implicit-int-float-conversion",
 	},
-	newWarningsDir: "/tmp/fatal_clang_warnings",
-	triciumNitsDir: "/tmp/linting_output/clang-tidy",
+	newWarningsDir:    "/tmp/fatal_clang_warnings",
+	triciumNitsDir:    "/tmp/linting_output/clang-tidy",
+	crashArtifactsDir: "/tmp/clang_crash_diagnostics",
 }
 
 var androidConfig = &config{
-	isHostWrapper:    false,
-	isAndroidWrapper: true,
-	rootRelPath:      "./",
-	commonFlags:      []string{},
-	gccFlags:         []string{},
-	clangFlags:       []string{},
-	clangPostFlags:   []string{},
-	newWarningsDir:   "",
-	triciumNitsDir:   "",
+	isHostWrapper:     false,
+	isAndroidWrapper:  true,
+	rootRelPath:       "./",
+	commonFlags:       []string{},
+	gccFlags:          []string{},
+	clangFlags:        []string{},
+	clangPostFlags:    []string{},
+	newWarningsDir:    "",
+	triciumNitsDir:    "",
+	crashArtifactsDir: "",
 }
diff --git a/compiler_wrapper/testutil_test.go b/compiler_wrapper/testutil_test.go
index d23a8434..21b7169d 100644
--- a/compiler_wrapper/testutil_test.go
+++ b/compiler_wrapper/testutil_test.go
@@ -142,6 +142,7 @@ func (ctx *testContext) updateConfig(cfg *config) {
 	*ctx.cfg = *cfg
 	ctx.cfg.newWarningsDir = filepath.Join(ctx.tempDir, "fatal_clang_warnings")
 	ctx.cfg.triciumNitsDir = filepath.Join(ctx.tempDir, "tricium_nits")
+	ctx.cfg.crashArtifactsDir = filepath.Join(ctx.tempDir, "clang_crash_diagnostics")
 }
 
 func (ctx *testContext) newCommand(path string, args ...string) *command {