stdliblist: return generated files (#3552)

The stdlib json currently is not populated with generated files. When
CGO is being used, this means that the file list returned does not
actually compile. This change adds `-compiled=true` to go list's output
which results in the full set required to compile stdlib.

Fixes #3523

Change-Id: Ic06243e7a5de68946d87ff9734156c522bfd8662

Co-authored-by: Thomas Rampelberg <thomas.rampelberg@airbnb.com>

2 files changed, 97 insertions, 33 deletions

diff --git a/go/private/actions/stdlib.bzl b/go/private/actions/stdlib.bzl
index a8097e16..35f552ac 100644
--- a/go/private/actions/stdlib.bzl
+++ b/go/private/actions/stdlib.bzl
@@ -68,16 +68,46 @@ def _build_stdlib_list_json(go):
     args = go.builder_args(go, "stdliblist")
     args.add("-sdk", go.sdk.root_file.dirname)
     args.add("-out", out)
+
+    inputs = go.sdk_files
+    if not go.mode.pure:
+        inputs += go.crosstool
+
     go.actions.run(
-        inputs = go.sdk_files,
+        inputs = inputs,
         outputs = [out],
         mnemonic = "GoStdlibList",
         executable = go.toolchain._builder,
         arguments = [args],
-        env = go.env,
+        env = _build_env(go),
     )
     return out
 
+def _build_env(go):
+    env = go.env
+
+    if go.mode.pure:
+        env.update({"CGO_ENABLED": "0"})
+        return env
+
+    # NOTE(#2545): avoid unnecessary dynamic link
+    # go std library doesn't use C++, so should not have -lstdc++
+    # Also drop coverage flags as nothing in the stdlib is compiled with
+    # coverage - we disable it for all CGo code anyway.
+    ldflags = [
+        option
+        for option in extldflags_from_cc_toolchain(go)
+        if option not in ("-lstdc++", "-lc++") and option not in COVERAGE_OPTIONS_DENYLIST
+    ]
+    env.update({
+        "CGO_ENABLED": "1",
+        "CC": go.cgo_tools.c_compiler_path,
+        "CGO_CFLAGS": " ".join(go.cgo_tools.c_compile_options),
+        "CGO_LDFLAGS": " ".join(ldflags),
+    })
+
+    return env
+
 def _sdk_stdlib(go):
     return GoStdLib(
         _list_json = _build_stdlib_list_json(go),
@@ -96,26 +126,9 @@ def _build_stdlib(go):
     if not go.mode.pure:
         args.add("-package", "runtime/cgo")
     args.add_all(link_mode_args(go.mode))
-    env = go.env
-    if go.mode.pure:
-        env.update({"CGO_ENABLED": "0"})
-    else:
-        # NOTE(#2545): avoid unnecessary dynamic link
-        # go std library doesn't use C++, so should not have -lstdc++
-        # Also drop coverage flags as nothing in the stdlib is compiled with
-        # coverage - we disable it for all CGo code anyway.
-        ldflags = [
-            option
-            for option in extldflags_from_cc_toolchain(go)
-            if option not in ("-lstdc++", "-lc++") and option not in COVERAGE_OPTIONS_DENYLIST
-        ]
-        env.update({
-            "CGO_ENABLED": "1",
-            "CC": go.cgo_tools.c_compiler_path,
-            "CGO_CFLAGS": " ".join(go.cgo_tools.c_compile_options),
-            "CGO_LDFLAGS": " ".join(ldflags),
-        })
+
     args.add("-gcflags", quote_opts(go.mode.gc_goopts))
+
     inputs = (go.sdk.srcs +
               go.sdk.headers +
               go.sdk.tools +
@@ -128,7 +141,7 @@ def _build_stdlib(go):
         mnemonic = "GoStdlib",
         executable = go.toolchain._builder,
         arguments = [args],
-        env = env,
+        env = _build_env(go),
     )
     return GoStdLib(
         _list_json = _build_stdlib_list_json(go),
diff --git a/go/tools/builders/stdliblist.go b/go/tools/builders/stdliblist.go
index 9caf37c9..f6a61442 100644
--- a/go/tools/builders/stdliblist.go
+++ b/go/tools/builders/stdliblist.go
@@ -116,20 +116,43 @@ func outputBasePath(cloneBase, p string) string {
 	return filepath.Join("__BAZEL_OUTPUT_BASE__", dir)
 }
 
-//  absoluteSourcesPaths replace cloneBase of the absolution
-//  paths with the label for all source files in a package
+// absoluteSourcesPaths replace cloneBase of the absolution
+// paths with the label for all source files in a package
 func absoluteSourcesPaths(cloneBase, pkgDir string, srcs []string) []string {
 	ret := make([]string, 0, len(srcs))
 	pkgDir = outputBasePath(cloneBase, pkgDir)
 	for _, src := range srcs {
-		ret = append(ret, filepath.Join(pkgDir, src))
+		absPath := src
+
+		// Generated files will already have an absolute path. These come from
+		// the compiler's cache.
+		if !filepath.IsAbs(src) {
+			absPath = filepath.Join(pkgDir, src)
+		}
+
+		ret = append(ret, absPath)
+	}
+	return ret
+}
+
+// filterGoFiles keeps only files either ending in .go or those without an
+// extension (which are from the cache). This is a work around for
+// https://golang.org/issue/28749: cmd/go puts assembly, C, and C++ files in
+// CompiledGoFiles.
+func filterGoFiles(srcs []string) []string {
+	ret := make([]string, 0, len(srcs))
+	for _, f := range srcs {
+		if ext := filepath.Ext(f); ext == ".go" || ext == "" {
+			ret = append(ret, f)
+		}
 	}
+
 	return ret
 }
 
 func flatPackageForStd(cloneBase string, pkg *goListPackage) *flatPackage {
-	// Don't use generated files from the stdlib
 	goFiles := absoluteSourcesPaths(cloneBase, pkg.Dir, pkg.GoFiles)
+	compiledGoFiles := absoluteSourcesPaths(cloneBase, pkg.Dir, pkg.CompiledGoFiles)
 
 	newPkg := &flatPackage{
 		ID:              stdlibPackageID(pkg.ImportPath),
@@ -139,13 +162,29 @@ func flatPackageForStd(cloneBase string, pkg *goListPackage) *flatPackage {
 		Imports:         map[string]string{},
 		Standard:        pkg.Standard,
 		GoFiles:         goFiles,
-		CompiledGoFiles: goFiles,
+		CompiledGoFiles: filterGoFiles(compiledGoFiles),
+	}
+
+	// imports
+	//
+	// Imports contains the IDs of all imported packages.
+	// ImportsMap records (path, ID) only where they differ.
+	ids := make(map[string]struct{})
+	for _, id := range pkg.Imports {
+		ids[id] = struct{}{}
+	}
+
+	for path, id := range pkg.ImportMap {
+		newPkg.Imports[path] = stdlibPackageID(id)
+		delete(ids, id)
 	}
-	for _, imp := range pkg.Imports {
-		newPkg.Imports[imp] = stdlibPackageID(imp)
+
+	for id := range ids {
+		if id != "C" {
+			newPkg.Imports[id] = stdlibPackageID(id)
+		}
 	}
-	// We don't support CGo for now
-	delete(newPkg.Imports, "C")
+
 	return newPkg
 }
 
@@ -200,12 +239,18 @@ func stdliblist(args []string) error {
 	}
 	os.Setenv("PATH", strings.Join(absPaths, string(os.PathListSeparator)))
 	os.Setenv("GOROOT", newGoRoot)
+
+	cgoEnabled := os.Getenv("CGO_ENABLED") == "1"
 	// Make sure we have an absolute path to the C compiler.
 	// TODO(#1357): also take absolute paths of includes and other paths in flags.
-	os.Setenv("CC", quotePathIfNeeded(abs(os.Getenv("CC"))))
+	ccEnv, ok := os.LookupEnv("CC")
+	if cgoEnabled && !ok {
+		return fmt.Errorf("CC must be set")
+	}
+	os.Setenv("CC", quotePathIfNeeded(abs(ccEnv)))
 
+	// We want to keep the cache around so that the processed files can be used by other tools.
 	cachePath := abs(*out + ".gocache")
-	defer os.RemoveAll(cachePath)
 	os.Setenv("GOCACHE", cachePath)
 	os.Setenv("GOMODCACHE", cachePath)
 	os.Setenv("GOPATH", cachePath)
@@ -214,6 +259,11 @@ func stdliblist(args []string) error {
 	if len(build.Default.BuildTags) > 0 {
 		listArgs = append(listArgs, "-tags", strings.Join(build.Default.BuildTags, ","))
 	}
+
+	if cgoEnabled {
+		listArgs = append(listArgs, "-compiled=true")
+	}
+
 	listArgs = append(listArgs, "-json", "builtin", "std", "runtime/cgo")
 
 	jsonFile, err := os.Create(*out)
@@ -226,6 +276,7 @@ func stdliblist(args []string) error {
 	if err := goenv.runCommandToFile(jsonData, os.Stderr, listArgs); err != nil {
 		return err
 	}
+
 	encoder := json.NewEncoder(jsonFile)
 	decoder := json.NewDecoder(jsonData)
 	for decoder.More() {