diff options
Diffstat (limited to 'go/tools/gopackagesdriver')
| -rw-r--r-- | go/tools/gopackagesdriver/BUILD.bazel | 39 | ||||
| -rw-r--r-- | go/tools/gopackagesdriver/aspect.bzl | 169 | ||||
| -rw-r--r-- | go/tools/gopackagesdriver/bazel.go | 164 | ||||
| -rw-r--r-- | go/tools/gopackagesdriver/bazel_json_builder.go | 250 | ||||
| -rw-r--r-- | go/tools/gopackagesdriver/build_context.go | 34 | ||||
| -rw-r--r-- | go/tools/gopackagesdriver/driver_request.go | 91 | ||||
| -rw-r--r-- | go/tools/gopackagesdriver/flatpackage.go | 159 | ||||
| -rw-r--r-- | go/tools/gopackagesdriver/json_packages_driver.go | 59 | ||||
| -rw-r--r-- | go/tools/gopackagesdriver/main.go | 126 | ||||
| -rw-r--r-- | go/tools/gopackagesdriver/packageregistry.go | 111 | ||||
| -rw-r--r-- | go/tools/gopackagesdriver/utils.go | 77 |
11 files changed, 1279 insertions, 0 deletions
diff --git a/go/tools/gopackagesdriver/BUILD.bazel b/go/tools/gopackagesdriver/BUILD.bazel new file mode 100644 index 00000000..542b75ad --- /dev/null +++ b/go/tools/gopackagesdriver/BUILD.bazel @@ -0,0 +1,39 @@ +load("//go:def.bzl", "go_binary", "go_library") +load(":aspect.bzl", "bazel_supports_canonical_label_literals") + +go_library( + name = "gopackagesdriver_lib", + srcs = [ + "bazel.go", + "bazel_json_builder.go", + "build_context.go", + "driver_request.go", + "flatpackage.go", + "json_packages_driver.go", + "main.go", + "packageregistry.go", + "utils.go", + ], + importpath = "github.com/bazelbuild/rules_go/go/tools/gopackagesdriver", + visibility = ["//visibility:private"], +) + +go_binary( + name = "gopackagesdriver", + embed = [":gopackagesdriver_lib"], + x_defs = { + # Determine the name of the rules_go repository as we need to specify it when invoking the + # aspect. + # If canonical label literals are supported, we can use a canonical label literal (starting + # with @@) to pass the repository_name() through repo mapping unchanged. + # If canonical label literals are not supported, then bzlmod is certainly not enabled and + # we can assume that the repository name is not affected by repo mappings. + # If run in the rules_go repo itself, repository_name() returns "@", which is equivalent to + # "@io_bazel_rules_go" since Bazel 6: + # https://github.com/bazelbuild/bazel/commit/7694cf75e6366b92e3905c2ad60234cda57627ee + # TODO: Once we drop support for Bazel 5, we can remove the feature detection logic and + # use "@" + repository_name(). + "rulesGoRepositoryName": "@" + repository_name() if bazel_supports_canonical_label_literals() else repository_name(), + }, + visibility = ["//visibility:public"], +) diff --git a/go/tools/gopackagesdriver/aspect.bzl b/go/tools/gopackagesdriver/aspect.bzl new file mode 100644 index 00000000..36703c75 --- /dev/null +++ b/go/tools/gopackagesdriver/aspect.bzl @@ -0,0 +1,169 @@ +# Copyright 2021 The Bazel Go Rules Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load( + "//go/private:providers.bzl", + "GoArchive", + "GoStdLib", +) +load( + "@bazel_skylib//lib:paths.bzl", + "paths", +) + +GoPkgInfo = provider() + +DEPS_ATTRS = [ + "deps", + "embed", +] + +PROTO_COMPILER_ATTRS = [ + "compiler", + "compilers", + "library", +] + +def bazel_supports_canonical_label_literals(): + return str(Label("//:bogus")).startswith("@@") + +def is_file_external(f): + return f.owner.workspace_root != "" + +def file_path(f): + prefix = "__BAZEL_WORKSPACE__" + if not f.is_source: + prefix = "__BAZEL_EXECROOT__" + elif is_file_external(f): + prefix = "__BAZEL_OUTPUT_BASE__" + return paths.join(prefix, f.path) + +def _go_archive_to_pkg(archive): + return struct( + ID = str(archive.data.label), + PkgPath = archive.data.importpath, + ExportFile = file_path(archive.data.export_file), + GoFiles = [ + file_path(src) + for src in archive.data.orig_srcs + if src.path.endswith(".go") + ], + CompiledGoFiles = [ + file_path(src) + for src in archive.data.srcs + if src.path.endswith(".go") + ], + OtherFiles = [ + file_path(src) + for src in archive.data.orig_srcs + if not src.path.endswith(".go") + ], + Imports = { + pkg.data.importpath: str(pkg.data.label) + for pkg in archive.direct + }, + ) + +def make_pkg_json(ctx, name, pkg_info): + pkg_json_file = ctx.actions.declare_file(name + ".pkg.json") + ctx.actions.write(pkg_json_file, content = pkg_info.to_json()) + return pkg_json_file + +def _go_pkg_info_aspect_impl(target, ctx): + # Fetch the stdlib JSON file from the inner most target + stdlib_json_file = None + + transitive_json_files = [] + transitive_export_files = [] + transitive_compiled_go_files = [] + + for attr in DEPS_ATTRS + PROTO_COMPILER_ATTRS: + deps = getattr(ctx.rule.attr, attr, []) or [] + + # Some attrs are not iterable, ensure that deps is always iterable. + if type(deps) != type([]): + deps = [deps] + + for dep in deps: + if GoPkgInfo in dep: + pkg_info = dep[GoPkgInfo] + transitive_json_files.append(pkg_info.pkg_json_files) + transitive_compiled_go_files.append(pkg_info.compiled_go_files) + transitive_export_files.append(pkg_info.export_files) + + # Fetch the stdlib json from the first dependency + if not stdlib_json_file: + stdlib_json_file = pkg_info.stdlib_json_file + + pkg_json_files = [] + compiled_go_files = [] + export_files = [] + + if GoArchive in target: + archive = target[GoArchive] + compiled_go_files.extend(archive.source.srcs) + export_files.append(archive.data.export_file) + pkg = _go_archive_to_pkg(archive) + pkg_json_files.append(make_pkg_json(ctx, archive.data.name, pkg)) + + if ctx.rule.kind == "go_test": + for dep_archive in archive.direct: + # find the archive containing the test sources + if archive.data.label == dep_archive.data.label: + pkg = _go_archive_to_pkg(dep_archive) + pkg_json_files.append(make_pkg_json(ctx, dep_archive.data.name, pkg)) + compiled_go_files.extend(dep_archive.source.srcs) + export_files.append(dep_archive.data.export_file) + break + + # If there was no stdlib json in any dependencies, fetch it from the + # current go_ node. + if not stdlib_json_file: + stdlib_json_file = ctx.attr._go_stdlib[GoStdLib]._list_json + + pkg_info = GoPkgInfo( + stdlib_json_file = stdlib_json_file, + pkg_json_files = depset( + direct = pkg_json_files, + transitive = transitive_json_files, + ), + compiled_go_files = depset( + direct = compiled_go_files, + transitive = transitive_compiled_go_files, + ), + export_files = depset( + direct = export_files, + transitive = transitive_export_files, + ), + ) + + return [ + pkg_info, + OutputGroupInfo( + go_pkg_driver_json_file = pkg_info.pkg_json_files, + go_pkg_driver_srcs = pkg_info.compiled_go_files, + go_pkg_driver_export_file = pkg_info.export_files, + go_pkg_driver_stdlib_json_file = depset([pkg_info.stdlib_json_file] if pkg_info.stdlib_json_file else []), + ), + ] + +go_pkg_info_aspect = aspect( + implementation = _go_pkg_info_aspect_impl, + attr_aspects = DEPS_ATTRS + PROTO_COMPILER_ATTRS, + attrs = { + "_go_stdlib": attr.label( + default = "//:stdlib", + ), + }, +) diff --git a/go/tools/gopackagesdriver/bazel.go b/go/tools/gopackagesdriver/bazel.go new file mode 100644 index 00000000..08da745d --- /dev/null +++ b/go/tools/gopackagesdriver/bazel.go @@ -0,0 +1,164 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const ( + toolTag = "gopackagesdriver" +) + +type Bazel struct { + bazelBin string + workspaceRoot string + bazelStartupFlags []string + info map[string]string +} + +// Minimal BEP structs to access the build outputs +type BEPNamedSet struct { + NamedSetOfFiles *struct { + Files []struct { + Name string `json:"name"` + URI string `json:"uri"` + } `json:"files"` + } `json:"namedSetOfFiles"` +} + +func NewBazel(ctx context.Context, bazelBin, workspaceRoot string, bazelStartupFlags []string) (*Bazel, error) { + b := &Bazel{ + bazelBin: bazelBin, + workspaceRoot: workspaceRoot, + bazelStartupFlags: bazelStartupFlags, + } + if err := b.fillInfo(ctx); err != nil { + return nil, fmt.Errorf("unable to query bazel info: %w", err) + } + return b, nil +} + +func (b *Bazel) fillInfo(ctx context.Context) error { + b.info = map[string]string{} + output, err := b.run(ctx, "info") + if err != nil { + return err + } + scanner := bufio.NewScanner(bytes.NewBufferString(output)) + for scanner.Scan() { + parts := strings.SplitN(strings.TrimSpace(scanner.Text()), ":", 2) + b.info[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1]) + } + return nil +} + +func (b *Bazel) run(ctx context.Context, command string, args ...string) (string, error) { + defaultArgs := []string{ + command, + "--tool_tag=" + toolTag, + "--ui_actions_shown=0", + } + cmd := exec.CommandContext(ctx, b.bazelBin, concatStringsArrays(b.bazelStartupFlags, defaultArgs, args)...) + fmt.Fprintln(os.Stderr, "Running:", cmd.Args) + cmd.Dir = b.WorkspaceRoot() + cmd.Stderr = os.Stderr + output, err := cmd.Output() + return string(output), err +} + +func (b *Bazel) Build(ctx context.Context, args ...string) ([]string, error) { + jsonFile, err := ioutil.TempFile("", "gopackagesdriver_bep_") + if err != nil { + return nil, fmt.Errorf("unable to create BEP JSON file: %w", err) + } + defer func() { + jsonFile.Close() + os.Remove(jsonFile.Name()) + }() + + args = append([]string{ + "--show_result=0", + "--build_event_json_file=" + jsonFile.Name(), + "--build_event_json_file_path_conversion=no", + }, args...) + if _, err := b.run(ctx, "build", args...); err != nil { + // Ignore a regular build failure to get partial data. + // See https://docs.bazel.build/versions/main/guide.html#what-exit-code-will-i-get on + // exit codes. + var exerr *exec.ExitError + if !errors.As(err, &exerr) || exerr.ExitCode() != 1 { + return nil, fmt.Errorf("bazel build failed: %w", err) + } + } + + files := make([]string, 0) + decoder := json.NewDecoder(jsonFile) + for decoder.More() { + var namedSet BEPNamedSet + if err := decoder.Decode(&namedSet); err != nil { + return nil, fmt.Errorf("unable to decode %s: %w", jsonFile.Name(), err) + } + + if namedSet.NamedSetOfFiles != nil { + for _, f := range namedSet.NamedSetOfFiles.Files { + fileUrl, err := url.Parse(f.URI) + if err != nil { + return nil, fmt.Errorf("unable to parse file URI: %w", err) + } + files = append(files, filepath.FromSlash(fileUrl.Path)) + } + } + } + + return files, nil +} + +func (b *Bazel) Query(ctx context.Context, args ...string) ([]string, error) { + output, err := b.run(ctx, "query", args...) + if err != nil { + return nil, fmt.Errorf("bazel query failed: %w", err) + } + + trimmedOutput := strings.TrimSpace(output) + if len(trimmedOutput) == 0 { + return nil, nil + } + + return strings.Split(trimmedOutput, "\n"), nil +} + +func (b *Bazel) WorkspaceRoot() string { + return b.workspaceRoot +} + +func (b *Bazel) ExecutionRoot() string { + return b.info["execution_root"] +} + +func (b *Bazel) OutputBase() string { + return b.info["output_base"] +} diff --git a/go/tools/gopackagesdriver/bazel_json_builder.go b/go/tools/gopackagesdriver/bazel_json_builder.go new file mode 100644 index 00000000..163be082 --- /dev/null +++ b/go/tools/gopackagesdriver/bazel_json_builder.go @@ -0,0 +1,250 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "context" + "errors" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "regexp" + "strings" +) + +type BazelJSONBuilder struct { + bazel *Bazel + includeTests bool +} + +var RulesGoStdlibLabel = rulesGoRepositoryName + "//:stdlib" + +var _defaultKinds = []string{"go_library", "go_test", "go_binary"} + +var externalRe = regexp.MustCompile(".*\\/external\\/([^\\/]+)(\\/(.*))?\\/([^\\/]+.go)") + +func (b *BazelJSONBuilder) fileQuery(filename string) string { + label := filename + + if filepath.IsAbs(filename) { + label, _ = filepath.Rel(b.bazel.WorkspaceRoot(), filename) + } else if strings.HasPrefix(filename, "./") { + label = strings.TrimPrefix(filename, "./") + } + + if matches := externalRe.FindStringSubmatch(filename); len(matches) == 5 { + // if filepath is for a third party lib, we need to know, what external + // library this file is part of. + matches = append(matches[:2], matches[3:]...) + label = fmt.Sprintf("@%s//%s", matches[1], strings.Join(matches[2:], ":")) + } + + relToBin, err := filepath.Rel(b.bazel.info["output_path"], filename) + if err == nil && !strings.HasPrefix(relToBin, "../") { + parts := strings.SplitN(relToBin, string(filepath.Separator), 3) + relToBin = parts[2] + // We've effectively converted filename from bazel-bin/some/path.go to some/path.go; + // Check if a BUILD.bazel files exists under this dir, if not walk up and repeat. + relToBin = filepath.Dir(relToBin) + _, err = os.Stat(filepath.Join(b.bazel.WorkspaceRoot(), relToBin, "BUILD.bazel")) + for errors.Is(err, os.ErrNotExist) && relToBin != "." { + relToBin = filepath.Dir(relToBin) + _, err = os.Stat(filepath.Join(b.bazel.WorkspaceRoot(), relToBin, "BUILD.bazel")) + } + + if err == nil { + // return package path found and build all targets (codegen doesn't fall under go_library) + // Otherwise fallback to default + if relToBin == "." { + relToBin = "" + } + label = fmt.Sprintf("//%s:all", relToBin) + additionalKinds = append(additionalKinds, "go_.*") + } + } + + kinds := append(_defaultKinds, additionalKinds...) + return fmt.Sprintf(`kind("%s", same_pkg_direct_rdeps("%s"))`, strings.Join(kinds, "|"), label) +} + +func (b *BazelJSONBuilder) getKind() string { + kinds := []string{"go_library"} + if b.includeTests { + kinds = append(kinds, "go_test") + } + + return strings.Join(kinds, "|") +} + +func (b *BazelJSONBuilder) localQuery(request string) string { + request = path.Clean(request) + if filepath.IsAbs(request) { + if relPath, err := filepath.Rel(workspaceRoot, request); err == nil { + request = relPath + } + } + + if !strings.HasSuffix(request, "...") { + request = fmt.Sprintf("%s:*", request) + } + + return fmt.Sprintf(`kind("%s", %s)`, b.getKind(), request) +} + +func (b *BazelJSONBuilder) packageQuery(importPath string) string { + if strings.HasSuffix(importPath, "/...") { + importPath = fmt.Sprintf(`^%s(/.+)?$`, strings.TrimSuffix(importPath, "/...")) + } + + return fmt.Sprintf( + `kind("%s", attr(importpath, "%s", deps(%s)))`, + b.getKind(), + importPath, + bazelQueryScope) +} + +func (b *BazelJSONBuilder) queryFromRequests(requests ...string) string { + ret := make([]string, 0, len(requests)) + for _, request := range requests { + result := "" + if strings.HasSuffix(request, ".go") { + f := strings.TrimPrefix(request, "file=") + result = b.fileQuery(f) + } else if isLocalPattern(request) { + result = b.localQuery(request) + } else if request == "builtin" || request == "std" { + result = fmt.Sprintf(RulesGoStdlibLabel) + } else if bazelQueryScope != "" { + result = b.packageQuery(request) + } + + if result != "" { + ret = append(ret, result) + } + } + if len(ret) == 0 { + return RulesGoStdlibLabel + } + return strings.Join(ret, " union ") +} + +func NewBazelJSONBuilder(bazel *Bazel, includeTests bool) (*BazelJSONBuilder, error) { + return &BazelJSONBuilder{ + bazel: bazel, + includeTests: includeTests, + }, nil +} + +func (b *BazelJSONBuilder) outputGroupsForMode(mode LoadMode) string { + og := "go_pkg_driver_json_file,go_pkg_driver_stdlib_json_file,go_pkg_driver_srcs" + if mode&NeedExportsFile != 0 { + og += ",go_pkg_driver_export_file" + } + return og +} + +func (b *BazelJSONBuilder) query(ctx context.Context, query string) ([]string, error) { + queryArgs := concatStringsArrays(bazelQueryFlags, []string{ + "--ui_event_filters=-info,-stderr", + "--noshow_progress", + "--order_output=no", + "--output=label", + "--nodep_deps", + "--noimplicit_deps", + "--notool_deps", + query, + }) + labels, err := b.bazel.Query(ctx, queryArgs...) + if err != nil { + return nil, fmt.Errorf("unable to query: %w", err) + } + + return labels, nil +} + +func (b *BazelJSONBuilder) Labels(ctx context.Context, requests []string) ([]string, error) { + labels, err := b.query(ctx, b.queryFromRequests(requests...)) + if err != nil { + return nil, fmt.Errorf("query failed: %w", err) + } + + if len(labels) == 0 { + return nil, fmt.Errorf("found no labels matching the requests") + } + + return labels, nil +} + +func (b *BazelJSONBuilder) Build(ctx context.Context, labels []string, mode LoadMode) ([]string, error) { + aspects := append(additionalAspects, goDefaultAspect) + + buildArgs := concatStringsArrays([]string{ + "--experimental_convenience_symlinks=ignore", + "--ui_event_filters=-info,-stderr", + "--noshow_progress", + "--aspects=" + strings.Join(aspects, ","), + "--output_groups=" + b.outputGroupsForMode(mode), + "--keep_going", // Build all possible packages + }, bazelBuildFlags) + + if len(labels) < 100 { + buildArgs = append(buildArgs, labels...) + } else { + // To avoid hitting MAX_ARGS length, write labels to a file and use `--target_pattern_file` + targetsFile, err := ioutil.TempFile("", "gopackagesdriver_targets_") + if err != nil { + return nil, fmt.Errorf("unable to create target pattern file: %w", err) + } + writer := bufio.NewWriter(targetsFile) + defer writer.Flush() + for _, l := range labels { + writer.WriteString(l + "\n") + } + if err := writer.Flush(); err != nil { + return nil, fmt.Errorf("unable to flush data to target pattern file: %w", err) + } + defer func() { + targetsFile.Close() + os.Remove(targetsFile.Name()) + }() + + buildArgs = append(buildArgs, "--target_pattern_file="+targetsFile.Name()) + } + files, err := b.bazel.Build(ctx, buildArgs...) + if err != nil { + return nil, fmt.Errorf("unable to bazel build %v: %w", buildArgs, err) + } + + ret := []string{} + for _, f := range files { + if strings.HasSuffix(f, ".pkg.json") { + ret = append(ret, f) + } + } + + return ret, nil +} + +func (b *BazelJSONBuilder) PathResolver() PathResolverFunc { + return func(p string) string { + p = strings.Replace(p, "__BAZEL_EXECROOT__", b.bazel.ExecutionRoot(), 1) + p = strings.Replace(p, "__BAZEL_WORKSPACE__", b.bazel.WorkspaceRoot(), 1) + p = strings.Replace(p, "__BAZEL_OUTPUT_BASE__", b.bazel.OutputBase(), 1) + return p + } +} diff --git a/go/tools/gopackagesdriver/build_context.go b/go/tools/gopackagesdriver/build_context.go new file mode 100644 index 00000000..dac786b9 --- /dev/null +++ b/go/tools/gopackagesdriver/build_context.go @@ -0,0 +1,34 @@ +package main + +import ( + "go/build" + "path/filepath" + "strings" +) + +var buildContext = makeBuildContext() + +func makeBuildContext() *build.Context { + bctx := build.Default + bctx.BuildTags = strings.Split(getenvDefault("GOTAGS", ""), ",") + + return &bctx +} + +func filterSourceFilesForTags(files []string) []string { + ret := make([]string, 0, len(files)) + + for _, f := range files { + dir, filename := filepath.Split(f) + ext := filepath.Ext(f) + + match, _ := buildContext.MatchFile(dir, filename) + // MatchFile filters out anything without a file extension. In the + // case of CompiledGoFiles (in particular gco processed files from + // the cache), we want them. + if match || ext == "" { + ret = append(ret, f) + } + } + return ret +} diff --git a/go/tools/gopackagesdriver/driver_request.go b/go/tools/gopackagesdriver/driver_request.go new file mode 100644 index 00000000..db572dcc --- /dev/null +++ b/go/tools/gopackagesdriver/driver_request.go @@ -0,0 +1,91 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "io" +) + +// From https://pkg.go.dev/golang.org/x/tools/go/packages#LoadMode +type LoadMode int + +// Only NeedExportsFile is needed in our case +const ( + // NeedName adds Name and PkgPath. + NeedName LoadMode = 1 << iota + + // NeedFiles adds GoFiles and OtherFiles. + NeedFiles + + // NeedCompiledGoFiles adds CompiledGoFiles. + NeedCompiledGoFiles + + // NeedImports adds Imports. If NeedDeps is not set, the Imports field will contain + // "placeholder" Packages with only the ID set. + NeedImports + + // NeedDeps adds the fields requested by the LoadMode in the packages in Imports. + NeedDeps + + // NeedExportsFile adds ExportFile. + NeedExportFile + + // NeedTypes adds Types, Fset, and IllTyped. + NeedTypes + + // NeedSyntax adds Syntax. + NeedSyntax + + // NeedTypesInfo adds TypesInfo. + NeedTypesInfo + + // NeedTypesSizes adds TypesSizes. + NeedTypesSizes + + // typecheckCgo enables full support for type checking cgo. Requires Go 1.15+. + // Modifies CompiledGoFiles and Types, and has no effect on its own. + typecheckCgo + + // NeedModule adds Module. + NeedModule +) + +// Deprecated: NeedExportsFile is a historical misspelling of NeedExportFile. +const NeedExportsFile = NeedExportFile + +// From https://github.com/golang/tools/blob/v0.1.0/go/packages/external.go#L32 +// Most fields are disabled since there is no need for them +type DriverRequest struct { + Mode LoadMode `json:"mode"` + // Env specifies the environment the underlying build system should be run in. + // Env []string `json:"env"` + // BuildFlags are flags that should be passed to the underlying build system. + // BuildFlags []string `json:"build_flags"` + // Tests specifies whether the patterns should also return test packages. + Tests bool `json:"tests"` + // Overlay maps file paths (relative to the driver's working directory) to the byte contents + // of overlay files. + // Overlay map[string][]byte `json:"overlay"` +} + +func ReadDriverRequest(r io.Reader) (*DriverRequest, error) { + req := &DriverRequest{} + if err := json.NewDecoder(r).Decode(&req); err != nil { + return nil, fmt.Errorf("unable to decode driver request: %w", err) + } + return req, nil +} diff --git a/go/tools/gopackagesdriver/flatpackage.go b/go/tools/gopackagesdriver/flatpackage.go new file mode 100644 index 00000000..9c22132a --- /dev/null +++ b/go/tools/gopackagesdriver/flatpackage.go @@ -0,0 +1,159 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "go/parser" + "go/token" + "os" + "strconv" + "strings" +) + +type ResolvePkgFunc func(importPath string) string + +// Copy and pasted from golang.org/x/tools/go/packages +type FlatPackagesError struct { + Pos string // "file:line:col" or "file:line" or "" or "-" + Msg string + Kind FlatPackagesErrorKind +} + +type FlatPackagesErrorKind int + +const ( + UnknownError FlatPackagesErrorKind = iota + ListError + ParseError + TypeError +) + +func (err FlatPackagesError) Error() string { + pos := err.Pos + if pos == "" { + pos = "-" // like token.Position{}.String() + } + return pos + ": " + err.Msg +} + +// FlatPackage is the JSON form of Package +// It drops all the type and syntax fields, and transforms the Imports +type FlatPackage struct { + ID string + Name string `json:",omitempty"` + PkgPath string `json:",omitempty"` + Errors []FlatPackagesError `json:",omitempty"` + GoFiles []string `json:",omitempty"` + CompiledGoFiles []string `json:",omitempty"` + OtherFiles []string `json:",omitempty"` + ExportFile string `json:",omitempty"` + Imports map[string]string `json:",omitempty"` + Standard bool `json:",omitempty"` +} + +type ( + PackageFunc func(pkg *FlatPackage) + PathResolverFunc func(path string) string +) + +func resolvePathsInPlace(prf PathResolverFunc, paths []string) { + for i, path := range paths { + paths[i] = prf(path) + } +} + +func WalkFlatPackagesFromJSON(jsonFile string, onPkg PackageFunc) error { + f, err := os.Open(jsonFile) + if err != nil { + return fmt.Errorf("unable to open package JSON file: %w", err) + } + defer f.Close() + + decoder := json.NewDecoder(f) + for decoder.More() { + pkg := &FlatPackage{} + if err := decoder.Decode(&pkg); err != nil { + return fmt.Errorf("unable to decode package in %s: %w", f.Name(), err) + } + + onPkg(pkg) + } + return nil +} + +func (fp *FlatPackage) ResolvePaths(prf PathResolverFunc) error { + resolvePathsInPlace(prf, fp.CompiledGoFiles) + resolvePathsInPlace(prf, fp.GoFiles) + resolvePathsInPlace(prf, fp.OtherFiles) + fp.ExportFile = prf(fp.ExportFile) + return nil +} + +// FilterFilesForBuildTags filters the source files given the current build +// tags. +func (fp *FlatPackage) FilterFilesForBuildTags() { + fp.GoFiles = filterSourceFilesForTags(fp.GoFiles) + fp.CompiledGoFiles = filterSourceFilesForTags(fp.CompiledGoFiles) +} + +func (fp *FlatPackage) IsStdlib() bool { + return fp.Standard +} + +func (fp *FlatPackage) ResolveImports(resolve ResolvePkgFunc) error { + // Stdlib packages are already complete import wise + if fp.IsStdlib() { + return nil + } + + fset := token.NewFileSet() + + for _, file := range fp.CompiledGoFiles { + f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) + if err != nil { + return err + } + // If the name is not provided, fetch it from the sources + if fp.Name == "" { + fp.Name = f.Name.Name + } + + for _, rawImport := range f.Imports { + imp, err := strconv.Unquote(rawImport.Path.Value) + if err != nil { + continue + } + // We don't handle CGo for now + if imp == "C" { + continue + } + if _, ok := fp.Imports[imp]; ok { + continue + } + + if pkgID := resolve(imp); pkgID != "" { + fp.Imports[imp] = pkgID + } + } + } + + return nil +} + +func (fp *FlatPackage) IsRoot() bool { + return strings.HasPrefix(fp.ID, "//") +} diff --git a/go/tools/gopackagesdriver/json_packages_driver.go b/go/tools/gopackagesdriver/json_packages_driver.go new file mode 100644 index 00000000..9bbf3408 --- /dev/null +++ b/go/tools/gopackagesdriver/json_packages_driver.go @@ -0,0 +1,59 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "go/types" +) + +type JSONPackagesDriver struct { + registry *PackageRegistry +} + +func NewJSONPackagesDriver(jsonFiles []string, prf PathResolverFunc) (*JSONPackagesDriver, error) { + jpd := &JSONPackagesDriver{ + registry: NewPackageRegistry(), + } + + for _, f := range jsonFiles { + if err := WalkFlatPackagesFromJSON(f, func(pkg *FlatPackage) { + jpd.registry.Add(pkg) + }); err != nil { + return nil, fmt.Errorf("unable to walk json: %w", err) + } + } + + if err := jpd.registry.ResolvePaths(prf); err != nil { + return nil, fmt.Errorf("unable to resolve paths: %w", err) + } + + if err := jpd.registry.ResolveImports(); err != nil { + return nil, fmt.Errorf("unable to resolve paths: %w", err) + } + + return jpd, nil +} + +func (b *JSONPackagesDriver) GetResponse(labels []string) *driverResponse { + rootPkgs, packages := b.registry.Match(labels) + + return &driverResponse{ + NotHandled: false, + Sizes: types.SizesFor("gc", "amd64").(*types.StdSizes), + Roots: rootPkgs, + Packages: packages, + } +} diff --git a/go/tools/gopackagesdriver/main.go b/go/tools/gopackagesdriver/main.go new file mode 100644 index 00000000..fea2d2c1 --- /dev/null +++ b/go/tools/gopackagesdriver/main.go @@ -0,0 +1,126 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "fmt" + "go/types" + "os" + "strings" +) + +type driverResponse struct { + // NotHandled is returned if the request can't be handled by the current + // driver. If an external driver returns a response with NotHandled, the + // rest of the driverResponse is ignored, and go/packages will fallback + // to the next driver. If go/packages is extended in the future to support + // lists of multiple drivers, go/packages will fall back to the next driver. + NotHandled bool + + // Sizes, if not nil, is the types.Sizes to use when type checking. + Sizes *types.StdSizes + + // Roots is the set of package IDs that make up the root packages. + // We have to encode this separately because when we encode a single package + // we cannot know if it is one of the roots as that requires knowledge of the + // graph it is part of. + Roots []string `json:",omitempty"` + + // Packages is the full set of packages in the graph. + // The packages are not connected into a graph. + // The Imports if populated will be stubs that only have their ID set. + // Imports will be connected and then type and syntax information added in a + // later pass (see refine). + Packages []*FlatPackage +} + +var ( + // Injected via x_defs. + + rulesGoRepositoryName string + goDefaultAspect = rulesGoRepositoryName + "//go/tools/gopackagesdriver:aspect.bzl%go_pkg_info_aspect" + bazelBin = getenvDefault("GOPACKAGESDRIVER_BAZEL", "bazel") + bazelStartupFlags = strings.Fields(os.Getenv("GOPACKAGESDRIVER_BAZEL_FLAGS")) + bazelQueryFlags = strings.Fields(os.Getenv("GOPACKAGESDRIVER_BAZEL_QUERY_FLAGS")) + bazelQueryScope = getenvDefault("GOPACKAGESDRIVER_BAZEL_QUERY_SCOPE", "") + bazelBuildFlags = strings.Fields(os.Getenv("GOPACKAGESDRIVER_BAZEL_BUILD_FLAGS")) + workspaceRoot = os.Getenv("BUILD_WORKSPACE_DIRECTORY") + additionalAspects = strings.Fields(os.Getenv("GOPACKAGESDRIVER_BAZEL_ADDTL_ASPECTS")) + additionalKinds = strings.Fields(os.Getenv("GOPACKAGESDRIVER_BAZEL_KINDS")) + emptyResponse = &driverResponse{ + NotHandled: true, + Sizes: types.SizesFor("gc", "amd64").(*types.StdSizes), + Roots: []string{}, + Packages: []*FlatPackage{}, + } +) + +func run() (*driverResponse, error) { + ctx, cancel := signalContext(context.Background(), os.Interrupt) + defer cancel() + + queries := os.Args[1:] + + request, err := ReadDriverRequest(os.Stdin) + if err != nil { + return emptyResponse, fmt.Errorf("unable to read request: %w", err) + } + + bazel, err := NewBazel(ctx, bazelBin, workspaceRoot, bazelStartupFlags) + if err != nil { + return emptyResponse, fmt.Errorf("unable to create bazel instance: %w", err) + } + + bazelJsonBuilder, err := NewBazelJSONBuilder(bazel, request.Tests) + if err != nil { + return emptyResponse, fmt.Errorf("unable to build JSON files: %w", err) + } + + labels, err := bazelJsonBuilder.Labels(ctx, queries) + if err != nil { + return emptyResponse, fmt.Errorf("unable to lookup package: %w", err) + } + + jsonFiles, err := bazelJsonBuilder.Build(ctx, labels, request.Mode) + if err != nil { + return emptyResponse, fmt.Errorf("unable to build JSON files: %w", err) + } + + driver, err := NewJSONPackagesDriver(jsonFiles, bazelJsonBuilder.PathResolver()) + if err != nil { + return emptyResponse, fmt.Errorf("unable to load JSON files: %w", err) + } + + // Note: we are returning all files required to build a specific package. + // For file queries (`file=`), this means that the CompiledGoFiles will + // include more than the only file being specified. + return driver.GetResponse(labels), nil +} + +func main() { + response, err := run() + if err := json.NewEncoder(os.Stdout).Encode(response); err != nil { + fmt.Fprintf(os.Stderr, "unable to encode response: %v", err) + } + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v", err) + // gopls will check the packages driver exit code, and if there is an + // error, it will fall back to go list. Obviously we don't want that, + // so force a 0 exit code. + os.Exit(0) + } +} diff --git a/go/tools/gopackagesdriver/packageregistry.go b/go/tools/gopackagesdriver/packageregistry.go new file mode 100644 index 00000000..05e620d5 --- /dev/null +++ b/go/tools/gopackagesdriver/packageregistry.go @@ -0,0 +1,111 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "strings" +) + +type PackageRegistry struct { + packagesByID map[string]*FlatPackage + stdlib map[string]string +} + +func NewPackageRegistry(pkgs ...*FlatPackage) *PackageRegistry { + pr := &PackageRegistry{ + packagesByID: map[string]*FlatPackage{}, + stdlib: map[string]string{}, + } + pr.Add(pkgs...) + return pr +} + +func (pr *PackageRegistry) Add(pkgs ...*FlatPackage) *PackageRegistry { + for _, pkg := range pkgs { + pr.packagesByID[pkg.ID] = pkg + + if pkg.IsStdlib() { + pr.stdlib[pkg.PkgPath] = pkg.ID + } + } + return pr +} + +func (pr *PackageRegistry) ResolvePaths(prf PathResolverFunc) error { + for _, pkg := range pr.packagesByID { + pkg.ResolvePaths(prf) + pkg.FilterFilesForBuildTags() + } + return nil +} + +// ResolveImports adds stdlib imports to packages. This is required because +// stdlib packages are not part of the JSON file exports as bazel is unaware of +// them. +func (pr *PackageRegistry) ResolveImports() error { + resolve := func(importPath string) string { + if pkgID, ok := pr.stdlib[importPath]; ok { + return pkgID + } + + return "" + } + + for _, pkg := range pr.packagesByID { + if err := pkg.ResolveImports(resolve); err != nil { + return err + } + } + + return nil +} + +func (pr *PackageRegistry) walk(acc map[string]*FlatPackage, root string) { + pkg := pr.packagesByID[root] + + acc[pkg.ID] = pkg + for _, pkgID := range pkg.Imports { + if _, ok := acc[pkgID]; !ok { + pr.walk(acc, pkgID) + } + } +} + +func (pr *PackageRegistry) Match(labels []string) ([]string, []*FlatPackage) { + roots := map[string]struct{}{} + + for _, label := range labels { + if !strings.HasPrefix(label, "@") { + label = fmt.Sprintf("@%s", label) + } + + roots[label] = struct{}{} + } + + walkedPackages := map[string]*FlatPackage{} + retRoots := make([]string, 0, len(roots)) + for rootPkg := range roots { + retRoots = append(retRoots, rootPkg) + pr.walk(walkedPackages, rootPkg) + } + + retPkgs := make([]*FlatPackage, 0, len(walkedPackages)) + for _, pkg := range walkedPackages { + retPkgs = append(retPkgs, pkg) + } + + return retRoots, retPkgs +} diff --git a/go/tools/gopackagesdriver/utils.go b/go/tools/gopackagesdriver/utils.go new file mode 100644 index 00000000..d5524fdd --- /dev/null +++ b/go/tools/gopackagesdriver/utils.go @@ -0,0 +1,77 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "go/build" + "os" + "os/signal" + "path" + "path/filepath" +) + +func getenvDefault(key, defaultValue string) string { + if v, ok := os.LookupEnv(key); ok { + return v + } + return defaultValue +} + +func concatStringsArrays(values ...[]string) []string { + ret := []string{} + for _, v := range values { + ret = append(ret, v...) + } + return ret +} + +func ensureAbsolutePathFromWorkspace(path string) string { + if filepath.IsAbs(path) { + return path + } + return filepath.Join(workspaceRoot, path) +} + +func signalContext(parentCtx context.Context, signals ...os.Signal) (ctx context.Context, stop context.CancelFunc) { + ctx, cancel := context.WithCancel(parentCtx) + ch := make(chan os.Signal, 1) + go func() { + select { + case <-ch: + cancel() + case <-ctx.Done(): + } + }() + signal.Notify(ch, signals...) + + return ctx, cancel +} + +func isLocalPattern(pattern string) bool { + return build.IsLocalImport(pattern) || filepath.IsAbs(pattern) +} + +func packageID(pattern string) string { + pattern = path.Clean(pattern) + if filepath.IsAbs(pattern) { + if relPath, err := filepath.Rel(workspaceRoot, pattern); err == nil { + pattern = relPath + } + } + + return fmt.Sprintf("//%s", pattern) +} |