diff options
author | Jingwen Chen <jingwen@google.com> | 2020-08-07 14:16:34 +0000 |
---|---|---|
committer | Jingwen Chen <jingwen@google.com> | 2020-08-21 11:45:46 +0000 |
commit | 69d4cbeb043137321afb1d1a904602295a8eccf5 (patch) | |
tree | ec3edfa1469d801a0656c60275dfeca40475a0ec /cmd/soong_build | |
parent | eaebec13127c17304ede6b9553fdbe327848fd3b (diff) | |
download | soong-69d4cbeb043137321afb1d1a904602295a8eccf5.tar.gz |
Surface module properties as Bazel BUILD target attributes in the Bazel overlay
This patchset changes bazel_overlay to generate soong_module as a macro,
instead of a rule, and generate module properties in the BUILD files as
kwargs to the soong_module macro.
Here's a sample of the new BUILD files with module properties:
bionic/libdl/BUILD.bazel:
https://paste.googleplex.com/6484466996346880?raw
art/build/apex/BUILD.bazel:
https://paste.googleplex.com/5461276001042432?raw
bionic/apex/BUILD.bazel:
https://paste.googleplex.com/4932795173437440?raw
soong_module is now a macro that conditionally expands to underlying
soong_<module type> rules with statically defined attributes. In this
CL, we are starting with a hardcoded filegroup rule definition to
demonstrate the conditional rule loading within the soong_module macro.
If the module_type matches an existing Bazel rule, soong_module forwards
the entire **kwargs into the rule, which Bazel typechecks.
Non-filegroup module types will be expanded into generic_soong_module,
but with the kwargs dropped.
This approach allows us to:
1) Programmtically generate soong_<module type> rules for all module
types available in Soong, together with the statically defined attribute
types in `attrs`.
2) Incrementally migrate and test individual module types from
generic_soong_module to their module rule shims.
3) Swap out the module rule shims to the actual Bazel rules (e.g
cc_library, java_library) and perform attribute manipulation in Starlark
itself.
Example of querying against the 'srcs' attribute in soong_filegroup:
```
$ bazel cquery 'kind(soong_filegroup, //...)' | wc -l
590
$ bazel cquery --output=build 'attr(srcs, "linker.cpp",
kind(soong_filegroup, //bionic/...))'
INFO: Analyzed 3907 targets (0 packages loaded, 0 targets configured).
INFO: Found 3907 targets...
/usr/local/google/home/jingwen/aosp/out/soong/bazel_overlay/bionic/linker/BUILD.bazel:4144:13
soong_filegroup(
name = "linker_sources",
generator_name = "linker_sources",
generator_function = "soong_module",
generator_location = "bionic/linker/BUILD.bazel:4144:13",
srcs = ["dlfcn.cpp", "linker.cpp", "linker_block_allocator.cpp",
"linker_dlwarning.cpp", "linker_cfi.cpp", "linker_config.cpp",
"linker_debug.cpp", "linker_gdb_support.cpp", "linker_globals.cpp",
"linker_libc_support.c", "linker_libcxx_support.cpp",
"linker_namespaces.cpp", "linker_logger.cpp",
"linker_mapped_file_fragment.cpp", "linker_phdr.cpp",
"linker_relocate.cpp", "linker_sdk_versions.cpp", "linker_soinfo.cpp",
"linker_tls.cpp", "linker_utils.cpp", "rt.cpp"],
deps = [],
)
/usr/local/google/home/jingwen/aosp/out/soong/bazel_overlay/soong_module.bzl:32:23
in <toplevel>
```
This CL is known to be lacking the following features, and will be looked at in follow up CLs:
1) Pretty printing reflect.Interface properties, like arch, multilib and
dists.
2) Generating module Bazel rule shims for all module types, instead of
hardcoding them like `soong_filegroup`.
Bug: 162720644
Test: bazel_overlay_test.go (soong build test)
Test: m bazel_overlay && cd out/soong/bazel_overlay && bazel cquery //...
Signed-off-by: Jingwen Chen <jingwen@google.com>
Change-Id: Ic1e448887eb540ed15a55bc4090cf75a4d832d41
Diffstat (limited to 'cmd/soong_build')
-rw-r--r-- | cmd/soong_build/Android.bp | 3 | ||||
-rw-r--r-- | cmd/soong_build/bazel_overlay.go | 350 | ||||
-rw-r--r-- | cmd/soong_build/bazel_overlay_test.go | 255 |
3 files changed, 571 insertions, 37 deletions
diff --git a/cmd/soong_build/Android.bp b/cmd/soong_build/Android.bp index 7b8352b6a..4ebbe6895 100644 --- a/cmd/soong_build/Android.bp +++ b/cmd/soong_build/Android.bp @@ -28,5 +28,8 @@ bootstrap_go_binary { "writedocs.go", "bazel_overlay.go", ], + testSrcs: [ + "bazel_overlay_test.go", + ], primaryBuilder: true, } diff --git a/cmd/soong_build/bazel_overlay.go b/cmd/soong_build/bazel_overlay.go index e37c163d4..e1d3d41bf 100644 --- a/cmd/soong_build/bazel_overlay.go +++ b/cmd/soong_build/bazel_overlay.go @@ -20,14 +20,17 @@ import ( "io/ioutil" "os" "path/filepath" + "reflect" "strings" "github.com/google/blueprint" + "github.com/google/blueprint/proptools" ) const ( soongModuleLoad = `package(default_visibility = ["//visibility:public"]) load("//:soong_module.bzl", "soong_module") + ` // A BUILD file target snippet representing a Soong module @@ -36,15 +39,11 @@ load("//:soong_module.bzl", "soong_module") module_name = "%s", module_type = "%s", module_variant = "%s", - deps = [ - %s - ], -) -` + deps = %s, +%s)` // The soong_module rule implementation in a .bzl file - soongModuleBzl = ` -SoongModuleInfo = provider( + soongModuleBzl = `SoongModuleInfo = provider( fields = { "name": "Name of module", "type": "Type of module", @@ -52,7 +51,17 @@ SoongModuleInfo = provider( }, ) -def _soong_module_impl(ctx): +def _merge_dicts(*dicts): + """Adds a list of dictionaries into a single dictionary.""" + + # If keys are repeated in multiple dictionaries, the latter one "wins". + result = {} + for d in dicts: + result.update(d) + + return result + +def _generic_soong_module_impl(ctx): return [ SoongModuleInfo( name = ctx.attr.module_name, @@ -61,21 +70,66 @@ def _soong_module_impl(ctx): ), ] -soong_module = rule( - implementation = _soong_module_impl, - attrs = { - "module_name": attr.string(mandatory = True), - "module_type": attr.string(mandatory = True), - "module_variant": attr.string(), - "deps": attr.label_list(providers = [SoongModuleInfo]), - }, +_COMMON_ATTRS = { + "module_name": attr.string(mandatory = True), + "module_type": attr.string(mandatory = True), + "module_variant": attr.string(), + "deps": attr.label_list(providers = [SoongModuleInfo]), +} + + +generic_soong_module = rule( + implementation = _generic_soong_module_impl, + attrs = _COMMON_ATTRS, ) + +# TODO(jingwen): auto generate Soong module shims +def _soong_filegroup_impl(ctx): + return [SoongModuleInfo(),] + +soong_filegroup = rule( + implementation = _soong_filegroup_impl, + # Matches https://cs.android.com/android/platform/superproject/+/master:build/soong/android/filegroup.go;l=25-40;drc=6a6478d49e78703ba22a432c41d819c8df79ef6c + attrs = _merge_dicts(_COMMON_ATTRS, { + "srcs": attr.string_list(doc = "srcs lists files that will be included in this filegroup"), + "exclude_srcs": attr.string_list(), + "path": attr.string(doc = "The base path to the files. May be used by other modules to determine which portion of the path to use. For example, when a filegroup is used as data in a cc_test rule, the base path is stripped off the path and the remaining path is used as the installation directory."), + "export_to_make_var": attr.string(doc = "Create a make variable with the specified name that contains the list of files in the filegroup, relative to the root of the source tree."), + }) +) + +soong_module_rule_map = { + "filegroup": soong_filegroup, +} + +# soong_module is a macro that supports arbitrary kwargs, and uses module_type to +# expand to the right underlying shim. +def soong_module(name, module_type, **kwargs): + soong_module_rule = soong_module_rule_map.get(module_type) + + if soong_module_rule == None: + # This module type does not have an existing rule to map to, so use the + # generic_soong_module rule instead. + generic_soong_module( + name = name, + module_type = module_type, + module_name = kwargs.pop("module_name", ""), + module_variant = kwargs.pop("module_variant", ""), + deps = kwargs.pop("deps", []), + ) + else: + soong_module_rule( + name = name, + module_type = module_type, + **kwargs, + ) ` ) func targetNameWithVariant(c *blueprint.Context, logicModule blueprint.Module) string { name := "" if c.ModuleSubDir(logicModule) != "" { + // TODO(b/162720883): Figure out a way to drop the "--" variant suffixes. name = c.ModuleName(logicModule) + "--" + c.ModuleSubDir(logicModule) } else { name = c.ModuleName(logicModule) @@ -95,6 +149,153 @@ func packagePath(c *blueprint.Context, logicModule blueprint.Module) string { return filepath.Dir(c.BlueprintFile(logicModule)) } +func escapeString(s string) string { + s = strings.ReplaceAll(s, "\\", "\\\\") + return strings.ReplaceAll(s, "\"", "\\\"") +} + +func makeIndent(indent int) string { + if indent < 0 { + panic(fmt.Errorf("indent column cannot be less than 0, but got %d", indent)) + } + return strings.Repeat(" ", indent) +} + +// prettyPrint a property value into the equivalent Starlark representation +// recursively. +func prettyPrint(propertyValue reflect.Value, indent int) (string, error) { + if isZero(propertyValue) { + // A property value being set or unset actually matters -- Soong does set default + // values for unset properties, like system_shared_libs = ["libc", "libm", "libdl"] at + // https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/linker.go;l=281-287;drc=f70926eef0b9b57faf04c17a1062ce50d209e480 + // + // In Bazel-parlance, we would use "attr.<type>(default = <default value>)" to set the default + // value of unset attributes. + return "", nil + } + + var ret string + switch propertyValue.Kind() { + case reflect.String: + ret = fmt.Sprintf("\"%v\"", escapeString(propertyValue.String())) + case reflect.Bool: + ret = strings.Title(fmt.Sprintf("%v", propertyValue.Interface())) + case reflect.Int, reflect.Uint, reflect.Int64: + ret = fmt.Sprintf("%v", propertyValue.Interface()) + case reflect.Ptr: + return prettyPrint(propertyValue.Elem(), indent) + case reflect.Slice: + ret = "[\n" + for i := 0; i < propertyValue.Len(); i++ { + indexedValue, err := prettyPrint(propertyValue.Index(i), indent+1) + if err != nil { + return "", err + } + + if indexedValue != "" { + ret += makeIndent(indent + 1) + ret += indexedValue + ret += ",\n" + } + } + ret += makeIndent(indent) + ret += "]" + case reflect.Struct: + ret = "{\n" + // Sort and print the struct props by the key. + structProps := extractStructProperties(propertyValue, indent) + for _, k := range android.SortedStringKeys(structProps) { + ret += makeIndent(indent + 1) + ret += "\"" + k + "\": " + ret += structProps[k] + ret += ",\n" + } + ret += makeIndent(indent) + ret += "}" + case reflect.Interface: + // TODO(b/164227191): implement pretty print for interfaces. + // Interfaces are used for for arch, multilib and target properties. + return "", nil + default: + return "", fmt.Errorf( + "unexpected kind for property struct field: %s", propertyValue.Kind()) + } + return ret, nil +} + +func extractStructProperties(structValue reflect.Value, indent int) map[string]string { + if structValue.Kind() != reflect.Struct { + panic(fmt.Errorf("Expected a reflect.Struct type, but got %s", structValue.Kind())) + } + + ret := map[string]string{} + structType := structValue.Type() + for i := 0; i < structValue.NumField(); i++ { + field := structType.Field(i) + if field.PkgPath != "" { + // Skip unexported fields. Some properties are + // internal to Soong only, and these fields do not have PkgPath. + continue + } + if proptools.HasTag(field, "blueprint", "mutated") { + continue + } + + fieldValue := structValue.Field(i) + if isZero(fieldValue) { + // Ignore zero-valued fields + continue + } + + propertyName := proptools.PropertyNameForField(field.Name) + prettyPrintedValue, err := prettyPrint(fieldValue, indent+1) + if err != nil { + panic( + fmt.Errorf( + "Error while parsing property: %q. %s", + propertyName, + err)) + } + if prettyPrintedValue != "" { + ret[propertyName] = prettyPrintedValue + } + } + + return ret +} + +func isStructPtr(t reflect.Type) bool { + return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct +} + +// Generically extract module properties and types into a map, keyed by the module property name. +func extractModuleProperties(aModule android.Module) map[string]string { + ret := map[string]string{} + + // Iterate over this android.Module's property structs. + for _, properties := range aModule.GetProperties() { + propertiesValue := reflect.ValueOf(properties) + // Check that propertiesValue is a pointer to the Properties struct, like + // *cc.BaseLinkerProperties or *java.CompilerProperties. + // + // propertiesValue can also be type-asserted to the structs to + // manipulate internal props, if needed. + if isStructPtr(propertiesValue.Type()) { + structValue := propertiesValue.Elem() + for k, v := range extractStructProperties(structValue, 0) { + ret[k] = v + } + } else { + panic(fmt.Errorf( + "properties must be a pointer to a struct, got %T", + propertiesValue.Interface())) + } + + } + + return ret +} + func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error { blueprintCtx := ctx.Context blueprintCtx.VisitAllModules(func(module blueprint.Module) { @@ -103,27 +304,7 @@ func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error { panic(err) } - // TODO(b/163018919): DirectDeps can have duplicate (module, variant) - // items, if the modules are added using different DependencyTag. Figure - // out the implications of that. - depLabels := map[string]bool{} - blueprintCtx.VisitDirectDeps(module, func(depModule blueprint.Module) { - depLabels[qualifiedTargetLabel(blueprintCtx, depModule)] = true - }) - - var depLabelList string - for depLabel, _ := range depLabels { - depLabelList += "\"" + depLabel + "\",\n " - } - buildFile.Write([]byte( - fmt.Sprintf( - soongModuleTarget, - targetNameWithVariant(blueprintCtx, module), - blueprintCtx.ModuleName(module), - blueprintCtx.ModuleType(module), - // misleading name, this actually returns the variant. - blueprintCtx.ModuleSubDir(module), - depLabelList))) + buildFile.Write([]byte(generateSoongModuleTarget(blueprintCtx, module) + "\n\n")) buildFile.Close() }) @@ -138,6 +319,70 @@ func createBazelOverlay(ctx *android.Context, bazelOverlayDir string) error { return writeReadOnlyFile(bazelOverlayDir, "soong_module.bzl", soongModuleBzl) } +var ignoredProps map[string]bool = map[string]bool{ + "name": true, // redundant, since this is explicitly generated for every target + "from": true, // reserved keyword + "in": true, // reserved keyword + "arch": true, // interface prop type is not supported yet. + "multilib": true, // interface prop type is not supported yet. + "target": true, // interface prop type is not supported yet. + "visibility": true, // Bazel has native visibility semantics. Handle later. +} + +func shouldGenerateAttribute(prop string) bool { + return !ignoredProps[prop] +} + +// props is an unsorted map. This function ensures that +// the generated attributes are sorted to ensure determinism. +func propsToAttributes(props map[string]string) string { + var attributes string + for _, propName := range android.SortedStringKeys(props) { + if shouldGenerateAttribute(propName) { + attributes += fmt.Sprintf(" %s = %s,\n", propName, props[propName]) + } + } + return attributes +} + +// Convert a module and its deps and props into a Bazel macro/rule +// representation in the BUILD file. +func generateSoongModuleTarget( + blueprintCtx *blueprint.Context, + module blueprint.Module) string { + + var props map[string]string + if aModule, ok := module.(android.Module); ok { + props = extractModuleProperties(aModule) + } + attributes := propsToAttributes(props) + + // TODO(b/163018919): DirectDeps can have duplicate (module, variant) + // items, if the modules are added using different DependencyTag. Figure + // out the implications of that. + depLabels := map[string]bool{} + blueprintCtx.VisitDirectDeps(module, func(depModule blueprint.Module) { + depLabels[qualifiedTargetLabel(blueprintCtx, depModule)] = true + }) + + depLabelList := "[\n" + for depLabel, _ := range depLabels { + depLabelList += " \"" + depLabelList += depLabel + depLabelList += "\",\n" + } + depLabelList += " ]" + + return fmt.Sprintf( + soongModuleTarget, + targetNameWithVariant(blueprintCtx, module), + blueprintCtx.ModuleName(module), + blueprintCtx.ModuleType(module), + blueprintCtx.ModuleSubDir(module), + depLabelList, + attributes) +} + func buildFileForModule(ctx *blueprint.Context, module blueprint.Module) (*os.File, error) { // Create nested directories for the BUILD file dirPath := filepath.Join(bazelOverlayDir, packagePath(ctx, module)) @@ -171,3 +416,34 @@ func writeReadOnlyFile(dir string, baseName string, content string) error { // 0444 is read-only return ioutil.WriteFile(workspaceFile, []byte(content), 0444) } + +func isZero(value reflect.Value) bool { + switch value.Kind() { + case reflect.Func, reflect.Map, reflect.Slice: + return value.IsNil() + case reflect.Array: + valueIsZero := true + for i := 0; i < value.Len(); i++ { + valueIsZero = valueIsZero && isZero(value.Index(i)) + } + return valueIsZero + case reflect.Struct: + valueIsZero := true + for i := 0; i < value.NumField(); i++ { + if value.Field(i).CanSet() { + valueIsZero = valueIsZero && isZero(value.Field(i)) + } + } + return valueIsZero + case reflect.Ptr: + if !value.IsNil() { + return isZero(reflect.Indirect(value)) + } else { + return true + } + default: + zeroValue := reflect.Zero(value.Type()) + result := value.Interface() == zeroValue.Interface() + return result + } +} diff --git a/cmd/soong_build/bazel_overlay_test.go b/cmd/soong_build/bazel_overlay_test.go new file mode 100644 index 000000000..67599c084 --- /dev/null +++ b/cmd/soong_build/bazel_overlay_test.go @@ -0,0 +1,255 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "android/soong/android" + "io/ioutil" + "os" + "testing" +) + +var buildDir string + +func setUp() { + var err error + buildDir, err = ioutil.TempDir("", "bazel_overlay_test") + if err != nil { + panic(err) + } +} + +func tearDown() { + os.RemoveAll(buildDir) +} + +func TestMain(m *testing.M) { + run := func() int { + setUp() + defer tearDown() + + return m.Run() + } + + os.Exit(run()) +} + +type customModule struct { + android.ModuleBase +} + +func (m *customModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // nothing for now. +} + +func customModuleFactory() android.Module { + module := &customModule{} + android.InitAndroidModule(module) + return module +} + +func TestGenerateBazelOverlayFromBlueprint(t *testing.T) { + testCases := []struct { + bp string + expectedBazelTarget string + }{ + { + bp: `custom { + name: "foo", +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + deps = [ + ], +)`, + }, + { + bp: `custom { + name: "foo", + ramdisk: true, +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + deps = [ + ], + ramdisk = True, +)`, + }, + { + bp: `custom { + name: "foo", + owner: "a_string_with\"quotes\"_and_\\backslashes\\\\", +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + deps = [ + ], + owner = "a_string_with\"quotes\"_and_\\backslashes\\\\", +)`, + }, + { + bp: `custom { + name: "foo", + required: ["bar"], +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + deps = [ + ], + required = [ + "bar", + ], +)`, + }, + { + bp: `custom { + name: "foo", + target_required: ["qux", "bazqux"], +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + deps = [ + ], + target_required = [ + "qux", + "bazqux", + ], +)`, + }, + { + bp: `custom { + name: "foo", + dist: { + targets: ["goal_foo"], + tag: ".foo", + }, + dists: [ + { + targets: ["goal_bar"], + tag: ".bar", + }, + ], +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + deps = [ + ], + dist = { + "tag": ".foo", + "targets": [ + "goal_foo", + ], + }, + dists = [ + { + "tag": ".bar", + "targets": [ + "goal_bar", + ], + }, + ], +)`, + }, + { + bp: `custom { + name: "foo", + required: ["bar"], + target_required: ["qux", "bazqux"], + ramdisk: true, + owner: "custom_owner", + dists: [ + { + tag: ".tag", + targets: ["my_goal"], + }, + ], +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + deps = [ + ], + dists = [ + { + "tag": ".tag", + "targets": [ + "my_goal", + ], + }, + ], + owner = "custom_owner", + ramdisk = True, + required = [ + "bar", + ], + target_required = [ + "qux", + "bazqux", + ], +)`, + }, + } + + for _, testCase := range testCases { + config := android.TestConfig(buildDir, nil, testCase.bp, nil) + ctx := android.NewTestContext() + ctx.RegisterModuleType("custom", customModuleFactory) + ctx.Register(config) + + _, errs := ctx.ParseFileList(".", []string{"Android.bp"}) + android.FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + android.FailIfErrored(t, errs) + + module := ctx.ModuleForTests("foo", "").Module().(*customModule) + blueprintCtx := ctx.Context.Context + + actualBazelTarget := generateSoongModuleTarget(blueprintCtx, module) + if actualBazelTarget != testCase.expectedBazelTarget { + t.Errorf( + "Expected generated Bazel target to be '%s', got '%s'", + testCase.expectedBazelTarget, + actualBazelTarget, + ) + } + } +} |