diff options
author | Liz Kammer <eakammer@google.com> | 2020-11-25 16:06:39 -0800 |
---|---|---|
committer | Liz Kammer <eakammer@google.com> | 2021-01-07 14:34:00 -0500 |
commit | 2dd9ca422d4745fac52d9d76e5e32fba3c555075 (patch) | |
tree | b3c65f92fe6eabfefff762d6029963dc5c9b5875 /bp2build | |
parent | 5bcf99a93a18030dbfa549dc99a41e8c762aa6d5 (diff) | |
download | soong-2dd9ca422d4745fac52d9d76e5e32fba3c555075.tar.gz |
Refactor queryview.
Splits queryview into queryview and bp2build. The latter runs as a
presingleton (could be converted to a singleton). This prevents needing
to run soong_ui a subsequent time to build the query. Queryview remains
as a separate step to prevent increasing runtime due to this generation
every time Soong runs.
Currently this is running as a presingleton as this gives a translation
of Android.bp files after only LoadHooks have run, no mutators.
Test: go tests
Test: m queryview && bazel query --config=queryview //...
Change-Id: If2ba28c7ef60fbd41f43bda6623d41c8c7d23a1b
Diffstat (limited to 'bp2build')
-rw-r--r-- | bp2build/Android.bp | 23 | ||||
-rw-r--r-- | bp2build/androidbp_to_build_templates.go | 124 | ||||
-rw-r--r-- | bp2build/bp2build.go | 77 | ||||
-rw-r--r-- | bp2build/build_conversion.go | 305 | ||||
-rw-r--r-- | bp2build/build_conversion_test.go | 221 | ||||
-rw-r--r-- | bp2build/bzl_conversion.go | 230 | ||||
-rw-r--r-- | bp2build/bzl_conversion_test.go | 208 | ||||
-rw-r--r-- | bp2build/conversion.go | 118 | ||||
-rw-r--r-- | bp2build/conversion_test.go | 73 | ||||
-rw-r--r-- | bp2build/testing.go | 136 |
10 files changed, 1515 insertions, 0 deletions
diff --git a/bp2build/Android.bp b/bp2build/Android.bp new file mode 100644 index 000000000..49587f488 --- /dev/null +++ b/bp2build/Android.bp @@ -0,0 +1,23 @@ +bootstrap_go_package { + name: "soong-bp2build", + pkgPath: "android/soong/bp2build", + srcs: [ + "androidbp_to_build_templates.go", + "bp2build.go", + "build_conversion.go", + "bzl_conversion.go", + "conversion.go", + ], + deps: [ + "soong-android", + ], + testSrcs: [ + "build_conversion_test.go", + "bzl_conversion_test.go", + "conversion_test.go", + "testing.go", + ], + pluginFor: [ + "soong_build", + ], +} diff --git a/bp2build/androidbp_to_build_templates.go b/bp2build/androidbp_to_build_templates.go new file mode 100644 index 000000000..75c3ccb9a --- /dev/null +++ b/bp2build/androidbp_to_build_templates.go @@ -0,0 +1,124 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +const ( + // The default `load` preamble for every generated BUILD file. + soongModuleLoad = `package(default_visibility = ["//visibility:public"]) +load("//build/bazel/queryview_rules:soong_module.bzl", "soong_module") + +` + + // A macro call in the BUILD file representing a Soong module, with space + // for expanding more attributes. + soongModuleTarget = `soong_module( + name = "%s", + module_name = "%s", + module_type = "%s", + module_variant = "%s", + module_deps = %s, +%s)` + + // A simple provider to mark and differentiate Soong module rule shims from + // regular Bazel rules. Every Soong module rule shim returns a + // SoongModuleInfo provider, and can only depend on rules returning + // SoongModuleInfo in the `module_deps` attribute. + providersBzl = `SoongModuleInfo = provider( + fields = { + "name": "Name of module", + "type": "Type of module", + "variant": "Variant of module", + }, +) +` + + // The soong_module rule implementation in a .bzl file. + soongModuleBzl = ` +%s + +load("//build/bazel/queryview_rules:providers.bzl", "SoongModuleInfo") + +def _generic_soong_module_impl(ctx): + return [ + SoongModuleInfo( + name = ctx.attr.module_name, + type = ctx.attr.module_type, + variant = ctx.attr.module_variant, + ), + ] + +generic_soong_module = rule( + implementation = _generic_soong_module_impl, + attrs = { + "module_name": attr.string(mandatory = True), + "module_type": attr.string(mandatory = True), + "module_variant": attr.string(), + "module_deps": attr.label_list(providers = [SoongModuleInfo]), + }, +) + +soong_module_rule_map = { +%s} + +_SUPPORTED_TYPES = ["bool", "int", "string"] + +def _is_supported_type(value): + if type(value) in _SUPPORTED_TYPES: + return True + elif type(value) == "list": + supported = True + for v in value: + supported = supported and type(v) in _SUPPORTED_TYPES + return supported + else: + return False + +# soong_module is a macro that supports arbitrary kwargs, and uses module_type to +# expand to the right underlying shim. +def soong_module(name, module_type, **kwargs): + soong_module_rule = soong_module_rule_map.get(module_type) + + if soong_module_rule == None: + # This module type does not have an existing rule to map to, so use the + # generic_soong_module rule instead. + generic_soong_module( + name = name, + module_type = module_type, + module_name = kwargs.pop("module_name", ""), + module_variant = kwargs.pop("module_variant", ""), + module_deps = kwargs.pop("module_deps", []), + ) + else: + supported_kwargs = dict() + for key, value in kwargs.items(): + if _is_supported_type(value): + supported_kwargs[key] = value + soong_module_rule( + name = name, + **supported_kwargs, + ) +` + + // A rule shim for representing a Soong module type and its properties. + moduleRuleShim = ` +def _%[1]s_impl(ctx): + return [SoongModuleInfo()] + +%[1]s = rule( + implementation = _%[1]s_impl, + attrs = %[2]s +) +` +) diff --git a/bp2build/bp2build.go b/bp2build/bp2build.go new file mode 100644 index 000000000..30f298a0c --- /dev/null +++ b/bp2build/bp2build.go @@ -0,0 +1,77 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "android/soong/android" + "os" +) + +// The Bazel bp2build singleton is responsible for writing .bzl files that are equivalent to +// Android.bp files that are capable of being built with Bazel. +func init() { + android.RegisterBazelConverterPreSingletonType("androidbp_to_build", AndroidBpToBuildSingleton) +} + +func AndroidBpToBuildSingleton() android.Singleton { + return &androidBpToBuildSingleton{ + name: "bp2build", + } +} + +type androidBpToBuildSingleton struct { + name string + outputDir android.OutputPath +} + +func (s *androidBpToBuildSingleton) GenerateBuildActions(ctx android.SingletonContext) { + s.outputDir = android.PathForOutput(ctx, s.name) + android.RemoveAllOutputDir(s.outputDir) + + if !ctx.Config().IsEnvTrue("CONVERT_TO_BAZEL") { + return + } + + ruleShims := CreateRuleShims(android.ModuleTypeFactories()) + + buildToTargets := GenerateSoongModuleTargets(ctx) + + filesToWrite := CreateBazelFiles(ruleShims, buildToTargets) + for _, f := range filesToWrite { + if err := s.writeFile(ctx, f); err != nil { + ctx.Errorf("Failed to write %q (dir %q) due to %q", f.Basename, f.Dir, err) + } + } +} + +func (s *androidBpToBuildSingleton) getOutputPath(ctx android.PathContext, dir string) android.OutputPath { + return s.outputDir.Join(ctx, dir) +} + +func (s *androidBpToBuildSingleton) writeFile(ctx android.PathContext, f BazelFile) error { + return writeReadOnlyFile(ctx, s.getOutputPath(ctx, f.Dir), f.Basename, f.Contents) +} + +// The auto-conversion directory should be read-only, sufficient for bazel query. The files +// are not intended to be edited by end users. +func writeReadOnlyFile(ctx android.PathContext, dir android.OutputPath, baseName, content string) error { + android.CreateOutputDirIfNonexistent(dir, os.ModePerm) + pathToFile := dir.Join(ctx, baseName) + + // 0444 is read-only + err := android.WriteFileToOutputDir(pathToFile, []byte(content), 0444) + + return err +} diff --git a/bp2build/build_conversion.go b/bp2build/build_conversion.go new file mode 100644 index 000000000..03296852f --- /dev/null +++ b/bp2build/build_conversion.go @@ -0,0 +1,305 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "android/soong/android" + "fmt" + "reflect" + "strings" + + "github.com/google/blueprint" + "github.com/google/blueprint/proptools" +) + +type BazelAttributes struct { + Attrs map[string]string +} + +type BazelTarget struct { + name string + content string +} + +type bpToBuildContext interface { + ModuleName(module blueprint.Module) string + ModuleDir(module blueprint.Module) string + ModuleSubDir(module blueprint.Module) string + ModuleType(module blueprint.Module) string + + VisitAllModulesBlueprint(visit func(blueprint.Module)) + VisitDirectDeps(module android.Module, visit func(android.Module)) +} + +// props is an unsorted map. This function ensures that +// the generated attributes are sorted to ensure determinism. +func propsToAttributes(props map[string]string) string { + var attributes string + for _, propName := range android.SortedStringKeys(props) { + if shouldGenerateAttribute(propName) { + attributes += fmt.Sprintf(" %s = %s,\n", propName, props[propName]) + } + } + return attributes +} + +func GenerateSoongModuleTargets(ctx bpToBuildContext) map[string][]BazelTarget { + buildFileToTargets := make(map[string][]BazelTarget) + ctx.VisitAllModulesBlueprint(func(m blueprint.Module) { + dir := ctx.ModuleDir(m) + t := generateSoongModuleTarget(ctx, m) + buildFileToTargets[ctx.ModuleDir(m)] = append(buildFileToTargets[dir], t) + }) + return buildFileToTargets +} + +// Convert a module and its deps and props into a Bazel macro/rule +// representation in the BUILD file. +func generateSoongModuleTarget(ctx bpToBuildContext, m blueprint.Module) BazelTarget { + props := getBuildProperties(ctx, m) + + // TODO(b/163018919): DirectDeps can have duplicate (module, variant) + // items, if the modules are added using different DependencyTag. Figure + // out the implications of that. + depLabels := map[string]bool{} + if aModule, ok := m.(android.Module); ok { + ctx.VisitDirectDeps(aModule, func(depModule android.Module) { + depLabels[qualifiedTargetLabel(ctx, depModule)] = true + }) + } + attributes := propsToAttributes(props.Attrs) + + depLabelList := "[\n" + for depLabel, _ := range depLabels { + depLabelList += fmt.Sprintf(" %q,\n", depLabel) + } + depLabelList += " ]" + + targetName := targetNameWithVariant(ctx, m) + return BazelTarget{ + name: targetName, + content: fmt.Sprintf( + soongModuleTarget, + targetName, + ctx.ModuleName(m), + canonicalizeModuleType(ctx.ModuleType(m)), + ctx.ModuleSubDir(m), + depLabelList, + attributes), + } +} + +func getBuildProperties(ctx bpToBuildContext, m blueprint.Module) BazelAttributes { + var allProps map[string]string + // TODO: this omits properties for blueprint modules (blueprint_go_binary, + // bootstrap_go_binary, bootstrap_go_package), which will have to be handled separately. + if aModule, ok := m.(android.Module); ok { + allProps = ExtractModuleProperties(aModule) + } + + return BazelAttributes{ + Attrs: allProps, + } +} + +// Generically extract module properties and types into a map, keyed by the module property name. +func ExtractModuleProperties(aModule android.Module) map[string]string { + ret := map[string]string{} + + // Iterate over this android.Module's property structs. + for _, properties := range aModule.GetProperties() { + propertiesValue := reflect.ValueOf(properties) + // Check that propertiesValue is a pointer to the Properties struct, like + // *cc.BaseLinkerProperties or *java.CompilerProperties. + // + // propertiesValue can also be type-asserted to the structs to + // manipulate internal props, if needed. + if isStructPtr(propertiesValue.Type()) { + structValue := propertiesValue.Elem() + for k, v := range extractStructProperties(structValue, 0) { + ret[k] = v + } + } else { + panic(fmt.Errorf( + "properties must be a pointer to a struct, got %T", + propertiesValue.Interface())) + } + } + + return ret +} + +func isStructPtr(t reflect.Type) bool { + return t.Kind() == reflect.Ptr && t.Elem().Kind() == reflect.Struct +} + +// prettyPrint a property value into the equivalent Starlark representation +// recursively. +func prettyPrint(propertyValue reflect.Value, indent int) (string, error) { + if isZero(propertyValue) { + // A property value being set or unset actually matters -- Soong does set default + // values for unset properties, like system_shared_libs = ["libc", "libm", "libdl"] at + // https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/linker.go;l=281-287;drc=f70926eef0b9b57faf04c17a1062ce50d209e480 + // + // In Bazel-parlance, we would use "attr.<type>(default = <default value>)" to set the default + // value of unset attributes. + return "", nil + } + + var ret string + switch propertyValue.Kind() { + case reflect.String: + ret = fmt.Sprintf("\"%v\"", escapeString(propertyValue.String())) + case reflect.Bool: + ret = strings.Title(fmt.Sprintf("%v", propertyValue.Interface())) + case reflect.Int, reflect.Uint, reflect.Int64: + ret = fmt.Sprintf("%v", propertyValue.Interface()) + case reflect.Ptr: + return prettyPrint(propertyValue.Elem(), indent) + case reflect.Slice: + ret = "[\n" + for i := 0; i < propertyValue.Len(); i++ { + indexedValue, err := prettyPrint(propertyValue.Index(i), indent+1) + if err != nil { + return "", err + } + + if indexedValue != "" { + ret += makeIndent(indent + 1) + ret += indexedValue + ret += ",\n" + } + } + ret += makeIndent(indent) + ret += "]" + case reflect.Struct: + ret = "{\n" + // Sort and print the struct props by the key. + structProps := extractStructProperties(propertyValue, indent) + for _, k := range android.SortedStringKeys(structProps) { + ret += makeIndent(indent + 1) + ret += fmt.Sprintf("%q: %s,\n", k, structProps[k]) + } + ret += makeIndent(indent) + ret += "}" + case reflect.Interface: + // TODO(b/164227191): implement pretty print for interfaces. + // Interfaces are used for for arch, multilib and target properties. + return "", nil + default: + return "", fmt.Errorf( + "unexpected kind for property struct field: %s", propertyValue.Kind()) + } + return ret, nil +} + +// Converts a reflected property struct value into a map of property names and property values, +// which each property value correctly pretty-printed and indented at the right nest level, +// since property structs can be nested. In Starlark, nested structs are represented as nested +// dicts: https://docs.bazel.build/skylark/lib/dict.html +func extractStructProperties(structValue reflect.Value, indent int) map[string]string { + if structValue.Kind() != reflect.Struct { + panic(fmt.Errorf("Expected a reflect.Struct type, but got %s", structValue.Kind())) + } + + ret := map[string]string{} + structType := structValue.Type() + for i := 0; i < structValue.NumField(); i++ { + field := structType.Field(i) + if shouldSkipStructField(field) { + continue + } + + fieldValue := structValue.Field(i) + if isZero(fieldValue) { + // Ignore zero-valued fields + continue + } + + propertyName := proptools.PropertyNameForField(field.Name) + prettyPrintedValue, err := prettyPrint(fieldValue, indent+1) + if err != nil { + panic( + fmt.Errorf( + "Error while parsing property: %q. %s", + propertyName, + err)) + } + if prettyPrintedValue != "" { + ret[propertyName] = prettyPrintedValue + } + } + + return ret +} + +func isZero(value reflect.Value) bool { + switch value.Kind() { + case reflect.Func, reflect.Map, reflect.Slice: + return value.IsNil() + case reflect.Array: + valueIsZero := true + for i := 0; i < value.Len(); i++ { + valueIsZero = valueIsZero && isZero(value.Index(i)) + } + return valueIsZero + case reflect.Struct: + valueIsZero := true + for i := 0; i < value.NumField(); i++ { + if value.Field(i).CanSet() { + valueIsZero = valueIsZero && isZero(value.Field(i)) + } + } + return valueIsZero + case reflect.Ptr: + if !value.IsNil() { + return isZero(reflect.Indirect(value)) + } else { + return true + } + default: + zeroValue := reflect.Zero(value.Type()) + result := value.Interface() == zeroValue.Interface() + return result + } +} + +func escapeString(s string) string { + s = strings.ReplaceAll(s, "\\", "\\\\") + return strings.ReplaceAll(s, "\"", "\\\"") +} + +func makeIndent(indent int) string { + if indent < 0 { + panic(fmt.Errorf("indent column cannot be less than 0, but got %d", indent)) + } + return strings.Repeat(" ", indent) +} + +func targetNameWithVariant(c bpToBuildContext, logicModule blueprint.Module) string { + name := "" + if c.ModuleSubDir(logicModule) != "" { + // TODO(b/162720883): Figure out a way to drop the "--" variant suffixes. + name = c.ModuleName(logicModule) + "--" + c.ModuleSubDir(logicModule) + } else { + name = c.ModuleName(logicModule) + } + + return strings.Replace(name, "//", "", 1) +} + +func qualifiedTargetLabel(c bpToBuildContext, logicModule blueprint.Module) string { + return fmt.Sprintf("//%s:%s", c.ModuleDir(logicModule), targetNameWithVariant(c, logicModule)) +} diff --git a/bp2build/build_conversion_test.go b/bp2build/build_conversion_test.go new file mode 100644 index 000000000..8230ad8c0 --- /dev/null +++ b/bp2build/build_conversion_test.go @@ -0,0 +1,221 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "android/soong/android" + "testing" +) + +func TestGenerateSoongModuleTargets(t *testing.T) { + testCases := []struct { + bp string + expectedBazelTarget string + }{ + { + bp: `custom { + name: "foo", +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + module_deps = [ + ], +)`, + }, + { + bp: `custom { + name: "foo", + ramdisk: true, +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + module_deps = [ + ], + ramdisk = True, +)`, + }, + { + bp: `custom { + name: "foo", + owner: "a_string_with\"quotes\"_and_\\backslashes\\\\", +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + module_deps = [ + ], + owner = "a_string_with\"quotes\"_and_\\backslashes\\\\", +)`, + }, + { + bp: `custom { + name: "foo", + required: ["bar"], +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + module_deps = [ + ], + required = [ + "bar", + ], +)`, + }, + { + bp: `custom { + name: "foo", + target_required: ["qux", "bazqux"], +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + module_deps = [ + ], + target_required = [ + "qux", + "bazqux", + ], +)`, + }, + { + bp: `custom { + name: "foo", + dist: { + targets: ["goal_foo"], + tag: ".foo", + }, + dists: [ + { + targets: ["goal_bar"], + tag: ".bar", + }, + ], +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + module_deps = [ + ], + dist = { + "tag": ".foo", + "targets": [ + "goal_foo", + ], + }, + dists = [ + { + "tag": ".bar", + "targets": [ + "goal_bar", + ], + }, + ], +)`, + }, + { + bp: `custom { + name: "foo", + required: ["bar"], + target_required: ["qux", "bazqux"], + ramdisk: true, + owner: "custom_owner", + dists: [ + { + tag: ".tag", + targets: ["my_goal"], + }, + ], +} + `, + expectedBazelTarget: `soong_module( + name = "foo", + module_name = "foo", + module_type = "custom", + module_variant = "", + module_deps = [ + ], + dists = [ + { + "tag": ".tag", + "targets": [ + "my_goal", + ], + }, + ], + owner = "custom_owner", + ramdisk = True, + required = [ + "bar", + ], + target_required = [ + "qux", + "bazqux", + ], +)`, + }, + } + + dir := "." + for _, testCase := range testCases { + config := android.TestConfig(buildDir, nil, testCase.bp, nil) + ctx := android.NewTestContext(config) + ctx.RegisterModuleType("custom", customModuleFactory) + ctx.Register() + + _, errs := ctx.ParseFileList(dir, []string{"Android.bp"}) + android.FailIfErrored(t, errs) + _, errs = ctx.PrepareBuildActions(config) + android.FailIfErrored(t, errs) + + bp2BuildCtx := bp2buildBlueprintWrapContext{ + bpCtx: ctx.Context.Context, + } + + bazelTargets := GenerateSoongModuleTargets(&bp2BuildCtx)[dir] + if g, w := len(bazelTargets), 1; g != w { + t.Fatalf("Expected %d bazel target, got %d", w, g) + } + + actualBazelTarget := bazelTargets[0] + if actualBazelTarget.content != testCase.expectedBazelTarget { + t.Errorf( + "Expected generated Bazel target to be '%s', got '%s'", + testCase.expectedBazelTarget, + actualBazelTarget, + ) + } + } +} diff --git a/bp2build/bzl_conversion.go b/bp2build/bzl_conversion.go new file mode 100644 index 000000000..04c45420b --- /dev/null +++ b/bp2build/bzl_conversion.go @@ -0,0 +1,230 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "android/soong/android" + "fmt" + "reflect" + "runtime" + "sort" + "strings" + + "github.com/google/blueprint/proptools" +) + +var ( + // An allowlist of prop types that are surfaced from module props to rule + // attributes. (nested) dictionaries are notably absent here, because while + // Soong supports multi value typed and nested dictionaries, Bazel's rule + // attr() API supports only single-level string_dicts. + allowedPropTypes = map[string]bool{ + "int": true, // e.g. 42 + "bool": true, // e.g. True + "string_list": true, // e.g. ["a", "b"] + "string": true, // e.g. "a" + } +) + +type rule struct { + name string + attrs string +} + +type RuleShim struct { + // The rule class shims contained in a bzl file. e.g. ["cc_object", "cc_library", ..] + rules []string + + // The generated string content of the bzl file. + content string +} + +// Create <module>.bzl containing Bazel rule shims for every module type available in Soong and +// user-specified Go plugins. +// +// This function reuses documentation generation APIs to ensure parity between modules-as-docs +// and modules-as-code, including the names and types of morule properties. +func CreateRuleShims(moduleTypeFactories map[string]android.ModuleFactory) map[string]RuleShim { + ruleShims := map[string]RuleShim{} + for pkg, rules := range generateRules(moduleTypeFactories) { + shim := RuleShim{ + rules: make([]string, 0, len(rules)), + } + shim.content = "load(\"//build/bazel/queryview_rules:providers.bzl\", \"SoongModuleInfo\")\n" + + bzlFileName := strings.ReplaceAll(pkg, "android/soong/", "") + bzlFileName = strings.ReplaceAll(bzlFileName, ".", "_") + bzlFileName = strings.ReplaceAll(bzlFileName, "/", "_") + + for _, r := range rules { + shim.content += fmt.Sprintf(moduleRuleShim, r.name, r.attrs) + shim.rules = append(shim.rules, r.name) + } + sort.Strings(shim.rules) + ruleShims[bzlFileName] = shim + } + return ruleShims +} + +// Generate the content of soong_module.bzl with the rule shim load statements +// and mapping of module_type to rule shim map for every module type in Soong. +func generateSoongModuleBzl(bzlLoads map[string]RuleShim) string { + var loadStmts string + var moduleRuleMap string + for _, bzlFileName := range android.SortedStringKeys(bzlLoads) { + loadStmt := "load(\"//build/bazel/queryview_rules:" + loadStmt += bzlFileName + loadStmt += ".bzl\"" + ruleShim := bzlLoads[bzlFileName] + for _, rule := range ruleShim.rules { + loadStmt += fmt.Sprintf(", %q", rule) + moduleRuleMap += " \"" + rule + "\": " + rule + ",\n" + } + loadStmt += ")\n" + loadStmts += loadStmt + } + + return fmt.Sprintf(soongModuleBzl, loadStmts, moduleRuleMap) +} + +func generateRules(moduleTypeFactories map[string]android.ModuleFactory) map[string][]rule { + // TODO: add shims for bootstrap/blueprint go modules types + + rules := make(map[string][]rule) + // TODO: allow registration of a bzl rule when registring a factory + for _, moduleType := range android.SortedStringKeys(moduleTypeFactories) { + factory := moduleTypeFactories[moduleType] + factoryName := runtime.FuncForPC(reflect.ValueOf(factory).Pointer()).Name() + pkg := strings.Split(factoryName, ".")[0] + attrs := `{ + "module_name": attr.string(mandatory = True), + "module_variant": attr.string(), + "module_deps": attr.label_list(providers = [SoongModuleInfo]), +` + attrs += getAttributes(factory) + attrs += " }," + + r := rule{ + name: canonicalizeModuleType(moduleType), + attrs: attrs, + } + + rules[pkg] = append(rules[pkg], r) + } + return rules +} + +type property struct { + name string + starlarkAttrType string + properties []property +} + +const ( + attributeIndent = " " +) + +func (p *property) attributeString() string { + if !shouldGenerateAttribute(p.name) { + return "" + } + + if _, ok := allowedPropTypes[p.starlarkAttrType]; !ok { + // a struct -- let's just comment out sub-props + s := fmt.Sprintf(attributeIndent+"# %s start\n", p.name) + for _, nestedP := range p.properties { + s += "# " + nestedP.attributeString() + } + s += fmt.Sprintf(attributeIndent+"# %s end\n", p.name) + return s + } + return fmt.Sprintf(attributeIndent+"%q: attr.%s(),\n", p.name, p.starlarkAttrType) +} + +func extractPropertyDescriptionsFromStruct(structType reflect.Type) []property { + properties := make([]property, 0) + for i := 0; i < structType.NumField(); i++ { + field := structType.Field(i) + if shouldSkipStructField(field) { + continue + } + + properties = append(properties, extractPropertyDescriptions(field.Name, field.Type)...) + } + return properties +} + +func extractPropertyDescriptions(name string, t reflect.Type) []property { + name = proptools.PropertyNameForField(name) + + // TODO: handle android:paths tags, they should be changed to label types + + starlarkAttrType := fmt.Sprintf("%s", t.Name()) + props := make([]property, 0) + + switch t.Kind() { + case reflect.Bool, reflect.String: + // do nothing + case reflect.Uint, reflect.Int, reflect.Int64: + starlarkAttrType = "int" + case reflect.Slice: + if t.Elem().Kind() != reflect.String { + // TODO: handle lists of non-strings (currently only list of Dist) + return []property{} + } + starlarkAttrType = "string_list" + case reflect.Struct: + props = extractPropertyDescriptionsFromStruct(t) + case reflect.Ptr: + return extractPropertyDescriptions(name, t.Elem()) + case reflect.Interface: + // Interfaces are used for for arch, multilib and target properties, which are handled at runtime. + // These will need to be handled in a bazel-specific version of the arch mutator. + return []property{} + } + + prop := property{ + name: name, + starlarkAttrType: starlarkAttrType, + properties: props, + } + + return []property{prop} +} + +func getPropertyDescriptions(props []interface{}) []property { + // there may be duplicate properties, e.g. from defaults libraries + propertiesByName := make(map[string]property) + for _, p := range props { + for _, prop := range extractPropertyDescriptionsFromStruct(reflect.ValueOf(p).Elem().Type()) { + propertiesByName[prop.name] = prop + } + } + + properties := make([]property, 0, len(propertiesByName)) + for _, key := range android.SortedStringKeys(propertiesByName) { + properties = append(properties, propertiesByName[key]) + } + + return properties +} + +func getAttributes(factory android.ModuleFactory) string { + attrs := "" + for _, p := range getPropertyDescriptions(factory().GetProperties()) { + attrs += p.attributeString() + } + return attrs +} diff --git a/bp2build/bzl_conversion_test.go b/bp2build/bzl_conversion_test.go new file mode 100644 index 000000000..8bea3f6cf --- /dev/null +++ b/bp2build/bzl_conversion_test.go @@ -0,0 +1,208 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "android/soong/android" + "io/ioutil" + "os" + "strings" + "testing" +) + +var buildDir string + +func setUp() { + var err error + buildDir, err = ioutil.TempDir("", "bazel_queryview_test") + if err != nil { + panic(err) + } +} + +func tearDown() { + os.RemoveAll(buildDir) +} + +func TestMain(m *testing.M) { + run := func() int { + setUp() + defer tearDown() + + return m.Run() + } + + os.Exit(run()) +} + +func TestGenerateModuleRuleShims(t *testing.T) { + moduleTypeFactories := map[string]android.ModuleFactory{ + "custom": customModuleFactoryBase, + "custom_test": customTestModuleFactoryBase, + "custom_defaults": customDefaultsModuleFactoryBasic, + } + ruleShims := CreateRuleShims(moduleTypeFactories) + + if len(ruleShims) != 1 { + t.Errorf("Expected to generate 1 rule shim, but got %d", len(ruleShims)) + } + + ruleShim := ruleShims["bp2build"] + expectedRules := []string{ + "custom", + "custom_defaults", + "custom_test_", + } + + if len(ruleShim.rules) != len(expectedRules) { + t.Errorf("Expected %d rules, but got %d", len(expectedRules), len(ruleShim.rules)) + } + + for i, rule := range ruleShim.rules { + if rule != expectedRules[i] { + t.Errorf("Expected rule shim to contain %s, but got %s", expectedRules[i], rule) + } + } + expectedBzl := `load("//build/bazel/queryview_rules:providers.bzl", "SoongModuleInfo") + +def _custom_impl(ctx): + return [SoongModuleInfo()] + +custom = rule( + implementation = _custom_impl, + attrs = { + "module_name": attr.string(mandatory = True), + "module_variant": attr.string(), + "module_deps": attr.label_list(providers = [SoongModuleInfo]), + "bool_prop": attr.bool(), + "bool_ptr_prop": attr.bool(), + "int64_ptr_prop": attr.int(), + # nested_props start +# "nested_prop": attr.string(), + # nested_props end + # nested_props_ptr start +# "nested_prop": attr.string(), + # nested_props_ptr end + "string_list_prop": attr.string_list(), + "string_prop": attr.string(), + "string_ptr_prop": attr.string(), + }, +) + +def _custom_defaults_impl(ctx): + return [SoongModuleInfo()] + +custom_defaults = rule( + implementation = _custom_defaults_impl, + attrs = { + "module_name": attr.string(mandatory = True), + "module_variant": attr.string(), + "module_deps": attr.label_list(providers = [SoongModuleInfo]), + "bool_prop": attr.bool(), + "bool_ptr_prop": attr.bool(), + "int64_ptr_prop": attr.int(), + # nested_props start +# "nested_prop": attr.string(), + # nested_props end + # nested_props_ptr start +# "nested_prop": attr.string(), + # nested_props_ptr end + "string_list_prop": attr.string_list(), + "string_prop": attr.string(), + "string_ptr_prop": attr.string(), + }, +) + +def _custom_test__impl(ctx): + return [SoongModuleInfo()] + +custom_test_ = rule( + implementation = _custom_test__impl, + attrs = { + "module_name": attr.string(mandatory = True), + "module_variant": attr.string(), + "module_deps": attr.label_list(providers = [SoongModuleInfo]), + "bool_prop": attr.bool(), + "bool_ptr_prop": attr.bool(), + "int64_ptr_prop": attr.int(), + # nested_props start +# "nested_prop": attr.string(), + # nested_props end + # nested_props_ptr start +# "nested_prop": attr.string(), + # nested_props_ptr end + "string_list_prop": attr.string_list(), + "string_prop": attr.string(), + "string_ptr_prop": attr.string(), + # test_prop start +# "test_string_prop": attr.string(), + # test_prop end + }, +) +` + + if ruleShim.content != expectedBzl { + t.Errorf( + "Expected the generated rule shim bzl to be:\n%s\nbut got:\n%s", + expectedBzl, + ruleShim.content) + } +} + +func TestGenerateSoongModuleBzl(t *testing.T) { + ruleShims := map[string]RuleShim{ + "file1": RuleShim{ + rules: []string{"a", "b"}, + content: "irrelevant", + }, + "file2": RuleShim{ + rules: []string{"c", "d"}, + content: "irrelevant", + }, + } + files := CreateBazelFiles(ruleShims, make(map[string][]BazelTarget)) + + var actualSoongModuleBzl BazelFile + for _, f := range files { + if f.Basename == "soong_module.bzl" { + actualSoongModuleBzl = f + } + } + + expectedLoad := `load("//build/bazel/queryview_rules:file1.bzl", "a", "b") +load("//build/bazel/queryview_rules:file2.bzl", "c", "d") +` + expectedRuleMap := `soong_module_rule_map = { + "a": a, + "b": b, + "c": c, + "d": d, +}` + if !strings.Contains(actualSoongModuleBzl.Contents, expectedLoad) { + t.Errorf( + "Generated soong_module.bzl:\n\n%s\n\n"+ + "Could not find the load statement in the generated soong_module.bzl:\n%s", + actualSoongModuleBzl.Contents, + expectedLoad) + } + + if !strings.Contains(actualSoongModuleBzl.Contents, expectedRuleMap) { + t.Errorf( + "Generated soong_module.bzl:\n\n%s\n\n"+ + "Could not find the module -> rule map in the generated soong_module.bzl:\n%s", + actualSoongModuleBzl.Contents, + expectedRuleMap) + } +} diff --git a/bp2build/conversion.go b/bp2build/conversion.go new file mode 100644 index 000000000..cdfb38b14 --- /dev/null +++ b/bp2build/conversion.go @@ -0,0 +1,118 @@ +package bp2build + +import ( + "android/soong/android" + "reflect" + "sort" + "strings" + + "github.com/google/blueprint/proptools" +) + +type BazelFile struct { + Dir string + Basename string + Contents string +} + +func CreateBazelFiles( + ruleShims map[string]RuleShim, + buildToTargets map[string][]BazelTarget) []BazelFile { + files := make([]BazelFile, 0, len(ruleShims)+len(buildToTargets)+numAdditionalFiles) + + // Write top level files: WORKSPACE and BUILD. These files are empty. + files = append(files, newFile("", "WORKSPACE", "")) + // Used to denote that the top level directory is a package. + files = append(files, newFile("", "BUILD", "")) + + files = append(files, newFile(bazelRulesSubDir, "BUILD", "")) + files = append(files, newFile(bazelRulesSubDir, "providers.bzl", providersBzl)) + + for bzlFileName, ruleShim := range ruleShims { + files = append(files, newFile(bazelRulesSubDir, bzlFileName+".bzl", ruleShim.content)) + } + files = append(files, newFile(bazelRulesSubDir, "soong_module.bzl", generateSoongModuleBzl(ruleShims))) + + files = append(files, createBuildFiles(buildToTargets)...) + + return files +} + +func createBuildFiles(buildToTargets map[string][]BazelTarget) []BazelFile { + files := make([]BazelFile, 0, len(buildToTargets)) + for _, dir := range android.SortedStringKeys(buildToTargets) { + content := soongModuleLoad + targets := buildToTargets[dir] + sort.Slice(targets, func(i, j int) bool { return targets[i].name < targets[j].name }) + for _, t := range targets { + content += "\n\n" + content += t.content + } + files = append(files, newFile(dir, "BUILD.bazel", content)) + } + return files +} + +func newFile(dir, basename, content string) BazelFile { + return BazelFile{ + Dir: dir, + Basename: basename, + Contents: content, + } +} + +const ( + bazelRulesSubDir = "build/bazel/queryview_rules" + + // additional files: + // * workspace file + // * base BUILD file + // * rules BUILD file + // * rules providers.bzl file + // * rules soong_module.bzl file + numAdditionalFiles = 5 +) + +var ( + // Certain module property names are blocklisted/ignored here, for the reasons commented. + ignoredPropNames = map[string]bool{ + "name": true, // redundant, since this is explicitly generated for every target + "from": true, // reserved keyword + "in": true, // reserved keyword + "arch": true, // interface prop type is not supported yet. + "multilib": true, // interface prop type is not supported yet. + "target": true, // interface prop type is not supported yet. + "visibility": true, // Bazel has native visibility semantics. Handle later. + "features": true, // There is already a built-in attribute 'features' which cannot be overridden. + } +) + +func shouldGenerateAttribute(prop string) bool { + return !ignoredPropNames[prop] +} + +func shouldSkipStructField(field reflect.StructField) bool { + if field.PkgPath != "" { + // Skip unexported fields. Some properties are + // internal to Soong only, and these fields do not have PkgPath. + return true + } + // fields with tag `blueprint:"mutated"` are exported to enable modification in mutators, etc + // but cannot be set in a .bp file + if proptools.HasTag(field, "blueprint", "mutated") { + return true + } + return false +} + +// FIXME(b/168089390): In Bazel, rules ending with "_test" needs to be marked as +// testonly = True, forcing other rules that depend on _test rules to also be +// marked as testonly = True. This semantic constraint is not present in Soong. +// To work around, rename "*_test" rules to "*_test_". +func canonicalizeModuleType(moduleName string) string { + if strings.HasSuffix(moduleName, "_test") { + return moduleName + "_" + } + + return moduleName +} diff --git a/bp2build/conversion_test.go b/bp2build/conversion_test.go new file mode 100644 index 000000000..a38fa6a55 --- /dev/null +++ b/bp2build/conversion_test.go @@ -0,0 +1,73 @@ +// Copyright 2020 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bp2build + +import ( + "sort" + "testing" +) + +func TestCreateBazelFiles_AddsTopLevelFiles(t *testing.T) { + files := CreateBazelFiles(map[string]RuleShim{}, map[string][]BazelTarget{}) + expectedFilePaths := []struct { + dir string + basename string + }{ + { + dir: "", + basename: "BUILD", + }, + { + dir: "", + basename: "WORKSPACE", + }, + { + dir: bazelRulesSubDir, + basename: "BUILD", + }, + { + dir: bazelRulesSubDir, + basename: "providers.bzl", + }, + { + dir: bazelRulesSubDir, + basename: "soong_module.bzl", + }, + } + + if g, w := len(files), len(expectedFilePaths); g != w { + t.Errorf("Expected %d files, got %d", w, g) + } + + sort.Slice(files, func(i, j int) bool { + if dir1, dir2 := files[i].Dir, files[j].Dir; dir1 == dir2 { + return files[i].Basename < files[j].Basename + } else { + return dir1 < dir2 + } + }) + + for i := range files { + if g, w := files[i], expectedFilePaths[i]; g.Dir != w.dir || g.Basename != w.basename { + t.Errorf("Did not find expected file %s/%s", g.Dir, g.Basename) + } else if g.Basename == "BUILD" || g.Basename == "WORKSPACE" { + if g.Contents != "" { + t.Errorf("Expected %s to have no content.", g) + } + } else if g.Contents == "" { + t.Errorf("Contents of %s unexpected empty.", g) + } + } +} diff --git a/bp2build/testing.go b/bp2build/testing.go new file mode 100644 index 000000000..160412de8 --- /dev/null +++ b/bp2build/testing.go @@ -0,0 +1,136 @@ +package bp2build + +import ( + "android/soong/android" + + "github.com/google/blueprint" +) + +type nestedProps struct { + Nested_prop string +} + +type customProps struct { + Bool_prop bool + Bool_ptr_prop *bool + // Ensure that properties tagged `blueprint:mutated` are omitted + Int_prop int `blueprint:"mutated"` + Int64_ptr_prop *int64 + String_prop string + String_ptr_prop *string + String_list_prop []string + + Nested_props nestedProps + Nested_props_ptr *nestedProps +} + +type customModule struct { + android.ModuleBase + + props customProps +} + +// OutputFiles is needed because some instances of this module use dist with a +// tag property which requires the module implements OutputFileProducer. +func (m *customModule) OutputFiles(tag string) (android.Paths, error) { + return android.PathsForTesting("path" + tag), nil +} + +func (m *customModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // nothing for now. +} + +func customModuleFactoryBase() android.Module { + module := &customModule{} + module.AddProperties(&module.props) + return module +} + +func customModuleFactory() android.Module { + m := customModuleFactoryBase() + android.InitAndroidModule(m) + return m +} + +type testProps struct { + Test_prop struct { + Test_string_prop string + } +} + +type customTestModule struct { + android.ModuleBase + + props customProps + test_props testProps +} + +func (m *customTestModule) GenerateAndroidBuildActions(ctx android.ModuleContext) { + // nothing for now. +} + +func customTestModuleFactoryBase() android.Module { + m := &customTestModule{} + m.AddProperties(&m.props) + m.AddProperties(&m.test_props) + return m +} + +func customTestModuleFactory() android.Module { + m := customTestModuleFactoryBase() + android.InitAndroidModule(m) + return m +} + +type customDefaultsModule struct { + android.ModuleBase + android.DefaultsModuleBase +} + +func customDefaultsModuleFactoryBase() android.DefaultsModule { + module := &customDefaultsModule{} + module.AddProperties(&customProps{}) + return module +} + +func customDefaultsModuleFactoryBasic() android.Module { + return customDefaultsModuleFactoryBase() +} + +func customDefaultsModuleFactory() android.Module { + m := customDefaultsModuleFactoryBase() + android.InitDefaultsModule(m) + return m +} + +type bp2buildBlueprintWrapContext struct { + bpCtx *blueprint.Context +} + +func (ctx *bp2buildBlueprintWrapContext) ModuleName(module blueprint.Module) string { + return ctx.bpCtx.ModuleName(module) +} + +func (ctx *bp2buildBlueprintWrapContext) ModuleDir(module blueprint.Module) string { + return ctx.bpCtx.ModuleDir(module) +} + +func (ctx *bp2buildBlueprintWrapContext) ModuleSubDir(module blueprint.Module) string { + return ctx.bpCtx.ModuleSubDir(module) +} + +func (ctx *bp2buildBlueprintWrapContext) ModuleType(module blueprint.Module) string { + return ctx.bpCtx.ModuleType(module) +} + +func (ctx *bp2buildBlueprintWrapContext) VisitAllModulesBlueprint(visit func(blueprint.Module)) { + ctx.bpCtx.VisitAllModules(visit) +} + +func (ctx *bp2buildBlueprintWrapContext) VisitDirectDeps(module android.Module, visit func(android.Module)) { + ctx.bpCtx.VisitDirectDeps(module, func(m blueprint.Module) { + if aModule, ok := m.(android.Module); ok { + visit(aModule) + } + }) +} |