diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2022-05-10 06:51:55 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2022-05-10 06:51:55 +0000 |
commit | 2ef4d783b9ef0a57613e2c75dd018e439b42ac5e (patch) | |
tree | 74393ec5dcc68491e5308c0f4d3bdc3fa1f58fcb | |
parent | 479e50d33ff6852449772ace5d54f9658dddf71e (diff) | |
parent | 92c1d863d4d44c61c934ddbb4392fcf3527f0639 (diff) | |
download | bazel-android13-mainline-wifi-release.tar.gz |
Snap for 8564071 from 92c1d863d4d44c61c934ddbb4392fcf3527f0639 to mainline-wifi-releaseaml_wif_331910020aml_wif_331810010aml_wif_331710030aml_wif_331613000aml_wif_331511020aml_wif_331414000aml_wif_331310070aml_wif_331112000aml_wif_331016070aml_wif_330910030aml_wif_330810040android13-mainline-wifi-release
Change-Id: I16ba12e150e30f1448a397198f7b2c0cdfbabf1f
163 files changed, 10298 insertions, 949 deletions
diff --git a/bazel.WORKSPACE b/bazel.WORKSPACE index 55f72624..4b889d10 100644 --- a/bazel.WORKSPACE +++ b/bazel.WORKSPACE @@ -1,16 +1,42 @@ -toplevel_output_directories(paths = ["out"]) - -load("//build/bazel/rules:lunch.bzl", "lunch") load("//build/bazel/rules:soong_injection.bzl", "soong_injection_repository") - -lunch() +load("//build/bazel/rules:make_injection.bzl", "make_injection_repository") register_toolchains( - "//prebuilts/clang/host/linux-x86:all" + "//prebuilts/build-tools:py_toolchain", + "//prebuilts/clang/host/linux-x86:all", ) +# This repository provides files that Soong emits during bp2build (other than +# converted BUILD files), mostly .bzl files containing constants to support the +# converted BUILD files. soong_injection_repository(name="soong_injection") +# This is a repository rule to allow Bazel builds to depend on Soong-built +# prebuilts for migration purposes. +make_injection_repository( + name = "make_injection", + binaries = [ + # APEX tools + "apex_compression_tool", + "apexer", + "conv_apex_manifest", + "deapexer", + "sefcontext_compile", + ], + target_module_files = { + # For APEX comparisons + "com.android.tzdata": ["system/apex/com.android.tzdata.apex"], + "com.android.runtime": ["system/apex/com.android.runtime.apex"], + "com.android.adbd": ["system/apex/com.android.adbd.capex"], + "build.bazel.examples.apex.minimal": ["system/product/apex/build.bazel.examples.apex.minimal.apex"], + }, + watch_android_bp_files = [ + "//:build/bazel/examples/apex/minimal/Android.bp", # for build.bazel.examples.apex.minimal + "//:packages/modules/adbd/apex/Android.bp", # for com.android.adbd + # TODO(b/210399979) - add the other .bp files to watch for the other modules built in these rule + ], +) + local_repository( name = "rules_cc", path = "build/bazel/rules_cc", @@ -18,7 +44,7 @@ local_repository( local_repository( name = "bazel_skylib", - path = "build/bazel/bazel_skylib", + path = "external/bazel-skylib", ) local_repository( @@ -33,6 +59,9 @@ register_toolchains( # For native android_binary "//prebuilts/sdk:android_sdk_tools_for_native_android_binary", + + # For APEX rules + "//build/bazel/rules/apex:all" ) bind( @@ -44,3 +73,19 @@ bind( name = "android/dx_jar_import", actual = "//prebuilts/sdk:dx_jar_import", ) + +# The r8.jar in prebuilts/r8 happens to have the d8 classes needed +# for Android app building, whereas the d8.jar in prebuilts/sdk/tools doesn't. +bind( + name = "android/d8_jar_import", + actual = "//prebuilts/r8:r8_jar_import", +) + +# TODO(b/201242197): Avoid downloading remote_coverage_tools (on CI) by creating +# a stub workspace. Test rules (e.g. sh_test) depend on this external dep, but +# we don't support coverage yet. Either vendor the external dep into AOSP, or +# cut the dependency from test rules to the external repo. +local_repository( + name = "remote_coverage_tools", + path = "build/bazel/rules/coverage/remote_coverage_tools", +) @@ -1,5 +1,7 @@ #!/bin/bash +set -eo pipefail + # TODO: Refactor build/make/envsetup.sh to make gettop() available elsewhere function gettop { @@ -56,19 +58,71 @@ case $(uname -s) in ANDROID_BAZEL_PATH="${TOP}/prebuilts/bazel/darwin-x86_64/bazel" ANDROID_BAZELRC_NAME="darwin.bazelrc" ANDROID_BAZEL_JDK_PATH="${TOP}/prebuilts/jdk/jdk11/darwin-x86" + + # Lock down PATH in action execution environment, thereby removing + # Bazel's default /bin, /usr/bin, /usr/local/bin and ensuring + # hermeticity from the system. + # + # The new PATH components are: + # + # - prebuilts/build-tools/path: contains checked-in tools that can be + # used as executables in actions. + # + # - out/.path: a special directory created by path_interposer with + # config from ui/build/paths/config.go for allowlisting specific + # binaries not in prebuilts/build-tools/path, but on the host system. + # If one runs Bazel without soong_ui, then this directory wouldn't + # exist, making standalone Bazel execution's PATH variable stricter than + # Bazel execution within soong_ui. + RESTRICTED_PATH="${TOP}/prebuilts/build-tools/path/darwin-x86:${TOP}/out/.path" ;; Linux) ANDROID_BAZEL_PATH="${TOP}/prebuilts/bazel/linux-x86_64/bazel" ANDROID_BAZELRC_NAME="linux.bazelrc" ANDROID_BAZEL_JDK_PATH="${TOP}/prebuilts/jdk/jdk11/linux-x86" + RESTRICTED_PATH="${TOP}/prebuilts/build-tools/path/linux-x86:${TOP}/out/.path" ;; *) - ANDROID_BAZEL_PATH= - ANDROID_BAZELRC_NAME= - ANDROID_BAZEL_JDK_PATH= + >&2 echo "Bazel is supported on Linux and Darwin only. Your OS is not supported for Bazel usage, based on 'uname -s': $(uname -s)" + exit 1 ;; esac +function verify_soong_outputs_exist() { + local to_check="${ABSOLUTE_OUT_DIR}/.path" + local no_soong=0 + if [[ ! -d "${to_check}" ]]; then + no_soong=1 + fi + + local bazel_configs=( + "bp2build" + "queryview" + ) + local valid_bazel_config=0 + for c in "${bazel_configs[@]}" + do + if [[ -d "${ABSOLUTE_OUT_DIR}/soong/""${c}" ]]; then + valid_bazel_config=1 + fi + done + + if [[ "${no_soong}" -eq "1" || "${valid_bazel_config}" -eq "0" ]]; then + >&2 echo "Error: missing generated Bazel files. Have you run bp2build or queryview?" + >&2 echo "Run bp2build with the command: m bp2build" + >&2 echo "Run queryview with the command: m queryview" + >&2 echo "Alternatively, for non-queryview applications, invoke Bazel using 'b' with the command: source envsetup.sh; b query/build/test <targets>" + exit 1 + fi +} + +function create_bazelrc() { + cat > "${ABSOLUTE_OUT_DIR}/bazel/path.bazelrc" <<EOF + # This file is generated by tools/bazel. Do not edit manually. +build --action_env=PATH=${RESTRICTED_PATH} +EOF +} + case "x${ANDROID_BAZELRC_PATH}" in x) # Path not provided, use default. @@ -111,13 +165,34 @@ else exit 1 fi ->&2 echo "WARNING: Bazel support for the Android Platform is experimental and is undergoing development." ->&2 echo "WARNING: Currently, build stability is not guaranteed. Thank you." ->&2 echo +ABSOLUTE_OUT_DIR="$(getoutdir)" + +# In order to be able to load JNI libraries, this directory needs to exist +mkdir -p "${ABSOLUTE_OUT_DIR}/bazel/javatmp" + +ADDITIONAL_FLAGS=() +if [[ "${STANDALONE_BAZEL}" =~ ^(true|TRUE|1)$ ]]; then + # STANDALONE_BAZEL is set. + >&2 echo "WARNING: Using Bazel in standalone mode. This mode is not integrated with Soong and Make, and is not supported" + >&2 echo "for Android Platform builds. Use this mode at your own risk." + >&2 echo +else + # STANDALONE_BAZEL is not set. + >&2 echo "WARNING: Bazel support for the Android Platform is experimental and is undergoing development." + >&2 echo "WARNING: Currently, build stability is not guaranteed. Thank you." + >&2 echo + + # Generate a bazelrc with dynamic content, like the absolute path to PATH variable values. + create_bazelrc + # Check that the Bazel synthetic workspace and other required inputs exist before handing over control to Bazel. + verify_soong_outputs_exist + ADDITIONAL_FLAGS+=("--bazelrc=${ABSOLUTE_OUT_DIR}/bazel/path.bazelrc") +fi -ABSOLUTE_OUT_DIR="$(getoutdir)" \ - "${ANDROID_BAZEL_PATH}" \ +JAVA_HOME="${ANDROID_BAZEL_JDK_PATH}" "${ANDROID_BAZEL_PATH}" \ --server_javabase="${ANDROID_BAZEL_JDK_PATH}" \ - --output_user_root="$(getoutdir)/bazel/output_user_root" \ + --output_user_root="${ABSOLUTE_OUT_DIR}/bazel/output_user_root" \ + --host_jvm_args=-Djava.io.tmpdir="${ABSOLUTE_OUT_DIR}/bazel/javatmp" \ --bazelrc="${ANDROID_BAZELRC_PATH}" \ + "${ADDITIONAL_FLAGS[@]}" \ "$@" diff --git a/bazel_skylib/BUILD b/bazel_skylib/BUILD deleted file mode 100644 index c29b0f3c..00000000 --- a/bazel_skylib/BUILD +++ /dev/null @@ -1,2 +0,0 @@ -# Divergence from bazel_skylib: Use a stub BUILD file, as there are reduced -# dependencies in this fork. diff --git a/bazel_skylib/README.md b/bazel_skylib/README.md deleted file mode 100644 index 577c05eb..00000000 --- a/bazel_skylib/README.md +++ /dev/null @@ -1,10 +0,0 @@ -This directory and its subdirectories are a partial fork of the -[`bazel-skylib`](https://github.com/bazelbuild/bazel-skylib) -github repository, for experimental use with Bazel builds. - -Not all files in `bazel-skylib` are included. When a file in this directory diverges -from `bazel-skylib`, add a comment containing `Divergence from bazel-skylib` which -explains the need for this divergence. - -It should be a goal to follow HEAD `bazel-skylib` as closely as possible, with -necessary changes made upstream ASAP. diff --git a/bazel_skylib/WORKSPACE b/bazel_skylib/WORKSPACE deleted file mode 100644 index d78cfad6..00000000 --- a/bazel_skylib/WORKSPACE +++ /dev/null @@ -1,78 +0,0 @@ -workspace(name = "bazel_skylib") - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") - -http_archive( - name = "rules_pkg", - urls = [ - "https://github.com/bazelbuild/rules_pkg/releases/download/0.2.5/rules_pkg-0.2.5.tar.gz", - "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.2.5/rules_pkg-0.2.5.tar.gz", - ], - sha256 = "352c090cc3d3f9a6b4e676cf42a6047c16824959b438895a76c2989c6d7c246a", -) -load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") -rules_pkg_dependencies() - -maybe( - name = "bazel_federation", - repo_rule = http_archive, - sha256 = "b10529fcf8a464591e845588348533981e948315b706183481e0d076afe2fa3c", - url = "https://github.com/bazelbuild/bazel-federation/releases/download/0.0.2/bazel_federation-0.0.2.tar.gz", -) - -load("@bazel_federation//:repositories.bzl", "bazel_skylib_deps", "rules_go") - -bazel_skylib_deps() - -rules_go() - -load("@bazel_federation//setup:bazel_skylib.bzl", "bazel_skylib_setup") - -bazel_skylib_setup() - -load("@bazel_federation//setup:rules_go.bzl", "rules_go_setup") - -rules_go_setup() - -# Below this line is for documentation generation only, -# and should thus not be included by dependencies on -# bazel-skylib. - -load("//:internal_deps.bzl", "bazel_skylib_internal_deps") - -bazel_skylib_internal_deps() - -load("//:internal_setup.bzl", "bazel_skylib_internal_setup") - -bazel_skylib_internal_setup() - -maybe( - name = "rules_cc", - repo_rule = http_archive, - sha256 = "b4b2a2078bdb7b8328d843e8de07d7c13c80e6c89e86a09d6c4b424cfd1aaa19", - strip_prefix = "rules_cc-cb2dfba6746bfa3c3705185981f3109f0ae1b893", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_cc/archive/cb2dfba6746bfa3c3705185981f3109f0ae1b893.zip", - "https://github.com/bazelbuild/rules_cc/archive/cb2dfba6746bfa3c3705185981f3109f0ae1b893.zip", - ], -) - -# Provide a repository hint for Gazelle to inform it that the go package -# github.com/bazelbuild/rules_go is available from io_bazel_rules_go and it -# doesn't need to duplicatively fetch it. -# gazelle:repository go_repository name=io_bazel_rules_go importpath=github.com/bazelbuild/rules_go -http_archive( - name = "bazel_gazelle", - sha256 = "bfd86b3cbe855d6c16c6fce60d76bd51f5c8dbc9cfcaef7a2bb5c1aafd0710e8", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.21.0/bazel-gazelle-v0.21.0.tar.gz", - "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.21.0/bazel-gazelle-v0.21.0.tar.gz", - ], -) -# Another Gazelle repository hint. -# gazelle:repository go_repository name=bazel_gazelle importpath=github.com/bazelbuild/bazel-gazelle/testtools - -load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") - -gazelle_dependencies() diff --git a/bazel_skylib/bzl_library.bzl b/bazel_skylib/bzl_library.bzl deleted file mode 100644 index 37a2329d..00000000 --- a/bazel_skylib/bzl_library.bzl +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2017 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Skylib module containing a library rule for aggregating rules files.""" - -StarlarkLibraryInfo = provider( - "Information on contained Starlark rules.", - fields = { - "srcs": "Top level rules files.", - "transitive_srcs": "Transitive closure of rules files required for " + - "interpretation of the srcs", - }, -) - -def _bzl_library_impl(ctx): - deps_files = [x.files for x in ctx.attr.deps] - all_files = depset(ctx.files.srcs, order = "postorder", transitive = deps_files) - return [ - # All dependent files should be listed in both `files` and in `runfiles`; - # this ensures that a `bzl_library` can be referenced as `data` from - # a separate program, or from `tools` of a genrule(). - DefaultInfo( - files = all_files, - runfiles = ctx.runfiles(transitive_files = all_files), - ), - - # We also define our own provider struct, for aggregation and testing. - StarlarkLibraryInfo( - srcs = ctx.files.srcs, - transitive_srcs = all_files, - ), - ] - -bzl_library = rule( - implementation = _bzl_library_impl, - attrs = { - "srcs": attr.label_list( - allow_files = [".bzl"], - doc = "List of `.bzl` files that are processed to create this target.", - ), - "deps": attr.label_list( - allow_files = [".bzl"], - providers = [ - [StarlarkLibraryInfo], - ], - doc = """List of other `bzl_library` targets that are required by the -Starlark files listed in `srcs`.""", - ), - }, - doc = """Creates a logical collection of Starlark .bzl files. -Example: - Suppose your project has the following structure: - ``` - [workspace]/ - WORKSPACE - BUILD - checkstyle/ - BUILD - checkstyle.bzl - lua/ - BUILD - lua.bzl - luarocks.bzl - ``` - In this case, you can have `bzl_library` targets in `checkstyle/BUILD` and - `lua/BUILD`: - `checkstyle/BUILD`: - ```python - load("@bazel_skylib//:bzl_library.bzl", "bzl_library") - bzl_library( - name = "checkstyle-rules", - srcs = ["checkstyle.bzl"], - ) - ``` - `lua/BUILD`: - ```python - load("@bazel_skylib//:bzl_library.bzl", "bzl_library") - bzl_library( - name = "lua-rules", - srcs = [ - "lua.bzl", - "luarocks.bzl", - ], - ) - ``` -""", -) diff --git a/bazel_skylib/rules/BUILD b/bazel_skylib/rules/BUILD deleted file mode 100644 index beb0baab..00000000 --- a/bazel_skylib/rules/BUILD +++ /dev/null @@ -1,2 +0,0 @@ -# Divergence from rules_cc: Use a stub BUILD file, as there are reduced -# dependencies in this fork. diff --git a/bazel_skylib/rules/common_settings.bzl b/bazel_skylib/rules/common_settings.bzl deleted file mode 100644 index 2e98d085..00000000 --- a/bazel_skylib/rules/common_settings.bzl +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2019 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Common build setting rules -These rules return a BuildSettingInfo with the value of the build setting. -For label-typed settings, use the native label_flag and label_setting rules. -More documentation on how to use build settings at -https://docs.bazel.build/versions/master/skylark/config.html#user-defined-build-settings -""" - -BuildSettingInfo = provider( - doc = "A singleton provider that contains the raw value of a build setting", - fields = { - "value": "The value of the build setting in the current configuration. " + - "This value may come from the command line or an upstream transition, " + - "or else it will be the build setting's default.", - }, -) - -def _impl(ctx): - return BuildSettingInfo(value = ctx.build_setting_value) - -int_flag = rule( - implementation = _impl, - build_setting = config.int(flag = True), - doc = "An int-typed build setting that can be set on the command line", -) - -int_setting = rule( - implementation = _impl, - build_setting = config.int(), - doc = "An int-typed build setting that cannot be set on the command line", -) - -bool_flag = rule( - implementation = _impl, - build_setting = config.bool(flag = True), - doc = "A bool-typed build setting that can be set on the command line", -) - -bool_setting = rule( - implementation = _impl, - build_setting = config.bool(), - doc = "A bool-typed build setting that cannot be set on the command line", -) - -string_list_flag = rule( - implementation = _impl, - build_setting = config.string_list(flag = True), - doc = "A string list-typed build setting that can be set on the command line", -) - -string_list_setting = rule( - implementation = _impl, - build_setting = config.string_list(), - doc = "A string list-typed build setting that cannot be set on the command line", -) - -def _string_impl(ctx): - allowed_values = ctx.attr.values - value = ctx.build_setting_value - if len(allowed_values) == 0 or value in ctx.attr.values: - return BuildSettingInfo(value = value) - else: - fail("Error setting " + str(ctx.label) + ": invalid value '" + value + "'. Allowed values are " + str(allowed_values)) - -string_flag = rule( - implementation = _string_impl, - build_setting = config.string(flag = True), - attrs = { - "values": attr.string_list( - doc = "The list of allowed values for this setting. An error is raised if any other value is given.", - ), - }, - doc = "A string-typed build setting that can be set on the command line", -) - -string_setting = rule( - implementation = _string_impl, - build_setting = config.string(), - attrs = { - "values": attr.string_list( - doc = "The list of allowed values for this setting. An error is raised if any other value is given.", - ), - }, - doc = "A string-typed build setting that cannot be set on the command line", -) diff --git a/ci/bp2build.sh b/ci/bp2build.sh index 474ec0bf..8f9e7ac9 100755 --- a/ci/bp2build.sh +++ b/ci/bp2build.sh @@ -1,9 +1,14 @@ #!/bin/bash -eux -# Verifies that bp2build-generated BUILD files for bionic (and its dependencies) -# result in successful Bazel builds. +# Verifies that bp2build-generated BUILD files result in successful Bazel +# builds. +# # This verification script is designed to be used for continuous integration # tests, though may also be used for manual developer verification. +####### +# Setup +####### + if [[ -z ${DIST_DIR+x} ]]; then echo "DIST_DIR not set. Using out/dist. This should only be used for manual developer testing." DIST_DIR="out/dist" @@ -11,40 +16,115 @@ fi # Generate BUILD files into out/soong/bp2build AOSP_ROOT="$(dirname $0)/../../.." -GENERATE_BAZEL_FILES=true "${AOSP_ROOT}/build/soong/soong_ui.bash" --make-mode nothing --skip-soong-tests +"${AOSP_ROOT}/build/soong/soong_ui.bash" --make-mode BP2BUILD_VERBOSE=1 --skip-soong-tests bp2build dist + +# Dist the entire workspace of generated BUILD files, rooted from +# out/soong/bp2build. This is done early so it's available even if builds/tests +# fail. +tar -czf "${DIST_DIR}/bp2build_generated_workspace.tar.gz" -C out/soong/bp2build . # Remove the ninja_build output marker file to communicate to buildbot that this is not a regular Ninja build, and its # output should not be parsed as such. rm -f out/ninja_build -# We could create .bazelrc files and use them on buildbots with --bazelrc, but -# it's simpler to use a list for now. -BUILD_FLAGS_LIST=( - --color=no - --curses=no - --show_progress_rate_limit=5 +# Before you add flags to this list, cosnider adding it to the "ci" bazelrc +# config instead of this list so that flags are not duplicated between scripts +# and bazelrc, and bazelrc is the Bazel-native way of organizing flags. +FLAGS_LIST=( --config=bp2build + --config=ci ) -BUILD_FLAGS="${BUILD_FLAGS_LIST[@]}" - -TEST_FLAGS_LIST=( - --keep_going - --test_output=errors -) -TEST_FLAGS="${TEST_FLAGS_LIST[@]}" +FLAGS="${FLAGS_LIST[@]}" -# Build targets for various architectures. +############### +# Build targets +############### BUILD_TARGETS_LIST=( + //art/... //bionic/... + //bootable/recovery/tools/recovery_l10n/... + //build/... + //cts/... + //development/... + //external/... + //frameworks/... + //libnativehelper/... + //packages/... + //prebuilts/clang/host/linux-x86:all //system/... - //external/arm-optimized-routines/... - //external/scudo/... + //tools/apksig/... + //tools/platform-compat/... + + # These tools only build for host currently + -//external/e2fsprogs/misc:all + -//external/e2fsprogs/resize:all + -//external/e2fsprogs/debugfs:all + -//external/e2fsprogs/e2fsck:all ) BUILD_TARGETS="${BUILD_TARGETS_LIST[@]}" -tools/bazel --max_idle_secs=5 build ${BUILD_FLAGS} --platforms //build/bazel/platforms:android_x86 -k ${BUILD_TARGETS} -tools/bazel --max_idle_secs=5 build ${BUILD_FLAGS} --platforms //build/bazel/platforms:android_x86_64 -k ${BUILD_TARGETS} -tools/bazel --max_idle_secs=5 build ${BUILD_FLAGS} --platforms //build/bazel/platforms:android_arm -k ${BUILD_TARGETS} -tools/bazel --max_idle_secs=5 build ${BUILD_FLAGS} --platforms //build/bazel/platforms:android_arm64 -k ${BUILD_TARGETS} +# Iterate over various architectures supported in the platform build. +tools/bazel --max_idle_secs=5 build ${FLAGS} --platforms //build/bazel/platforms:android_x86 -k -- ${BUILD_TARGETS} +tools/bazel --max_idle_secs=5 build ${FLAGS} --platforms //build/bazel/platforms:android_x86_64 -k -- ${BUILD_TARGETS} +tools/bazel --max_idle_secs=5 build ${FLAGS} --platforms //build/bazel/platforms:android_arm -k -- ${BUILD_TARGETS} +tools/bazel --max_idle_secs=5 build ${FLAGS} --platforms //build/bazel/platforms:android_arm64 -k -- ${BUILD_TARGETS} + +HOST_INCOMPATIBLE_TARGETS=( + # TODO(b/217756861): Apex toolchain is incompatible with host arches but apex modules do + # not have this restriction + -//build/bazel/examples/apex/... + -//packages/modules/adb/apex:com.android.adbd + -//system/timezone/apex:com.android.tzdata + -//build/bazel/tests/apex/... + -//build/bazel/ci/dist/... + + # TODO(b/217927043): Determine how to address targets that are device only + -//system/core/libpackagelistparser:all + -//external/icu/libicu:all + //external/icu/libicu:libicu + -//external/icu/icu4c/source/tools/ctestfw:all + + # TODO(b/217926427): determine why these host_supported modules do not build on host + -//packages/modules/adb:all + -//packages/modules/adb/pairing_connection:all +) + +# build for host +tools/bazel --max_idle_secs=5 build ${FLAGS} \ + --platforms //build/bazel/platforms:linux_x86_64 \ + -- ${BUILD_TARGETS} "${HOST_INCOMPATIBLE_TARGETS[@]}" + +########### +# Run tests +########### +tools/bazel --max_idle_secs=5 test ${FLAGS} //build/bazel/tests/... //build/bazel/rules/apex/... //build/bazel/scripts/... + +########### +# Dist mainline modules +########### +tools/bazel --max_idle_secs=5 run //build/bazel/ci/dist:mainline_modules ${FLAGS} --platforms=//build/bazel/platforms:android_x86 -- --dist_dir="${DIST_DIR}/mainline_modules_x86" +tools/bazel --max_idle_secs=5 run //build/bazel/ci/dist:mainline_modules ${FLAGS} --platforms=//build/bazel/platforms:android_x86_64 -- --dist_dir="${DIST_DIR}/mainline_modules_x86_64" +tools/bazel --max_idle_secs=5 run //build/bazel/ci/dist:mainline_modules ${FLAGS} --platforms=//build/bazel/platforms:android_arm -- --dist_dir="${DIST_DIR}/mainline_modules_arm" +tools/bazel --max_idle_secs=5 run //build/bazel/ci/dist:mainline_modules ${FLAGS} --platforms=//build/bazel/platforms:android_arm64 -- --dist_dir="${DIST_DIR}/mainline_modules_arm64" + +################### +# bp2build-progress +################### + +# Generate bp2build progress reports and graphs for these modules into the dist +# dir so that they can be downloaded from the CI artifact list. +BP2BUILD_PROGRESS_MODULES=( + com.android.runtime + com.android.neuralnetworks + com.android.media.swcodec +) +bp2build_progress_script="${AOSP_ROOT}/build/bazel/scripts/bp2build-progress/bp2build-progress.py" +bp2build_progress_output_dir="${DIST_DIR}/bp2build-progress" +mkdir -p "${bp2build_progress_output_dir}" + +report_args="" +for m in "${BP2BUILD_PROGRESS_MODULES[@]}"; do + report_args="$report_args -m ""${m}" + "${bp2build_progress_script}" graph -m "${m}" --use_queryview=true > "${bp2build_progress_output_dir}/${m}_graph.dot" +done -# Run tests. -tools/bazel --max_idle_secs=5 test ${BUILD_FLAGS} ${TEST_FLAGS} //build/bazel/tests/... +"${bp2build_progress_script}" report ${report_args} --use_queryview=true > "${bp2build_progress_output_dir}/progress_report.txt" diff --git a/ci/diffs.sh b/ci/diffs.sh new file mode 100755 index 00000000..210b61b5 --- /dev/null +++ b/ci/diffs.sh @@ -0,0 +1,88 @@ +#!/bin/bash -eu +# checks the diff between legacy Soong built artifacts and their counterparts +# built with bazel/mixed build +export TARGET_PRODUCT=aosp_arm64 +export TARGET_BUILD_VARIANT=userdebug + +build/soong/soong_ui.bash \ + --build-mode \ + --all-modules \ + --dir="$(pwd)" \ + bp2build +tools/bazel build --config=bp2build //build/bazel/scripts/difftool:collect_zip +tools/bazel build --config=bp2build //build/bazel/scripts/difftool:difftool_zip + +# the following 2 arrays must be of the same size +MODULES=( + libnativehelper +) +OUTPUTS=( + JNIHelp.o +) +PATH_FILTERS=( + "linux_glibc_x86_shared/\|linux_x86-fastbuild" + "linux_glibc_x86_64_shared/\|linux_x86_64-fastbuild" + "android_arm64[-_]" +# "android_arm[-_]" TODO(usta) investigate why there is a diff for this +) +readonly AOSP_ROOT="$(readlink -f "$(dirname "$0")"/../../..)" +#TODO(usta): absolute path isn't compatible with collect.py and ninja +readonly LEGACY_OUTPUT_SEARCH_TREE="out/soong/.intermediates/libnativehelper" +readonly MIXED_OUTPUT_SEARCH_TREE="out/bazel/output/execroot/__main__/bazel-out" +readonly NINJA_FILE="$AOSP_ROOT/out/combined-$TARGET_PRODUCT.ninja" +# python is expected in PATH but used only to start a zipped python archive, +# which bundles its own interpreter. We could also simply use `tools/bazel run` +# instead however that sets the working directly differently and collect.py +# won't work because it expects paths relative to $OUT_DIR +# TODO(usta) make collect.py work with absolute paths and maybe consider +# using `tools/bazel run` on the `py_binary` target directly instead of using +# the python_zip_file filegroup's output +readonly stub_python=python3 +readonly LEGACY_COLLECTION="$AOSP_ROOT/out/diff_metadata/legacy" +readonly MIXED_COLLECTION="$AOSP_ROOT/out/diff_metadata/mixed" +mkdir -p "$LEGACY_COLLECTION" +mkdir -p "$MIXED_COLLECTION" + +function findIn() { + result=$(find "$1" -name "$3" | grep "$2") + count=$(echo "$result" | wc -l) + if [ "$count" != 1 ]; then + printf "multiple files found instead of exactly ONE:\n%s\n" "$result" 1>&2 + exit 1 + fi + echo "$result" +} + +for ((i = 0; i < ${#MODULES[@]}; i++)); do + MODULE=${MODULES[$i]} + echo "Building $MODULE for comparison" + build/soong/soong_ui.bash --make-mode "$MODULE" + $stub_python "bazel-bin/build/bazel/scripts/difftool/collect.zip" \ + "$NINJA_FILE" "$LEGACY_COLLECTION" + build/soong/soong_ui.bash \ + --make-mode \ + USE_BAZEL_ANALYSIS=1 \ + BAZEL_STARTUP_ARGS="--max_idle_secs=5" \ + BAZEL_BUILD_ARGS="--color=no --curses=no --noshow_progress" \ + "$MODULE" + $stub_python "bazel-bin/build/bazel/scripts/difftool/collect.zip" \ + "$NINJA_FILE" "$MIXED_COLLECTION" + OUTPUT=${OUTPUTS[$i]} + for ((j = 0; j < ${#PATH_FILTERS[@]}; j++)); do + PATH_FILTER=${PATH_FILTERS[$j]} + LEGACY_OUTPUT=$(findIn "$LEGACY_OUTPUT_SEARCH_TREE" "$PATH_FILTER" "$OUTPUT") + MIXED_OUTPUT=$(findIn "$MIXED_OUTPUT_SEARCH_TREE" "$PATH_FILTER" "$OUTPUT") + + LEGACY_COLLECTION_DIR=$(dirname "$LEGACY_COLLECTION/$LEGACY_OUTPUT") + mkdir -p "$LEGACY_COLLECTION_DIR" + cp "$LEGACY_OUTPUT" "$LEGACY_COLLECTION_DIR" + MIXED_COLLECTION_DIR=$(dirname "$MIXED_COLLECTION/$MIXED_OUTPUT") + mkdir -p "$MIXED_COLLECTION_DIR" + cp "$MIXED_OUTPUT" "$MIXED_COLLECTION_DIR" + + $stub_python "bazel-bin/build/bazel/scripts/difftool/difftool.zip" \ + --level=SEVERE -v "$LEGACY_COLLECTION" "$MIXED_COLLECTION" \ + -l="$LEGACY_OUTPUT" -r="$MIXED_OUTPUT" + done +done + diff --git a/ci/dist/BUILD b/ci/dist/BUILD new file mode 100644 index 00000000..c4cd15ee --- /dev/null +++ b/ci/dist/BUILD @@ -0,0 +1,12 @@ +load("//build/bazel_common_rules/dist:dist.bzl", "copy_to_dist_dir") + +# bazel run --package_path=out/soong/workspace //build/bazel/ci/dist:mainline_modules -- --dist_dir=/tmp/dist +# TODO(jingwen): use a split transition on --platforms to dist all 4 architectures in a single invocation. +copy_to_dist_dir( + name = "mainline_modules", + data = [ + "//system/timezone/apex:com.android.tzdata.apex", + "//packages/modules/adb/apex:com.android.adbd.apex", + ], + flat = True, +) diff --git a/ci/mixed_droid.sh b/ci/mixed_droid.sh index 002afc51..06dcf39d 100755 --- a/ci/mixed_droid.sh +++ b/ci/mixed_droid.sh @@ -9,11 +9,20 @@ if [[ -z ${DIST_DIR+x} ]]; then fi # Run a mixed build of "droid" -build/soong/soong_ui.bash --make-mode USE_BAZEL_ANALYSIS=1 BAZEL_STARTUP_ARGS="--max_idle_secs=5" BAZEL_BUILD_ARGS="--color=no --curses=no --show_progress_rate_limit=5" TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug droid dist DIST_DIR=$DIST_DIR +build/soong/soong_ui.bash --make-mode \ + --mk-metrics \ + BP2BUILD_VERBOSE=1 \ + USE_BAZEL_ANALYSIS=1 \ + BAZEL_STARTUP_ARGS="--max_idle_secs=5" \ + BAZEL_BUILD_ARGS="--color=no --curses=no --show_progress_rate_limit=5" \ + TARGET_PRODUCT=aosp_arm64 \ + TARGET_BUILD_VARIANT=userdebug \ + droid platform_tests \ + dist DIST_DIR=$DIST_DIR # Verify there are artifacts under the out directory that originated from bazel. echo "Verifying OUT_DIR contains bazel-out..." -if find out/ | grep bazel-out &>/dev/null; then +if find out/ -type d -name bazel-out &>/dev/null; then echo "bazel-out found." else echo "bazel-out not found. This may indicate that mixed builds are silently not running." diff --git a/ci/mixed_libc.sh b/ci/mixed_libc.sh index 2b4a822f..e939a4ed 100755 --- a/ci/mixed_libc.sh +++ b/ci/mixed_libc.sh @@ -8,19 +8,34 @@ if [[ -z ${DIST_DIR+x} ]]; then DIST_DIR="out/dist" fi +TARGETS=( + libbacktrace + libfdtrack + libsimpleperf + com.android.adbd + com.android.runtime + bluetoothtbd + framework-minus-apex +) + # Run a mixed build of "libc" -build/soong/soong_ui.bash --make-mode USE_BAZEL_ANALYSIS=1 BAZEL_STARTUP_ARGS="--max_idle_secs=5" BAZEL_BUILD_ARGS="--color=no --curses=no --show_progress_rate_limit=5" TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug libc dist DIST_DIR=$DIST_DIR +build/soong/soong_ui.bash --make-mode \ + --mk-metrics \ + BP2BUILD_VERBOSE=1 \ + USE_BAZEL_ANALYSIS=1 \ + BAZEL_STARTUP_ARGS="--max_idle_secs=5" \ + BAZEL_BUILD_ARGS="--color=no --curses=no --show_progress_rate_limit=5" \ + TARGET_PRODUCT=aosp_arm64 \ + TARGET_BUILD_VARIANT=userdebug \ + "${TARGETS[@]}" \ + dist DIST_DIR=$DIST_DIR # Verify there are artifacts under the out directory that originated from bazel. echo "Verifying OUT_DIR contains bazel-out..." -if find out/ | grep bazel-out &>/dev/null; then +if find out/ -type d -name bazel-out &>/dev/null; then echo "bazel-out found." else echo "bazel-out not found. This may indicate that mixed builds are silently not running." exit 1 fi -# Run a mixed build of "libbacktrace" -# This is a small module which uses propagated includes from libc; thus will -# fail if includes are not propagated appropriately from bazel-built libc. -build/soong/soong_ui.bash --make-mode USE_BAZEL_ANALYSIS=1 BAZEL_STARTUP_ARGS="--max_idle_secs=5" BAZEL_BUILD_ARGS="--color=no --curses=no --show_progress_rate_limit=5" TARGET_PRODUCT=aosp_arm64 TARGET_BUILD_VARIANT=userdebug libbacktrace dist DIST_DIR=$DIST_DIR diff --git a/ci/rbc_dashboard.py b/ci/rbc_dashboard.py new file mode 100755 index 00000000..2e3ef1b9 --- /dev/null +++ b/ci/rbc_dashboard.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +"""Generates a dashboard for the current RBC product/board config conversion status.""" +# pylint: disable=line-too-long + +import argparse +import asyncio +import dataclasses +import datetime +import os +import re +import shutil +import socket +import subprocess +import sys +import time +from typing import List, Tuple +import xml.etree.ElementTree as ET + +_PRODUCT_REGEX = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)(?:-(user|userdebug|eng))?') + + +@dataclasses.dataclass(frozen=True) +class Product: + """Represents a TARGET_PRODUCT and TARGET_BUILD_VARIANT.""" + product: str + variant: str + + def __post_init__(self): + if not _PRODUCT_REGEX.match(str(self)): + raise ValueError(f'Invalid product name: {self}') + + def __str__(self): + return self.product + '-' + self.variant + + +@dataclasses.dataclass(frozen=True) +class ProductResult: + baseline_success: bool + product_success: bool + board_success: bool + product_has_diffs: bool + board_has_diffs: bool + + def success(self) -> bool: + return not self.baseline_success or ( + self.product_success and self.board_success + and not self.product_has_diffs and not self.board_has_diffs) + + +@dataclasses.dataclass(frozen=True) +class Directories: + out: str + out_baseline: str + out_product: str + out_board: str + results: str + + +def get_top() -> str: + path = '.' + while not os.path.isfile(os.path.join(path, 'build/soong/soong_ui.bash')): + if os.path.abspath(path) == '/': + sys.exit('Could not find android source tree root.') + path = os.path.join(path, '..') + return os.path.abspath(path) + + +def get_build_var(variable, product: Product) -> str: + """Returns the result of the shell command get_build_var.""" + env = { + **os.environ, + 'TARGET_PRODUCT': product.product, + 'TARGET_BUILD_VARIANT': product.variant, + } + return subprocess.run([ + 'build/soong/soong_ui.bash', + '--dumpvar-mode', + variable + ], check=True, capture_output=True, env=env, text=True).stdout.strip() + + +async def run_jailed_command(args: List[str], out_dir: str, env=None) -> bool: + """Runs a command, saves its output to out_dir/build.log, and returns if it succeeded.""" + with open(os.path.join(out_dir, 'build.log'), 'wb') as f: + result = await asyncio.create_subprocess_exec( + 'prebuilts/build-tools/linux-x86/bin/nsjail', + '-q', + '--cwd', + os.getcwd(), + '-e', + '-B', + '/', + '-B', + f'{os.path.abspath(out_dir)}:{os.path.abspath("out")}', + '--time_limit', + '0', + '--skip_setsid', + '--keep_caps', + '--disable_clone_newcgroup', + '--disable_clone_newnet', + '--rlimit_as', + 'soft', + '--rlimit_core', + 'soft', + '--rlimit_cpu', + 'soft', + '--rlimit_fsize', + 'soft', + '--rlimit_nofile', + 'soft', + '--proc_rw', + '--hostname', + socket.gethostname(), + '--', + *args, stdout=f, stderr=subprocess.STDOUT, env=env) + return await result.wait() == 0 + + +async def run_build(flags: List[str], out_dir: str) -> bool: + return await run_jailed_command([ + 'build/soong/soong_ui.bash', + '--make-mode', + *flags, + '--skip-ninja', + 'nothing' + ], out_dir) + + +async def run_config(product: Product, rbc_product: bool, rbc_board: bool, out_dir: str) -> bool: + """Runs config.mk and saves results to out/rbc_variable_dump.txt.""" + env = { + 'OUT_DIR': 'out', + 'TMPDIR': 'tmp', + 'BUILD_DATETIME_FILE': 'out/build_date.txt', + 'CALLED_FROM_SETUP': 'true', + 'TARGET_PRODUCT': product.product, + 'TARGET_BUILD_VARIANT': product.variant, + 'RBC_PRODUCT_CONFIG': 'true' if rbc_product else '', + 'RBC_BOARD_CONFIG': 'true' if rbc_board else '', + 'RBC_DUMP_CONFIG_FILE': 'out/rbc_variable_dump.txt', + } + return await run_jailed_command([ + 'prebuilts/build-tools/linux-x86/bin/ckati', + '-f', + 'build/make/core/config.mk' + ], out_dir, env=env) + + +async def has_diffs(success: bool, file_pairs: List[Tuple[str]], results_folder: str) -> bool: + """Returns true if the two out folders provided have differing ninja files.""" + if not success: + return False + results = [] + for pair in file_pairs: + name = 'soong_build.ninja' if pair[0].endswith('soong/build.ninja') else os.path.basename(pair[0]) + with open(os.path.join(results_folder, name)+'.diff', 'wb') as f: + results.append((await asyncio.create_subprocess_exec( + 'diff', + pair[0], + pair[1], + stdout=f, stderr=subprocess.STDOUT)).wait()) + + for return_code in await asyncio.gather(*results): + if return_code != 0: + return True + return False + + +def generate_html_row(num: int, product: Product, results: ProductResult): + def generate_status_cell(success: bool, diffs: bool) -> str: + message = 'Success' + if diffs: + message = 'Results differed' + if not success: + message = 'Build failed' + return f'<td style="background-color: {"lightgreen" if success and not diffs else "salmon"}">{message}</td>' + + return f''' + <tr> + <td>{num}</td> + <td>{product if results.success() and results.baseline_success else f'<a href="{product}/">{product}</a>'}</td> + {generate_status_cell(results.baseline_success, False)} + {generate_status_cell(results.product_success, results.product_has_diffs)} + {generate_status_cell(results.board_success, results.board_has_diffs)} + </tr> + ''' + + +def get_branch() -> str: + try: + tree = ET.parse('.repo/manifests/default.xml') + default_tag = tree.getroot().find('default') + return default_tag.get('remote') + '/' + default_tag.get('revision') + except Exception as e: # pylint: disable=broad-except + print(str(e), file=sys.stderr) + return 'Unknown' + + +def cleanup_empty_files(path): + if os.path.isfile(path): + if os.path.getsize(path) == 0: + os.remove(path) + elif os.path.isdir(path): + for subfile in os.listdir(path): + cleanup_empty_files(os.path.join(path, subfile)) + if not os.listdir(path): + os.rmdir(path) + + +async def test_one_product(product: Product, dirs: Directories) -> ProductResult: + """Runs the builds and tests for differences for a single product.""" + baseline_success, product_success, board_success = await asyncio.gather( + run_build([ + f'TARGET_PRODUCT={product.product}', + f'TARGET_BUILD_VARIANT={product.variant}', + ], dirs.out_baseline), + run_build([ + f'TARGET_PRODUCT={product.product}', + f'TARGET_BUILD_VARIANT={product.variant}', + 'RBC_PRODUCT_CONFIG=1', + ], dirs.out_product), + run_build([ + f'TARGET_PRODUCT={product.product}', + f'TARGET_BUILD_VARIANT={product.variant}', + 'RBC_BOARD_CONFIG=1', + ], dirs.out_board), + ) + + product_dashboard_folder = os.path.join(dirs.results, str(product)) + os.mkdir(product_dashboard_folder) + os.mkdir(product_dashboard_folder+'/baseline') + os.mkdir(product_dashboard_folder+'/product') + os.mkdir(product_dashboard_folder+'/board') + + if not baseline_success: + shutil.copy2(os.path.join(dirs.out_baseline, 'build.log'), + f'{product_dashboard_folder}/baseline/build.log') + if not product_success: + shutil.copy2(os.path.join(dirs.out_product, 'build.log'), + f'{product_dashboard_folder}/product/build.log') + if not board_success: + shutil.copy2(os.path.join(dirs.out_board, 'build.log'), + f'{product_dashboard_folder}/board/build.log') + + files = [f'build-{product.product}.ninja', f'build-{product.product}-package.ninja', 'soong/build.ninja'] + product_files = [(os.path.join(dirs.out_baseline, x), os.path.join(dirs.out_product, x)) for x in files] + board_files = [(os.path.join(dirs.out_baseline, x), os.path.join(dirs.out_board, x)) for x in files] + product_has_diffs, board_has_diffs = await asyncio.gather( + has_diffs(baseline_success and product_success, product_files, product_dashboard_folder+'/product'), + has_diffs(baseline_success and board_success, board_files, product_dashboard_folder+'/board')) + + # delete files that contain the product name in them to save space, + # otherwise the ninja files end up filling up the whole harddrive + for out_folder in [dirs.out_baseline, dirs.out_product, dirs.out_board]: + for subfolder in ['', 'soong']: + folder = os.path.join(out_folder, subfolder) + for file in os.listdir(folder): + if os.path.isfile(os.path.join(folder, file)) and product.product in file: + os.remove(os.path.join(folder, file)) + + cleanup_empty_files(product_dashboard_folder) + + return ProductResult(baseline_success, product_success, board_success, product_has_diffs, board_has_diffs) + + +async def test_one_product_quick(product: Product, dirs: Directories) -> ProductResult: + """Runs the builds and tests for differences for a single product.""" + baseline_success, product_success, board_success = await asyncio.gather( + run_config( + product, + False, + False, + dirs.out_baseline), + run_config( + product, + True, + False, + dirs.out_product), + run_config( + product, + False, + True, + dirs.out_board), + ) + + product_dashboard_folder = os.path.join(dirs.results, str(product)) + os.mkdir(product_dashboard_folder) + os.mkdir(product_dashboard_folder+'/baseline') + os.mkdir(product_dashboard_folder+'/product') + os.mkdir(product_dashboard_folder+'/board') + + if not baseline_success: + shutil.copy2(os.path.join(dirs.out_baseline, 'build.log'), + f'{product_dashboard_folder}/baseline/build.log') + if not product_success: + shutil.copy2(os.path.join(dirs.out_product, 'build.log'), + f'{product_dashboard_folder}/product/build.log') + if not board_success: + shutil.copy2(os.path.join(dirs.out_board, 'build.log'), + f'{product_dashboard_folder}/board/build.log') + + files = ['rbc_variable_dump.txt'] + product_files = [(os.path.join(dirs.out_baseline, x), os.path.join(dirs.out_product, x)) for x in files] + board_files = [(os.path.join(dirs.out_baseline, x), os.path.join(dirs.out_board, x)) for x in files] + product_has_diffs, board_has_diffs = await asyncio.gather( + has_diffs(baseline_success and product_success, product_files, product_dashboard_folder+'/product'), + has_diffs(baseline_success and board_success, board_files, product_dashboard_folder+'/board')) + + cleanup_empty_files(product_dashboard_folder) + + return ProductResult(baseline_success, product_success, board_success, product_has_diffs, board_has_diffs) + + +async def main(): + parser = argparse.ArgumentParser( + description='Generates a dashboard of the starlark product configuration conversion.') + parser.add_argument('products', nargs='*', + help='list of products to test. If not given, all ' + + 'products will be tested. ' + + 'Example: aosp_arm64-userdebug') + parser.add_argument('--quick', action='store_true', + help='Run a quick test. This will only run config.mk and ' + + 'diff the make variables at the end of it, instead of ' + + 'diffing the full ninja files.') + parser.add_argument('--exclude', nargs='+', default=[], + help='Exclude these producs from the build. Useful if not ' + + 'supplying a list of products manually.') + parser.add_argument('--results-directory', + help='Directory to store results in. Defaults to $(OUT_DIR)/rbc_dashboard. ' + + 'Warning: will be cleared!') + args = parser.parse_args() + + if args.results_directory: + args.results_directory = os.path.abspath(args.results_directory) + + os.chdir(get_top()) + + def str_to_product(p: str) -> Product: + match = _PRODUCT_REGEX.fullmatch(p) + if not match: + sys.exit(f'Invalid product name: {p}. Example: aosp_arm64-userdebug') + return Product(match.group(1), match.group(2) if match.group(2) else 'userdebug') + + products = [str_to_product(p) for p in args.products] + + if not products: + products = list(map(lambda x: Product(x, 'userdebug'), get_build_var( + 'all_named_products', Product('aosp_arm64', 'userdebug')).split())) + + excluded = [str_to_product(p) for p in args.exclude] + products = [p for p in products if p not in excluded] + + for i, product in enumerate(products): + for j, product2 in enumerate(products): + if i != j and product.product == product2.product: + sys.exit(f'Product {product.product} cannot be repeated.') + + out_dir = get_build_var('OUT_DIR', Product('aosp_arm64', 'userdebug')) + + dirs = Directories( + out=out_dir, + out_baseline=os.path.join(out_dir, 'rbc_out_baseline'), + out_product=os.path.join(out_dir, 'rbc_out_product'), + out_board=os.path.join(out_dir, 'rbc_out_board'), + results=args.results_directory if args.results_directory else os.path.join(out_dir, 'rbc_dashboard')) + + for folder in [dirs.out_baseline, dirs.out_product, dirs.out_board, dirs.results]: + # delete and recreate the out directories. You can't reuse them for + # a particular product, because after we delete some product-specific + # files inside the out dir to save space, the build will fail if you + # try to build the same product again. + shutil.rmtree(folder, ignore_errors=True) + os.makedirs(folder) + + # When running in quick mode, we still need to build + # mk2rbc/rbcrun/AndroidProducts.mk.list, so run a get_build_var command to do + # that in each folder. + if args.quick: + commands = [] + for folder in [dirs.out_baseline, dirs.out_product, dirs.out_board]: + commands.append(run_jailed_command([ + 'build/soong/soong_ui.bash', + '--dumpvar-mode', + 'TARGET_PRODUCT' + ], folder)) + for success in await asyncio.gather(*commands): + if not success: + sys.exit('Failed to setup output directories') + + with open(os.path.join(dirs.results, 'index.html'), 'w') as f: + f.write(f''' + <body> + <h2>RBC Product/Board conversion status</h2> + Generated on {datetime.date.today()} for branch {get_branch()} + <table> + <tr> + <th>#</th> + <th>product</th> + <th>baseline</th> + <th>RBC product config</th> + <th>RBC board config</th> + </tr>\n''') + f.flush() + + all_results = [] + start_time = time.time() + print(f'{"Current product":31.31} | {"Time Elapsed":>16} | {"Per each":>8} | {"ETA":>16} | Status') + print('-' * 91) + for i, product in enumerate(products): + if i > 0: + elapsed_time = time.time() - start_time + time_per_product = elapsed_time / i + eta = time_per_product * (len(products) - i) + elapsed_time_str = str(datetime.timedelta(seconds=int(elapsed_time))) + time_per_product_str = str(datetime.timedelta(seconds=int(time_per_product))) + eta_str = str(datetime.timedelta(seconds=int(eta))) + print(f'{f"{i+1}/{len(products)} {product}":31.31} | {elapsed_time_str:>16} | {time_per_product_str:>8} | {eta_str:>16} | ', end='', flush=True) + else: + print(f'{f"{i+1}/{len(products)} {product}":31.31} | {"":>16} | {"":>8} | {"":>16} | ', end='', flush=True) + + if not args.quick: + result = await test_one_product(product, dirs) + else: + result = await test_one_product_quick(product, dirs) + + all_results.append(result) + + if result.success(): + print('Success') + else: + print('Failure') + + f.write(generate_html_row(i+1, product, result)) + f.flush() + + baseline_successes = len([x for x in all_results if x.baseline_success]) + product_successes = len([x for x in all_results if x.product_success and not x.product_has_diffs]) + board_successes = len([x for x in all_results if x.board_success and not x.board_has_diffs]) + f.write(f''' + <tr> + <td></td> + <td># Successful</td> + <td>{baseline_successes}</td> + <td>{product_successes}</td> + <td>{board_successes}</td> + </tr> + <tr> + <td></td> + <td># Failed</td> + <td>N/A</td> + <td>{baseline_successes - product_successes}</td> + <td>{baseline_successes - board_successes}</td> + </tr> + </table> + Finished running successfully. + </body>\n''') + + print('Success!') + print('file://'+os.path.abspath(os.path.join(dirs.results, 'index.html'))) + + for result in all_results: + if result.baseline_success and not result.success(): + print('There were one or more failing products. See the html report for details.') + sys.exit(1) + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/ci/rbc_regression_test.sh b/ci/rbc_regression_test.sh new file mode 100755 index 00000000..b1f4e575 --- /dev/null +++ b/ci/rbc_regression_test.sh @@ -0,0 +1,92 @@ +#!/bin/bash -u +# Regression test for the product and/or board configuration converter. +# +# Builds 'nothing' for a given product-variant twice: with product/board +# config makefiles converted to Starlark, and without such conversion. +# The generated Ninja files should be the same. +set -u + +function die() { + echo $@ >&2 + exit 1 +} + +function usage() { + cat <<EOF >&2 +Usage: $myname [-p] [-b] [-q] [-r] <product-variant> [product-variant ...] + -p: Test RBC product configuration. This is implied if -b is not supplied + -b: Test RBC board configuration. This is implied if -p is not supplied + -q: Quiet. Suppress all output other than a failure message + -r: Retain Ninja files +EOF + exit 1 +} + +function build() { + local -r flavor="$1" + local -r product="$2" + local -r variant="$3" + shift 3 + command="build/soong/soong_ui.bash --make-mode TARGET_PRODUCT=$product TARGET_BUILD_VARIANT=$variant $@ nothing" + if ! ANDROID_QUIET_BUILD=$quiet $command; then + printf "%s-%s: %s build failed, actual command:\n %s\n" $product $variant $flavor "$command" >&2 + exit 1 + fi +} + +mypath=$(realpath "$0") +declare -r mydir=${mypath%/*/*/*/*} +declare -r myname=${mypath#${mydir}/} + +flags_rbc=() +quiet= +while getopts "bkpqr" o; do + case "${o}" in + k) ;; # backward compatibility to be removed later + q) quiet=true ;; + b) flags_rbc+=(RBC_BOARD_CONFIG=true) ;; + p) flags_rbc+=(RBC_PRODUCT_CONFIG=true) ;; + r) retain_files=t ;; + *) usage ;; + esac +done +shift $((OPTIND-1)) +[[ $# -gt 0 ]] || usage +((${#flags_rbc[@]})) || flags_rbc+=(RBC_PRODUCT_CONFIG=true RBC_BOARD_CONFIG=true) + +cd $mydir +rc=0 +for arg in $@; do + [[ "$arg" =~ ^([a-zA-Z0-9_]+)-([a-zA-Z0-9_]+)$ ]] || \ + die "Invalid product name: $arg. Example: aosp_arm64-userdebug" + product="${BASH_REMATCH[1]}" + variant="${BASH_REMATCH[2]}" + ninja_files=(soong/build.ninja build-${product}.ninja build-${product}-package.ninja) + + # Build with converter, save Ninja files, build without it. + saved_ninja_dir=out/ninja_rbc/${product}-${variant} + build RBC $product $variant ${flags_rbc[@]} && \ + rm -rf $saved_ninja_dir && mkdir -p $saved_ninja_dir/soong && \ + (for f in ${ninja_files[@]}; do mv -f out/$f $saved_ninja_dir/$f || exit 1; done) && \ + build baseline $product $variant + rc=$? + + # Compare Ninja files + if ((rc==0)); then + for f in "${ninja_files[@]}"; do + diff_file=$(mktemp) + diff out/$f $saved_ninja_dir/$f | head >& $diff_file + if [[ -s $diff_file ]]; then + echo ${product}-${variant}: "$f" is different '< make, > RBC):' >&2 + cat $diff_file >&2 + echo ... + rc=1 + fi + rm $diff_file + done + fi + [[ -n "${retain_files:-}" ]] || rm -rf $saved_ninja_dir +done + +((rc==0)) || printf "In order to reproduce the failures above, run\n %s <product>-<variant>\n" $myname >&2 +exit $rc diff --git a/common.bazelrc b/common.bazelrc index 2ed069c3..368efa21 100644 --- a/common.bazelrc +++ b/common.bazelrc @@ -1,13 +1,20 @@ # Platforms and toolchains for AOSP. # -# Set a default target platform for builds. -build --platforms //build/bazel/platforms:android_x86_64 +# Set default target platform for builds to rely on product config's arch and os variables +build --platforms //build/bazel/platforms:android_target -# # Use toolchain resolution to find the cc toolchain. +# Use the target platform (android_x86, android_arm) in the bazel-out/ output +# directory name fragment instead of the CPU (darwin, k8). This avoids +# thrashing the output directory when switching between top level target +# --platforms values. +build --experimental_platform_in_output_dir + +# Use toolchain resolution to find the cc toolchain. build --incompatible_enable_cc_toolchain_resolution # Ensure that the host_javabase always use @local_jdk, the checked-in JDK. build --tool_java_runtime_version=local_jdk +build --java_runtime_version=local_jdk # Lock down the PATH variable in actions to /usr/bin and /usr/local/bin. build --experimental_strict_action_env @@ -15,25 +22,28 @@ build --experimental_strict_action_env # Explicitly allow unresolved symlinks (it's an experimental Bazel feature) build --experimental_allow_unresolved_symlinks -# Enable usage of cc_shared_library build APIs +# Enable usage of experimental cc-related build APIs build --experimental_cc_shared_library +build --experimental_starlark_cc_import # Do not tokenize copts, other than strings that consist of a single Make # variable. This prevents the need to double-escape characters like backslashes # and quotes in copts. build --features no_copts_tokenization -# Disable middleman actions -build --noexperimental_enable_aggregating_middleman - # Disable local cpp toolchain detection, as it is explicitly declared in AOSP. build --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 +build --proto_compiler=//external/protobuf:aprotoc + # Disable sandboxing for CppCompile actions, as headers are not fully specified. # TODO(b/186116353): This is a temporary fix, as appropriately-sandboxed actions # are a long term goal. build --strategy=CppCompile=standalone +# Enable use of the implementation_deps attribute in native cc rules +build --experimental_cc_implementation_deps + # Enable building targets in //external:__subpackages__. common --experimental_sibling_repository_layout common --experimental_disable_external_package @@ -59,6 +69,14 @@ common:queryview --package_path=%workspace%/out/soong/queryview # Note that this hardcodes the output dir. It will not work if $OUT_DIR != out. common:bp2build --package_path=%workspace%/out/soong/workspace +# Configurations specific to CI builds, generally to improve signal-to-noise ratio in server logs. +common:ci --color=no +common:ci --curses=no +common:ci --show_progress_rate_limit=5 +common:ci --noshow_loading_progress +test:ci --keep_going +test:ci --test_output=errors + # Support a local user-specific bazelrc file. try-import %workspace%/user.bazelrc @@ -68,6 +86,15 @@ build --experimental_google_legacy_api build --incompatible_java_common_parameters build --android_databinding_use_v3_4_args build --experimental_android_databinding_v2 +build --define=android_incremental_dexing_tool=d8_dexbuilder +build --define=android_dexmerger_tool=d8_dexmerger +build --nouse_workers_with_dexbuilder +build --fat_apk_cpu=k8 + +# TODO(b/199038020): Use a python_toolchain when we have Starlark rules_python. +# This also means all python scripts are using py3 runtime. +build --python_top=//prebuilts/build-tools:python3 +build --noincompatible_use_python_toolchains # Developer instance for result storage. This only works if you have access # to the Bazel GCP project. Follow the GCP gcloud client's auth instructions to diff --git a/docs/concepts.md b/docs/concepts.md index 3a7c3a96..b9f8c246 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -4,6 +4,9 @@ This document provides high level explanations and mapping of the internal build system components and concepts of the Android build system and Bazel, and how components communicate with each other. +For implementation concepts, see: +https://android.googlesource.com/platform/build/bazel/+/refs/heads/master/docs/internal_concepts.md. + ## High level components This table provides a high level overview of the components in the current diff --git a/docs/internal_concepts.md b/docs/internal_concepts.md new file mode 100644 index 00000000..03dd40a4 --- /dev/null +++ b/docs/internal_concepts.md @@ -0,0 +1,616 @@ +# Soong-Bazel equivalents + +This doc aims to describe *internal*-facing implementation concepts. For +external-facing, see +https://android.googlesource.com/platform/build/bazel/+/refs/heads/master/docs/concepts.md. + +[TOC] + +## Overview + +Soong/Ninja | Bazel | Remarks +--------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | ------- +make phony goal, e.g. "dist", "sdk", "apps_only", "droidcore" | Top level `filegroup` rule target | [Details](#phony-goal) +Ninja build target (phony) | (readable) alias to a file target | +Ninja build target (non-phony) | File target | +`ModuleFactory` | `RuleConfiguredTargetFactory` | +`Module type` (e.g. `cc_library`) | Rule class (e.g. `cc_library`) | +Module object instance | Target (instance of a rule) | [Details](#instance) +Module properties | [Rule attributes](https://docs.bazel.build/versions/main/skylark/rules.html#attributes) | [Details](#props) +Module name | Target label | +Module variant | (Split) configured target | +[LoadHooks](#loadhooks) | [macros (ish)](https://docs.bazel.build/versions/main/skylark/macros.html) | +Top-down mutators on modules | Split configuration on targets | Allows building multiple "variants" of the same build artifact in the same build. +Bottom-up mutators on modules | [Aspects](https://docs.bazel.build/versions/main/skylark/aspects.html) on targets | +[Build statement (Ninja)](#ninja-build-statement) | Action (result of ctx.actions.run) | +[Rule statement (Ninja)](#ninja-rules) | [ctx.actions.run() API](https://docs.bazel.build/versions/main/skylark/lib/actions.html) | +`out/soong/build.ninja` and `out/build-<target>.ninja` | Action graph (serialized) | +Pool (ninja) | Thread pools / `ExecutorService` | +Blueprint's Registration and Parse, `ResolveDependencies` phase | Loading phase | +Blueprint's [Generate and Write phases](#blueprint-analysis) | Analysis Phase | +Ninja execution | Execution phase | +Blueprints/`Android.bp` files | `BUILD`/`BUILD.bazel` files | +[Namespaces](#namespaces) | [Packages](#pkgs) | Most Soong modules are within the global namespace +[Mutators](#mutators) | Configuration keys (ish) | +[Variation](#variation) | Configuration value | +[Singleton](#singleton) | Aspect-ish | +Target (system + vendor + product) | [Platform](https://docs.bazel.build/versions/main/platforms.html) | +Bash scripts e.g. envsetup functions, `soong_ui.bash`) | Repository rule | +Product and board configuration makefile and env variables | Configuration in Bazel (ish) | [Details](#config) +[Dependency Tags](#deptags) | Provider names | + +## Remarks + +### Phony goals {#phony-goal} + +Soong maintains the make terminology of +[goals](https://www.gnu.org/software/make/manual/html_node/Goals.html) to denote +what should be built. All modules can be specified by name as a goal, in +addition, phony goals are supported. + +A Phony goal creates a Make-style phony rule, a rule with no commands that can +depend on other phony rules or real files. Phony can be called on the same name +multiple times to add additional dependencies. These are often used to build +many targets at once. The default goal for Android's build system is `droid`. +Some other common phony goals include: `nothing` (perform loading/analysis), +`docs`, `checkbuild`, `apps_only`. + +Some common phony goals are defined in +[`build/make/core/main.mk`](http://cs.android.com/android/platform/superproject/+/master:build/make/core/main.mk) +The purpose is to help `soong_ui` to determine what top level files to build. + +### Module/Target {#instance} + +When a Module is instantiated by Blueprint (which calls the appropriate +`ModuleFactory`), the [property structs](#props) are populated by Blueprint. + +Blueprint performs no additional operations on these properties, such that +dependencies on other modules and references to source files are unresolved +initially. [`Mutators`](#mutators) then introspect the values of properties to +[specify dependencies](https://cs.android.com/android/platform/superproject/+/master:build/blueprint/module_ctx.go;l=871-886,918-960;drc=030150d8f9d164783ea661f07793c45198739cca) +between modules, which +[Blueprint resolves](https://cs.android.com/android/platform/superproject/+/master:build/blueprint/context.go;l=1630,1667;drc=5c4abb15e3b84ab0bcedfa119e2feb397d1fb106). +Source files (including globs) and output paths for references to other modules +are resolved during [blueprint analysis](#blueprint-analysis) via the various +`Path[s]ForModuleSrc[Excludes]` functions within +[build/soong/android/paths.go](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/paths.go). + +For a Bazel target instance, the dependencies and source file references within +[`attrs`](#attributes) have been resolved by Bazel. + +Bazel +[implementation](https://github.com/bazelbuild/bazel/blob/a20b32690a71caf712d1d241f01fef16649562ba/src/main/java/com/google/devtools/build/lib/skyframe/TransitiveBaseTraversalFunction.java#L113-L140) +to collect deps. + +### Properties/Attributes {#props} + +#### Properties + +Within Soong/Blueprint, properties are represented as Go structs, which can be +nested, with no depth limit. Properties can be primitive or pointer types, but +they must be one of these types: `int64`, `string`, `bool`, `list`. + +These properties can be defined from various structs within the module type +factory itself (via +[AddProperties](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=1276;drc=8631cc7327919845c9d9037188cbd483d22ba077)) +or from common helper functions such as: + +* `InitAndroidModule`: + [specifies](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=1042-1045;drc=8631cc7327919845c9d9037188cbd483d22ba077) + name-related, common, and dist properties. +* `InitAndroidArchModule`: adds + [host/device properies](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=1077;drc=8631cc7327919845c9d9037188cbd483d22ba077) + +Go comments for a property will be treated as documentation to describe the +property. In some cases, these comments describe a default value for the +property. However, the default value is not based on the comment or field +definition but resolved somewhere within the module's mutators or build. These +defaults are often determined using Blueprint +[`proptools`](https://cs.android.com/android/platform/superproject/+/master:build/blueprint/proptools/proptools.go) +`*Default` functions. For example, `cc` modules have a property +[`include_build_directory`](https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/compiler.go;l=265;drc=135bf55281d79576f33469ce4f9abc517a614af5), +which is described in the comments. The default value is +[resolved](https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/compiler.go;l=265;drc=135bf55281d79576f33469ce4f9abc517a614af5) +when compiler flags are being determined. + +In general, these can be set in an Android.bp file. However, if the property is +tagged with `` `blueprint:"mutated"` ``, it can only be set programmatically +within Blueprint/Soong. Additionally, `mutated` tagged properties also support +`map` and `int` types in addition to those mentioned above. These `mutated` +properties are used to propagate data that gets set during mutations, which +ensures that the information is copied successfully to module variants during +mutation. + +Soong supports additional property tags to provide additional +functionality/information about a property: + +* `` `android:arch_variant` ``: This specifies that a property can be + configured for different architectures, operating systems, targets, etc. The + [arch mutator](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/arch.go;l=597;drc=135bf55281d79576f33469ce4f9abc517a614af5), + will merge target-specific properties into the correct variant for + properties with this tag. + + Note: if a nested property is arch-variant, all recursively nesting structs + that can be specified in an Android.bp file must also be tagged as + arch-variant. + +* `` `android:variant_prepend` ``: When merging properties for the arch + variant, the arch-specific values should be *prepended* rather than appended + to existing property values. + +* `` `android:path` ``: This specifies that this property will contain some + combination of: + + * module-relative paths + * references to other modules in the form: + * `":<name>{.<tag>}"`, where `{.<tag>}` is optional to specify a + non-default output file, specific to the module type + * `"<namespace>:<name>{.<tag>}""` + + Note: Dependencies to other modules for these properties will be + automatically added by the + [pathdeps mutator](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/path_properties.go;l=40;drc=40131a3f9e5ac974a44d3bd1293d31d585dc3a07). + +#### Attributes + +Similar to properties, +[attributes](https://docs.bazel.build/versions/main/skylark/lib/attr.html) only +support a few types. The difference is that Bazel attributes cannot be nested . + +Some attributes are +[common](https://docs.bazel.build/versions/2.1.0/be/common-definitions.html#common-attributes) +across many/all rule classes, including (but not limited to) `name`, `tag`, +`visibility`. + +The definition of an attribute can contain settings, such as: its default value, +whether it is mandatory ot have a value, and its documentation. + +To specify a source file or reference to another module, use `label` or +`label_list` attribute types (rather than regular `string` or `string_list` +types). These support additional restrictions (as compared to `string*` types), +such as: + +* whether files are supported +* the providers that must be given by a dependency +* whether the dependency should be executable +* the configuration (host, target) +* aspects + +Unlike Soong, when accessing this attribute within the rule's implementation (at +anlysis time), the label(s) will be resolved to the file or target they refer +to. + +Attributes do not need to specify whether they accept +[configurable attribute](https://docs.bazel.build/versions/main/configurable-attributes.html). +However, the rule definition can specify the configuration or specify a +[configuration transition](https://docs.bazel.build/versions/main/skylark/lib/transition.html). + +However, not all target definitions within a `BUILD` file are invoking a rule. +Instead, they may invoke a Starlark macro, which is a load-time wrapper around +rules. Arguments for a macro are not typed. If macros are used, their arguments +would have to be wrangled into an attribute-compatible type. + +### LoadHooks + +[LoadHooks](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/hooks.go;l=24-36;drc=07656410df1836a70bea3054e50bb410ecbf8e07) +provide access to : + +* append/prepend additional properties to the module + (`AppendProperties`/`PrependProperties`) +* create a new module `CreateModule` + +`LoadHooks` make it easier to extend existing module factories to always specify +certain properties or to split a single `Android.bp` definition into multiple +Module instances . + +### Build Statement (ninja) {#ninja-build-statement} + +[Ninja build statements](https://ninja-build.org/manual.html#_build_statements) can be +expanded from [Ninja rules](https://ninja-build.org/manual.html#_rules), which are like +templates. + +``` +# rule +rule cattool + depfile = out/test/depfile.d + command = ${in} ${out} + +# build statement +build out/test/output.txt: cattool test/cattool.sh test/one test/two + +# build statement +build out/test/other_output.txt: cattool test/cattool.sh test/three test/four +``` + +Rules for `Android.mk` modules (`out/build-<target>.ninja`) and build statements +are 1:1. That is every rule is only used once by a single build statement. + +Soong (`out/soong/build.ninja`) rules are reused extensively in build statements +(1:many). For example the `Cp` rule is a commonly used rule for creating build +statements which copy files. + +### Ninja Rules in Soong {#ninja-rules} + +In Soong, Ninja rules can be defined in two ways: + +* [rule_builder](http://cs.android.com/android/platform/superproject/+/master:build/soong/android/rule_builder.go) +* [package_ctx](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/package_ctx.go;l=102-293;drc=77cdcfdeafd383ef1f1214226c47eb20c902a28f) + +### Blueprint Generate & Write phase {#blueprint-analysis} + +1. [`ResolveDependencies`](https://cs.android.com/android/platform/superproject/+/master:build/blueprint/context.go;l=1547;drc=5c4abb15e3b84ab0bcedfa119e2feb397d1fb106) + Running a series of Mutators, to add dependencies, split modules with + variations, etc +1. [`PrepareBuildActions`](https://cs.android.com/android/platform/superproject/+/master:build/blueprint/context.go;l=2367;drc=5c4abb15e3b84ab0bcedfa119e2feb397d1fb106): + + 1. Running Modules’ `GenerateBuildActions` to generate Ninja statements, + which in turn calls each module's + [`GenerateAndroidBuildActions`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=445-448;drc=8631cc7327919845c9d9037188cbd483d22ba077). + 1. Running Singletons to generate Ninja statements that generate docs, + android.mk statements, etc + +### Soong namespaces {#namespace} + +Module +[Namespaces](https://android.googlesource.com/platform/build/soong/+/master/README.md#namespaces) +can import other namespaces, and there’s a module name lookup algorithm which +terminates in the global namespace. + +Note: this is not widely used and most Soong modules are in the global +namespace. + +### Bazel packages {#pkgs} + +[Packages](https://docs.bazel.build/versions/main/build-ref.html#packages) can +nest subpackages recursively, but they are independent containers of Bazel +targets. This means that Bazel target names only need to be unique within a +package. + +### Mutators + +blueprint invokes mutators are invoking in the order they are registered (e.g. +top-down and bottom-up can be interleaved). Each mutator applys a single +visitation to every module in the graph. + +Mutators visiting module can parallelized, while maintaining their ordering, by +calling `.Parallel()`. + +While top-down and bottom-up mutators differ in their purposes, the interface +available to each contains many similarities. Both have access to: +[`BaseModuleContext`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=139;drc=8631cc7327919845c9d9037188cbd483d22ba077) +and +[`BaseMutatorContext`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/mutator.go;l=246;drc=2ada09a5463a0108d713773679c5ba2c35450fa4). + +In addition to the registration order, Soong supports phase-based ordering of +mutators: + +1. Pre-Arch: mutators that need to run before arch-variation. For example, + defaults are handled at this stage such properties from defaults are + correctly propagated to arch-variants later. + +1. (Hard-coded) + [`archMutator`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/arch.go;l=597;drc=135bf55281d79576f33469ce4f9abc517a614af5) + splits a module into the appropriate target(s). Next, the arch- and + OS-specific properties are merged into the appropriate variant. + +1. Pre-Deps: mutators that can/need to run before deps have been resolved, for + instance, creating variations that have an impact on dependency resolution. + +1. (Hard-coded) + [`depsMutator`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/mutator.go;l=502;drc=2ada09a5463a0108d713773679c5ba2c35450fa4), + which calls the `DepsMutator` function that *must* be part of a Soong + `Module`'s interface. + +1. Post-Deps: mutators that need to run after deps have been resolved + +1. Final-Deps like post-deps but variations cannot be created + +#### Top-down Mutator + +A top-down mutator is invoked on a module before its dependencies. + +The general purpose is to propagate dependency info from a module to its +dependencies. + +#### Bottom-up Mutator + +A bottom-up mutator is invoked on a module only after the mutator has been +invoked on all its dependencies. + +The general purpose of a bottom-up mutator is to split modules into variants. + +### Soong/Blueprint Variation {#variation} + +A tuple (name of mutator, variation / config value) passed to +`CreateVariations`. + +### Configuration {#config} + +Soong's config process encompasses both *what* should build and *how* it should +build. This section focuses on the *how* aspect. + +We do not cover how Soong's configuration will be implemented in Bazel, but the +general capabilities of Bazel to configure builds. + +#### Soong + +Android users can configure their builds based on: + +* Specifying a target (via lunch, banchan, tapas, or Soong’s command line + options) +* Environment variables + +Some environment variables or command line options are used directly to alter +the build. However, specification of target product encompasses many aspects of +both *what* and *how* things are built. This configuration is currently handled +within Make but is in the process of being migrated to Starlark. + +Soong +[invokes Kati](https://cs.android.com/android/platform/superproject/+/master:build/soong/ui/build/dumpvars.go;drc=7ae80a704494bbb934dced97ed97eb55a21a9a00) +to run in a "config" mode, also commonly known as "product config". This mode +limits the scope of what `.mk` files are parsed. The product-specific handlers +are largely in: + +* [`product_config.mk`](https://cs.android.com/android/platform/superproject/+/master:build/make/core/product_config.mk;drc=d189ab71f3505ea28324ebfaced2466af5eb0af7): + this subset of functionality is also commonly referred to as "product + config" +* [`board_config.mk`](https://cs.android.com/android/platform/superproject/+/master:build/make/core/board_config.mk) + +However, these cover only a subset of +[`config.mk`](https://cs.android.com/android/platform/superproject/+/master:build/make/core/config.mk). +This ensures that all values have appropriate defaults and specify details +necessary to the build. Some examples: + +* [handling of version defaults](https://cs.android.com/android/platform/superproject/+/master:build/make/core/version_defaults.mk) +* [rbe setup](https://cs.android.com/android/platform/superproject/+/master:build/make/core/rbe.mk) +* [user-defined config](https://cs.android.com/android/platform/superproject/+/master:build/make/core/config.mk;l=300-308;drc=ee20ae1a8dcdfe7b843d65099000708800d9b93a): + [buildspec.mk](http://cs.android.com/android/platform/superproject/+/master:build/make/buildspec.mk.default) + is similar to + [`.bazelrc`](https://docs.bazel.build/versions/main/guide.html#bazelrc-the-bazel-configuration-file) + file. +* ensuring + [`PRODUCT_SHIPPING_API_LEVEL`](https://cs.android.com/android/platform/superproject/+/master:build/make/core/config.mk;l=729-745;drc=ee20ae1a8dcdfe7b843d65099000708800d9b93a) + is defaulted if not specified by the target. + +Finally, Kati dumps variables to be consumed by Soong: + +* environment variables specifically requested by Soong +* writes + [`soong.variables`](http://cs.android.com/android/platform/superproject/+/master:build/make/core/soong_config.mk), + a JSON file + +Throughout Soong, environment variables can be accessed to alter the build via +the `Config`: + +* [`GetEnv`](http://cs.android.com/search?q=f:soong%20%5C.GetEnv%5C%28%20-f:%2Fui%2F%20-f:%2Fcmd%2F&sq=) +* [`GetEnvWithDefault`](http://cs.android.com/search?q=f:soong%20%5C.GetEnvWithDefault%5C%28%20-f:%2Fui%2F%20-f:%2Fcmd%2F&sq=) +* [`IsEnvTrue`](http://cs.android.com/search?q=f:soong%20%5C.IsEnvTrue%5C%28%20-f:%2Fui%2F%20-f:%2Fcmd%2F&sq=) +* [`IsEnvFalse`](http://cs.android.com/search?q=f:soong%20%5C.IsEnvFalse%5C%28%20-f:%2Fui%2F%20-f:%2Fcmd%2F&sq=) + +Soong +[loads the `soong.variables`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/config.go;l=174;drc=b078ade28d94c85cec78e9776eb31948a5647070) +config file, stored as +[`productVariables`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/variable.go;l=163;drc=16e77a9b303a71018eb6630f12f1414cd6ad615c). +These variables are used in three ways: + +* Direct access from `Config`, for example: paths can be + [opted out](https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/sanitize.go;l=364,371,393;drc=582fc2d1dde6c70687e6a0bea192f2a2ef67bbd5) + of specific sanitizers +* In limited cases, users can use these within their `Android.bp` file to + control what is built or perform variable replacement. + [`variableProperties`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/variable.go;l=38;drc=16e77a9b303a71018eb6630f12f1414cd6ad615c) + limits which configuration variables can be specified within an `Android.bp` + file and which properties they can apply to. The values specified within an + `Android.bp` file, are merged/replaced by the + [`VariableMutator`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/variable.go;l=539;drc=16e77a9b303a71018eb6630f12f1414cd6ad615c), + which appends performs string replacement if requested and merges the + properties into the modules. +* Through + [Soong Config Variables](https://android.googlesource.com/platform/build/soong/+/refs/heads/master/README.md#soong-config-variables): + which allow users to specify additional configuration variables that can be + used within an `Android.bp` file for the module type and properties they + request. Soong config variable structs are + [dynamically generated](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/soongconfig/modules.go;l=257;drc=997f27aa0353dabf76d063d78ee5d4495da85651) + via reflection. In the + [factory](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/soong_config_modules.go;l=423;drc=18fd09998223d004a926b02938e4cb588e4cc934), + the properties to merge into the module instance are + [identified](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/soongconfig/modules.go;l=416;drc=997f27aa0353dabf76d063d78ee5d4495da85651) + based on the config variable's type. + +The product configuration also provides information about architecture and +operating system, both for target(s) and host. This is used within the +[`archMutator`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/arch.go;l=569-597;drc=135bf55281d79576f33469ce4f9abc517a614af5) +to split a module into the required variants and merge target-specific +properties into the appropriate variant. Only properties which have been tagged +with `android:"arch_variant"` can be specified within an `Android.bp` as +arch/os/target-specific. For example: + +```go +type properties struct { + // this property will be arch-variant + Arch_variant_not_nested *string `android:"arch_variant"` + + Nested_with_arch_variant struct { + // this property is arch-variant + Arch_variant_nested *string `android:"arch_variant"` + + // this property is **not** arch-variant + Not_arch_variant_nested *string + } `android:"arch_variant"` + + Nested_no_arch_variant struct { + // this property is **NOT** arch-variant + No_arch_variant_nested_not_arch_variant *string `android:"arch_variant"` + + // this property is **not** arch-variant + No_arch_variant_nested *string + } +} +``` + +The arch/os/target-specific structs are +[dynamically generated](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/arch.go;l=780-787;drc=135bf55281d79576f33469ce4f9abc517a614af5) +based on the tags using reflection. + +#### Bazel + +Bazel documentation covers configurable builds fairly extensively, so this is a +short overview that primarily links to existing Bazel documentation rather than +repeating it here. + +[Configurable attributes](https://docs.bazel.build/versions/main/configurable-attributes.html), +(aka `select()`) allows users to toggle values of build rule attributes on the +command line. + +Within a `rule`, the value of a `select` will have been resolved based on the +configuration at analysis phase. However, within a macro (at loading phase, +before analysis phase), a `select()` is an opaque type that cannot be inspected. +This restricts what operations are possible on the arguments passed to a macro. + +The conditions within a `select` statement are one of: + +* [`config_setting`](https://docs.bazel.build/versions/main/be/general.html#config_setting) +* [`constraint_value`](https://docs.bazel.build/versions/main/be/platform.html#constraint_value) + +A `config_setting` is a collection of build settings, whether defined by Bazel, +or user-defined. + +User-defined +[build settings](https://docs.bazel.build/versions/main/skylark/config.html#defining-build-settings) +allow users to specify additional configuration, which *optionally* can be +specified as a flag. In addition to specifying build settings within a +`config_setting`, rules can depend directly on them. + +In addition, Bazel supports +[`platform`s](https://docs.bazel.build/versions/main/be/platform.html#platform), +which is a named collection of constraints. Both a target and host platform can +be specified on the command line. +[More about platforms](https://docs.bazel.build/versions/main/platforms.html). + +## Communicating between modules/targets + +### Soong communication + +There are many mechanisms to communicate between Soong modules. Because of this, +it can be difficult to trace the information communicated between modules. + +#### Dependency Tags {#deptags} + +Dependency tags are the primary way to filter module dependencies by what +purpose the dependency serves. For example, to filter for annotation processor +plugins in the deps of a Java library module, use `ctx.VisitDirectDeps` and +check the tags: + +``` +ctx.VisitDirectDeps(func(module android.Module) { + tag := ctx.OtherModuleDependencyTag(module) + if tag == pluginTag { patchPaths += ":" + strings.Split(ctx.OtherModuleDir(module), "/")[0] } + } +) +``` + +At this point the module managing the dependency, may have enough information to +cast it to a specific type or interface and perform more specific operations. + +For instance, shared libraries and executables have +[special handling](http://cs.android.com/android/platform/superproject/+/master:build/soong/cc/cc.go;l=2771-2776;drc=5df7bd33f7b64e2b880856e3193419697a8fb693) +for static library dependencies: where the coverage files and source based ABI +dump files are needed explicitly. Based on the dependency tag, the module is +cast to a concrete type, like `cc.Module`, where internal fields are accessed +and used to obtain the desired data. + +Usage of dependency tags can be more evident when used between module types +representing different langauges, as the functions must be exported in Go due to +Soong's language-based package layout. For example, rust uses `cc` module's +[`HasStubVariants`](http://cs.android.com/android/platform/superproject/+/master:build/soong/rust/rust.go;l=1457-1458;drc=9f59e8db270f58a3f2e4fe5bc041f84363a5877e). + +#### Interfaces + +A common mechanism for a module to communicate information about itself is to +define or implement a Go interface. + +Some interfaces are common throughout Soong: + +* [`SourceFileProducer`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=2967;drc=8707cd74bf083fe4a31e5f5aa5e74bd2a47e9e58), + by implementing `Srcs() Paths` +* [`OutputFileProducer`](http://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=2974;drc=8707cd74bf083fe4a31e5f5aa5e74bd2a47e9e58) + by implementing `OutputFiles(string) (Paths, error)` +* [`HostToolProvider`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=3032;drc=8707cd74bf083fe4a31e5f5aa5e74bd2a47e9e58) + by implementing `HostToolPath() OptionalPath` + +`SourceFileProducer` and `OutputFileProducer` are used to resolve references to +other modules via `android:"path"` references. + +Modules may define additional interfaces. For example, `genrule` defines a +[`SourceFileGenerator` interface](http://cs.android.com/android/platform/superproject/+/master:build/soong/genrule/genrule.go;l=98-102;drc=2ada09a5463a0108d713773679c5ba2c35450fa4). + +#### Providers + +Soong has Bazel-inspired providers, but providers are not used in all cases yet. + +Usages of providers are the easiest, simplest, and cleanest communication +approach in Soong. + +In the module providing information, these are specified via +[`SetProvider`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=212;drc=5a34ffb350fb295780e5c373fd1c78430fa4e3ed) +and +[`SetVariationProvider`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/mutator.go;l=719;drc=5a34ffb350fb295780e5c373fd1c78430fa4e3ed). + +In the module retrieving information, +[`HasProvider`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=205-206;drc=8631cc7327919845c9d9037188cbd483d22ba077) +and +[`Provider`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=198-203;drc=8631cc7327919845c9d9037188cbd483d22ba077) +or +[`OtherModuleHasProvider`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=195-196;drc=8631cc7327919845c9d9037188cbd483d22ba077) +and +[`OtherModuleProvider`](https://cs.android.com/android/platform/superproject/+/master:build/soong/android/module.go;l=189-193;drc=8631cc7327919845c9d9037188cbd483d22ba077) +are used to test existence and retrieve a provider. + +### Bazel communication + +Targets primarily communicate with each other via providers in Bazel rule +implementations. All rules have access to any of the providers but rules will +pick and choose which ones to access based on their needs. For example, all +rules can access `JavaInfo` provider, which provides information about compile +and rolled-up runtime jars for javac and java invocations downstream. However, +the `JavaInfo` provider is only useful to `java_*` rules or rules that need jvm +information. + +#### Starlark rules + +[Providers](https://docs.bazel.build/versions/main/skylark/rules.html#providers) +are pieces of information exposed to other modules. + +One such provider is `DefaultInfo`, which contains the default output files and +[`runfiles`](https://docs.bazel.build/versions/main/skylark/rules.html#runfiles). + +Rule authors can also create +[custom providers](https://docs.bazel.build/versions/main/skylark/lib/Provider.html#modules.Provider) +or implement existing providers to communicate information specific to their +rule logic. For instance, in Android Starlark +[`cc_object`](http://cs/android/build/bazel/rules/cc_object.bzl?l=86-87&rcl=42607e831f8ff73c82825b663609cafb777c18e1) +rule implementation, we return a +[`CcInfo`](https://docs.bazel.build/versions/main/skylark/lib/CcInfo.html) +provider and a custom +[`CcObjectInfo`](http://cs/android/build/bazel/rules/cc_object.bzl?l=17-21&rcl=42607e831f8ff73c82825b663609cafb777c18e1) +provider. + +#### Native rules + +For implementation of native rules in Java, +[`ruleContext.getPrerequisite`](https://github.com/bazelbuild/bazel/blob/a20b32690a71caf712d1d241f01fef16649562ba/src/main/java/com/google/devtools/build/lib/analysis/RuleContext.java#L911-L983) +is used to extract providers from dependencies. + +#### `depset` construction + +[`depset`](https://docs.bazel.build/versions/main/glossary.html#depset) are used +in conjunction with providers to accumulate data efficiently from transitive +dependencies. used to accumulate data from transitive dependencies. + +#### `exports` + +Some target have an `exports` attribute by convention, like +[`java_library.exports`](https://docs.bazel.build/versions/main/be/java.html#java_import.exports). +This attribute is commonly used to propagate transitive dependencies to the +dependent as though the dependent has a direct edge to the transitive +dependencies. diff --git a/examples/android_app/java/com/app/BUILD b/examples/android_app/java/com/app/BUILD index 5f70697e..786bfbdb 100644 --- a/examples/android_app/java/com/app/BUILD +++ b/examples/android_app/java/com/app/BUILD @@ -1,4 +1,7 @@ -load("@rules_android//rules:rules.bzl", "android_binary", "android_library") +load("//build/bazel/rules/android:android_binary.bzl", "android_binary") +load("//build/bazel/rules/cc:cc_library_static.bzl", "cc_library_static") +load("//build/bazel/rules/cc:cc_library_shared.bzl", "cc_library_shared") +load("@rules_android//rules:rules.bzl", "android_library") android_binary( name = "app", @@ -8,18 +11,36 @@ android_binary( ], ) +android_binary( + name = "app-cert-string", + certificate_name = "platform", + manifest = "AndroidManifest.xml", + deps = [ + ":applib", + ], +) + +android_binary( + name = "app-cert-module", + certificate = "//build/make/target/product/security:aosp-testkey", + manifest = "AndroidManifest.xml", + deps = [ + ":applib", + ], +) + android_library( - name = "applib", - srcs = [ - "MainActivity.java", - #"Jni.java", # TODO: integrate JNI - ], - resource_files = glob(["res/**"]), - manifest = "AndroidManifest.xml", - deps = [ - ":lib", - #":jni", # TODO: integrate JNI - ] + name = "applib", + srcs = [ + "Jni.java", + "MainActivity.java", + ], + manifest = "AndroidManifest.xml", + resource_files = glob(["res/**"]), + deps = [ + ":jni", + ":lib", + ], ) android_library( @@ -27,14 +48,15 @@ android_library( srcs = ["Lib.java"], ) -cc_library( +cc_library_shared( name = "jni", srcs = ["jni.cc"], deps = [":jni_dep"], ) -cc_library( +cc_library_static( name = "jni_dep", srcs = ["jni_dep.cc"], hdrs = ["jni_dep.h"], + deps = ["//libnativehelper:jni_headers"], ) diff --git a/examples/android_app/java/com/app/Jni.java b/examples/android_app/java/com/app/Jni.java index 5466b01f..2aea68c3 100644 --- a/examples/android_app/java/com/app/Jni.java +++ b/examples/android_app/java/com/app/Jni.java @@ -19,4 +19,3 @@ package com.app; public class Jni { public static native String hello(); } - diff --git a/examples/android_app/java/com/app/jni.cc b/examples/android_app/java/com/app/jni.cc index a3996b67..99ef4830 100644 --- a/examples/android_app/java/com/app/jni.cc +++ b/examples/android_app/java/com/app/jni.cc @@ -17,10 +17,10 @@ #include <jni.h> #include <string> -#include "java/app/jni_dep.h" +#include "build/bazel/examples/android_app/java/com/app/jni_dep.h" extern "C" JNIEXPORT jstring JNICALL -Java_app_Jni_hello(JNIEnv *env, jclass clazz) { +Java_com_app_Jni_hello(JNIEnv *env, jclass clazz) { std::string hello = "Hello"; std::string jni = "JNI"; return NewStringLatin1(env, (hello + " " + jni).c_str()); diff --git a/examples/android_app/java/com/app/jni_dep.cc b/examples/android_app/java/com/app/jni_dep.cc index fed44c4e..6a6414ff 100644 --- a/examples/android_app/java/com/app/jni_dep.cc +++ b/examples/android_app/java/com/app/jni_dep.cc @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "java/app/jni_dep.h" +#include "build/bazel/examples/android_app/java/com/app/jni_dep.h" #include <stdlib.h> #include <string.h> diff --git a/examples/apex/minimal/Android.bp b/examples/apex/minimal/Android.bp new file mode 100644 index 00000000..e18120fc --- /dev/null +++ b/examples/apex/minimal/Android.bp @@ -0,0 +1,133 @@ +// Copyright (C) 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This is a minimal apex that contains no files. +// Build with `m build.bazel.examples.apex.minimal`. +// +// Generated by system/apex/tools/create_apex_skeleton.sh. + +// WARNING: These keys are for test and dev purposes only. +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +apex_key { + name: "build.bazel.examples.apex.minimal.key", + public_key: "build.bazel.examples.apex.minimal.avbpubkey", + private_key: "build.bazel.examples.apex.minimal.pem", +} + +android_app_certificate { + name: "build.bazel.examples.apex.minimal.certificate", + certificate: "build.bazel.examples.apex.minimal", +} + +filegroup { + name: "build.bazel.examples.apex.minimal-file_contexts", + srcs: [ + "file_contexts", + ], +} + +cc_library { + name: "build.bazel.examples.apex.minimal_dummy_cc_lib", + + srcs: ["dummy_cc_lib.cc"], + + apex_available: [ + "build.bazel.examples.apex.minimal", + "build.bazel.examples.apex.minimal_compressed", + ], + + // Because the APEX sets this + product_specific: true, + + // Because the APEX sets this + min_sdk_version: "30", +} + +prebuilt_etc { + name: "build.bazel.examples.apex.minimal_dummy_named_prebuilt_etc", + src: "dummy_prebuilt_etc_data_1", + filename: "dummy_prebuilt_etc_data_1_renamed", + sub_dir: "dummy_sub_dir", +} + +prebuilt_etc { + name: "build.bazel.examples.apex.minimal_dummy_unnamed_prebuilt_etc", + src: "dummy_prebuilt_etc_data_2", + sub_dir: "dummy_sub_dir", +} + +prebuilt_etc { + name: "build.bazel.examples.apex.minimal_dummy_prebuilt_etc_without_subdir", + src: "dummy_prebuilt_etc_data_3", +} + +cc_binary { + name: "build.bazel.examples.apex.cc_binary", + srcs: ["main.cc"], + + apex_available: [ + "build.bazel.examples.apex.minimal", + "build.bazel.examples.apex.minimal_compressed" + ], + + // Because the APEX sets these + product_specific: true, + min_sdk_version: "30", +} + +apex_defaults { + name: "build.bazel.examples.apex.minimal_defaults", + manifest: "manifest.json", + file_contexts: ":build.bazel.examples.apex.minimal-file_contexts", + + // So that we aren't considered a "platform APEX" and can use a file_context that lives outside of system/sepolicy/apex + product_specific: true, + + key: "build.bazel.examples.apex.minimal.key", + min_sdk_version: "30", + + native_shared_libs: [ + "build.bazel.examples.apex.minimal_dummy_cc_lib", + ], + + prebuilts: [ + "build.bazel.examples.apex.minimal_dummy_named_prebuilt_etc", + "build.bazel.examples.apex.minimal_dummy_unnamed_prebuilt_etc", + "build.bazel.examples.apex.minimal_dummy_prebuilt_etc_without_subdir", + ], + + binaries: [ + "build.bazel.examples.apex.cc_binary", + ], + + certificate: ":build.bazel.examples.apex.minimal.certificate", +} + +apex { + name: "build.bazel.examples.apex.minimal", + defaults: [ + "build.bazel.examples.apex.minimal_defaults", + ] +} + +apex { + name: "build.bazel.examples.apex.minimal_compressed", + compressible: true, + defaults: [ + "build.bazel.examples.apex.minimal_defaults", + ] +} diff --git a/examples/apex/minimal/build.bazel.examples.apex.minimal.avbpubkey b/examples/apex/minimal/build.bazel.examples.apex.minimal.avbpubkey Binary files differnew file mode 100644 index 00000000..e6ffe589 --- /dev/null +++ b/examples/apex/minimal/build.bazel.examples.apex.minimal.avbpubkey diff --git a/examples/apex/minimal/build.bazel.examples.apex.minimal.pem b/examples/apex/minimal/build.bazel.examples.apex.minimal.pem new file mode 100644 index 00000000..28a36ae8 --- /dev/null +++ b/examples/apex/minimal/build.bazel.examples.apex.minimal.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEA3kVUTZJFAc0jOjcBQeikubCeYb6AGWzowgGurkwkX7zGY0rN +s30r8vTuFtmQreU1vjp3e5mOTas7TE9A6BevTSZGgaZEHPujolgf0hfPaCLlZw3p +BwEwE+lQppbRV/PEWLWM5suHUtFgAmch7o3dXO+5Kxy3T1wdivUT7rgIsWZZLwt7 +JNtTkm0YucmSBsN3QLhbevqb8msZrt8GgvJyd2jbGlTsP4j0R0DgSehkWwNFsxzn +abdT1Uk+uwEY0p3uHhlLxkIDoOV0LmKmrwTmfEDJFMqlhdJNWfZ80sYG+vYcb1YF ++5YwqnGiZwPUb4ARzdXwg7kjZVnKegi9tuTbVdkCT3MbS5r/uLQ1/SyD1aCg0iHS +pltJhqa5VddvuvNjuWgjDrvPJlub8DMfn3YSX6qaPSEBqDCDhbERzpAIGhAV8rUL +VbSS1E5F+YNvfNwGXVxd9u4D6HqS0sxA30RERYwYiBl8f8g7n9Kyw/pKNQ+ii5q7 +MtXcN7mBUPUwj4NpFnMyXjPWje1r2T0ZdwxXnU+hw1Kzgp5o/r5WJsh/GhJq52wt +WcMR3AnNf4IB1UAVKhlY9/6zHLqNDH/dRnuYkhLvUqheh/rVrdbwPnPhbSnxDE4D +WGXVM/dwoB5cqt2ZOQnIm4Ro4R5NR2mJ354Mene51pTx5qenZUKvo/UUC3UCAwEA +AQKCAgEAyezOA5LIYjSMtkxWNhw12gQcPswj7/VDF00T9oBx4w/KY1YHvfIPRS9C +RTvt9izvQBw5g+4im2jd/Btb6f3qYfpNv2bfJj1tkQTiE6lR4VcidRbsBlML7Grf +vBfzoyVv0O9OTDXGgHR59nTfHKuA2Pdnj0UNO2mB5UV0kEBclV1X1Cdn/jnKmJHI +DrCSmtZktkVFxll55ydpM7alYb0ERHrb4fYYkjfHRBwqJrqawRwd6/RsP1wvKurZ +tGXwuRaExo9QiZJbXYZWn/U7XHDlOyhYBS18ZjCB2SLWj10b5k5D0tmhocf1+kI1 +ucR+77S3d/nOALzm7SI/kdHzF/6dSpQEptNeYaBxfNnDe+tjTseuINN0L+q/gU/j +tt2+HtAKzEwNsZTbbU6CHkMXIndiKzMRNtQQvqL0st+5EY3LNzAhq81xPpIEk+lR +3/lh1utDExr5APIzGzreNTthmZ/mUBtb412retgtvqZenUqmOw6bE/HsesuvNcT5 +kwPgwv6pXnuVUJpofBmq3iH9q1oNfNapLWMVSzsqYncMG0rJx98WcZzcuXFrpYic +fnDLJ7f38PguX5IUTAVj4vxbAmPizDz4pByW6pwKtHegSHf35Zj7dZovV7dsjHg5 +KKOpEi86bo4jBlGtAQ41/mZi38yrnd2WZlYHb542gimDWPJb3FkCggEBAP5mqdJw +NCDDCKlp/TMiP1MfgyLIgy4AELuU8AV6pSxYSd6bJZmEe6IWxgrzmBn4HdtL1W4B +asSjRTKeTxcStB3Kang5OAYWFP/IoWprNXcNWdHZmfLXcK4kU4JoXxGZawsBwGip +2ZlXdvWcAc/FABVGOoj0SHDT6/9GCXBw5+6u9aQLJ05E75hd3t6kMZAXaKfL1di+ +EzGhkT7bTIWDBfAGWVfAv9fbWki3/Tr4S3TCRZdDry3oLtum9f9dvdRch3Zmx3WS +wKCVsD5cAb1HMc1wc6ftb0AHRaeC/Ff5mBjXASPf6FrcJoqn0xvIwrjO+haUlsJa ++evcIY+ZzEzMGscCggEBAN+q976fvGwCNIV+aocE5FBJlwTvPaAGTnUx54VZLD9Y +b0IlZVOHQjS+33kl35WqTAm2byziWn05YN6Twgmz8V18GBg72Ow6JOKKV1F/1f77 +aVnX1uIY4XGzQAyUNRmmzZkFhINPdfeKwVDij4bJdnxZz9TC6emRkqFFF2xygRm3 +sIeFEbJAKU8VFQPqDl89rHAnS99ey6uZsmiaRh0Fiz4zz0sMD7b95FRrhF7ISF0w +2f+75tsud/JIVg00O6HAY1NN7uNBHUrVe6WUDmC5zX1dgfKOrLTs8y9SBMHUmvDw +xMYBp16LBeC2AX1QH2H4eqw8Un2KfcZsKREQNS3AS+MCggEBAJ3JfzsObMaFMTIi +kvDMJQqhWOySAQre74Ho9pXvY9MFT4vKHqABE82M4niYbZZt1dbWSiJYrqgXvCuR +FOzypNR1X2QB2UWtHIkpHzSqGhclKLiHhnygMztPPJx9r7lytnm1NGm2L0h0f0wG +3vjG7y5CyLt5CBUy3AUQo3oiDTh1O9XrNQ3Oo+yVDE56+GKuojKwsookGjOGSlf6 +HJQSl/Qve+p5moN/gZfoxh91MRfSBuezC0wl3ipOe/VPZxX37ez6AmxiASeYsB99 +YXbMWY9aV+Gn2oCUAhfLnkfSfwupDwP5NFxmf2EwhjeNZXjKp0KqFnelha1Mc2Rl +t9lKHrECggEAINhieUPLz7CjNddSnXFCdefYnYjka9OBQgSKEzL9JHBAJl6HerSq +LES7XHcLVlcjw2le+iBjWXBLoWPhYrth/nByuVxQhqAjOoFGQVmce01AM0lW7Ozq +bl/8T9yTz6iHGKmOxBmvISLnMJGUsh5zn98wffYBk6Mk/TnALtTB0J0A92W7K4nw +hwR5iZGdeq7725xpr+ujzRdM879z/F8MC55gGo0y4ZR3K05+qcyPk9/Qo5VcWWOB +ibrFqDq7Zw1xiJIxOn4dZxfiiHYthlbCrl5E7vG0uvzw2UoaGLy35Bn3S3yCr4eK +3Wzc2yIg+aobmZ9iHb1wqurHPRI8PPO4MQKCAQBU7kJwRhJ1BUd0mG59aJSIwWOn +PBl7oMBwGGrKp7/lquK+2VAR/Q0uFnGtYkjFg4n2kzi72F1IkLS02BTo5QR5Jwfr +oJmH9ZldB+StNuqA4Zhg53dSW0WSkVN/xBGIWBY99b+yJi0NONrIL11xbBwPTMMp +3CI0kfNnl9iWuNq/7q0eilf1S2P+L4uaE+N5fmfS2VVRTV1ChiSZcZEHBKBp32MC +e04kkElHzDGaK5Jsi6AlPppH2TYsewmbMwseBUTH/pst0HJK2pTLpbsNQKA7e2v9 +4O3DzvgWLrXNqdvwSnd4efahsm+aoE9eyj3rIMj1JKsvAtc5WEmXhEke22dX +-----END RSA PRIVATE KEY----- diff --git a/examples/apex/minimal/build.bazel.examples.apex.minimal.pk8 b/examples/apex/minimal/build.bazel.examples.apex.minimal.pk8 Binary files differnew file mode 100644 index 00000000..96a1b6a3 --- /dev/null +++ b/examples/apex/minimal/build.bazel.examples.apex.minimal.pk8 diff --git a/examples/apex/minimal/build.bazel.examples.apex.minimal.x509.pem b/examples/apex/minimal/build.bazel.examples.apex.minimal.x509.pem new file mode 100644 index 00000000..a70fb31a --- /dev/null +++ b/examples/apex/minimal/build.bazel.examples.apex.minimal.x509.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF5TCCA80CFG12QQebWMaR+Kj0TNs0Y7VzDDVEMA0GCSqGSIb3DQEBCwUAMIGt +MQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNTW91 +bnRhaW4gVmlldzEQMA4GA1UECgwHQW5kcm9pZDEQMA4GA1UECwwHQW5kcm9pZDEi +MCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lkLmNvbTEpMCcGA1UEAwwgYnVp +bGQuYmF6ZWwuZXhhbXBsZXMubWluaW1hbGFwZXgwIBcNMjEwNzE2MDU1MTExWhgP +NDc1OTA2MTIwNTUxMTFaMIGtMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZv +cm5pYTEWMBQGA1UEBwwNTW91bnRhaW4gVmlldzEQMA4GA1UECgwHQW5kcm9pZDEQ +MA4GA1UECwwHQW5kcm9pZDEiMCAGCSqGSIb3DQEJARYTYW5kcm9pZEBhbmRyb2lk +LmNvbTEpMCcGA1UEAwwgYnVpbGQuYmF6ZWwuZXhhbXBsZXMubWluaW1hbGFwZXgw +ggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCu0WkwooTpX9wzJoM79KjD +ZNj1tl74H/v2Iac/j4QYZiYBtcZelPCHISY5CWYPjkA5HY3dka6zjMeh7gUjRLre +7pYiFWwdeQmv6sAO3q0bYEK4+WqLdiYQR9tV8bkSPWy+3DUqKkHEfbNCPip0tlYw +h4rnZ0b5YZPGRLHsFtX1SS+RRtlg91eiAgpWdhhL4fELJUv2jD2o/DKYZI+4fwwx +P/sgoiS18IAxiHvN6a3AcC70YvWape+GhVtptlYBaFNNQ7jmVLXsXIyCMrfFypbT +jopjMhuk/pnazA/3+eK1Yay3OPtPM9NGE2VVBvHDqwCeQT0EIe8hIsrHIUVy067q +/byNk3hVM3tK9z8/OPYRDlLyzxshfdt0Q0JpkTdzXXBT2t5i0bow6+9e1rJyw7xM +zRczYxfmyKti4KZ9ZrykENW8P2vYQNj10ZANvC1WGM8nFSet5I61xnxuQMy4wZwz +4a3yz3PPdD1iFxjw0r57z1/8k9qSsRCVXipLOo7ZxweoGyyEgUuTcvrel6fAcvct +V+m0oBNLWG6+uAViPKWdNr0Wl0odamjj2IOowaw5/1QHWGADpXGob6Dy7k0fVL8Z +U/3VeNA/e293MHM9M7eWRxrMduUGCDYicyTUsHKPgFVpk5A9kHe4b6iQb/6fztyy +IZ5lz/jGQOITXe0HE4CRywIDAQABMA0GCSqGSIb3DQEBCwUAA4ICAQA9gnyIrSW3 +2/oOfApLdPcKLXxOKF0MGykV4OQjWJTCwIOojmPVmFfd2nDFdAy6yW5kSVas0Om0 +CABQqMHqfayCzgECgNkzdNRwwczKgPMBijt+SpSqpA9ma9KTTyWI20vZfXj/5d4G +3cZXCJ20hYP5eCjz2JoqEKuHvzcf4k4U6hPOyGHw7Zj8XQuoHHUJzpWZzn6/8Qh5 +ESAmQ6JJWT2JrpKVvHcaO+SNQmox1+s0+4e3L6WqtoMAHkaKSNGq/8/VH1A+0qg8 +BQGTycPQb0lhV3laDxkKVnBNC5tB1qi+i8mHxQnTGh7GsZWgvFwyPMdqchJa/c69 +oZX06Ip99UZSQpMCh6GOxs7KoD7idN9mmsKtE7ycu6mYr4/tEGbAv0/rBiZvxIXU +K22GfQlGKpFRd2rxOcguRj+KavdM8N+zokT9i+k+w0xJQWnHWp6faw/oYBkahAkr +GKh60mALHcU6L7SGM3TAJV4Xsy+wy6KwhkJZVFgMGKsdcx9aj7tSwmgW8RMiYrpQ +B6j8s4jxmRQn0yFBkmjRr9dgWFQh8I2hoMt+Wu4AuIH6Ui4PvE7gcV0h2ws+QMsA ++64a702ESExURkhtamWlxiKbnmy4rcDycdE1rb2XnZM1hic1R0PYmZV9jZo1tPk+ +YAqGsO9wGSNNs9Lej85K8DzErmYEVQZuTQ== +-----END CERTIFICATE----- diff --git a/examples/apex/minimal/dummy_cc_lib.cc b/examples/apex/minimal/dummy_cc_lib.cc new file mode 100644 index 00000000..36a07a0a --- /dev/null +++ b/examples/apex/minimal/dummy_cc_lib.cc @@ -0,0 +1,9 @@ +int main(int argc, char** argv) { + // Unused + (void)argc; + + // Unused + (void)argv; + + return 0; +} diff --git a/examples/apex/minimal/dummy_prebuilt_etc_data_1 b/examples/apex/minimal/dummy_prebuilt_etc_data_1 new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/examples/apex/minimal/dummy_prebuilt_etc_data_1 @@ -0,0 +1 @@ +1 diff --git a/examples/apex/minimal/dummy_prebuilt_etc_data_2 b/examples/apex/minimal/dummy_prebuilt_etc_data_2 new file mode 100644 index 00000000..0cfbf088 --- /dev/null +++ b/examples/apex/minimal/dummy_prebuilt_etc_data_2 @@ -0,0 +1 @@ +2 diff --git a/examples/apex/minimal/dummy_prebuilt_etc_data_3 b/examples/apex/minimal/dummy_prebuilt_etc_data_3 new file mode 100644 index 00000000..00750edc --- /dev/null +++ b/examples/apex/minimal/dummy_prebuilt_etc_data_3 @@ -0,0 +1 @@ +3 diff --git a/examples/apex/minimal/file_contexts b/examples/apex/minimal/file_contexts new file mode 100644 index 00000000..3619cc83 --- /dev/null +++ b/examples/apex/minimal/file_contexts @@ -0,0 +1,5 @@ +/bin/apex_test_preInstallHook u:object_r:apex_test_prepostinstall_exec:s0 +/bin/apex_test_postInstallHook u:object_r:apex_test_prepostinstall_exec:s0 +/bin/surfaceflinger u:object_r:surfaceflinger_exec:s0 +/lib(64)?(/.*)? u:object_r:system_lib_file:s0 +(/.*)? u:object_r:system_file:s0 diff --git a/examples/apex/minimal/main.cc b/examples/apex/minimal/main.cc new file mode 100644 index 00000000..76e81970 --- /dev/null +++ b/examples/apex/minimal/main.cc @@ -0,0 +1 @@ +int main() { return 0; } diff --git a/examples/apex/minimal/manifest.json b/examples/apex/minimal/manifest.json new file mode 100644 index 00000000..68b4e42c --- /dev/null +++ b/examples/apex/minimal/manifest.json @@ -0,0 +1,4 @@ +{ + "name": "build.bazel.examples.apex.minimal", + "version": 1 +} diff --git a/examples/java/com/bazel/BUILD.bazel b/examples/java/com/bazel/BUILD.bazel new file mode 100644 index 00000000..6f96f9d3 --- /dev/null +++ b/examples/java/com/bazel/BUILD.bazel @@ -0,0 +1,11 @@ +java_binary( + name = "hello_java", + srcs = ["example/HelloWorld.java"], + main_class = "com.bazel.example.HelloWorld", + deps = [":hello_java_lib"], +) + +java_library( + name = "hello_java_lib", + srcs = ["example_lib/HelloLib.java"], +) diff --git a/examples/java/com/bazel/example/HelloWorld.java b/examples/java/com/bazel/example/HelloWorld.java new file mode 100644 index 00000000..d74bb89b --- /dev/null +++ b/examples/java/com/bazel/example/HelloWorld.java @@ -0,0 +1,10 @@ +package com.bazel.example; + +import com.bazel.example_lib.HelloLib; + +public class HelloWorld { + public static void main(String[] args) { + System.out.println("Hello world!"); + System.out.println("Library says: " + HelloLib.libValue()); + } +} diff --git a/examples/java/com/bazel/example_lib/HelloLib.java b/examples/java/com/bazel/example_lib/HelloLib.java new file mode 100644 index 00000000..8937060b --- /dev/null +++ b/examples/java/com/bazel/example_lib/HelloLib.java @@ -0,0 +1,7 @@ +package com.bazel.example_lib; + +public class HelloLib { + public static String libValue() { + return "Hello Library!"; + } +} diff --git a/examples/soong_config_variables/Android.bp b/examples/soong_config_variables/Android.bp new file mode 100644 index 00000000..e863e820 --- /dev/null +++ b/examples/soong_config_variables/Android.bp @@ -0,0 +1,101 @@ +// Use local copy of the soong_config_module_type dep graph to keep this Android.bp +// self-contained. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +soong_config_string_variable { + name: "fake_library_linking_strategy", + values: [ + "prefer_static", + ], +} + +soong_config_module_type { + name: "fake_library_linking_strategy_cc_defaults", + module_type: "cc_defaults", + config_namespace: "bp2build", + variables: ["fake_library_linking_strategy"], + properties: [ + "shared_libs", + "static_libs", + ], +} + +// Fake cc_library modules for testing +cc_library { + name: "bp2build_foo", + srcs: ["main.cpp"], +} + +cc_library { + name: "bp2build_bar", + srcs: ["main.cpp"], +} + +cc_library { + name: "bp2build_baz", + srcs: ["main.cpp"], +} + +cc_library { + name: "bp2build_qux", + srcs: ["main.cpp"], +} + +cc_library { + name: "bp2build_quux", + srcs: ["main.cpp"], +} + +fake_library_linking_strategy_cc_defaults { + name: "fake_libadbd_binary_dependencies", + static_libs: [ + "bp2build_foo", + ], + + shared_libs: [ + "bp2build_bar", + ], + + soong_config_variables:{ + fake_library_linking_strategy: { + prefer_static: { + static_libs: [ + "bp2build_baz", + ], + }, + conditions_default: { + shared_libs: [ + "bp2build_qux", + ], + }, + }, + }, + + target: { + android: { + shared_libs: ["bp2build_quux"], + }, + linux_glibc: { + enabled: false, + }, + linux_musl: { + enabled: false, + }, + linux_bionic: { + enabled: false, + }, + }, +} + +// Experimental "stub" adbd for bp2build development +cc_binary { + name: "bp2build_adbd", + defaults: ["adbd_defaults", "host_adbd_supported", "fake_libadbd_binary_dependencies"], + srcs: [ + "main.cpp", + ], + use_version_lib: false, +} diff --git a/examples/soong_config_variables/main.cpp b/examples/soong_config_variables/main.cpp new file mode 100644 index 00000000..143b6385 --- /dev/null +++ b/examples/soong_config_variables/main.cpp @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +int main() {return 0; } diff --git a/json_module_graph/README.md b/json_module_graph/README.md index a374bff1..f6d0f0a8 100644 --- a/json_module_graph/README.md +++ b/json_module_graph/README.md @@ -1,32 +1,39 @@ # JSON module graph queries This directory contains `jq` scripts that query Soong's module graph. - -It uses the JSON module graph that Soongs dumps when the -`SOONG_DUMP_JSON_MODULE_GRAPH` environment variable is set. +`jq` may be installed through your distribution's repository. Usage: ``` -SOONG_DUMP_JSON_MODULE_GRAPH=<some file> m nothing -query.sh [-C] <command> <some file> [argument] +m json-module-graph +query.sh [-C] <command> <base-of-your-tree>/out/soong/module-graph.json [argument] ``` The following commands are available: -* `printModule` prints all variations of a given module +* `directDeps` prints the names of the direct dependencies of the given module +* `distanceFromLeaves` prints the longest distance each module has from a leaf + in the module graph within the transitive closure of given module * `filterSubtree` dumps only those modules that are in the given subtree of the source tree -* `directDeps` prints the names of the direct dependencies of the given module +* `fullTransitiveDeps` returns the full transitive dependencies of the given + module +* `moduleTypeStats`: returns of a summary of the module types present on the + input +* `modulesOfType`: returns the names of modules of the input type +* `printModule` prints all variations of a given module +* `printModule`: returns a slightly more consise view of the input module +* `properties`: returns the properties set in the input module, includes + properties set via defaults * `transitiveDeps` prints the names of the transitive dependencies of the given module -* `fullTransitiveDeps` returns the full transitive dependencies of the given - module -* `distanceFromLeaves` prints the longest distance each module has from a leaf - in the module graph within the transitive closure of given module * `usedVariations` returns a map that shows which variations are used in the input and what values they take * `variantTransitions` summarizes the variant transitions in the transitive closure of the given module +* `fullTransitiveDepsProperties` returns the properties set (including via + defaults) grouped by module type of the modules in the transitive closure of + the given module It's best to filter the full module graph to the part you are interested in because `jq` isn't too fast on the full graph. diff --git a/json_module_graph/directDeps.jq b/json_module_graph/directDeps.jq index b246671b..9c82a8d6 100644 --- a/json_module_graph/directDeps.jq +++ b/json_module_graph/directDeps.jq @@ -1,3 +1,5 @@ +# CMD: Returns the names of the direct dependencies of the module named $arg + include "library"; [.[] | select(.Name == $arg) | .Deps | map(.Name)] | flatten | unique | sort
\ No newline at end of file diff --git a/json_module_graph/distanceFromLeaves.jq b/json_module_graph/distanceFromLeaves.jq index 9a488dc1..d48fa674 100644 --- a/json_module_graph/distanceFromLeaves.jq +++ b/json_module_graph/distanceFromLeaves.jq @@ -1,3 +1,5 @@ +# CMD: Returns the maximum distance from a leaf for each module + include "library"; def onlyDeps: diff --git a/json_module_graph/filterSubtree.jq b/json_module_graph/filterSubtree.jq index 9d5d6a2d..16c9fcc5 100644 --- a/json_module_graph/filterSubtree.jq +++ b/json_module_graph/filterSubtree.jq @@ -1,3 +1,5 @@ +# CMD: Returns modules defined under the directory $arg + include "library"; def isBlueprint($p): .Blueprint | index($p) != null diff --git a/json_module_graph/findModulesCrossPkgBoundary.jq b/json_module_graph/findModulesCrossPkgBoundary.jq new file mode 100644 index 00000000..5ebfc79b --- /dev/null +++ b/json_module_graph/findModulesCrossPkgBoundary.jq @@ -0,0 +1,61 @@ +# CMD: Finds all modules whose input files cross package boundaries. + +include "library"; + +def getBlueprintDirPaths: +[.[] | .Blueprint | getDirPath] | sort_by(.) | unique | map({(.):""}) | add +; + +def getNonNullActionModules: +[.[] | select(nonNullAction)] +; + +def getOutputsOfModule: +[.Module.Actions | .[] | .Outputs | if . == null then [] else . end | .[]] +; + +def getOutputsOfModules($nonNullActionModules): +$nonNullActionModules | map({(.Name):getOutputsOfModule}) | add +; + +def getDepOutputs($outputsOfModules): +. as $depName | +if in($outputsOfModules) then ($outputsOfModules | ."\($depName)") +else [] end | .[] +; + +def getDepOutputsOfModule($outputsOfModules): +[.Deps | .[] | .Name | getDepOutputs($outputsOfModules)] +| map({(.):""}) | add +; + +def isDirPathMatch($blueprintDirPath; $allBlueprintDirPaths): + def _isDirPathMatch($blueprintDirPath; $allBlueprintDirPaths): + # True if there's a Blueprint file in the path and the path isn't + # equal to $blueprintDirPath of the module. + if in($allBlueprintDirPaths) and . != $blueprintDirPath then true + # Stops checking if the current path is already the $blueprintDirPath. + elif . == $blueprintDirPath then false + # Usually it should not hit this logic as it stops when the path is + # equal to $blueprintDirPath. + elif (contains("/") | not) then false + else (getDirPath | _isDirPathMatch($blueprintDirPath; $allBlueprintDirPaths)) + end + ; + _isDirPathMatch($blueprintDirPath; $allBlueprintDirPaths) +; + +def isActionInputMatch($outputsOfModules; $allBlueprintDirPaths): +. as $moduleVariant | .Blueprint | getDirPath as $blueprintDirPath | +$moduleVariant | getDepOutputsOfModule($outputsOfModules) as $depOutputs | +$moduleVariant | getActionInputs | select(in($depOutputs) | not) | +select(startswith($blueprintDirPath)) | getDirPath | +isDirPathMatch($blueprintDirPath; $allBlueprintDirPaths) +; + +getBlueprintDirPaths as $allBlueprintDirPaths | +getNonNullActionModules as $nonNullActionModules | +getOutputsOfModules($nonNullActionModules) as $outputsOfModules | +[$nonNullActionModules | .[] | +select(isActionInputMatch($outputsOfModules; $allBlueprintDirPaths)) | +.Name] | sort_by(.) | unique diff --git a/json_module_graph/findModulesWithNameSrcCollision.jq b/json_module_graph/findModulesWithNameSrcCollision.jq new file mode 100644 index 00000000..f3907d96 --- /dev/null +++ b/json_module_graph/findModulesWithNameSrcCollision.jq @@ -0,0 +1,16 @@ +# CMD: Finds all modules whose name is equal to the name of one of its input +# files. + +include "library"; + +def isActionInputMatch($name; $blueprintDirPath): . as $actionInput | +getDirPath as $inputDirPath | $actionInput | split("/") | +last | . == $name and $inputDirPath == $blueprintDirPath +; + +def isActionInputsMatch($name; $blueprint): getActionInputs as $actionInputs | +$blueprint | getDirPath as $blueprintDirPath | $actionInputs | +isActionInputMatch($name; $blueprintDirPath) +; + +[.[] | select(nonNullAction) | select(isActionInputsMatch(.Name; .Blueprint)) | .Name] | sort_by(.) | unique diff --git a/json_module_graph/findModulesWithProperty.jq b/json_module_graph/findModulesWithProperty.jq new file mode 100644 index 00000000..79f9b1ae --- /dev/null +++ b/json_module_graph/findModulesWithProperty.jq @@ -0,0 +1,13 @@ +# CMD: Returns the modules of type $arg that have property $arg2 + +def hasPropertyWithName($a): + map(select(.Name == $a)) | + length | + . > 0 +; + +[.[] | +select(.Type == $arg) | +select(.Module.Android.SetProperties | + hasPropertyWithName($arg2)) | +.Name] | unique | sort | .[] diff --git a/json_module_graph/findModulesWithSrcType.jq b/json_module_graph/findModulesWithSrcType.jq new file mode 100644 index 00000000..cd088baf --- /dev/null +++ b/json_module_graph/findModulesWithSrcType.jq @@ -0,0 +1,8 @@ +# CMD: Finds all modules whose input files contain the specific file type $arg. + +include "library"; + +def isActionInputMatch($fileType): getActionInputs | split(".") | last | . == $fileType +; + +[.[] | select(nonNullAction) | select(isActionInputMatch($arg)) | .Name] | sort_by(.) | unique diff --git a/json_module_graph/fullTransitiveDeps.jq b/json_module_graph/fullTransitiveDeps.jq index 1f982cfe..39e12b7d 100644 --- a/json_module_graph/fullTransitiveDeps.jq +++ b/json_module_graph/fullTransitiveDeps.jq @@ -1,10 +1,7 @@ +# CMD: Returns the modules in the transitive closure of module $arg + include "library"; -[((moduleGraphNoVariants | removeSelfEdges) as $m | - [$arg] | - transitiveDeps($m)) as $names | - .[] | - select (IN(.Name; $names | .[]))] | - sort_by(.Name) +fullTransitiveDeps([$arg]) diff --git a/json_module_graph/fullTransitiveDepsProperties.jq b/json_module_graph/fullTransitiveDepsProperties.jq new file mode 100644 index 00000000..ca28d359 --- /dev/null +++ b/json_module_graph/fullTransitiveDepsProperties.jq @@ -0,0 +1,16 @@ +# CMD: Returns the properties of module types in the transitive closure of module $arg + +include "library"; + +[((moduleGraphNoVariants | removeSelfEdges) as $m | + [$arg] | + transitiveDeps($m)) as $names | + .[] | + select (IN(.Name; $names | .[]))] | + group_by(.Type) | + map({Type: .[0].Type, + Props: map(.Module.Android.SetProperties) | flatten | map(.Name) | unique | sort }) | + sort_by(.Type) + + + diff --git a/json_module_graph/fullTransitiveModuleTypeDeps.jq b/json_module_graph/fullTransitiveModuleTypeDeps.jq new file mode 100644 index 00000000..3b3ea32c --- /dev/null +++ b/json_module_graph/fullTransitiveModuleTypeDeps.jq @@ -0,0 +1,7 @@ +# CMD: Returns all modules of type $arg and all modules in their transitive closure. + +include "library"; + +fullTransitiveDeps(modulesOfType($arg)) + + diff --git a/json_module_graph/library.jq b/json_module_graph/library.jq index 9fe5be4b..6550e1af 100644 --- a/json_module_graph/library.jq +++ b/json_module_graph/library.jq @@ -116,3 +116,30 @@ def findEdge($from;$to): .Deps |= [.[] | select(.Name == $to)] | select((.Deps | length) > 0) ; + +def nonNullAction: .Module.Actions != null +; + +def getActionInputs: .Module.Actions | .[] | + .Inputs | if . == null then [] else . end | .[] +; + +# Gets the directory path by the given file path. +def getDirPath: sub("(?<p>.*)\\/.*"; "\(.p)") +; + +# Returns the names of modules of type $arg +def modulesOfType($arg): + [.[] | select(.Type == $arg) | .Name] | unique +; + +# Returns the modules in the transitive closure of $arg. +# $arg must be an array of modules names +def fullTransitiveDeps($arg): + [((moduleGraphNoVariants | removeSelfEdges) as $m | + $arg | + transitiveDeps($m)) as $names | + .[] | + select (IN(.Name; $names | .[]))] | + sort_by(.Name) +; diff --git a/json_module_graph/moduleTypeStats.jq b/json_module_graph/moduleTypeStats.jq new file mode 100644 index 00000000..ef40a25e --- /dev/null +++ b/json_module_graph/moduleTypeStats.jq @@ -0,0 +1,15 @@ +# CMD: Returns a summary of the module types present on the input + +include "library"; + +def moduleTypeStats($arg): + group_by(.Type) | + map({ + Type: .[0].Type, + Count: map(.Name) | unique | length, + VariantCount: length, + }) | + sort_by(.Count) + ; + +moduleTypeStats($arg)
\ No newline at end of file diff --git a/json_module_graph/modulesOfType.jq b/json_module_graph/modulesOfType.jq new file mode 100644 index 00000000..6e796bf1 --- /dev/null +++ b/json_module_graph/modulesOfType.jq @@ -0,0 +1,5 @@ +# CMD: Returns the names of modules of type $arg + +include "library"; + +modulesOfType($arg)
\ No newline at end of file diff --git a/json_module_graph/printModule.jq b/json_module_graph/printModule.jq index 7ef4fccb..194a0bd1 100644 --- a/json_module_graph/printModule.jq +++ b/json_module_graph/printModule.jq @@ -1,3 +1,5 @@ +# CMD: Prints the module named $arg in a slightly more concise way + include "library"; def printModule($mod): diff --git a/json_module_graph/properties.jq b/json_module_graph/properties.jq new file mode 100644 index 00000000..337b5d58 --- /dev/null +++ b/json_module_graph/properties.jq @@ -0,0 +1,7 @@ +# CMD: Returns the names of properties used by $arg + +[.[] | + select (.Name == $arg) | + .Module.Android.SetProperties | + map(.Name)] | + flatten | unique | sort diff --git a/json_module_graph/query.sh b/json_module_graph/query.sh index c720b5ac..fb3b6c09 100755 --- a/json_module_graph/query.sh +++ b/json_module_graph/query.sh @@ -1,13 +1,25 @@ #!/bin/bash -eu -JQARGS="" +LIBDIR="$(dirname "$(readlink -f "$0")")" function print_usage() { echo "Usage: query.sh [-C] <command> <graph JSON> [argument]" 1>&2 echo " -C: colorized output" 1>&2 + echo + echo "Commands": + for jq in "$LIBDIR"/*.jq; do + if ! grep -q "^# CMD:" "$jq"; then + continue + fi + + local CMD="$(echo $(basename "$jq") | sed 's/\..*$//')" + echo " $CMD": $(cat "$jq" | grep "^# CMD:" | head -n 1 | sed 's/^# CMD://') + done exit 1 } +JQARGS="" + while getopts "C" arg; do case "$arg" in C) @@ -16,6 +28,7 @@ while getopts "C" arg; do ;; *) print_usage + ;; esac done @@ -32,6 +45,10 @@ else ARG="" fi -LIBDIR="$(dirname "$(readlink -f "$0")")" +if [[ "$#" -gt 3 ]]; then + ARG2="$4" +else + ARG2="" +fi -jq $JQARGS -L "$LIBDIR" -f "$LIBDIR/$COMMAND".jq "$GRAPH" --arg arg "$ARG" +jq $JQARGS -L "$LIBDIR" -f "$LIBDIR/$COMMAND".jq "$GRAPH" --arg arg "$ARG" --arg arg2 "$ARG2" diff --git a/json_module_graph/transitiveDeps.jq b/json_module_graph/transitiveDeps.jq index c9a5e43f..d0a55e55 100644 --- a/json_module_graph/transitiveDeps.jq +++ b/json_module_graph/transitiveDeps.jq @@ -1,3 +1,5 @@ +# CMD: Returns the names of the transitive dependencies of the module named $arg + include "library"; (moduleGraphNoVariants | removeSelfEdges) as $m | diff --git a/json_module_graph/usedVariations.jq b/json_module_graph/usedVariations.jq index f6104439..7544358e 100644 --- a/json_module_graph/usedVariations.jq +++ b/json_module_graph/usedVariations.jq @@ -1,3 +1,5 @@ +# CMD: Prints the set of variations and their values used in the input + [[.[] | .Variations | select(. != null) | to_entries] | flatten | group_by(.key) | diff --git a/json_module_graph/variantTransitions.jq b/json_module_graph/variantTransitions.jq index 1f0ddc48..ebf57564 100644 --- a/json_module_graph/variantTransitions.jq +++ b/json_module_graph/variantTransitions.jq @@ -1,3 +1,5 @@ +# CMD: Groups outgoing dependency edges by the differences in variants + include "library"; # This filters out modules with "interesting" deps diff --git a/linux.bazelrc b/linux.bazelrc index 824daf5d..4fb75ab9 100644 --- a/linux.bazelrc +++ b/linux.bazelrc @@ -1,3 +1,10 @@ import %workspace%/build/bazel/common.bazelrc build --host_platform //build/bazel/platforms:linux_x86_64 + +# Workaround JVM segfault issue as suggested at +# https://github.com/bazelbuild/bazel/issues/3236#issuecomment-310656266 +build --sandbox_tmpfs_path=/tmp/ + +# Create a build number that will be injected later. +build --workspace_status_command=build/bazel/scripts/gen_build_number.sh
\ No newline at end of file diff --git a/mk2rbc/apply_scripted_change.sh b/mk2rbc/apply_scripted_change.sh new file mode 100755 index 00000000..5d98d71a --- /dev/null +++ b/mk2rbc/apply_scripted_change.sh @@ -0,0 +1,45 @@ +#! /bin/bash +# Run given scripted change and commit the changes. +# +# Assumes that the current directory is the top-level directory of +# the Android source code, created with 'repo init', and that 'repo' +# tool is on the path. +# For each of the given repo git repositories: +# 1. Checks there are neither modified not untracked files in it. +# 2. Runs the given script, which is supposed to change some files +# 3. Creates a development branch if necessary +# 4. Commits changed files. The commit message is extracted from the +# script and contains all the lines in it starting with ##CL +# +# As an example, running +# build/bazel/mk2rbc/apply_scripted_change.sh build/bazel/mk2rbc/replace_is_board_platform.sh hardware/qcom/media +# replaces the old is-board-platform calls with the new is-board-platform2 calls. + +set -eu + +function die() { format=$1; shift; printf "$format\n" $@; exit 1; } +function usage() { die "Usage: %s script git-repo ..." ${0##*/}; } + +(($#>=2)) || usage +declare -r script=$(realpath $1); shift +rc=0 + +[[ -x $script ]] || die "%s is not an executable script" $script +declare -r bugid="$(sed -nr 's/^##CL (Bug:|Fixes:) +([0-9]+)$/\2/p' $script)" +[[ -n "$bugid" ]] || die "$script contains neither '##CL Bug: ' nor '##CL Fixes: 'tag" + + +for gr in $@; do + [[ -d $gr/.git ]] || { echo $gr is not a Git directory; rc=1; continue; } + out=$(git -C $gr status --porcelain --untracked-files=no) || { die "so skipping $gr because of the above"; rc=1; continue; } + [[ -z "$out" ]] || { echo $gr contains uncommitted changes:; echo "$out" >&2; rc=1; continue; } + (cd $gr && $script) + modified="$(git -C $gr status --porcelain | sed -nr 's/^ M (.*)/\1/p')" + [[ -n "$modified" ]] || { echo no files changed in $gr; continue; } + [[ -z "$(repo status -q $gr 2>/dev/null)" ]] || repo start b$bugid $gr + git -C $gr add $modified + git -C $gr commit -q \ + -F <(sed -nr 's/^##CL *//p' $script; echo -e '\nThis change has been generated by the following script:\n\n```'; grep -vP '^##CL' $script; echo '```') +done + +exit $rc diff --git a/mk2rbc/replace_dynamic_qcpath.sh b/mk2rbc/replace_dynamic_qcpath.sh new file mode 100755 index 00000000..e1e743f1 --- /dev/null +++ b/mk2rbc/replace_dynamic_qcpath.sh @@ -0,0 +1,11 @@ +#! /bin/bash +##CL Provide location hint for the dynamically calculated paths. +##CL +##CL For the paths using QC_PROP_PATH or QC_PROP_ROOT +##CL Bug: 203582721 +##CL Test: treehugger +declare -r files="$(grep -rlP '^ *(\$\(call inherit-product|-?include).*\$\(QC_PROP_(PATH|ROOT)\)' --include 'BoardConfig*.mk')" +[[ -z "$files" ]] || sed -i -r -f <(cat <<"EOF" +/^ *(\$\(call inherit-product|-?include).*\$\(QC_PROP_(PATH|ROOT)\)/i#RBC# include_top vendor/qcom +EOF +) $files diff --git a/mk2rbc/replace_is_board_platform.sh b/mk2rbc/replace_is_board_platform.sh new file mode 100755 index 00000000..1f758b71 --- /dev/null +++ b/mk2rbc/replace_is_board_platform.sh @@ -0,0 +1,31 @@ +#! /bin/bash +##CL Replace is-board-platform[-in-list] with is-board-platform[-in-list]2 +##CL +##CL The regular is-board-platform[-in-list] functions are defined in +##CL some product/board configuration makefiles, and sometimes also +##CL used in Android.mk files. When the product/board configuration +##CL is converted to starlark, the functions will no longer be defined +##CL for the Android.mk files to use. Switch to using a version of +##CL these functions that is defined inside the core build system +##CL makefiles, so it will still be defined when the configuration +##CL is in Starlark. +##CL +##CL The new function returns either an empty string or the matching +##CL platform, while the old one returned either an empty string or true. +##CL So now if statements are compared against an empty string instead of +##CL true. +##CL +##CL Bug: 201477826 +##CL Test: treehugger +declare -r files="$(grep -rlP '^[^#]*call +is-board-platform' --include '*.mk' --exclude 'utils_test.mk' --exclude 'utils_sample_usage.mk')" +[[ -z "$files" ]] || sed -i -r -f <(cat <<"EOF" +s/ifeq +\(\$\(call is-board-platform,(.*)\), *true\)/ifneq (,$(call is-board-platform2,\1))/ +s/ifeq +\(\$\(call is-board-platform,(.*)\), *\)/ifeq (,$(call is-board-platform2,\1))/ +s/ifneq +\(\$\(call is-board-platform,(.*)\), *true\)/ifeq (,$(call is-board-platform2,\1))/ +s/ifeq +\(\$\(call is-board-platform-in-list,(.*)\), *true\)/ifneq (,$(call is-board-platform-in-list2,\1))/ +s/ifeq +\(\$\(call is-board-platform-in-list,(.*)\), *\)/ifeq (,$(call is-board-platform-in-list2,\1))/ +s/ifeq +\(\$\(call is-board-platform-in-list,(.*)\), *false\)/ifeq (,T) # TODO: remove useless check/ +s/ifneq +\(\$\(call is-board-platform-in-list,(.*)\), *true\)/ifeq (,$(call is-board-platform-in-list2,\1))/ +s/\$\(call is-board-platform,(.*)\)/$(call is-board-platform2,\1)/ +EOF +) $files diff --git a/mk2rbc/replace_is_platform_sdk_version_at_least.sh b/mk2rbc/replace_is_platform_sdk_version_at_least.sh new file mode 100755 index 00000000..4f4d4c82 --- /dev/null +++ b/mk2rbc/replace_is_platform_sdk_version_at_least.sh @@ -0,0 +1,16 @@ +#! /bin/bash +##CL Replace is-platform-sdk-version-at-least calls with checking IS_AT_LEAST_xxx. +##CL +##CL Bug: 201477826 +##CL Test: treehugger +declare -r files="$(grep -rlP '^[^#]*call +is-platform-sdk-version-at-least' --include '*.mk')" +[[ -z "$files" ]] || sed -i -r -f <(cat <<"EOF" +s/^([^#]*ifn?eq) +\(\$\(call is-platform-sdk-version-at-least, *(16|17|18|19|20|21|22|23|24|25)\), *true\)/\1 \(T,T\) \# TODO: Obsolete, please remove/ +s/^([^#]*)if(n?)eq +\(\$\(call is-platform-sdk-version-at-least, *26\), *true\)/\1if\2def IS_AT_LEAST_OPR1/ +s/^([^#]*)if(n?)eq +\(\$\(call is-platform-sdk-version-at-least, *27\), *true\)/\1if\2def IS_AT_LEAST_OPM1/ +s/^([^#]*)if(n?)eq +\(\$\(call is-platform-sdk-version-at-least, *28\), *true\)/\1if\2def IS_AT_LEAST_PPR1/ +s/^([^#]*)if(n?)eq +\(\$\(call is-platform-sdk-version-at-least, *29\), *true\)/\1if\2def IS_AT_LEAST_QP1A/ +s/^([^#]*)if(n?)eq +\(\$\(call is-platform-sdk-version-at-least, *30\), *true\)/\1if\2def IS_AT_LEAST_RP1A/ +s/^([^#]*)if(n?)eq +\(\$\(call is-platform-sdk-version-at-least, *31\), *true\)/\1if\2def IS_AT_LEAST_SP1A/ +EOF +) $files diff --git a/mk2rbc/replace_is_vendor_board_platform.sh b/mk2rbc/replace_is_vendor_board_platform.sh new file mode 100755 index 00000000..ea15e8d5 --- /dev/null +++ b/mk2rbc/replace_is_vendor_board_platform.sh @@ -0,0 +1,11 @@ +#! /bin/bash +##CL Replace is-vendor-board-platform with is-vendor-board-qcom. +##CL +##CL Bug: 201477826 +##CL Test: treehugger +declare -r files="$(grep -rlP '^[^#]*call +is-vendor-board-platform' --include '*.mk')" +[[ -z "$files" ]] || sed -i -r -f <(cat <<"EOF" +s/ifeq \(\$\(call is-vendor-board-platform,QCOM\),true\)/ifneq (,$(call is-vendor-board-qcom))/ +s/ifneq \(\$\(call is-vendor-board-platform,QCOM\),true\)/ifeq (,$(call is-vendor-board-qcom))/ +EOF +) $files diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index c6b193b8..24829e23 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -11,100 +11,125 @@ # # These model after the arch and OS definitions in build/soong/android/arch.go. +load("@soong_injection//product_config:product_variables.bzl", "product_vars") +load("//build/bazel/platforms:product_variables/product_platform.bzl", "android_platform", "product_variable_config") +load("//build/bazel/platforms/arch/variants:constants.bzl", "constants") +load( + "//prebuilts/clang/host/linux-x86:cc_toolchain_constants.bzl", + "arch_to_variants", + "variant_constraints", + "variant_name", +) + package(default_visibility = ["//visibility:public"]) +product_variable_config( + name = "android_target", + product_config_vars = product_vars, +) + # Linux is the OS # for the Linux kernel plus the glibc runtime. -platform( +android_platform( name = "linux_x86", constraint_values = [ "//build/bazel/platforms/arch:x86", "//build/bazel/platforms/os:linux", ], + product = ":android_target", ) -platform( +android_platform( name = "linux_x86_64", constraint_values = [ "//build/bazel/platforms/arch:x86_64", "//build/bazel/platforms/os:linux", ], + product = ":android_target", ) # linux_bionic is the OS for the Linux kernel plus the Bionic libc runtime, but # without the rest of Android. -platform( +android_platform( name = "linux_bionic_arm64", constraint_values = [ "//build/bazel/platforms/arch:arm64", "//build/bazel/platforms/os:linux_bionic", ], + product = ":android_target", ) -platform( +android_platform( name = "linux_bionic_x86_64", constraint_values = [ "//build/bazel/platforms/arch:x86_64", "//build/bazel/platforms/os:linux_bionic", ], + product = ":android_target", ) # Darwin is the OS for MacOS host machines. -platform( +android_platform( + name = "darwin_arm64", + constraint_values = [ + "//build/bazel/platforms/arch:arm64", + "//build/bazel/platforms/os:darwin", + ], + product = ":android_target", +) + +android_platform( name = "darwin_x86_64", constraint_values = [ "//build/bazel/platforms/arch:x86_64", - "//build/bazel/platforms/os:osx", + "//build/bazel/platforms/os:darwin", ], + product = ":android_target", ) # Windows is the OS for Windows host machines. -platform( +android_platform( name = "windows_x86", constraint_values = [ "//build/bazel/platforms/arch:x86", "//build/bazel/platforms/os:windows", ], + product = ":android_target", ) -platform( +android_platform( name = "windows_x86_64", constraint_values = [ "//build/bazel/platforms/arch:x86_64", "//build/bazel/platforms/os:windows", ], + product = ":android_target", ) -# Android is the OS for target devices that run all of Android, including the Linux kernel -# and the Bionic libc runtime. -platform( +alias( name = "android_arm", - constraint_values = [ - "//build/bazel/platforms/arch:arm", - "//build/bazel/platforms/os:android", - ], + actual = ":android_arm_armv7-a-neon", # default to armv7-a-neon ) -platform( +alias( name = "android_arm64", - constraint_values = [ - "//build/bazel/platforms/arch:arm64", - "//build/bazel/platforms/os:android", - ], + actual = ":android_arm64_armv8-a", # default to armv8-a ) -platform( - name = "android_x86", - constraint_values = [ - "//build/bazel/platforms/arch:x86", - "//build/bazel/platforms/os:android", - ], -) - -platform( - name = "android_x86_64", - constraint_values = [ - "//build/bazel/platforms/arch:x86_64", - "//build/bazel/platforms/os:android", - ], -) +[ + [ + android_platform( + name = "android_" + arch + variant_name(variant), + constraint_values = [ + "//build/bazel/platforms/arch:" + arch, + "//build/bazel/platforms/os:android", + ] + variant_constraints( + variant, + constants.AndroidArchToVariantToFeatures[arch], + ), + product = ":android_target", + ) + for variant in variants + ] + for arch, variants in arch_to_variants.items() +] diff --git a/platforms/arch/BUILD b/platforms/arch/BUILD index c0fdf7cd..35df294f 100644 --- a/platforms/arch/BUILD +++ b/platforms/arch/BUILD @@ -1,4 +1,5 @@ # Standard cpu name constraint_setting and constraint_values + licenses(["notice"]) package( diff --git a/platforms/arch/variants/BUILD b/platforms/arch/variants/BUILD new file mode 100644 index 00000000..0b722b6b --- /dev/null +++ b/platforms/arch/variants/BUILD @@ -0,0 +1,89 @@ +# Cpu/Arch Variants and features + +load("//build/bazel/product_variables:constants.bzl", _product_variable_constants = "constants") +load(":constants.bzl", "constants") + +constraint_setting( + name = "arch_variant_constraint", +) + +licenses(["notice"]) + +package( + default_visibility = ["//visibility:public"], +) + +[ + constraint_value( + name = arch_variant, + constraint_setting = "arch_variant_constraint", + ) + for arch_variant in constants.AvailableArchVariants +] + +[ + [ + config_setting( + name = variant + "-" + arch, + constraint_values = [ + _product_variable_constants.ArchVariantToConstraints[arch], + ":" + variant, + ], + ) + for variant in variants + ] + for arch, variants in constants.ArchToVariants.items() +] + +constraint_setting( + name = "cpu_variant_constraint", +) + +[ + constraint_value( + name = cpu_variant, + constraint_setting = "cpu_variant_constraint", + ) + for cpu_variant in constants.AvailableCpuVariants +] + +[ + [ + config_setting( + name = variant + "-" + arch, + constraint_values = [ + _product_variable_constants.ArchVariantToConstraints[arch], + ":" + variant, + ], + ) + for variant in variants + ] + for arch, variants in constants.CpuToVariants.items() +] + +[ + ( + constraint_setting( + name = "arch_feature_constraint_" + arch_feature, + ), + constraint_value( + name = arch_feature, + constraint_setting = "arch_feature_constraint_" + arch_feature, + ), + ) + for arch_feature in constants.AvailableArchFeatures +] + +[ + [ + config_setting( + name = feature + "-" + arch, + constraint_values = [ + _product_variable_constants.ArchVariantToConstraints[arch], + ":" + feature, + ], + ) + for feature in features + ] + for arch, features in constants.ArchToFeatures.items() +] diff --git a/platforms/arch/variants/constants.bzl b/platforms/arch/variants/constants.bzl new file mode 100644 index 00000000..1e5feb94 --- /dev/null +++ b/platforms/arch/variants/constants.bzl @@ -0,0 +1,44 @@ +"""Constants for arch/cpu variants/features.""" + +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load( + "@soong_injection//product_config:arch_configuration.bzl", + _android_arch_feature_for_arch_variant = "android_arch_feature_for_arch_variants", + _arch_to_cpu_variants = "arch_to_cpu_variants", + _arch_to_features = "arch_to_features", + _arch_to_variants = "arch_to_variants", +) + +def _flatten_string_list_dict_to_set(string_list_dict): + ret = {} + for l in string_list_dict.values(): + for i in l: + ret[i] = True + return ret + +_arch_variants = _flatten_string_list_dict_to_set(_arch_to_variants) +_cpu_variants = _flatten_string_list_dict_to_set(_arch_to_cpu_variants) +_arch_features = _flatten_string_list_dict_to_set(_arch_to_features) + +constants = struct( + AvailableArchVariants = _arch_variants, + AvailableCpuVariants = _cpu_variants, + AvailableArchFeatures = _arch_features, + ArchToVariants = _arch_to_variants, + CpuToVariants = _arch_to_cpu_variants, + ArchToFeatures = _arch_to_features, + AndroidArchToVariantToFeatures = _android_arch_feature_for_arch_variant, +) diff --git a/platforms/os/BUILD b/platforms/os/BUILD index 9f543d1c..bc8759f8 100644 --- a/platforms/os/BUILD +++ b/platforms/os/BUILD @@ -1,4 +1,7 @@ # Standard constraint_setting and constraint_values to be used in platforms. + +load("@bazel_skylib//lib:selects.bzl", "selects") + licenses(["notice"]) package( @@ -10,6 +13,13 @@ constraint_value( constraint_setting = "@platforms//os:os", ) +config_setting( + name = "android_config_setting", + constraint_values = [ + ":android", + ], +) + # Alias to the local_jdk's toolchain constraint to make local_jdk resolve # correctly with --tool_java_runtime_version=local_jdk and the checked-in JDK. alias( @@ -17,17 +27,42 @@ alias( actual = "@platforms//os:linux", ) +alias( + name = "linux_glibc", + actual = "@platforms//os:linux", +) + +constraint_value( + name = "linux_musl", + constraint_setting = "@platforms//os:os", +) + constraint_value( name = "linux_bionic", constraint_setting = "@platforms//os:os", ) +config_setting( + name = "linux_bionic_config_setting", + constraint_values = [ + ":linux_bionic", + ], +) + constraint_value( name = "windows", constraint_setting = "@platforms//os:os", ) constraint_value( - name = "fuchsia", + name = "darwin", constraint_setting = "@platforms//os:os", ) + +selects.config_setting_group( + name = "bionic", + match_any = [ + ":android_config_setting", + ":linux_bionic_config_setting", + ], +) diff --git a/platforms/os_arch/BUILD.bazel b/platforms/os_arch/BUILD.bazel new file mode 100644 index 00000000..d9c0ebe7 --- /dev/null +++ b/platforms/os_arch/BUILD.bazel @@ -0,0 +1,111 @@ +config_setting( + name = "android_arm", + constraint_values = [ + "//build/bazel/platforms/arch:arm", + "//build/bazel/platforms/os:android", + ], +) + +config_setting( + name = "android_arm64", + constraint_values = [ + "//build/bazel/platforms/arch:arm64", + "//build/bazel/platforms/os:android", + ], +) + +config_setting( + name = "android_x86", + constraint_values = [ + "//build/bazel/platforms/arch:x86", + "//build/bazel/platforms/os:android", + ], +) + +config_setting( + name = "android_x86_64", + constraint_values = [ + "//build/bazel/platforms/arch:x86_64", + "//build/bazel/platforms/os:android", + ], +) + +config_setting( + name = "darwin_arm64", + constraint_values = [ + "//build/bazel/platforms/arch:arm64", + "//build/bazel/platforms/os:darwin", + ], +) + +config_setting( + name = "darwin_x86_64", + constraint_values = [ + "//build/bazel/platforms/arch:x86_64", + "//build/bazel/platforms/os:darwin", + ], +) + +config_setting( + name = "linux_glibc_x86", + constraint_values = [ + "//build/bazel/platforms/arch:x86", + "//build/bazel/platforms/os:linux_glibc", + ], +) + +config_setting( + name = "linux_glibc_x86_64", + constraint_values = [ + "//build/bazel/platforms/arch:x86_64", + "//build/bazel/platforms/os:linux_glibc", + ], +) + +config_setting( + name = "linux_bionic_arm64", + constraint_values = [ + "//build/bazel/platforms/arch:arm64", + "//build/bazel/platforms/os:linux_bionic", + ], +) + +config_setting( + name = "linux_bionic_x86_64", + constraint_values = [ + "//build/bazel/platforms/arch:x86_64", + "//build/bazel/platforms/os:linux_bionic", + ], +) + +config_setting( + name = "linux_musl_x86", + constraint_values = [ + "//build/bazel/platforms/arch:x86", + "//build/bazel/platforms/os:linux_musl", + ], +) + +config_setting( + name = "linux_musl_x86_64", + constraint_values = [ + "//build/bazel/platforms/arch:x86_64", + "//build/bazel/platforms/os:linux_musl", + ], +) + +config_setting( + name = "windows_x86", + constraint_values = [ + "//build/bazel/platforms/arch:x86", + "//build/bazel/platforms/os:windows", + ], +) + +config_setting( + name = "windows_x86_64", + constraint_values = [ + "//build/bazel/platforms/arch:x86_64", + "//build/bazel/platforms/os:windows", + ], +) diff --git a/platforms/product_variables/product_platform.bzl b/platforms/product_variables/product_platform.bzl new file mode 100644 index 00000000..f2c6f91f --- /dev/null +++ b/platforms/product_variables/product_platform.bzl @@ -0,0 +1,148 @@ +"""Parallels variable.go to provide variables and create a platform based on converted config.""" + +load("//build/bazel/product_variables:constants.bzl", "constants") +load("//prebuilts/clang/host/linux-x86:cc_toolchain_constants.bzl", "variant_name") + +def _product_variables_providing_rule_impl(ctx): + return [ + platform_common.TemplateVariableInfo(ctx.attr.product_vars), + ] + +# Provides product variables for templated string replacement. +product_variables_providing_rule = rule( + implementation = _product_variables_providing_rule_impl, + attrs = { + "product_vars": attr.string_dict(), + }, +) + +_arch_os_only_suffix = "_arch_os" +_product_only_suffix = "_product" + +def add_providing_var(providing_vars, typ, var, value): + if typ == "bool": + providing_vars[var] = "1" if value else "0" + elif typ == "list": + providing_vars[var] = ",".join(value) + elif typ == "int": + providing_vars[var] = str(value) + elif typ == "string": + providing_vars[var] = value + +def product_variable_config(name, product_config_vars): + constraints = [] + + local_vars = dict(product_config_vars) + + # Native_coverage is not set within soong.variables, but is hardcoded + # within config.go NewConfig + local_vars["Native_coverage"] = ( + local_vars.get("ClangCoverage", False) or + local_vars.get("GcovCoverage", False) + ) + + providing_vars = {} + + # Generate constraints for Soong config variables (bool, value, string typed). + vendor_vars = local_vars.pop("VendorVars", default = {}) + for (namespace, variables) in vendor_vars.items(): + for (var, value) in variables.items(): + # All vendor vars are Starlark string-typed, even though they may be + # boxed bools/strings/arbitrary printf'd values, like numbers, so + # we'll need to do some translation work here by referring to + # soong_injection's generated data. + + if value == "": + # Variable is not set so skip adding this as a constraint. + continue + + # Create the identifier for the constraint var (or select key) + config_var = namespace + "__" + var + + # List of all soong_config_module_type variables. + if not config_var in constants.SoongConfigVariables: + continue + + # Normalize all constraint vars (i.e. select keys) to be lowercased. + constraint_var = config_var.lower() + + if config_var in constants.SoongConfigBoolVariables: + constraints.append("//build/bazel/product_variables:" + constraint_var) + elif config_var in constants.SoongConfigStringVariables: + # The string value is part of the the select key. + constraints.append("//build/bazel/product_variables:" + constraint_var + "__" + value.lower()) + elif config_var in constants.SoongConfigValueVariables: + # For value variables, providing_vars add support for substituting + # the value using TemplateVariableInfo. + constraints.append("//build/bazel/product_variables:" + constraint_var) + add_providing_var(providing_vars, "string", constraint_var, value) + + for (var, value) in local_vars.items(): + # TODO(b/187323817): determine how to handle remaining product + # variables not used in product_variables + constraint_var = var.lower() + if not constants.ProductVariables.get(constraint_var): + continue + + # variable.go excludes nil values + add_constraint = (value != None) + add_providing_var(providing_vars, type(value), var, value) + if type(value) == "bool": + # variable.go special cases bools + add_constraint = value + + if add_constraint: + constraints.append("//build/bazel/product_variables:" + constraint_var) + + native.platform( + name = name + _product_only_suffix, + constraint_values = constraints, + ) + + arch = local_vars.get("DeviceArch") + arch_variant = local_vars.get("DeviceArchVariant") + cpu_variant = local_vars.get("DeviceCpuVariant") + + os = "android" + + native.alias( + name = name, + actual = "{os}_{arch}{variant}".format(os = os, arch = arch, variant = _variant_name(arch, arch_variant, cpu_variant)), + ) + + arch = local_vars.get("DeviceSecondaryArch") + arch_variant = local_vars.get("DeviceSecondaryArchVariant") + cpu_variant = local_vars.get("DeviceSecondaryCpuVariant") + + if arch: + native.alias( + name = name + "_secondary", + actual = "{os}_{arch}{variant}".format(os = os, arch = arch, variant = _variant_name(arch, arch_variant, cpu_variant)), + ) + + product_variables_providing_rule( + name = name + "_product_vars", + product_vars = providing_vars, + ) + +def _is_variant_default(arch, variant): + return variant == None or variant in (arch, "generic") + +def _variant_name(arch, arch_variant, cpu_variant): + if _is_variant_default(arch, arch_variant): + arch_variant = "" + if _is_variant_default(arch, cpu_variant): + cpu_variant = "" + variant = struct( + arch_variant = arch_variant, + cpu_variant = cpu_variant, + ) + return variant_name(variant) + +def android_platform(name = None, constraint_values = [], product = None): + """ android_platform creates a platform with the specified constraint_values and product constraints.""" + native.platform( + name = name, + constraint_values = constraint_values, + parents = [product + _product_only_suffix], + ) diff --git a/platforms/rule_utilities.bzl b/platforms/rule_utilities.bzl new file mode 100644 index 00000000..97481b37 --- /dev/null +++ b/platforms/rule_utilities.bzl @@ -0,0 +1,47 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utilities for rule implementations to interact with platform definitions.""" + +# Merge ARCH_CONSTRAINT_ATTRS with the rule attrs to use get_arch(ctx). +ARCH_CONSTRAINT_ATTRS = { + "_x86_constraint": attr.label(default = Label("//build/bazel/platforms/arch:x86")), + "_x86_64_constraint": attr.label(default = Label("//build/bazel/platforms/arch:x86_64")), + "_arm_constraint": attr.label(default = Label("//build/bazel/platforms/arch:arm")), + "_arm64_constraint": attr.label(default = Label("//build/bazel/platforms/arch:arm64")), +} + +# get_arch takes a rule context with ARCH_CONSTRAINT_ATTRS and returns the string representation +# of the target platform by executing the target_platform_has_constraint boilerplate. +def get_arch(ctx): + if not hasattr(ctx.attr, "_x86_constraint") or \ + not hasattr(ctx.attr, "_x86_64_constraint") or \ + not hasattr(ctx.attr, "_arm_constraint") or \ + not hasattr(ctx.attr, "_arm64_constraint"): + fail("Could not get the target architecture of this rule due to missing constraint attrs.", + "Have you merged ARCH_CONSTRAINT_ATTRS into this rule's attributes?") + + x86_constraint = ctx.attr._x86_constraint[platform_common.ConstraintValueInfo] + x86_64_constraint = ctx.attr._x86_64_constraint[platform_common.ConstraintValueInfo] + arm_constraint = ctx.attr._arm_constraint[platform_common.ConstraintValueInfo] + arm64_constraint = ctx.attr._arm64_constraint[platform_common.ConstraintValueInfo] + + if ctx.target_platform_has_constraint(x86_constraint): + return "x86" + elif ctx.target_platform_has_constraint(x86_64_constraint): + return "x86_64" + elif ctx.target_platform_has_constraint(arm_constraint): + return "arm" + elif ctx.target_platform_has_constraint(arm64_constraint): + return "arm64" diff --git a/product_variables/BUILD.bazel b/product_variables/BUILD.bazel index f8462f51..6313d1f6 100644 --- a/product_variables/BUILD.bazel +++ b/product_variables/BUILD.bazel @@ -1,8 +1,78 @@ -# Hard-coded product_variables until product product_variables is available. -load(":product_variables.bzl", "product_variables_providing_rule") +"""Constraints corresponding to product variables.""" + +load(":constants.bzl", "constants") +load(":settings.bzl", "soong_config_variables") package(default_visibility = ["//visibility:public"]) -product_variables_providing_rule( - name = "android_product_variables", +# Unlike product config variables below, these are dynamically generated from +# Soong, since the list of config variables are dynamically defined in +# Android.bp files and not hardcoded into Soong. +soong_config_variables( + bool_vars = constants.SoongConfigBoolVariables, + string_vars = constants.SoongConfigStringVariables, + value_vars = constants.SoongConfigValueVariables, ) + +# Generate one constraint_value for each product_variable +# The constraint_value for <var> can be within a select() to specify an +# attribute value for the same conditions product_variable.<var>, for most +# cases, that is when the value of <var> is true. For example, +# +# product_variables: { +# debuggable: { +# cflags: ["debug_flag1", "debug_flag2"], +# }, +# } +# +# translates into: +# +# cflags = select({ +# "//build/bazel/product_variables:debuggable": ["debug_flag1", "debug_flag2"], +# "//conditions:default": [], +# }), +[ + ( + constraint_setting(name = product_variable + "_constraint"), + constraint_value( + name = product_variable, + constraint_setting = product_variable + "_constraint", + ), + ) + for product_variable in constants.ProductVariables +] + +# Caution: do not use these arch-variant product variables directly. +# If you have a complex combination of product variable and architecture/os/etc, +# prefer instead to craft an appropriate configuration in your BUILD file. +# See: https://docs.bazel.build/versions/master/configurable-attributes.html +# Within bp2build, :safestack-android should be used when an attribute value is +# conditional on both safestack:true and the os is android. +# +# e.g. +# target: { +# android: { +# product_variables: { +# safestack: { +# cflags: ["-Dsafestack-android"], +# }, +# }, +# }, +# }, +# +# would translate to: +# +# cflags = select({ +# "//build/bazel/product_variables:safestack-android": ["-Dsafestack-android"], +# "//conditions:default": [], +# }), +[ + [config_setting( + name = product_variable + "-" + variant, + constraint_values = [ + ":" + product_variable, + variantConstraint, + ], + ) for variant, variantConstraint in constants.ArchVariantToConstraints.items()] + for product_variable in constants.ArchVariantProductVariables +] diff --git a/product_variables/constants.bzl b/product_variables/constants.bzl new file mode 100644 index 00000000..09fd3142 --- /dev/null +++ b/product_variables/constants.bzl @@ -0,0 +1,49 @@ +"""Constants for product variables based on information in variable.go""" + +load( + "@soong_injection//product_config:soong_config_variables.bzl", + _soong_config_bool_variables = "soong_config_bool_variables", + _soong_config_string_variables = "soong_config_string_variables", + _soong_config_value_variables = "soong_config_value_variables", +) +load( + "@soong_injection//product_config:product_variables.bzl", + _arch_variant_product_var_constraints = "arch_variant_product_var_constraints", + _product_var_constraints = "product_var_constraints", +) + +_soong_config_variables = _soong_config_bool_variables.keys() + \ + _soong_config_string_variables.keys() + \ + _soong_config_value_variables.keys() + +_product_variables = { + var: True + for var in _product_var_constraints +} + +_arch_variant_product_variables = { + var: True + for var in _arch_variant_product_var_constraints +} + +_arch_variant_to_constraints = { + "arm": "//build/bazel/platforms/arch:arm", + "arm64": "//build/bazel/platforms/arch:arm64", + "x86": "//build/bazel/platforms/arch:x86", + "x86_64": "//build/bazel/platforms/arch:x86_64", + "android": "//build/bazel/platforms/os:android", + "darwin": "//build/bazel/platforms/os:darwin", + "linux": "//build/bazel/platforms/os:linux", + "linux_bionic": "//build/bazel/platforms/os:linux_bionic", + "windows": "//build/bazel/platforms/os:windows", +} + +constants = struct( + SoongConfigVariables = _soong_config_variables, + SoongConfigBoolVariables = _soong_config_bool_variables, + SoongConfigStringVariables = _soong_config_string_variables, + SoongConfigValueVariables = _soong_config_value_variables, + ProductVariables = _product_variables, + ArchVariantProductVariables = _arch_variant_product_variables, + ArchVariantToConstraints = _arch_variant_to_constraints, +) diff --git a/product_variables/product_variables.bzl b/product_variables/product_variables.bzl deleted file mode 100644 index e9b59e84..00000000 --- a/product_variables/product_variables.bzl +++ /dev/null @@ -1,29 +0,0 @@ -"""Hard-coded rules for product variables until product config files are -converted and available.""" - -def _product_variables_providing_rule_impl(ctx): - return [ - platform_common.TemplateVariableInfo({ - "Platform_version_name": ctx.attr.platform_version_name, - "Platform_sdk_version": str(ctx.attr.platform_sdk_version), - "Platform_sdk_codename": ctx.attr.platform_sdk_codename, - "Platform_sdk_final": str(1 if ctx.attr.platform_sdk_final else 0), - "Platform_version_active_codenames": ",".join(ctx.attr.platform_version_active_codenames), - "Platform_vndk_version": ctx.attr.platform_vndk_version, - }), - ] - -# Values taken from https://cs.android.com/android/platform/superproject/+/master:build/soong/android/variable.go;l=412;drc=7b85eeb41ef3e6d2cf44558d3f54f9ed1b247036 -product_variables_providing_rule = rule( - implementation = _product_variables_providing_rule_impl, - attrs = { - "platform_version_name": attr.string(default="S"), - "platform_sdk_version": attr.int(default=30), - "platform_sdk_codename": attr.string(default="S"), - "platform_sdk_final": attr.bool(default=False), - "platform_version_active_codenames": attr.string_list(default=["S"]), - "platform_vndk_version": attr.string(default="S"), - }, -) - - diff --git a/product_variables/settings.bzl b/product_variables/settings.bzl new file mode 100644 index 00000000..3b241966 --- /dev/null +++ b/product_variables/settings.bzl @@ -0,0 +1,22 @@ +"""Macros to generate constraint settings and values for Soong variables.""" + +def soong_config_variables(bool_vars, value_vars, string_vars): + for variable in bool_vars.keys() + value_vars.keys(): + variable = variable.lower() + native.constraint_setting( + name = variable + "_constraint", + ) + native.constraint_value( + name = variable, + constraint_setting = variable + "_constraint", + ) + for variable, choices in string_vars.items(): + for choice in choices: + var_with_choice = (variable + "__" + choice).lower() + native.constraint_setting( + name = var_with_choice + "_constraint", + ) + native.constraint_value( + name = var_with_choice, + constraint_setting = var_with_choice + "_constraint", + ) diff --git a/rules/README.md b/rules/README.md new file mode 100644 index 00000000..d6a1b8a7 --- /dev/null +++ b/rules/README.md @@ -0,0 +1,17 @@ +# Bazel rules for Android Platform. + +This directory contains Starlark extensions for building the Android Platform with Bazel. + +## APEX + +Run the following command to build a miminal APEX example. + +``` +$ b build //build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal +``` + +Verify the contents of the APEX with `zipinfo`: + +``` +$ zipinfo bazel-bin/build/bazel/examples/apex/minimal/build.bazel.examples.apex.minimal.apex +``` diff --git a/rules/android/BUILD.bazel b/rules/android/BUILD.bazel new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/rules/android/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/rules/android/android_app_certificate.bzl b/rules/android/android_app_certificate.bzl new file mode 100644 index 00000000..d3f3f54e --- /dev/null +++ b/rules/android/android_app_certificate.bzl @@ -0,0 +1,49 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +AndroidAppCertificateInfo = provider( + "Info needed for Android app certificates", + fields = { + "pem": "Certificate .pem file", + "pk8": "Certificate .pk8 file", + }, +) + +def _android_app_certificate_rule_impl(ctx): + return [ + AndroidAppCertificateInfo(pem = ctx.file.pem, pk8 = ctx.file.pk8), + ] + +_android_app_certificate = rule( + implementation = _android_app_certificate_rule_impl, + attrs = { + "pem": attr.label(mandatory = True, allow_single_file = [".pem"]), + "pk8": attr.label(mandatory = True, allow_single_file = [".pk8"]), + }, +) + +def android_app_certificate( + name, + certificate, + **kwargs): + "Bazel macro to correspond with the Android app certificate Soong module." + + _android_app_certificate( + name = name, + pem = certificate + ".x509.pem", + pk8 = certificate + ".pk8", + **kwargs + ) diff --git a/rules/android/android_app_keystore.bzl b/rules/android/android_app_keystore.bzl new file mode 100644 index 00000000..32088cec --- /dev/null +++ b/rules/android/android_app_keystore.bzl @@ -0,0 +1,131 @@ +""" +Copyright (C) 2022 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("android_app_certificate.bzl", "AndroidAppCertificateInfo") + +AndroidAppKeystoreInfo = provider( + "Info needed for Android app keystores", + fields = { + "keystore": "JKS .keystore file housing certificate info", + }, +) + +def _pk8_to_private_pem(ctx, openssl, pk8_file, private_pem_file): + """Converts a .pk8 private key file in DER format to a .pem private key file in PEM format.""" + args = ctx.actions.args() + args.add("pkcs8") + args.add_all(["-in", pk8_file]) + args.add_all(["-inform", "DER"]) + args.add_all(["-outform", "PEM"]) + args.add_all(["-out", private_pem_file]) + args.add("-nocrypt") # don't bother encrypting this private key since it is just an intermediate file + + ctx.actions.run( + inputs = [pk8_file], + executable = openssl, + outputs = [private_pem_file], + arguments = [args], + mnemonic = "CreatePrivPEM", + ) + +def _pem_to_pk12(ctx, openssl, certificate_pem, private_key_pem, pk12_file): + """Converts an X.509 certificate and private key pair of PEM files to a single PKCS12 keystore file.""" + args = ctx.actions.args() + args.add("pkcs12") + args.add("-export") + args.add_all(["-in", certificate_pem]) + args.add_all(["-inkey", private_key_pem]) + args.add_all(["-out", pk12_file]) + args.add_all(["-name", "android"]) + # openssl requires a password and will request a + # password from STDIN if we don't supply one here + args.add_all(["-passout", "pass:android"]) + + ctx.actions.run( + inputs = [ + certificate_pem, + private_key_pem, + ], + executable = openssl, + outputs = [pk12_file], + arguments = [args], + mnemonic = "CreatePK12", + ) + +def _pk12_to_keystore(ctx, keytool, pk12_file, keystore_file): + """Converts a PKCS12 keystore file to a JKS keystore file.""" + args = ctx.actions.args() + args.add("-importkeystore") + args.add_all(["-destkeystore", keystore_file]) + args.add_all(["-srckeystore", pk12_file]) + args.add_all(["-srcstoretype", "PKCS12"]) + args.add_all(["-srcstorepass", "android"]) + # apksigner expects keystores provided by the debug_signing_keys attribute + # to be secured with the password "android" + args.add_all(["-deststorepass", "android"]) + + ctx.actions.run( + inputs = [pk12_file], + executable = keytool, + outputs = [keystore_file], + arguments = [args], + mnemonic = "CreateKeystore", + ) + +def _android_app_keystore_rule_impl(ctx): + openssl = ctx.executable._openssl + keytool = ctx.executable._keytool + + private_pem = ctx.actions.declare_file(ctx.attr.name + ".priv.pem") + pk12 = ctx.actions.declare_file(ctx.attr.name + ".pk12") + keystore = ctx.actions.declare_file(ctx.attr.name + ".keystore") + + pk8_file = ctx.attr.certificate[AndroidAppCertificateInfo].pk8 + pem_file = ctx.attr.certificate[AndroidAppCertificateInfo].pem + _pk8_to_private_pem(ctx, openssl, pk8_file, private_pem) + _pem_to_pk12(ctx, openssl, pem_file, private_pem, pk12) + _pk12_to_keystore(ctx, keytool, pk12, keystore) + + return [ + AndroidAppKeystoreInfo( + keystore = keystore, + ), + DefaultInfo(files = depset(direct = [keystore])) + ] + +"""Converts an android_app_certificate (i.e. pem/pk8 pair) into a JKS keystore""" +android_app_keystore = rule( + implementation = _android_app_keystore_rule_impl, + attrs = { + "certificate": attr.label(mandatory = True, providers = [AndroidAppCertificateInfo]), + "_openssl": attr.label( + default = Label("//prebuilts/build-tools:linux-x86/bin/openssl"), + allow_single_file = True, + executable = True, + cfg = "exec", + doc = "An OpenSSL compatible tool." + ), + "_keytool": attr.label( + default = Label("//prebuilts/jdk/jdk11:linux-x86/bin/keytool"), + allow_single_file = True, + executable = True, + cfg = "exec", + doc = "The keytool binary." + ), + }, + provides = [AndroidAppKeystoreInfo], +) diff --git a/rules/android/android_binary.bzl b/rules/android/android_binary.bzl new file mode 100644 index 00000000..3504e845 --- /dev/null +++ b/rules/android/android_binary.bzl @@ -0,0 +1,99 @@ +""" +Copyright (C) 2022 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@rules_android//rules:rules.bzl", _android_binary = "android_binary") +load("@soong_injection//product_config:product_variables.bzl", "product_vars") + +load("android_app_certificate.bzl", "android_app_certificate") +load("android_app_keystore.bzl", "android_app_keystore") + + +def _default_cert_prod_var(): + return product_vars["DefaultAppCertificate"] + +def _default_app_certificate_package(): + default_cert = _default_cert_prod_var() + if default_cert: + return "//" + paths.dirname(default_cert) + # if product variable is not set, default to Soong default: + return "//build/make/target/product/security" + +def _default_app_certificate(): + default_cert = _default_cert_prod_var() + if default_cert: + return default_cert + return _default_app_certificate_package() + ":testkey" + +def _android_app_certificate_with_default_cert(name, cert_name): + + if cert_name: + # if a specific certificate name is given, check the default directory + # for that certificate + certificate = _default_app_certificate_package() + ":" + cert_name + else: + certificate = _default_app_certificate() + + android_app_certificate( + name = name, + certificate = certificate, + ) + +def android_binary( + name, + certificate = None, + certificate_name = None, + **kwargs): + """Bazel macro to find and create a keystore to use for debug_signing_keys + with @rules_android android_binary. + + This module emulates the Soong behavior which allows a developer to specify + a specific module name for the android_app_certificate or the name of a + .pem/.pk8 certificate/key pair in a directory specified by the + DefaultAppCertificate product variable. In either case, we convert the specified + .pem/.pk8 certificate/key pair to a JKS .keystore file before passing it to the + android_binary rule. + + Arguments: + certificate: Bazel target + certificate_name: string, name of private key file in default certificate directory + **kwargs: map, additional args to pass to android_binary + """ + + if certificate and certificate_name: + fail("Cannot use both certificate_name and certificate attributes together. Use only one of them.") + + debug_signing_keys = kwargs.pop("debug_signing_keys", []) + + if certificate or certificate_name: + if certificate_name: + app_cert_name = name + "_app_certificate" + _android_app_certificate_with_default_cert(app_cert_name, certificate_name) + certificate = ":" + app_cert_name + + app_keystore_name = name + "_keystore" + android_app_keystore( + name = app_keystore_name, + certificate = certificate + ) + + debug_signing_keys.append(app_keystore_name) + + _android_binary( + name = name, + debug_signing_keys = debug_signing_keys, + **kwargs + ) diff --git a/rules/apex.bzl b/rules/apex.bzl new file mode 100644 index 00000000..e5fc11ef --- /dev/null +++ b/rules/apex.bzl @@ -0,0 +1,432 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load(":apex_key.bzl", "ApexKeyInfo") +load(":prebuilt_file.bzl", "PrebuiltFileInfo") +load(":sh_binary.bzl", "ShBinaryInfo") +load("//build/bazel/rules/cc:stripped_cc_common.bzl", "StrippedCcBinaryInfo") +load("//build/bazel/rules/android:android_app_certificate.bzl", "AndroidAppCertificateInfo") +load("//build/bazel/rules/apex:transition.bzl", "apex_transition", "shared_lib_transition_32", "shared_lib_transition_64") +load("//build/bazel/rules/apex:cc.bzl", "ApexCcInfo", "apex_cc_aspect") + +DIR_LIB = "lib" +DIR_LIB64 = "lib64" + +ApexInfo = provider( + "ApexInfo has no field currently and is used by apex rule dependents to ensure an attribute is a target of apex rule.", + fields = {}, +) + +# Prepare the input files info for bazel_apexer_wrapper to generate APEX filesystem image. +def _prepare_apexer_wrapper_inputs(ctx): + # dictionary to return in the format: + # apex_manifest[(image_file_dirname, image_file_basename)] = bazel_output_file + apex_manifest = {} + + x86_constraint = ctx.attr._x86_constraint[platform_common.ConstraintValueInfo] + x86_64_constraint = ctx.attr._x86_64_constraint[platform_common.ConstraintValueInfo] + arm_constraint = ctx.attr._arm_constraint[platform_common.ConstraintValueInfo] + arm64_constraint = ctx.attr._arm64_constraint[platform_common.ConstraintValueInfo] + + if ctx.target_platform_has_constraint(x86_constraint): + _add_libs_32_target(ctx, "x86", apex_manifest) + elif ctx.target_platform_has_constraint(x86_64_constraint): + _add_libs_64_target(ctx, "x86", "x86_64", apex_manifest) + elif ctx.target_platform_has_constraint(arm_constraint): + _add_libs_32_target(ctx, "arm", apex_manifest) + elif ctx.target_platform_has_constraint(arm64_constraint): + _add_libs_64_target(ctx, "arm", "arm64", apex_manifest) + + # Handle prebuilts + for dep in ctx.attr.prebuilts: + prebuilt_file_info = dep[PrebuiltFileInfo] + if prebuilt_file_info.filename: + filename = prebuilt_file_info.filename + else: + filename = dep.label.name + apex_manifest[(prebuilt_file_info.dir, filename)] = prebuilt_file_info.src + + # Handle binaries + for dep in ctx.attr.binaries: + if ShBinaryInfo in dep: + # sh_binary requires special handling on directory/filename construction. + sh_binary_info = dep[ShBinaryInfo] + default_info = dep[DefaultInfo] + if sh_binary_info != None: + directory = "bin" + if sh_binary_info.sub_dir != None and sh_binary_info.sub_dir != "": + directory = "/".join([directory, sh_binary_info.sub_dir]) + + if sh_binary_info.filename != None and sh_binary_info.filename != "": + filename = sh_binary_info.filename + else: + filename = dep.label.name + + apex_manifest[(directory, filename)] = default_info.files_to_run.executable + elif CcInfo in dep: + # cc_binary just takes the final executable from the runfiles. + apex_manifest[("bin", dep.label.name)] = dep[DefaultInfo].files_to_run.executable + + apex_content_inputs = [] + + bazel_apexer_wrapper_manifest = ctx.actions.declare_file("%s_bazel_apexer_wrapper_manifest" % ctx.attr.name) + file_lines = [] + + # Store the apex file target directory, file name and the path in the source tree in a file. + # This file will be read by the bazel_apexer_wrapper to create the apex input directory. + # Here is an example: + # {etc/tz,tz_version,system/timezone/output_data/version/tz_version} + for (apex_dirname, apex_basename), bazel_input_file in apex_manifest.items(): + apex_content_inputs.append(bazel_input_file) + file_lines += [",".join([apex_dirname, apex_basename, bazel_input_file.path])] + + ctx.actions.write(bazel_apexer_wrapper_manifest, "\n".join(file_lines)) + + return apex_content_inputs, bazel_apexer_wrapper_manifest + +def _add_libs_32_target(ctx, key, apex_manifest): + if len(ctx.split_attr.native_shared_libs_32.keys()) > 0: + _add_lib_file(DIR_LIB, ctx.split_attr.native_shared_libs_32[key], apex_manifest) + +def _add_libs_64_target(ctx, key_32, key_64, apex_manifest): + _add_libs_32_target(ctx, key_32, apex_manifest) + if len(ctx.split_attr.native_shared_libs_64.keys()) > 0: + _add_lib_file(DIR_LIB64, ctx.split_attr.native_shared_libs_64[key_64], apex_manifest) + +def _add_lib_file(dir, libs, apex_manifest): + for dep in libs: + apex_cc_info = dep[ApexCcInfo] + for lib_file in apex_cc_info.transitive_shared_libs.to_list(): + apex_manifest[(dir, lib_file.basename)] = lib_file + +# conv_apex_manifest - Convert the JSON APEX manifest to protobuf, which is needed by apexer. +def _convert_apex_manifest_json_to_pb(ctx, apex_toolchain): + apex_manifest_json = ctx.file.manifest + apex_manifest_pb = ctx.actions.declare_file("apex_manifest.pb") + + ctx.actions.run( + outputs = [apex_manifest_pb], + inputs = [ctx.file.manifest], + executable = apex_toolchain.conv_apex_manifest, + arguments = [ + "proto", + apex_manifest_json.path, + "-o", + apex_manifest_pb.path, + ], + mnemonic = "ConvApexManifest", + ) + + return apex_manifest_pb + +# apexer - generate the APEX file. +def _run_apexer(ctx, apex_toolchain, apex_content_inputs, bazel_apexer_wrapper_manifest, apex_manifest_pb): + # Inputs + file_contexts = ctx.file.file_contexts + apex_key_info = ctx.attr.key[ApexKeyInfo] + privkey = apex_key_info.private_key + pubkey = apex_key_info.public_key + android_jar = apex_toolchain.android_jar + android_manifest = ctx.file.android_manifest + + # Outputs + apex_output_file = ctx.actions.declare_file(ctx.attr.name + ".apex.unsigned") + + # Arguments + args = ctx.actions.args() + args.add_all(["--manifest", apex_manifest_pb.path]) + args.add_all(["--file_contexts", file_contexts.path]) + args.add_all(["--key", privkey.path]) + args.add_all(["--pubkey", pubkey.path]) + min_sdk_version = ctx.attr.min_sdk_version + + # TODO(b/215339575): This is a super rudimentary way to convert "current" to a numerical number. + # Generalize this to API level handling logic in a separate Starlark utility, preferably using + # API level maps dumped from api_levels.go + if min_sdk_version == "current": + min_sdk_version = "10000" + args.add_all(["--min_sdk_version", min_sdk_version]) + args.add_all(["--bazel_apexer_wrapper_manifest", bazel_apexer_wrapper_manifest]) + args.add_all(["--apexer_path", apex_toolchain.apexer]) + + # apexer needs the list of directories containing all auxilliary tools invoked during + # the creation of an apex + avbtool_files = apex_toolchain.avbtool[DefaultInfo].files_to_run + e2fsdroid_files = apex_toolchain.e2fsdroid[DefaultInfo].files_to_run + mke2fs_files = apex_toolchain.mke2fs[DefaultInfo].files_to_run + resize2fs_files = apex_toolchain.resize2fs[DefaultInfo].files_to_run + apexer_tool_paths = [ + # These are built by make_injection + apex_toolchain.apexer.dirname, + + # These are real Bazel targets + apex_toolchain.aapt2.dirname, + avbtool_files.executable.dirname, + e2fsdroid_files.executable.dirname, + mke2fs_files.executable.dirname, + resize2fs_files.executable.dirname, + ] + + args.add_all(["--apexer_tool_path", ":".join(apexer_tool_paths)]) + args.add_all(["--apex_output_file", apex_output_file]) + + if android_manifest != None: + args.add_all(["--android_manifest", android_manifest.path]) + + inputs = apex_content_inputs + [ + bazel_apexer_wrapper_manifest, + apex_manifest_pb, + file_contexts, + privkey, + pubkey, + android_jar, + ] + + tools = [ + avbtool_files, + e2fsdroid_files, + mke2fs_files, + resize2fs_files, + apex_toolchain.aapt2, + + apex_toolchain.apexer, + apex_toolchain.sefcontext_compile, + ] + + if android_manifest != None: + inputs.append(android_manifest) + + ctx.actions.run( + inputs = inputs, + tools = tools, + outputs = [apex_output_file], + executable = ctx.executable._bazel_apexer_wrapper, + arguments = [args], + mnemonic = "BazelApexerWrapper", + ) + + return apex_output_file + +# Sign a file with signapk. +def _run_signapk(ctx, unsigned_file, signed_file, private_key, public_key, mnemonic): + # Inputs + inputs = [ + unsigned_file, + private_key, + public_key, + ctx.executable._signapk, + ] + + # Outputs + outputs = [signed_file] + + # Arguments + args = ctx.actions.args() + args.add_all(["-a", 4096]) + args.add_all(["--align-file-size"]) + args.add_all([public_key, private_key]) + args.add_all([unsigned_file, signed_file]) + + ctx.actions.run( + inputs = inputs, + outputs = outputs, + executable = ctx.executable._signapk, + arguments = [args], + mnemonic = mnemonic, + ) + + return signed_file + +# Compress a file with apex_compression_tool. +def _run_apex_compression_tool(ctx, apex_toolchain, input_file, output_file_name): + # Inputs + inputs = [ + input_file, + ] + + avbtool_files = apex_toolchain.avbtool[DefaultInfo].files_to_run + tools = [ + avbtool_files, + apex_toolchain.apex_compression_tool, + apex_toolchain.soong_zip, + ] + + # Outputs + compressed_file = ctx.actions.declare_file(output_file_name) + outputs = [compressed_file] + + # Arguments + args = ctx.actions.args() + args.add_all(["compress"]) + tool_dirs = [apex_toolchain.soong_zip.dirname, avbtool_files.executable.dirname] + args.add_all(["--apex_compression_tool", ":".join(tool_dirs)]) + args.add_all(["--input", input_file]) + args.add_all(["--output", compressed_file]) + + ctx.actions.run( + inputs = inputs, + tools = tools, + outputs = outputs, + executable = apex_toolchain.apex_compression_tool, + arguments = [args], + mnemonic = "BazelApexCompressing", + ) + return compressed_file + +# See the APEX section in the README on how to use this rule. +def _apex_rule_impl(ctx): + apex_toolchain = ctx.toolchains["//build/bazel/rules/apex:apex_toolchain_type"].toolchain_info + + apex_content_inputs, bazel_apexer_wrapper_manifest = _prepare_apexer_wrapper_inputs(ctx) + apex_manifest_pb = _convert_apex_manifest_json_to_pb(ctx, apex_toolchain) + + unsigned_apex_output_file = _run_apexer(ctx, apex_toolchain, apex_content_inputs, bazel_apexer_wrapper_manifest, apex_manifest_pb) + + apex_cert_info = ctx.attr.certificate[AndroidAppCertificateInfo] + private_key = apex_cert_info.pk8 + public_key = apex_cert_info.pem + + signed_apex = ctx.outputs.apex_output + _run_signapk(ctx, unsigned_apex_output_file, signed_apex, private_key, public_key, "BazelApexSigning") + output_file = signed_apex + + if ctx.attr.compressible: + compressed_apex_output_file = _run_apex_compression_tool(ctx, apex_toolchain, signed_apex, ctx.attr.name + ".capex.unsigned") + signed_capex = ctx.outputs.capex_output + _run_signapk(ctx, compressed_apex_output_file, signed_capex, private_key, public_key, "BazelCompressedApexSigning") + + files_to_build = depset([output_file]) + return [DefaultInfo(files = files_to_build), ApexInfo()] + +_apex = rule( + implementation = _apex_rule_impl, + attrs = { + "manifest": attr.label(allow_single_file = [".json"]), + "android_manifest": attr.label(allow_single_file = [".xml"]), + "file_contexts": attr.label(allow_single_file = True, mandatory = True), + "key": attr.label(providers = [ApexKeyInfo]), + "certificate": attr.label(providers = [AndroidAppCertificateInfo]), + "min_sdk_version": attr.string(default = "current"), + "updatable": attr.bool(default = True), + "installable": attr.bool(default = True), + "compressible": attr.bool(default = False), + "native_shared_libs_32": attr.label_list( + providers = [ApexCcInfo], + aspects = [apex_cc_aspect], + cfg = shared_lib_transition_32, + doc = "The libs compiled for 32-bit", + ), + "native_shared_libs_64": attr.label_list( + providers = [ApexCcInfo], + aspects = [apex_cc_aspect], + cfg = shared_lib_transition_64, + doc = "The libs compiled for 64-bit", + ), + "binaries": attr.label_list( + providers = [ + # The dependency must produce _all_ of the providers in _one_ of these lists. + [ShBinaryInfo], # sh_binary + [StrippedCcBinaryInfo, CcInfo], # cc_binary (stripped) + ], + cfg = apex_transition, + ), + "prebuilts": attr.label_list(providers = [PrebuiltFileInfo], cfg = apex_transition), + "apex_output": attr.output(doc = "signed .apex output"), + "capex_output": attr.output(doc = "signed .capex output"), + + # Required to use apex_transition. This is an acknowledgement to the risks of memory bloat when using transitions. + "_allowlist_function_transition": attr.label(default = "@bazel_tools//tools/allowlists/function_transition_allowlist"), + "_bazel_apexer_wrapper": attr.label( + cfg = "host", + doc = "The apexer wrapper to avoid the problem where symlinks are created inside apex image.", + executable = True, + default = "//build/bazel/rules/apex:bazel_apexer_wrapper", + ), + "_signapk": attr.label( + cfg = "host", + doc = "The signapk tool.", + executable = True, + default = "//build/make/tools/signapk", + ), + "_x86_constraint": attr.label( + default = Label("//build/bazel/platforms/arch:x86"), + ), + "_x86_64_constraint": attr.label( + default = Label("//build/bazel/platforms/arch:x86_64"), + ), + "_arm_constraint": attr.label( + default = Label("//build/bazel/platforms/arch:arm"), + ), + "_arm64_constraint": attr.label( + default = Label("//build/bazel/platforms/arch:arm64"), + ), + }, + toolchains = ["//build/bazel/rules/apex:apex_toolchain_type"], + fragments = ["platform"], +) + +def apex( + name, + manifest = "apex_manifest.json", + android_manifest = None, + file_contexts = None, + key = None, + certificate = None, + min_sdk_version = None, + updatable = True, + installable = True, + compressible = False, + native_shared_libs_32 = [], + native_shared_libs_64 = [], + binaries = [], + prebuilts = [], + **kwargs): + "Bazel macro to correspond with the APEX bundle Soong module." + + # If file_contexts is not specified, then use the default from //system/sepolicy/apex. + # https://cs.android.com/android/platform/superproject/+/master:build/soong/apex/builder.go;l=259-263;drc=b02043b84d86fe1007afef1ff012a2155172215c + if file_contexts == None: + file_contexts = "//system/sepolicy/apex:" + name + "-file_contexts" + + apex_output = name + ".apex" + capex_output = None + if compressible: + capex_output = name + ".capex" + + _apex( + name = name, + manifest = manifest, + android_manifest = android_manifest, + file_contexts = file_contexts, + key = key, + certificate = certificate, + min_sdk_version = min_sdk_version, + updatable = updatable, + installable = installable, + compressible = compressible, + native_shared_libs_32 = native_shared_libs_32, + native_shared_libs_64 = native_shared_libs_64, + binaries = binaries, + prebuilts = prebuilts, + + # Enables predeclared output builds from command line directly, e.g. + # + # $ bazel build //path/to/module:com.android.module.apex + # $ bazel build //path/to/module:com.android.module.capex + apex_output = apex_output, + capex_output = capex_output, + **kwargs + ) diff --git a/rules/apex/BUILD b/rules/apex/BUILD new file mode 100644 index 00000000..ef3998cd --- /dev/null +++ b/rules/apex/BUILD @@ -0,0 +1,79 @@ +load("//build/bazel/rules/apex:toolchain.bzl", "apex_toolchain") +load("@bazel_skylib//rules:common_settings.bzl", "string_setting", "string_list_setting") + +string_setting( + name = "apex_name", + build_setting_default = "", + visibility = ["//visibility:public"], +) + +string_setting( + name = "min_sdk_version", + build_setting_default = "", + visibility = ["//visibility:public"], +) + +string_list_setting( + name = "apex_direct_deps", + build_setting_default = [], + visibility = ["//visibility:public"], +) + +toolchain_type(name = "apex_toolchain_type") + +apex_toolchain( + name = "prebuilt_apex_toolchain", + aapt2 = "//prebuilts/sdk/tools:linux/bin/aapt2", + avbtool = "//external/avb:avbtool", + apexer = "@make_injection//:host/linux-x86/bin/apexer", + mke2fs = "//external/e2fsprogs/misc:mke2fs", + resize2fs = "//external/e2fsprogs/resize:resize2fs", + e2fsdroid = "//external/e2fsprogs/contrib/android:e2fsdroid", + sefcontext_compile = "@make_injection//:host/linux-x86/bin/sefcontext_compile", + conv_apex_manifest = "@make_injection//:host/linux-x86/bin/conv_apex_manifest", + android_jar = "//prebuilts/sdk/current:public/android.jar", + apex_compression_tool = "@make_injection//:host/linux-x86/bin/apex_compression_tool", + soong_zip = "//prebuilts/build-tools:linux-x86/bin/soong_zip", +) + +toolchain( + name = "prebuilt_apex_toolchain_def", + exec_compatible_with = [ + "//build/bazel/platforms/arch:x86_64", + "//build/bazel/platforms/os:linux", + ], + target_compatible_with = [ + "//build/bazel/platforms/os:android", + ], + toolchain = ":prebuilt_apex_toolchain", + toolchain_type = "//build/bazel/rules/apex:apex_toolchain_type", +) + +py_binary( + name = "bazel_apexer_wrapper", + srcs = ["bazel_apexer_wrapper.py"], + visibility = ["//visibility:public"], +) + +sh_test( + name = "bazel_apexer_wrapper_test", + srcs = ["bazel_apexer_wrapper_test.sh"], + deps = ["@bazel_tools//tools/bash/runfiles"], + data = [ + ":bazel_apexer_wrapper", + "test.pem", + "//external/avb:avbtool", + "//external/e2fsprogs/contrib/android:e2fsdroid", + "//external/e2fsprogs/misc:mke2fs", + "//external/e2fsprogs/resize:resize2fs", + "//external/e2fsprogs/debugfs:debugfs", + "//prebuilts/build-tools:linux-x86/bin/soong_zip", + "//prebuilts/sdk/tools:linux/bin/aapt2", + "@make_injection//:host/linux-x86/bin/apex_compression_tool", + "@make_injection//:host/linux-x86/bin/apexer", + "@make_injection//:host/linux-x86/bin/conv_apex_manifest", + "@make_injection//:host/linux-x86/bin/deapexer", + "@make_injection//:host/linux-x86/bin/sefcontext_compile", + "//prebuilts/sdk/current:public/android.jar", + ] +) diff --git a/rules/apex/bazel_apexer_wrapper.py b/rules/apex/bazel_apexer_wrapper.py new file mode 100644 index 00000000..f1c14dbd --- /dev/null +++ b/rules/apex/bazel_apexer_wrapper.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python +# +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile + +def _create_apex(args, work_dir): + + image_apex_dir = "image.apex" + + # Used for creating canned_fs_config, since every file and dir in the APEX are represented + # by an entry in the fs_config. + apex_subdirs = [] + apex_filepaths = [] + + input_dir = os.path.join(work_dir, image_apex_dir) + os.makedirs(input_dir, exist_ok=True) + bazel_apexer_wrapper_manifest = open(args.bazel_apexer_wrapper_manifest, 'r') + file_lines = bazel_apexer_wrapper_manifest.readlines() + for line in file_lines: + line = line.strip() + if (len(line) == 0): + continue + apex_dirname, apex_filename, bazel_input_file = line.split(",") + full_apex_dirname = "/".join([input_dir, apex_dirname]) + os.makedirs(full_apex_dirname, exist_ok=True) + + apex_filepath = "/".join([apex_dirname, apex_filename]) + apex_filepaths.append(apex_filepath) + apex_subdirs.append(apex_dirname) + + full_apex_filepath = "/".join([input_dir, apex_filepath]) + # Because Bazel execution root is a symlink forest, all the input files are symlinks, these + # include the dependency files declared in the BUILD files as well as the files declared + # and created in the bzl files. For sandbox runs the former are two or more level symlinks and + # latter are one level symlinks. For non-sandbox runs, the former are one level symlinks + # and the latter are actual files. Here are some examples: + # + # Two level symlinks: + # system/timezone/output_data/version/tz_version -> + # /usr/local/google/home/...out/bazel/output_user_root/b1ed7e1e9af3ebbd1403e9cf794e4884/ + # execroot/__main__/system/timezone/output_data/version/tz_version -> + # /usr/local/google/home/.../system/timezone/output_data/version/tz_version + # + # Three level symlinks: + # bazel-out/android_x86_64-fastbuild-ST-4ecd5e98bfdd/bin/external/boringssl/libcrypto.so -> + # /usr/local/google/home/yudiliu/android/aosp/master/out/bazel/output_user_root/b1ed7e1e9af3ebbd1403e9cf794e4884/ + # execroot/__main__/bazel-out/android_x86_64-fastbuild-ST-4ecd5e98bfdd/bin/external/boringssl/libcrypto.so -> + # /usr/local/google/home/yudiliu/android/aosp/master/out/bazel/output_user_root/b1ed7e1e9af3ebbd1403e9cf794e4884/ + # execroot/__main__/bazel-out/android_x86_64-fastbuild-ST-4ecd5e98bfdd/bin/external/boringssl/ + # liblibcrypto_stripped.so -> + # /usr/local/google/home/yudiliu/android/aosp/master/out/bazel/output_user_root/b1ed7e1e9af3ebbd1403e9cf794e4884/ + # execroot/__main__/bazel-out/android_x86_64-fastbuild-ST-4ecd5e98bfdd/bin/external/boringssl/ + # liblibcrypto_unstripped.so + # + # One level symlinks: + # bazel-out/android_target-fastbuild/bin/system/timezone/apex/apex_manifest.pb -> + # /usr/local/google/home/.../out/bazel/output_user_root/b1ed7e1e9af3ebbd1403e9cf794e4884/ + # execroot/__main__/bazel-out/android_target-fastbuild/bin/system/timezone/apex/ + # apex_manifest.pb + + if os.path.islink(bazel_input_file): + bazel_input_file = os.readlink(bazel_input_file) + + # For sandbox run these are the 2nd level symlinks and we need to resolve + while os.path.islink(bazel_input_file) and 'execroot/__main__' in bazel_input_file: + bazel_input_file = os.readlink(bazel_input_file) + + shutil.copyfile(bazel_input_file, full_apex_filepath, follow_symlinks=False) + + # Make sure subdirs are unique + apex_subdirs_set = set() + for d in apex_subdirs: + apex_subdirs_set.add(d) + + # Make sure all the parent dirs of the current subdir are in the set, too + dirs = d.split("/") + for i in range(0, len(dirs)): + apex_subdirs_set.add("/".join(dirs[:i])) + + canned_fs_config = _generate_canned_fs_config(work_dir, apex_subdirs_set, apex_filepaths) + + # Construct the main apexer command. + cmd = [args.apexer_path] + cmd.append('--verbose') + cmd.append('--force') + cmd.append('--include_build_info') + cmd.extend(['--file_contexts', args.file_contexts]) + cmd.extend(['--canned_fs_config', canned_fs_config]) + cmd.extend(['--key', args.key]) + cmd.extend(['--payload_type', 'image']) + cmd.extend(['--target_sdk_version', '10000']) + cmd.extend(['--payload_fs_type', 'ext4']) + cmd.extend(['--apexer_tool_path', args.apexer_tool_paths]) + + if args.android_manifest != None: + cmd.extend(['--android_manifest', args.android_manifest]) + + if args.pubkey != None: + cmd.extend(['--pubkey', args.pubkey]) + + if args.manifest != None: + cmd.extend(['--manifest', args.manifest]) + + if args.min_sdk_version != None: + cmd.extend(['--min_sdk_version', args.min_sdk_version]) + + if args.android_jar_path != None: + cmd.extend(['--android_jar_path', args.android_jar_path]) + + cmd.append(input_dir) + cmd.append(args.apex_output_file) + + popen = subprocess.Popen(cmd) + popen.wait() + + return True + +# Generate filesystem config. This encodes the filemode, uid, and gid of each +# file in the APEX, including apex_manifest.json and apex_manifest.pb. +# +# NOTE: every file must have an entry. +def _generate_canned_fs_config(work_dir, dirs, filepaths): + with tempfile.NamedTemporaryFile(mode = 'w+', dir=work_dir, delete=False) as canned_fs_config: + config_lines = [] + config_lines += ["/ 1000 1000 0755"] + config_lines += ["/apex_manifest.json 1000 1000 0644"] + config_lines += ["/apex_manifest.pb 1000 1000 0644"] + config_lines += ["/" + filepath + " 1000 1000 0644" for filepath in filepaths] + config_lines += ["/" + d + " 0 2000 0755" for d in dirs] + canned_fs_config.write("\n".join(config_lines)) + + return canned_fs_config.name + +def _parse_args(argv): + parser = argparse.ArgumentParser(description='Build an APEX file') + + parser.add_argument( + '--manifest', + help='path to the APEX manifest file (.pb)') + parser.add_argument( + '--apex_output_file', + required=True, + help='path to the APEX image file') + parser.add_argument( + '--bazel_apexer_wrapper_manifest', + required=True, + help='path to the manifest file that stores the info about the files to be packaged by apexer') + parser.add_argument( + '--android_manifest', + help='path to the AndroidManifest file. If omitted, a default one is created and used') + parser.add_argument( + '--file_contexts', + required=True, + help='selinux file contexts file.') + parser.add_argument( + '--key', + required=True, + help='path to the private key file.') + parser.add_argument( + '--pubkey', + help='path to the public key file. Used to bundle the public key in APEX for testing.') + parser.add_argument( + '--apexer_path', + required=True, + help='Path to the apexer binary.') + parser.add_argument( + '--apexer_tool_paths', + required=True, + help='Directories containing all the tools used by apexer, separated by ":" character.') + parser.add_argument( + '--min_sdk_version', + help='Default Min SDK version to use for AndroidManifest.xml') + parser.add_argument( + '--android_jar_path', + help='path to use as the source of the android API.') + + return parser.parse_args(argv) + +def main(argv): + args = _parse_args(argv) + + with tempfile.TemporaryDirectory() as work_dir: + success = _create_apex(args, work_dir) + + if not success: + sys.exit(1) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/rules/apex/bazel_apexer_wrapper_test.sh b/rules/apex/bazel_apexer_wrapper_test.sh new file mode 100755 index 00000000..1cf03fa9 --- /dev/null +++ b/rules/apex/bazel_apexer_wrapper_test.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -xeuo pipefail + +apexer_tool_path="${RUNFILES_DIR}/__main__/external/make_injection/host/linux-x86/bin" +avb_tool_path="${RUNFILES_DIR}/__main__/external/avb" +android_jar="${RUNFILES_DIR}/__main__/prebuilts/sdk/current/public/android.jar" + +input_dir=$(mktemp -d) +output_dir=$(mktemp -d) + +function cleanup { + rm -rf ${input_dir} + rm -rf ${output_dir} +} + +trap cleanup ERR +############################################# +# prepare the inputs +############################################# +# Create the input directory with +# 1. a file with random bits +# 2. a file installed sub dir with random bits +# 3. a one-level symlink +# 4. a two-level symlink with "execroot/__main__" in the path +# 5. a two-level sumlink without "execroot/__main__" in the path +# 6. a three-level symlink with "execroot/__main__" in the path +echo "test file1" > "${input_dir}/file1" +echo "test file2" > "${input_dir}/file2" +mkdir -p "${input_dir}/execroot/__main__" +ln -s "${input_dir}/file1" "${input_dir}/one_level_sym" +ln -s "${input_dir}/file2" "${input_dir}/execroot/__main__/middle_sym" +ln -s "${input_dir}/execroot/__main__/middle_sym" "${input_dir}/two_level_sym_in_execroot" +ln -s "${input_dir}/one_level_sym" "${input_dir}/two_level_sym_not_in_execroot" +ln -s "${input_dir}/two_level_sym_in_execroot" "${input_dir}/three_level_sym_in_execroot" + +# Create the APEX manifest file +manifest_dir=$(mktemp -d) +manifest_file="${manifest_dir}/apex_manifest.pb" +echo '{"name": "com.android.example.apex", "version": 1}' > "${manifest_dir}/apex_manifest.json" +"${apexer_tool_path}/conv_apex_manifest" proto "${manifest_dir}/apex_manifest.json" -o ${manifest_file} + +# Create the file_contexts file +file_contexts_file=$(mktemp) +echo ' +(/.*)? u:object_r:root_file:s0 +/execroot(/.*)? u:object_r:execroot_file:s0 +' > ${file_contexts_file} + +output_file="${output_dir}/test.apex" + +# Create the wrapper manifest file +bazel_apexer_wrapper_manifest_file=$(mktemp) +echo " +dir1,file1,"${input_dir}/file1" +dir2/dir3,file2,"${input_dir}/file2" +dir4,one_level_sym,"${input_dir}/one_level_sym" +dir5,two_level_sym_in_execroot,"${input_dir}/two_level_sym_in_execroot" +dir6,two_level_sym_not_in_execroot,"${input_dir}/two_level_sym_not_in_execroot" +dir7,three_level_sym_in_execroot,"${input_dir}/three_level_sym_in_execroot" +" > ${bazel_apexer_wrapper_manifest_file} + +############################################# +# run bazel_apexer_wrapper +############################################# +"${RUNFILES_DIR}/__main__/build/bazel/rules/apex/bazel_apexer_wrapper" \ + --manifest ${manifest_file} \ + --file_contexts ${file_contexts_file} \ + --key "${RUNFILES_DIR}/__main__/build/bazel/rules/apex/test.pem" \ + --apexer_path ${apexer_tool_path} \ + --apexer_tool_paths ${apexer_tool_path}:${avb_tool_path} \ + --apex_output_file ${output_file} \ + --bazel_apexer_wrapper_manifest ${bazel_apexer_wrapper_manifest_file} \ + --android_jar_path ${android_jar} + +############################################# +# check the result +############################################# +"${apexer_tool_path}/deapexer" --debugfs_path="${apexer_tool_path}/debugfs" extract ${output_file} ${output_dir} + +# The expected mounted tree should be something like this: +# /tmp/tmp.9u7ViPlMr7 +# ├── apex_manifest.pb +# ├── apex_payload.img +# ├── mnt +# │  ├── apex_manifest.pb +# │  ├── dir1 +# │  │  └── file1 +# │  ├── dir2 +# │  │  └── dir3 +# │  │  └── file2 +# │  ├── dir4 +# │  │  └── one_level_sym +# (one level symlinks always resolve) +# │  ├── dir5 +# │  │  └── two_level_sym_in_execroot +# (two level symlink resolve if the path contains execroot/__main__) +# │  ├── dir6 +# │  │  └── two_level_sym_not_in_execroot -> /tmp/tmp.evJh21oYGG/file1 +# (two level symlink resolve only one level otherwise) +# │  ├── dir7 +# │  │  └── three_level_sym_in_execroot +# (three level symlink resolve if the path contains execroot/__main__) +# └── test.apex + +# b/215129834: +# https://android-review.googlesource.com/c/platform/system/apex/+/1944264 made +# it such that the hash of non-payload files in the APEX (like +# AndroidManifest.xml) will be included as part of the apex_manifest.pb via the +# apexContainerFilesHash string to ensure that changes to AndroidManifest.xml +# results in changes in content hash for the apex_payload.img. Since this is +# potentially fragile, we skip diffing the apex_manifest.pb, and just check that +# it exists. +test -f "${output_dir}/apex_manifest.pb" || echo "expected apex_manifest.pb to exist" + +# check the contents with diff for the rest of the files +diff ${input_dir}/file1 ${output_dir}/dir1/file1 +diff ${input_dir}/file2 ${output_dir}/dir2/dir3/file2 +diff ${input_dir}/file1 ${output_dir}/dir4/one_level_sym +diff ${input_dir}/file2 ${output_dir}/dir5/two_level_sym_in_execroot +[ `readlink ${output_dir}/dir6/two_level_sym_not_in_execroot` = "${input_dir}/file1" ] +diff ${input_dir}/file2 ${output_dir}/dir7/three_level_sym_in_execroot + +cleanup + +echo "Passed for all test cases" diff --git a/rules/apex/cc.bzl b/rules/apex/cc.bzl new file mode 100644 index 00000000..2ef4bda0 --- /dev/null +++ b/rules/apex/cc.bzl @@ -0,0 +1,130 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load("//build/bazel/rules/cc:cc_library_shared.bzl", "CcStubLibrariesInfo") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") + +ApexCcInfo = provider( + "Info needed to use CC targets in APEXes", + fields = { + "transitive_shared_libs": "File references to transitive .so libs produced by the CC targets and should be included in the APEX.", + }, +) + +# Return True if this target provides stubs that is equal to, or below, the +# APEX's min_sdk_level. +# +# These stable ABI libraries are intentionally omitted from APEXes as they are +# provided from another APEX or the platform. By omitting them from APEXes, we +# ensure that there are no multiple copies of such libraries on a device. +def has_cc_stubs(target, ctx): + if ctx.rule.kind != "_cc_library_shared_proxy": + # only _cc_library_shared_proxy contains merged CcStubLibrariesInfo providers + # (a provider aggregating CcStubInfo and CcSharedLibraryInfo) + return False + + if len(target[CcStubLibrariesInfo].infos) == 0: + # Not all shared library targets have stubs + return False + + # Minimum SDK version supported by the APEX that transitively depends on + # this target. + min_sdk_version = ctx.attr._min_sdk_version[BuildSettingInfo].value + apex_name = ctx.attr._apex_name[BuildSettingInfo].value + + available_versions = [] + + # Check that the shared library has stubs built for (at least) the + # min_sdk_version of the APEX + for stub_info in target[CcStubLibrariesInfo].infos: + stub_version = stub_info["CcStubInfo"].version + available_versions.append(stub_version) + if stub_version <= min_sdk_version: + return True + + fail("cannot find a stub lib version for min_sdk_level %s (%s apex)\navailable versions: %s (%s)" % + (min_sdk_version, apex_name, available_versions, target.label)) + +# Check if this target is specified as a direct dependency of the APEX, +# as opposed to a transitive dependency, as the transitivity impacts +# the files that go into an APEX. +def is_apex_direct_dep(target, ctx): + apex_direct_deps = ctx.attr._apex_direct_deps[BuildSettingInfo].value + return str(target.label) in apex_direct_deps + +def _apex_cc_aspect_impl(target, ctx): + # Whether this dep is a direct dep of an APEX or makes a difference in dependency + # traversal, and aggregation of libs that are required from the platform/other APEXes, + # and libs that this APEX will provide to others. + is_direct_dep = is_apex_direct_dep(target, ctx) + + if has_cc_stubs(target, ctx): + if is_direct_dep: + # TODO(b/215500321): Mark these libraries as "stub-providing" exports + # of this APEX, which the system and other APEXes can depend on, + # and propagate this list. + pass + else: + # If this is not a direct dep, and stubs are available, don't propagate + # the libraries. + # + # TODO(b/215500321): In a bundled build, ensure that these libraries are + # available on the system either via the system partition, or another APEX + # and propagate this list. + return [ApexCcInfo(transitive_shared_libs = depset())] + + shared_object_files = [] + + # Transitive deps containing shared libraries to be propagated the apex. + transitive_deps = [] + rules_propagate_src = ["_bssl_hash_injection", "stripped_shared_library", "versioned_shared_library"] + + # Exclude the stripped and unstripped so files + if ctx.rule.kind == "_cc_library_shared_proxy": + for output_file in target[DefaultInfo].files.to_list(): + if output_file.extension == "so": + shared_object_files.append(output_file) + if hasattr(ctx.rule.attr, "shared"): + transitive_deps.append(ctx.rule.attr.shared) + elif ctx.rule.kind == "cc_shared_library" and hasattr(ctx.rule.attr, "dynamic_deps"): + # Propagate along the dynamic_deps edge + for dep in ctx.rule.attr.dynamic_deps: + transitive_deps.append(dep) + elif ctx.rule.kind in rules_propagate_src and hasattr(ctx.rule.attr, "src"): + # Propagate along the src edge + transitive_deps.append(ctx.rule.attr.src) + + return [ + ApexCcInfo( + # TODO: Rely on a split transition across arches to happen earlier + transitive_shared_libs = depset( + shared_object_files, + transitive = [dep[ApexCcInfo].transitive_shared_libs for dep in transitive_deps], + ), + ), + ] + +# This aspect is intended to be applied on a apex.native_shared_libs attribute +apex_cc_aspect = aspect( + implementation = _apex_cc_aspect_impl, + attrs = { + "_min_sdk_version": attr.label(default = "//build/bazel/rules/apex:min_sdk_version"), + "_apex_name": attr.label(default = "//build/bazel/rules/apex:apex_name"), + "_apex_direct_deps": attr.label(default = "//build/bazel/rules/apex:apex_direct_deps"), + }, + attr_aspects = ["dynamic_deps", "shared", "src"], + # TODO: Have this aspect also propagate along attributes of native_shared_libs? +) diff --git a/rules/apex/mainline_modules.bzl b/rules/apex/mainline_modules.bzl new file mode 100644 index 00000000..c17782e2 --- /dev/null +++ b/rules/apex/mainline_modules.bzl @@ -0,0 +1,260 @@ +""" +Copyright (C) 2022 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +load("//build/bazel/rules:apex.bzl", "ApexInfo") + +def _arch_transition_impl(settings, attr): + """Implementation of arch_transition. + Four archs are included for mainline modules: x86, x86_64, arm and arm64. + """ + return { + "x86": { + "//command_line_option:platforms": "//build/bazel/platforms:android_x86", + }, + "x86_64": { + "//command_line_option:platforms": "//build/bazel/platforms:android_x86_64", + }, + "arm": { + "//command_line_option:platforms": "//build/bazel/platforms:android_arm", + }, + "arm64": { + "//command_line_option:platforms": "//build/bazel/platforms:android_arm64", + }, + } + +# Multi-arch transition. +arch_transition = transition( + implementation = _arch_transition_impl, + inputs = [], + outputs = [ + "//command_line_option:platforms", + ], +) + +# Arch to ABI map +_arch_abi_map = { + "arm64": "arm64-v8a", + "arm": "armeabi-v7a", + "x86_64": "x86_64", + "x86": "x86", +} + +def _apex_proto_convert(ctx, arch, module_name, apex_file): + """Run 'aapt2 convert' to convert resource files to protobuf format.""" + # Inputs + inputs = [ + apex_file, + ctx.executable._aapt2, + ] + + # Outputs + filename = apex_file.basename + pos_dot = filename.rindex(".") + proto_convert_file = ctx.actions.declare_file("/".join([ + module_name, + arch, + filename[:pos_dot] + ".pb" + filename[pos_dot:]])) + outputs = [proto_convert_file] + + # Arguments + args = ctx.actions.args() + args.add_all(["convert"]) + args.add_all(["--output-format", "proto"]) + args.add_all([apex_file]) + args.add_all(["-o", proto_convert_file.path]) + + ctx.actions.run( + inputs = inputs, + outputs = outputs, + executable = ctx.executable._aapt2, + arguments = [args], + mnemonic = "ApexProtoConvert", + ) + return proto_convert_file + +def _apex_base_file(ctx, arch, module_name, apex_proto_file): + """Run zip2zip to transform the apex file the expected directory structure + with all files that will be included in the base module of aab file.""" + + # Inputs + inputs = [ + apex_proto_file, + ctx.executable._zip2zip, + ] + + # Outputs + base_file = ctx.actions.declare_file("/".join([module_name, arch, module_name + ".base"])) + outputs = [base_file] + + # Arguments + args = ctx.actions.args() + args.add_all(["-i", apex_proto_file]) + args.add_all(["-o", base_file]) + abi = _arch_abi_map[arch] + args.add_all([ + "apex_payload.img:apex/%s.img" % abi, + "apex_build_info.pb:apex/%s.build_info.pb" % abi, + "apex_manifest.json:root/apex_manifest.json", + "apex_manifest.pb:root/apex_manifest.pb", + "AndroidManifest.xml:manifest/AndroidManifest.xml", + "assets/NOTICE.html.gz:assets/NOTICE.html.gz", + ]) + + ctx.actions.run( + inputs = inputs, + outputs = outputs, + executable = ctx.executable._zip2zip, + arguments = [args], + mnemonic = "ApexBaseFile", + ) + return base_file + +def _build_bundle_config(ctx, arch, module_name): + """Create bundle_config.json as configuration for running bundletool.""" + file_content = { + "compression": { + "uncompressed_glob": [ + "apex_payload.img", + "apex_manifest.*", + ], + }, + "apex_config": {}, + } + bundle_config_file = ctx.actions.declare_file("/".join([module_name, "bundle_config.json"])) + ctx.actions.write(bundle_config_file, json.encode(file_content)) + + return bundle_config_file + +def _merge_base_files(ctx, module_name, base_files): + """Run merge_zips to merge all files created for each arch by _apex_base_file.""" + + # Inputs + inputs = base_files + [ctx.executable._merge_zips] + + # Outputs + merged_base_file = ctx.actions.declare_file(module_name + "/" + module_name + ".zip") + outputs = [merged_base_file] + + # Arguments + args = ctx.actions.args() + args.add_all(["--ignore-duplicates"]) + args.add_all([merged_base_file]) + args.add_all(base_files) + + ctx.actions.run( + inputs = inputs, + outputs = outputs, + executable = ctx.executable._merge_zips, + arguments = [args], + mnemonic = "ApexMergeBaseFiles", + ) + return merged_base_file + +def _apex_bundle(ctx, module_name, merged_base_file, bundle_config_file): + """Run bundletool to create the aab file.""" + + # Inputs + inputs = [ + bundle_config_file, + merged_base_file, + ctx.executable._bundletool, + ] + + # Outputs + bundle_file = ctx.actions.declare_file(module_name + "/" + module_name + ".aab") + outputs = [bundle_file] + + # Arguments + args = ctx.actions.args() + args.add_all(["build-bundle"]) + args.add_all(["--config", bundle_config_file]) + args.add_all(["--modules", merged_base_file]) + args.add_all(["--output", bundle_file]) + + ctx.actions.run( + inputs = inputs, + outputs = outputs, + executable = ctx.executable._bundletool, + arguments = [args], + mnemonic = "ApexBundleFile", + ) + return bundle_file + +def _apex_aab_impl(ctx): + """Implementation of apex_aab rule, which drives the process of creating aab + file from apex files created for each arch.""" + apex_base_files = [] + bundle_config_file = None + module_name = ctx.attr.mainline_module[0].label.name + for arch in ctx.split_attr.mainline_module: + apex_file = ctx.split_attr.mainline_module[arch].files.to_list()[0] + proto_convert_file = _apex_proto_convert(ctx, arch, module_name, apex_file) + base_file = _apex_base_file(ctx, arch, module_name, proto_convert_file) + apex_base_files.append(base_file) + # It is assumed that the bundle config is the same for all products. + if bundle_config_file == None: + bundle_config_file = _build_bundle_config(ctx, arch, module_name) + + merged_base_file = _merge_base_files(ctx, module_name, apex_base_files) + bundle_file = _apex_bundle(ctx, module_name, merged_base_file, bundle_config_file) + + return [DefaultInfo(files = depset([bundle_file]))] + +# apex_aab rule creates Android Apk Bundle (.aab) file of the APEX specified in mainline_module. +# There is no equivalent Soong module, and it is currently done in shell script by +# invoking Soong multiple times. +apex_aab = rule( + implementation = _apex_aab_impl, + attrs = { + "mainline_module": attr.label( + mandatory = True, + cfg = arch_transition, + providers = [ApexInfo], + doc = "The label of a mainline module target", + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + doc = "Allow transition.", + ), + "_zipper": attr.label( + cfg = "host", + executable = True, + default = "@bazel_tools//tools/zip:zipper", + ), + "_aapt2": attr.label( + allow_single_file = True, + cfg = "host", + executable = True, + default = "//prebuilts/sdk/tools:linux/bin/aapt2", + ), + "_merge_zips": attr.label( + allow_single_file = True, + cfg = "host", + executable = True, + default = "//prebuilts/build-tools:linux-x86/bin/merge_zips", + ), + "_zip2zip": attr.label( + allow_single_file = True, + cfg = "host", + executable = True, + default = "//prebuilts/build-tools:linux-x86/bin/zip2zip", + ), + "_bundletool": attr.label( + cfg = "host", + executable = True, + default = "//prebuilts/bundletool", + ), + }, +) diff --git a/rules/apex/test.pem b/rules/apex/test.pem new file mode 100644 index 00000000..bd567789 --- /dev/null +++ b/rules/apex/test.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKgIBAAKCAgEAt4iSfTF+e2khGQf0bUzTMwWFsgaiQbwQB3cvyBlE9XekFXUt +GdOEhC2J0p+930UoF6gjjRRrgGF+8K5iV1m3oEbB3qGz6UUOurvVkt4tq96e/Q5a +ogCOZEuWHjZfs2tQUVNJJtptIp9+0cM768vdf+qnK2JNFIhBqSY0FhjVljKevMcM +w2tWFRZnKPQ3JoRnWqi5CIauQtBcWRFKIApyf41uHGMjpQRd8aTGeLXBRTi/yD73 +HltuKwSF2SXpj1F+9j4stqskQvipjQnid/Wb+nN3CNgyrGuRrtGvz71WWYcK3DLM +jvGLOl06QrN6a7ZfLUN4qQjJ6Is5SLTSw/sfFE7Fpcbg6/Geh+jSvChuo6EUtzoX +Qu42HsVXhrJLQ9/AVTWNmGc9IDr4PMtDiQc4FN8MOpUtR6V/zwrZFoeR3PHl9Z7v +uTxLIcQLIott0mAjPhbNgbFBs5HP1Z8TfFcyZWpShlx+aM1V2mzYQ7sgsWjFKMSQ +wIUk/YZ9QK/H5WKjC5M0yxueCU0ocvWFaAZ4RyS/r/SUyQpvyNXNwUsdp1a8sNxp +LP9U7FG64C+T791yoQJ0sKVbts5SEu/Tojw6miYbH6Fspdo2xxfCbrv6SAbkjlct +afOnEepgTlHet0G+y0N7OZRJ9WRGyLJNgGjmmDy9XSYGAykwwe4Fv348D0cCAwEA +AQKCAgBuFra/78NNpXbb++CK+20oCqTyb3Y+dd8rizuXDElH8Fb1JA9EkZLIckRc +mcMbvPDal9mTU29UV6b8Ga4VdVRnCGpb76TqRKkcK3Vlnm3IzUWSx1xoFmtTD9/h +CX6IMdPApHOZoaWbAg7hJfm4a9XWV9ukc1eG/GBeZPMTWhwr9vsugztNsQG2rnR8 +pVi7eupAADrVOWwn2bG7H1rWM04Q4rXswy7rWd48BzmhyGxA6FRpehNjGzbPCOx8 +n3gkpp7Ad/T8MVYT8fJKDmbQy/ue1EnPfVeQAwok0dRiiNDV7OH/yVzYVVzNSoSa +4+uH1qHqlbE3u3TZT0GyMfzG38f4scsbvG/AhH1fuPsy4QcWyLlMV6KUnk3KPc3Q +yOeRR82qndQMTYQ5/PFiilk7cNbTU0OBjuNpu/t1LIE2J2gGZ5Jw+g2NGtM/xsgC +jOahpRYvZB8fZ/bSjirwwmSSU+v0ZoPDHtt75R/QxqwPG2jai8kaGr7GEXWJfrfv +CktMnb6LoCyNiiiZSMUgdDHOQEkVNmt9fxiVaxsaIL4BygropwlD4WbuyRMevfYz +EffvvmaqC24zJi8WzDszCNLgP/piNhXDyxZX+KaQXj0Do/tzWBBkO0OO6mVGOkX2 +6dadXfhOIggWO8K2lKCUKwWMO9LaKwSwZ4gzcc1a+U9rpE8kUQKCAQEA8lBGLzOL +Ht8+d13SY+NdPbL6qGvoqsKd5BfIhaNbH04Cp2zQs2TWySxmV47df03pGUpQOCKn +tFRxoczUrf1gfFDCCC95+A/crls8QJHG+MScTBH5U8Q0s9ReUo/0xaa55u77x5uS +0fAtdnOdqP8/pf1fSXUJvyLW85LWdkge1c7jk7I5MnWVO2Ak9/GkuRgITSSgVdBa +kr8nU1BCzDY0gOTWo5J1+NqqVH2eYfEI621iD4SAE3n2JrCC4K/Nt2enEJwup2TR +ym15g9nClicUQP5Y67eDfqTZu1d0I0Ezl1tL8UPxcLI+ucN4V6KL8RvqTVMnGX/R +s1FwkPVMQ6dKaQKCAQEAweZeggcSFukr+tTbnzDAHxg4YqiR+30wo7i8NadGu6W/ +EiAdcCdmZYMI9KKc+B/N3cuFqBnaSd7VM7XvINdwZRanRj56Ya8LvQMi0S9YPiRn +T4TXC3EeewN5+SSO0Dkw83tW1PLqgSINy5ijBs5lGoIYMCC+GSA2DuRBiPpcfhqJ +kmC9uFQvrsge8CC8Sb1wHCr0Wz34qhPoTff6ZV8wm11Jkb5+tT7PMS5Ft0sEBsxV +R1JFtLNs0k/YpMb4/OrZFZZSIFCTUVPvHQ1/5BwumVnolBC4LORCaSk1xUOydU9h +bZd4qzIpFteGLGGRT6nEWC1YejLAvcFHVJiKs1F2LwKCAQEAzgnwA8bCLvgIt5rx +gLod2I7NkFRhPIHLm92VRf0HSHEe1Jo0Q7Yk5F56j00NjmgDItwLpg/hpfZ/wOLY +nTFrz4kj0636+jESprcxXn4WQAV+GTjXVqDpZ1fW9EEwEriYLoNbV/kzOIwPPD9G ++iJATrZJRb7dEMdhGy/qaB0fCxKmdDoBZKSSxjAUfzfbpv+GX4IbS5ykx07+81q1 +0crtjgQHdoLdCUN1ve4qtIEt4nHaBfPWq7jy0ycXwlH6jE74wajsCq4xrPy1bKXH +TcHg+PrNRXF/wDoQYboVKL0ST0r0IixxqjAGIhLRy0KN1/CypBlmj8od12oSW1AZ +DxW6sQKCAQEAtIMW8M5MVO/2dam8XFMySMBvncl5PjuqEIFnFjwIaaFAZEtpnIPR +nCeFKtpIb+aL7TQP1hNbWPIOYfm6CUUH6dRRHeAEZvRjZS+KNlxxNkkFtM3itVA2 +JCd0YjFakxbrL4FfsRgEoPtnBGexPiDflvIOOqAA2btXGD3/lNofSXbDJHbTqMsX +KQw9YSfYon2t5UtH+bmTyiKGXi/B+KXJxpnuZ7SEmY9DrHF7jcxUj0+jBKbfJf70 +DEcxVRW3rx2jw6kSA+t/enM9ZDqxGVfzOeit0UpPa9uEyAoJeQAxH20rMq+VMyub +fRxgWOjsMtHFbKGqgPjG3uEU2vi4B4CLGQKCAQEA2Mr5f2AXPR8jca1+Id+CxZpU +bgMML7gW31L4lGX9Teo9z+zSdN7sIwqe42Zla1N9wda8p5ribnJxwRdxcPL8bid5 +LLlls4xXD/jQCQCFL90X59Tm6VD6tm1VyCjL44nRwAqP4vJObSB5rTqJYtkfVmnp +KERF5P0i5yv4Oox0ZOsThou9jtyl1dS50Td0Urhp4LhPdmpDPUq25K1sDDfnGFm6 +IcMPkVznRPUoKQCG9DSQcQqttkSV9Po+qfLa3aHtdndfe88Gd9uom8bsAMTZAfSZ +D4YhqBHSLWrxvtQ8GxkaPITJv7hocwssdFRUj5/UJKJBgUXPBXEXh+fxlDaGQQ== +-----END RSA PRIVATE KEY----- diff --git a/rules/apex/toolchain.bzl b/rules/apex/toolchain.bzl new file mode 100644 index 00000000..04c7f4d8 --- /dev/null +++ b/rules/apex/toolchain.bzl @@ -0,0 +1,70 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +ApexToolchainInfo = provider( + doc = "APEX toolchain", + fields = [ + "aapt2", + "avbtool", + "apexer", + "mke2fs", + "resize2fs", + "e2fsdroid", + "sefcontext_compile", + "conv_apex_manifest", + "android_jar", + "apex_compression_tool", + "soong_zip", + ], +) + +def _apex_toolchain_impl(ctx): + toolchain_info = platform_common.ToolchainInfo( + toolchain_info = ApexToolchainInfo( + aapt2 = ctx.file.aapt2, + avbtool = ctx.attr.avbtool, + apexer = ctx.file.apexer, + mke2fs = ctx.attr.mke2fs, + resize2fs = ctx.attr.resize2fs, + e2fsdroid = ctx.attr.e2fsdroid, + sefcontext_compile = ctx.file.sefcontext_compile, + conv_apex_manifest = ctx.file.conv_apex_manifest, + android_jar = ctx.file.android_jar, + apex_compression_tool = ctx.file.apex_compression_tool, + soong_zip = ctx.file.soong_zip, + ), + ) + return [toolchain_info] + +apex_toolchain = rule( + implementation = _apex_toolchain_impl, + attrs = { + "aapt2": attr.label(allow_single_file = True, cfg = "host", executable = True), + "avbtool": attr.label(cfg = "host", executable = True), + "apexer": attr.label(allow_single_file = True, cfg = "host", executable = True), + "mke2fs": attr.label(cfg = "host", executable = True), + "resize2fs": attr.label(cfg = "host", executable = True), + "e2fsdroid": attr.label(cfg = "host", executable = True), + "sefcontext_compile": attr.label(allow_single_file = True, cfg = "host", executable = True), + "conv_apex_manifest": attr.label(allow_single_file = True, cfg = "host", executable = True), + "android_jar": attr.label(allow_single_file = True, cfg = "host"), + "apex_compression_tool": attr.label(allow_single_file = True, cfg = "host", executable = True), + # soong_zip is added as a dependency of apex_compression_tool which uses + # soong_zip to compress APEX files. avbtool is also used in apex_compression tool + # and has been added to apex toolchain previously. + "soong_zip": attr.label(allow_single_file = True, cfg = "host", executable = True), + }, +) diff --git a/rules/apex/transition.bzl b/rules/apex/transition.bzl new file mode 100644 index 00000000..e6ffa374 --- /dev/null +++ b/rules/apex/transition.bzl @@ -0,0 +1,113 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Configuration transitions for APEX rules. +# +# Transitions are a Bazel mechanism to analyze/build dependencies in a different +# configuration (i.e. options and flags). The APEX transition is applied from a +# top level APEX rule to its dependencies via an outgoing edge, so that the +# dependencies can be built specially for APEXes (vs the platform). +# +# e.g. if an apex A depends on some target T, building T directly as a top level target +# will use a different configuration from building T indirectly as a dependency of A. The +# latter will contain APEX specific configuration settings that its rule or an aspect can +# use to create different actions or providers for APEXes specifically.. +# +# The outgoing transitions are similar to ApexInfo propagation in Soong's +# top-down ApexInfoMutator: +# https://cs.android.com/android/platform/superproject/+/master:build/soong/apex/apex.go;l=948-962;drc=539d41b686758eeb86236c0e0dcf75478acb77f3 + +load("@bazel_skylib//lib:dicts.bzl", "dicts") + +def _create_apex_configuration(attr, additional = {}): + return dicts.add({ + "//build/bazel/rules/apex:apex_name": attr.name, # Name of the APEX + "//build/bazel/rules/apex:min_sdk_version": attr.min_sdk_version, # Min SDK version of the APEX + }, additional) + +def _impl(settings, attr): + # Perform a transition to apply APEX specific build settings on the + # destination target (i.e. an APEX dependency). + return _create_apex_configuration(attr) + +apex_transition = transition( + implementation = _impl, + inputs = [], + outputs = [ + "//build/bazel/rules/apex:apex_name", + "//build/bazel/rules/apex:min_sdk_version", + ], +) + +def _impl_shared_lib_transition_32(settings, attr): + # Perform a transition to apply APEX specific build settings on the + # destination target (i.e. an APEX dependency). + + direct_deps = [str(dep) for dep in attr.native_shared_libs_32] + + # TODO: We need to check if this is a x86 or arm arch then only set one platform + # instead of this 1:2 split to avoid performance hit. + return { + "x86": _create_apex_configuration(attr, { + "//command_line_option:platforms": "//build/bazel/platforms:android_x86", + "//build/bazel/rules/apex:apex_direct_deps": direct_deps, + }), + "arm": _create_apex_configuration(attr, { + "//command_line_option:platforms": "//build/bazel/platforms:android_arm", + "//build/bazel/rules/apex:apex_direct_deps": direct_deps, + }), + } + +shared_lib_transition_32 = transition( + implementation = _impl_shared_lib_transition_32, + inputs = [], + outputs = [ + "//build/bazel/rules/apex:apex_name", + "//build/bazel/rules/apex:min_sdk_version", + "//build/bazel/rules/apex:apex_direct_deps", + "//command_line_option:platforms", + ], +) + +def _impl_shared_lib_transition_64(settings, attr): + # Perform a transition to apply APEX specific build settings on the + # destination target (i.e. an APEX dependency). + + direct_deps = [str(dep) for dep in attr.native_shared_libs_64] + + # TODO: We need to check if this is a x86 or arm arch then only set one platform + # instead of this 1:2 split to avoid performance hit. + return { + "x86_64": _create_apex_configuration(attr, { + "//command_line_option:platforms": "//build/bazel/platforms:android_x86_64", + "//build/bazel/rules/apex:apex_direct_deps": direct_deps, + }), + "arm64": _create_apex_configuration(attr, { + "//command_line_option:platforms": "//build/bazel/platforms:android_arm64", + "//build/bazel/rules/apex:apex_direct_deps": direct_deps, + }), + } + +shared_lib_transition_64 = transition( + implementation = _impl_shared_lib_transition_64, + inputs = [], + outputs = [ + "//build/bazel/rules/apex:apex_name", + "//build/bazel/rules/apex:min_sdk_version", + "//build/bazel/rules/apex:apex_direct_deps", + "//command_line_option:platforms", + ], +) diff --git a/rules/apex_key.bzl b/rules/apex_key.bzl new file mode 100644 index 00000000..be31da67 --- /dev/null +++ b/rules/apex_key.bzl @@ -0,0 +1,36 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +ApexKeyInfo = provider( + "Info needed to sign APEX bundles", + fields = { + "public_key": "File containing the public_key", + "private_key": "File containing the private key", + }, +) + +def _apex_key_rule_impl(ctx): + return [ + ApexKeyInfo(public_key = ctx.file.public_key, private_key = ctx.file.private_key), + ] + +apex_key = rule( + implementation = _apex_key_rule_impl, + attrs = { + "public_key": attr.label(mandatory = True, allow_single_file = True), + "private_key": attr.label(mandatory = True, allow_single_file = True), + }, +) diff --git a/rules/cc/BUILD.bazel b/rules/cc/BUILD.bazel new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/rules/cc/BUILD.bazel @@ -0,0 +1 @@ + diff --git a/rules/cc/cc_binary.bzl b/rules/cc/cc_binary.bzl new file mode 100644 index 00000000..8ba78590 --- /dev/null +++ b/rules/cc/cc_binary.bzl @@ -0,0 +1,157 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load( + ":cc_library_common.bzl", + "add_lists_defaulting_to_none", + "parse_sdk_version", + "system_dynamic_deps_defaults", + "system_static_deps_defaults", +) +load(":cc_library_static.bzl", "cc_library_static") +load(":stl.bzl", "shared_stl_deps", "static_binary_stl_deps") +load(":stripped_cc_common.bzl", "stripped_binary") +load(":versioned_cc_common.bzl", "versioned_binary") + +def cc_binary( + name, + dynamic_deps = [], + srcs = [], + srcs_c = [], + srcs_as = [], + copts = [], + cppflags = [], + conlyflags = [], + asflags = [], + deps = [], + whole_archive_deps = [], + system_deps = None, + export_includes = [], + export_system_includes = [], + local_includes = [], + absolute_includes = [], + linkshared = True, + linkopts = [], + rtti = False, + use_libcrt = True, + stl = "", + cpp_std = "", + additional_linker_inputs = None, + strip = {}, + features = [], + target_compatible_with = [], + sdk_version = "", + min_sdk_version = "", + use_version_lib = False, + **kwargs): + "Bazel macro to correspond with the cc_binary Soong module." + + root_name = name + "_root" + unstripped_name = name + "_unstripped" + + toolchain_features = [] + toolchain_features += features + + if linkshared: + toolchain_features.extend(["dynamic_executable", "dynamic_linker"]) + else: + toolchain_features.extend(["-dynamic_executable", "-dynamic_linker", "static_executable", "static_flag"]) + + if not use_libcrt: + toolchain_features += ["-use_libcrt"] + + if min_sdk_version: + toolchain_features += [ + "sdk_version_" + parse_sdk_version(min_sdk_version), + "-sdk_version_default", + ] + + system_dynamic_deps = [] + system_static_deps = [] + if system_deps == None: + if linkshared: + system_deps = system_dynamic_deps_defaults + else: + system_deps = system_static_deps_defaults + + if linkshared: + system_dynamic_deps = system_deps + else: + system_static_deps = system_deps + + stl_static, stl_shared = [], [] + + if linkshared: + stl_static, stl_shared = shared_stl_deps(stl) + else: + stl_static = static_binary_stl_deps(stl) + + # The static library at the root of the shared library. + # This may be distinct from the static version of the library if e.g. + # the static-variant srcs are different than the shared-variant srcs. + cc_library_static( + name = root_name, + absolute_includes = absolute_includes, + alwayslink = True, + asflags = asflags, + conlyflags = conlyflags, + copts = copts, + cpp_std = cpp_std, + cppflags = cppflags, + deps = deps + whole_archive_deps + stl_static + system_static_deps, + dynamic_deps = dynamic_deps, + features = toolchain_features, + local_includes = local_includes, + rtti = rtti, + srcs = srcs, + srcs_as = srcs_as, + srcs_c = srcs_c, + stl = stl, + system_dynamic_deps = system_dynamic_deps, + target_compatible_with = target_compatible_with, + use_version_lib = use_version_lib, + ) + + binary_dynamic_deps = add_lists_defaulting_to_none( + dynamic_deps, + system_dynamic_deps, + stl_shared, + ) + + native.cc_binary( + name = unstripped_name, + deps = [root_name] + deps + system_static_deps + stl_static, + dynamic_deps = binary_dynamic_deps, + features = toolchain_features, + linkopts = linkopts, + additional_linker_inputs = additional_linker_inputs, + target_compatible_with = target_compatible_with, + **kwargs + ) + + versioned_name = name + "_versioned" + versioned_binary( + name = versioned_name, + src = unstripped_name, + stamp_build_number = use_version_lib, + ) + + stripped_binary( + name = name, + src = versioned_name, + target_compatible_with = target_compatible_with, + ) + diff --git a/rules/cc_constants.bzl b/rules/cc/cc_constants.bzl index b86417cb..26f56b88 100644 --- a/rules/cc_constants.bzl +++ b/rules/cc/cc_constants.bzl @@ -1,10 +1,29 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + # Constants for cc_* rules. # To use, load the constants struct: # # load("//build/bazel/rules:cc_constants.bzl", "constants") # Supported hdr extensions in Soong. Keep this consistent with hdrExts in build/soong/cc/snapshot_utils.go _HDR_EXTS = ["h", "hh", "hpp", "hxx", "h++", "inl", "inc", "ipp", "h.generic"] -_SRC_EXTS = ["c", "cc", "cpp", "S"] +_C_SRC_EXTS = ["c"] +_CPP_SRC_EXTS = ["cc", "cpp"] +_AS_SRC_EXTS = ["S"] +_SRC_EXTS = _C_SRC_EXTS + _CPP_SRC_EXTS + _AS_SRC_EXTS _ALL_EXTS = _SRC_EXTS + _HDR_EXTS _HDR_EXTS_WITH_DOT = ["." + ext for ext in _HDR_EXTS] _SRC_EXTS_WITH_DOT = ["." + ext for ext in _SRC_EXTS] @@ -16,6 +35,9 @@ _GLOBAL_INCLUDE_DIRS_COPTS_ONLY_USED_FOR_SOONG_COMPATIBILITY_DO_NOT_ADD_MORE = [ ] constants = struct( hdr_exts = _HDR_EXTS, + c_src_exts = _C_SRC_EXTS, + cpp_src_exts = _CPP_SRC_EXTS, + as_src_exts = _AS_SRC_EXTS, src_exts = _SRC_EXTS, all_exts = _ALL_EXTS, hdr_dot_exts = _HDR_EXTS_WITH_DOT, diff --git a/rules/cc/cc_library_common.bzl b/rules/cc/cc_library_common.bzl new file mode 100644 index 00000000..1431ff3c --- /dev/null +++ b/rules/cc/cc_library_common.bzl @@ -0,0 +1,132 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load("//build/bazel/product_variables:constants.bzl", "constants") +load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cpp_toolchain") +load("@soong_injection//api_levels:api_levels.bzl", "api_levels") + +_bionic_targets = ["//bionic/libc", "//bionic/libdl", "//bionic/libm"] +_static_bionic_targets = ["//bionic/libc:libc_bp2build_cc_library_static", "//bionic/libdl:libdl_bp2build_cc_library_static", "//bionic/libm:libm_bp2build_cc_library_static"] + +# The default system_dynamic_deps value for cc libraries. This value should be +# used if no value for system_dynamic_deps is specified. +system_dynamic_deps_defaults = select({ + constants.ArchVariantToConstraints["linux_bionic"]: _bionic_targets, + constants.ArchVariantToConstraints["android"]: _bionic_targets, + "//conditions:default": [], +}) + +system_static_deps_defaults = select({ + constants.ArchVariantToConstraints["linux_bionic"]: _static_bionic_targets, + constants.ArchVariantToConstraints["android"]: _static_bionic_targets, + "//conditions:default": [], +}) + +def add_lists_defaulting_to_none(*args): + """Adds multiple lists, but is well behaved with a `None` default.""" + combined = None + for arg in args: + if arg != None: + if combined == None: + combined = [] + combined += arg + + return combined + +# By default, crtbegin/crtend linking is enabled for shared libraries and cc_binary. +def disable_crt_link(features): + return features + ["-link_crt"] + +# get_includes_paths expects a rule context, a list of directories, and +# whether the directories are package-relative and returns a list of exec +# root-relative paths. This handles the need to search for files both in the +# source tree and generated files. +def get_includes_paths(ctx, dirs, package_relative = True): + execution_relative_dirs = [] + for rel_dir in dirs: + if rel_dir == ".": + rel_dir = "" + execution_rel_dir = rel_dir + if package_relative: + execution_rel_dir = ctx.label.package + if len(rel_dir) > 0: + execution_rel_dir = execution_rel_dir + "/" + rel_dir + execution_relative_dirs.append(execution_rel_dir) + + # to support generated files, we also need to export includes relatives to the bin directory + if not execution_rel_dir.startswith("/"): + execution_relative_dirs.append(ctx.bin_dir.path + "/" + execution_rel_dir) + return execution_relative_dirs + +def create_ccinfo_for_includes( + ctx, + includes = [], + absolute_includes = [], + system_includes = [], + deps = []): + cc_toolchain = find_cpp_toolchain(ctx) + + # Create a compilation context using the string includes of this target. + compilation_context = cc_common.create_compilation_context( + includes = depset( + get_includes_paths(ctx, includes) + + get_includes_paths(ctx, absolute_includes, package_relative = False), + ), + system_includes = depset(get_includes_paths(ctx, system_includes)), + ) + + # Combine this target's compilation context with those of the deps; use only + # the compilation context of the combined CcInfo. + cc_infos = [dep[CcInfo] for dep in deps] + cc_infos += [CcInfo(compilation_context = compilation_context)] + combined_info = cc_common.merge_cc_infos(cc_infos = cc_infos) + + return CcInfo(compilation_context = combined_info.compilation_context) + + +def is_external_directory(package_name): + if package_name.startswith('external'): + return True + if package_name.startswith('hardware'): + paths = package_name.split("/") + if len(paths) < 2: + return True + secondary_path = paths[1] + if secondary_path in ["google", "interfaces", "ril"]: + return True + return secondary_path.startswith("libhardware") + if package_name.startswith("vendor"): + paths = package_name.split("/") + if len(paths) < 2: + return True + secondary_path = paths[1] + return secondary_path.contains("google") + return False + +def parse_sdk_version(version): + future_version = "10000" + + if version == "" or version == "current": + return future_version + elif version.isdigit() and int(version) in api_levels.values(): + return version + elif version in api_levels.keys(): + return str(api_levels[version]) + # We need to handle this case properly later + elif version == "apex_inherit": + return future_version + else: + fail("Unknown sdk version: %s" % (version)) diff --git a/rules/cc/cc_library_headers.bzl b/rules/cc/cc_library_headers.bzl new file mode 100644 index 00000000..c54c1413 --- /dev/null +++ b/rules/cc/cc_library_headers.bzl @@ -0,0 +1,48 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load(":cc_library_static.bzl", "cc_library_static") + +def cc_library_headers( + name, + implementation_deps = [], + deps = [], + hdrs = [], + export_includes = [], + export_absolute_includes = [], + export_system_includes = [], + native_bridge_supported = False, # TODO: not supported yet. + sdk_version = "", + min_sdk_version = "", + **kwargs): + "Bazel macro to correspond with the cc_library_headers Soong module." + + cc_library_static( + name = name, + implementation_deps = implementation_deps, + deps = deps, + export_includes = export_includes, + export_absolute_includes = export_absolute_includes, + export_system_includes = export_system_includes, + hdrs = hdrs, + native_bridge_supported = native_bridge_supported, + # do not automatically add libcrt dependency to header libraries + use_libcrt = False, + stl = "none", + sdk_version = sdk_version, + min_sdk_version = min_sdk_version, + **kwargs + ) diff --git a/rules/cc/cc_library_shared.bzl b/rules/cc/cc_library_shared.bzl new file mode 100644 index 00000000..b4367e50 --- /dev/null +++ b/rules/cc/cc_library_shared.bzl @@ -0,0 +1,449 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load( + ":cc_library_common.bzl", + "add_lists_defaulting_to_none", + "disable_crt_link", + "parse_sdk_version", + "system_dynamic_deps_defaults", +) +load(":cc_library_static.bzl", "cc_library_static") +load(":cc_stub_library.bzl", "CcStubInfo", "cc_stub_gen") +load(":generate_toc.bzl", "shared_library_toc", _CcTocInfo = "CcTocInfo") +load(":stl.bzl", "shared_stl_deps") +load(":stripped_cc_common.bzl", "stripped_shared_library") +load(":versioned_cc_common.bzl", "versioned_shared_library") +load("@rules_cc//examples:experimental_cc_shared_library.bzl", "cc_shared_library", _CcSharedLibraryInfo = "CcSharedLibraryInfo") +load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cpp_toolchain") + +CcTocInfo = _CcTocInfo +CcSharedLibraryInfo = _CcSharedLibraryInfo + +def cc_library_shared( + name, + # Common arguments between shared_root and the shared library + features = [], + dynamic_deps = [], + implementation_dynamic_deps = [], + linkopts = [], + target_compatible_with = [], + # Ultimately _static arguments for shared_root production + srcs = [], + srcs_c = [], + srcs_as = [], + copts = [], + cppflags = [], + conlyflags = [], + asflags = [], + hdrs = [], + implementation_deps = [], + deps = [], + whole_archive_deps = [], + system_dynamic_deps = None, + export_includes = [], + export_absolute_includes = [], + export_system_includes = [], + local_includes = [], + absolute_includes = [], + rtti = False, + use_libcrt = True, # FIXME: Unused below? + stl = "", + cpp_std = "", + c_std = "", + link_crt = True, + additional_linker_inputs = None, + + # Purely _shared arguments + strip = {}, + soname = "", + + # TODO(b/202299295): Handle data attribute. + data = [], + use_version_lib = False, + stubs_symbol_file = None, + stubs_versions = [], + inject_bssl_hash = False, + sdk_version = "", + min_sdk_version = "", + **kwargs): + "Bazel macro to correspond with the cc_library_shared Soong module." + + shared_root_name = name + "_root" + unstripped_name = name + "_unstripped" + stripped_name = name + "_stripped" + toc_name = name + "_toc" + + if system_dynamic_deps == None: + system_dynamic_deps = system_dynamic_deps_defaults + + # Force crtbegin and crtend linking unless explicitly disabled (i.e. bionic + # libraries do this) + if link_crt == False: + features = disable_crt_link(features) + + if min_sdk_version: + features = features + [ + "sdk_version_" + parse_sdk_version(min_sdk_version), + "-sdk_version_default", + ] + + # The static library at the root of the shared library. + # This may be distinct from the static version of the library if e.g. + # the static-variant srcs are different than the shared-variant srcs. + cc_library_static( + name = shared_root_name, + hdrs = hdrs, + srcs = srcs, + srcs_c = srcs_c, + srcs_as = srcs_as, + copts = copts, + cppflags = cppflags, + conlyflags = conlyflags, + asflags = asflags, + export_includes = export_includes, + export_absolute_includes = export_absolute_includes, + export_system_includes = export_system_includes, + local_includes = local_includes, + absolute_includes = absolute_includes, + rtti = rtti, + stl = stl, + cpp_std = cpp_std, + c_std = c_std, + dynamic_deps = dynamic_deps, + implementation_deps = implementation_deps, + implementation_dynamic_deps = implementation_dynamic_deps, + system_dynamic_deps = system_dynamic_deps, + deps = deps + whole_archive_deps, + features = features, + use_version_lib = use_version_lib, + target_compatible_with = target_compatible_with, + ) + + stl_static, stl_shared = shared_stl_deps(stl) + + # implementation_deps and deps are to be linked into the shared library via + # --no-whole-archive. In order to do so, they need to be dependencies of + # a "root" of the cc_shared_library, but may not be roots themselves. + # Below we define stub roots (which themselves have no srcs) in order to facilitate + # this. + imp_deps_stub = name + "_implementation_deps" + deps_stub = name + "_deps" + native.cc_library( + name = imp_deps_stub, + deps = implementation_deps + stl_static, + target_compatible_with = target_compatible_with, + ) + native.cc_library( + name = deps_stub, + deps = deps, + target_compatible_with = target_compatible_with, + ) + + shared_dynamic_deps = add_lists_defaulting_to_none( + dynamic_deps, + system_dynamic_deps, + implementation_dynamic_deps, + stl_shared, + ) + + if len(soname) == 0: + soname = name + ".so" + soname_flag = "-Wl,-soname," + soname + + cc_shared_library( + name = unstripped_name, + user_link_flags = linkopts + [soname_flag], + # b/184806113: Note this is a workaround so users don't have to + # declare all transitive static deps used by this target. It'd be great + # if a shared library could declare a transitive exported static dep + # instead of needing to declare each target transitively. + static_deps = ["//:__subpackages__"] + [shared_root_name, imp_deps_stub, deps_stub], + dynamic_deps = shared_dynamic_deps, + additional_linker_inputs = additional_linker_inputs, + roots = [shared_root_name, imp_deps_stub, deps_stub] + whole_archive_deps, + features = features, + target_compatible_with = target_compatible_with, + **kwargs + ) + + hashed_name = name + "_hashed" + _bssl_hash_injection( + name = hashed_name, + src = unstripped_name, + inject_bssl_hash = inject_bssl_hash, + ) + + versioned_name = name + "_versioned" + versioned_shared_library( + name = versioned_name, + src = hashed_name, + stamp_build_number = use_version_lib, + ) + + stripped_shared_library( + name = stripped_name, + src = versioned_name, + target_compatible_with = target_compatible_with, + **strip + ) + + shared_library_toc( + name = toc_name, + src = stripped_name, + target_compatible_with = target_compatible_with, + ) + + # Emit the stub version of this library (e.g. for libraries that are + # provided by the NDK) + stub_shared_libraries = [] + if stubs_symbol_file and len(stubs_versions) > 0: + # TODO(b/193663198): This unconditionally creates stubs for every version, but + # that's not always true depending on whether this module is available + # on the host, ramdisk, vendor ramdisk. We currently don't have + # information about the image variant yet, so we'll create stub targets + # for all shared libraries with the stubs property for now. + # + # See: https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/library.go;l=2316-2377;drc=3d3b35c94ed2a3432b2e5e7e969a3a788a7a80b5 + for version in stubs_versions: + stubs_library_name = "_".join([name, version, "stubs"]) + cc_stub_library_shared( + name = stubs_library_name, + stubs_symbol_file = stubs_symbol_file, + version = version, + target_compatible_with = target_compatible_with, + features = features, + ) + stub_shared_libraries.append(stubs_library_name) + + _cc_library_shared_proxy( + name = name, + shared = stripped_name, + root = shared_root_name, + table_of_contents = toc_name, + output_file = soname, + target_compatible_with = target_compatible_with, + stub_shared_libraries = stub_shared_libraries, + ) + +# cc_stub_library_shared creates a cc_library_shared target, but using stub C source files generated +# from a library's .map.txt files and ndkstubgen. The top level target returns the same +# providers as a cc_library_shared, with the addition of a CcStubInfo +# containing metadata files and versions of the stub library. +def cc_stub_library_shared(name, stubs_symbol_file, version, target_compatible_with, features): + # Call ndkstubgen to generate the stub.c source file from a .map.txt file. These + # are accessible in the CcStubInfo provider of this target. + cc_stub_gen( + name = name + "_files", + symbol_file = stubs_symbol_file, + version = version, + target_compatible_with = target_compatible_with, + ) + + # The static library at the root of the stub shared library. + cc_library_static( + name = name + "_root", + srcs_c = [name + "_files"], # compile the stub.c file + features = disable_crt_link(features) + \ + [ + # Enable the stub library compile flags + "stub_library", + # Disable all include-related features to avoid including any headers + # that may cause conflicting type errors with the symbols in the + # generated stubs source code. + # e.g. + # double acos(double); // in header + # void acos() {} // in the generated source code + # See https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/library.go;l=942-946;drc=d8a72d7dc91b2122b7b10b47b80cf2f7c65f9049 + "-toolchain_include_directories", + "-includes", + "-include_paths", + ], + target_compatible_with = target_compatible_with, + stl = "none", + system_dynamic_deps = [], + ) + + # Create a .so for the stub library. This library is self contained, has + # no deps, and doesn't link against crt. + cc_shared_library( + name = name + "_so", + roots = [name + "_root"], + features = disable_crt_link(features), + target_compatible_with = target_compatible_with, + ) + + # Create a target with CcSharedLibraryInfo and CcStubInfo providers. + _cc_stub_library_shared( + name = name, + stub_target = name + "_files", + library_target = name + "_so", + ) + +def _cc_stub_library_shared_impl(ctx): + return [ + ctx.attr.library_target[DefaultInfo], + ctx.attr.library_target[CcSharedLibraryInfo], + ctx.attr.stub_target[CcStubInfo], + ] + +_cc_stub_library_shared = rule( + implementation = _cc_stub_library_shared_impl, + doc = "Top level rule to merge CcStubInfo and CcSharedLibraryInfo into a single target", + attrs = { + "stub_target": attr.label(mandatory = True), + "library_target": attr.label(mandatory = True), + }, +) + +def _swap_shared_linker_input(ctx, shared_info, new_output): + old_library_to_link = shared_info.linker_input.libraries[0] + + cc_toolchain = find_cpp_toolchain(ctx) + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + ) + + new_library_to_link = cc_common.create_library_to_link( + actions = ctx.actions, + dynamic_library = new_output, + feature_configuration = feature_configuration, + cc_toolchain = cc_toolchain, + ) + + new_linker_input = cc_common.create_linker_input( + owner = shared_info.linker_input.owner, + libraries = depset([new_library_to_link]), + ) + + return CcSharedLibraryInfo( + dynamic_deps = shared_info.dynamic_deps, + exports = shared_info.exports, + link_once_static_libs = shared_info.link_once_static_libs, + linker_input = new_linker_input, + preloaded_deps = shared_info.preloaded_deps, + ) + +CcStubLibrariesInfo = provider( + fields = { + "infos": "A list of dict, where each dict contains the CcStubInfo, CcSharedLibraryInfo and DefaultInfo of a version of a stub library.", + }, +) + +def _cc_library_shared_proxy_impl(ctx): + root_files = ctx.attr.root[DefaultInfo].files.to_list() + shared_files = ctx.attr.shared[DefaultInfo].files.to_list() + + if len(shared_files) != 1: + fail("Expected only one shared library file") + + shared_lib = shared_files[0] + + ctx.actions.symlink( + output = ctx.outputs.output_file, + target_file = shared_lib, + ) + + files = root_files + [ctx.outputs.output_file, ctx.files.table_of_contents[0]] + + stub_library_infos = [] + for stub_library in ctx.attr.stub_shared_libraries: + providers = { + "CcStubInfo": stub_library[CcStubInfo], + "CcSharedLibraryInfo": stub_library[CcSharedLibraryInfo], + "DefaultInfo": stub_library[DefaultInfo], + } + stub_library_infos.append(providers) + + return [ + DefaultInfo( + files = depset(direct = files), + runfiles = ctx.runfiles(files = [ctx.outputs.output_file]), + ), + _swap_shared_linker_input(ctx, ctx.attr.shared[CcSharedLibraryInfo], ctx.outputs.output_file), + ctx.attr.table_of_contents[CcTocInfo], + # Propagate only includes from the root. Do not re-propagate linker inputs. + CcInfo(compilation_context = ctx.attr.root[CcInfo].compilation_context), + CcStubLibrariesInfo(infos = stub_library_infos), + ] + +_cc_library_shared_proxy = rule( + implementation = _cc_library_shared_proxy_impl, + attrs = { + "shared": attr.label(mandatory = True, providers = [CcSharedLibraryInfo]), + "root": attr.label(mandatory = True, providers = [CcInfo]), + "output_file": attr.output(mandatory = True), + "table_of_contents": attr.label( + mandatory = True, + # TODO(b/217908237): reenable allow_single_file + # allow_single_file = True, + providers = [CcTocInfo], + ), + "stub_shared_libraries": attr.label_list(providers = [CcStubInfo, CcSharedLibraryInfo]), + }, + fragments = ["cpp"], + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], +) + +def _bssl_hash_injection_impl(ctx): + if len(ctx.files.src) != 1: + fail("Expected only one shared library file") + + hashed_file = ctx.files.src[0] + if ctx.attr.inject_bssl_hash: + hashed_file = ctx.actions.declare_file("lib" + ctx.attr.name + ".so") + args = ctx.actions.args() + args.add_all(["-sha256"]) + args.add_all(["-in-object", ctx.files.src[0]]) + args.add_all(["-o", hashed_file]) + + ctx.actions.run( + inputs = ctx.files.src, + outputs = [hashed_file], + executable = ctx.executable._bssl_inject_hash, + arguments = [args], + tools = [ctx.executable._bssl_inject_hash], + mnemonic = "BsslInjectHash", + ) + + return [ + DefaultInfo(files = depset([hashed_file])), + ctx.attr.src[CcSharedLibraryInfo], + ] + +_bssl_hash_injection = rule( + implementation = _bssl_hash_injection_impl, + attrs = { + "src": attr.label( + mandatory = True, + # TODO(b/217908237): reenable allow_single_file + # allow_single_file = True, + providers = [CcSharedLibraryInfo], + ), + "inject_bssl_hash": attr.bool( + default = False, + doc = "Whether inject BSSL hash", + ), + "_bssl_inject_hash": attr.label( + cfg = "exec", + doc = "The BSSL hash injection tool.", + executable = True, + default = "//prebuilts/build-tools:linux-x86/bin/bssl_inject_hash", + allow_single_file = True, + ), + }, +) diff --git a/rules/cc/cc_library_static.bzl b/rules/cc/cc_library_static.bzl new file mode 100644 index 00000000..f994490c --- /dev/null +++ b/rules/cc/cc_library_static.bzl @@ -0,0 +1,314 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load( + ":cc_library_common.bzl", + "create_ccinfo_for_includes", + "is_external_directory", + "parse_sdk_version", + "system_dynamic_deps_defaults", +) +load(":stl.bzl", "static_stl_deps") +load("@bazel_skylib//lib:collections.bzl", "collections") +load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cpp_toolchain") +load("@rules_cc//examples:experimental_cc_shared_library.bzl", "CcSharedLibraryInfo") +load("//build/bazel/product_variables:constants.bzl", "constants") + +CcStaticLibraryInfo = provider(fields = ["root_static_archive", "objects"]) + +def cc_library_static( + name, + deps = [], + implementation_deps = [], + dynamic_deps = [], + implementation_dynamic_deps = [], + whole_archive_deps = [], + implementation_whole_archive_deps = [], + system_dynamic_deps = None, + export_absolute_includes = [], + export_includes = [], + export_system_includes = [], + local_includes = [], + absolute_includes = [], + hdrs = [], + native_bridge_supported = False, # TODO: not supported yet. + use_libcrt = True, + rtti = False, + stl = "", + cpp_std = "", + c_std = "", + # Flags for C and C++ + copts = [], + # C++ attributes + srcs = [], + cppflags = [], + # C attributes + srcs_c = [], + conlyflags = [], + # asm attributes + srcs_as = [], + asflags = [], + features = [], + alwayslink = None, + target_compatible_with = [], + # TODO(b/202299295): Handle data attribute. + data = [], + sdk_version = "", + min_sdk_version = "", + use_version_lib = False): + "Bazel macro to correspond with the cc_library_static Soong module." + + exports_name = "%s_exports" % name + locals_name = "%s_locals" % name + cpp_name = "%s_cpp" % name + c_name = "%s_c" % name + asm_name = "%s_asm" % name + + toolchain_features = [] + toolchain_features += features + + if is_external_directory(native.package_name()): + toolchain_features += [ + "-non_external_compiler_flags", + "external_compiler_flags", + ] + + if use_version_lib: + libbuildversionLabel = "//build/soong/cc/libbuildversion:libbuildversion" + whole_archive_deps = whole_archive_deps + [libbuildversionLabel] + + if rtti: + toolchain_features += ["rtti"] + if not use_libcrt: + toolchain_features += ["use_libcrt"] + if cpp_std: + toolchain_features += [cpp_std, "-cpp_std_default"] + if c_std: + toolchain_features += [c_std, "-c_std_default"] + + if min_sdk_version: + toolchain_features += [ + "sdk_version_" + parse_sdk_version(min_sdk_version), + "-sdk_version_default", + ] + + if system_dynamic_deps == None: + system_dynamic_deps = system_dynamic_deps_defaults + + _cc_includes( + name = exports_name, + includes = export_includes, + absolute_includes = export_absolute_includes, + system_includes = export_system_includes, + # whole archive deps always re-export their includes, etc + deps = deps + whole_archive_deps + dynamic_deps, + target_compatible_with = target_compatible_with, + ) + + _cc_includes( + name = locals_name, + includes = local_includes, + absolute_includes = absolute_includes, + deps = implementation_deps + implementation_dynamic_deps + system_dynamic_deps + static_stl_deps(stl) + implementation_whole_archive_deps, + target_compatible_with = target_compatible_with, + ) + + # Silently drop these attributes for now: + # - native_bridge_supported + common_attrs = dict( + [ + # TODO(b/199917423): This may be superfluous. Investigate and possibly remove. + ("linkstatic", True), + ("hdrs", hdrs), + # Add dynamic_deps to implementation_deps, as the include paths from the + # dynamic_deps are also needed. + ("implementation_deps", [locals_name]), + ("deps", [exports_name]), + ("features", toolchain_features), + ("toolchains", ["//build/bazel/platforms:android_target_product_vars"]), + ("alwayslink", alwayslink), + ("target_compatible_with", target_compatible_with), + ], + ) + + native.cc_library( + name = cpp_name, + srcs = srcs, + copts = copts + cppflags, + **common_attrs + ) + native.cc_library( + name = c_name, + srcs = srcs_c, + copts = copts + conlyflags, + **common_attrs + ) + native.cc_library( + name = asm_name, + srcs = srcs_as, + copts = asflags, + **common_attrs + ) + + # Root target to handle combining of the providers of the language-specific targets. + _cc_library_combiner( + name = name, + deps = [cpp_name, c_name, asm_name] + whole_archive_deps + implementation_whole_archive_deps, + target_compatible_with = target_compatible_with, + ) + +# Returns a CcInfo object which combines one or more CcInfo objects, except that all +# linker inputs owned by owners in `old_owner_labels` are relinked and owned by the current target. +# +# This is useful in the "macro with proxy rule" pattern, as some rules upstream +# may expect they are depending directly on a target which generates linker inputs, +# as opposed to a proxy target which is a level of indirection to such a target. +def _cc_library_combiner_impl(ctx): + old_owner_labels = [] + cc_infos = [] + for dep in ctx.attr.deps: + old_owner_labels.append(dep.label) + cc_infos.append(dep[CcInfo]) + combined_info = cc_common.merge_cc_infos(cc_infos = cc_infos) + + objects_to_link = [] + + # This is not ideal, as it flattens a depset. + for old_linker_input in combined_info.linking_context.linker_inputs.to_list(): + if old_linker_input.owner in old_owner_labels: + for lib in old_linker_input.libraries: + # These objects will be recombined into the root archive. + objects_to_link.extend(lib.objects) + else: + # Android macros don't handle transitive linker dependencies because + # it's unsupported in legacy. We may want to change this going forward, + # but for now it's good to validate that this invariant remains. + fail("cc_static_library %s given transitive linker dependency from %s" % (ctx.label, old_linker_input.owner)) + + cc_toolchain = find_cpp_toolchain(ctx) + CPP_LINK_STATIC_LIBRARY_ACTION_NAME = "c++-link-static-library" + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + requested_features = ctx.features, + unsupported_features = ctx.disabled_features + ["linker_flags"], + ) + + output_file = ctx.actions.declare_file("lib" + ctx.label.name + ".a") + linker_input = cc_common.create_linker_input( + owner = ctx.label, + libraries = depset(direct = [ + cc_common.create_library_to_link( + actions = ctx.actions, + feature_configuration = feature_configuration, + cc_toolchain = cc_toolchain, + static_library = output_file, + objects = objects_to_link, + ), + ]), + ) + + linking_context = cc_common.create_linking_context(linker_inputs = depset(direct = [linker_input])) + + archiver_path = cc_common.get_tool_for_action( + feature_configuration = feature_configuration, + action_name = CPP_LINK_STATIC_LIBRARY_ACTION_NAME, + ) + archiver_variables = cc_common.create_link_variables( + feature_configuration = feature_configuration, + cc_toolchain = cc_toolchain, + output_file = output_file.path, + is_using_linker = False, + ) + command_line = cc_common.get_memory_inefficient_command_line( + feature_configuration = feature_configuration, + action_name = CPP_LINK_STATIC_LIBRARY_ACTION_NAME, + variables = archiver_variables, + ) + args = ctx.actions.args() + args.add_all(command_line) + args.add_all(objects_to_link) + + ctx.actions.run( + executable = archiver_path, + arguments = [args], + inputs = depset( + direct = objects_to_link, + transitive = [ + cc_toolchain.all_files, + ], + ), + outputs = [output_file], + ) + return [ + DefaultInfo(files = depset(direct = [output_file]), data_runfiles = ctx.runfiles(files = [output_file])), + CcInfo(compilation_context = combined_info.compilation_context, linking_context = linking_context), + CcStaticLibraryInfo(root_static_archive = output_file, objects = objects_to_link), + ] + +# A rule which combines objects of oen or more cc_library targets into a single +# static linker input. This outputs a single archive file combining the objects +# of its direct deps, and propagates Cc providers describing that these objects +# should be linked for linking rules upstream. +# This rule is useful for maintaining the illusion that the target's deps are +# comprised by a single consistent rule: +# - A single archive file is always output by this rule. +# - A single linker input struct is always output by this rule, and it is 'owned' +# by this rule. +_cc_library_combiner = rule( + implementation = _cc_library_combiner_impl, + attrs = { + "deps": attr.label_list(providers = [CcInfo]), + "_cc_toolchain": attr.label( + default = Label("@local_config_cc//:toolchain"), + providers = [cc_common.CcToolchainInfo], + ), + }, + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], + provides = [CcInfo], + fragments = ["cpp"], +) + +def _cc_includes_impl(ctx): + return [create_ccinfo_for_includes( + ctx, + includes = ctx.attr.includes, + absolute_includes = ctx.attr.absolute_includes, + system_includes = ctx.attr.system_includes, + deps = ctx.attr.deps, + )] + +# Bazel's native cc_library rule supports specifying include paths two ways: +# 1. non-exported includes can be specified via copts attribute +# 2. exported -isystem includes can be specified via includes attribute +# +# In order to guarantee a correct inclusion search order, we need to export +# includes paths for both -I and -isystem; however, there is no native Bazel +# support to export both of these, this rule provides a CcInfo to propagate the +# given package-relative include/system include paths as exec root relative +# include/system include paths. +_cc_includes = rule( + implementation = _cc_includes_impl, + attrs = { + "absolute_includes": attr.string_list(doc = "List of exec-root relative or absolute search paths for headers, usually passed with -I"), + "includes": attr.string_list(doc = "Package-relative list of search paths for headers, usually passed with -I"), + "system_includes": attr.string_list(doc = "Package-relative list of search paths for headers, usually passed with -isystem"), + "deps": attr.label_list(doc = "Re-propagates the includes obtained from these dependencies.", providers = [CcInfo]), + }, + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], + fragments = ["cpp"], + provides = [CcInfo], +) diff --git a/rules/cc/cc_object.bzl b/rules/cc/cc_object.bzl new file mode 100644 index 00000000..a9b32551 --- /dev/null +++ b/rules/cc/cc_object.bzl @@ -0,0 +1,220 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cpp_toolchain") +load( + ":cc_library_common.bzl", + "get_includes_paths", + "is_external_directory", + "system_dynamic_deps_defaults", + "parse_sdk_version") +load(":cc_constants.bzl", "constants") +load(":stl.bzl", "static_stl_deps") + +# "cc_object" module copts, taken from build/soong/cc/object.go +_CC_OBJECT_COPTS = ["-fno-addrsig"] + +# partialLd module link opts, taken from build/soong/cc/builder.go +# https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/builder.go;l=87;drc=f2be52c4dcc2e3d743318e106633e61de0ad2afd +_CC_OBJECT_LINKOPTS = [ + "-fuse-ld=lld", + "-nostdlib", + "-no-pie", + "-Wl,-r", +] + +CcObjectInfo = provider(fields = [ + # The merged compilation outputs for this cc_object and its transitive + # dependencies. + "objects", +]) + +def split_srcs_hdrs(files): + headers = [] + non_headers_as = [] + non_headers_c = [] + for f in files: + if f.extension in constants.hdr_exts: + headers += [f] + elif f.extension in constants.as_src_exts: + non_headers_as += [f] + else: + non_headers_c += [f] + return non_headers_c, non_headers_as, headers + +def _cc_object_impl(ctx): + cc_toolchain = ctx.toolchains["//prebuilts/clang/host/linux-x86:nocrt_toolchain"].cc + + extra_features = [] + + extra_disabled_features = [ + "disable_pack_relocations", + "dynamic_executable", + "dynamic_linker", + "linker_flags", + "no_undefined_symbols", + "pack_dynamic_relocations", + "strip_debug_symbols", + # TODO(cparsons): Look into disabling this feature for nocrt toolchain? + "use_libcrt", + ] + if is_external_directory(ctx.label.package): + extra_disabled_features.append("non_external_compiler_flags") + extra_features.append("external_compiler_flags") + + if ctx.attr.min_sdk_version: + extra_disabled_features.append("sdk_version_default") + extra_features.append("sdk_version_" + parse_sdk_version(ctx.attr.min_sdk_version)) + + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + requested_features = ctx.features + extra_features, + unsupported_features = ctx.disabled_features + extra_disabled_features, + ) + + compilation_contexts = [] + deps_objects = [] + for obj in ctx.attr.deps: + compilation_contexts.append(obj[CcInfo].compilation_context) + deps_objects.append(obj[CcObjectInfo].objects) + for includes_dep in ctx.attr.includes_deps: + compilation_contexts.append(includes_dep[CcInfo].compilation_context) + + product_variables = ctx.attr._android_product_variables[platform_common.TemplateVariableInfo] + asflags = [ctx.expand_make_variables("asflags", flag, product_variables.variables) for flag in ctx.attr.asflags] + + srcs_c, srcs_as, private_hdrs = split_srcs_hdrs(ctx.files.srcs) + + (compilation_context, compilation_outputs_c) = cc_common.compile( + name = ctx.label.name, + actions = ctx.actions, + feature_configuration = feature_configuration, + cc_toolchain = cc_toolchain, + srcs = srcs_c, + includes = get_includes_paths(ctx, ctx.attr.local_includes) + get_includes_paths(ctx, ctx.attr.absolute_includes, package_relative = False), + public_hdrs = ctx.files.hdrs, + private_hdrs = private_hdrs, + user_compile_flags = ctx.attr.copts, + compilation_contexts = compilation_contexts, + ) + + (compilation_context, compilation_outputs_as) = cc_common.compile( + name = ctx.label.name, + actions = ctx.actions, + feature_configuration = feature_configuration, + cc_toolchain = cc_toolchain, + srcs = srcs_as, + includes = get_includes_paths(ctx, ctx.attr.local_includes) + get_includes_paths(ctx, ctx.attr.absolute_includes, package_relative = False), + public_hdrs = ctx.files.hdrs, + private_hdrs = private_hdrs, + user_compile_flags = ctx.attr.copts + asflags, + compilation_contexts = compilation_contexts, + ) + + # do not propagate includes + compilation_context = cc_common.create_compilation_context( + headers = compilation_context.headers, + defines = compilation_context.defines, + local_defines = compilation_context.local_defines, + ) + + objects_to_link = cc_common.merge_compilation_outputs(compilation_outputs = deps_objects + [compilation_outputs_c, compilation_outputs_as]) + + user_link_flags = [] + user_link_flags.extend(_CC_OBJECT_LINKOPTS) + additional_inputs = [] + + if ctx.attr.linker_script != None: + linker_script = ctx.files.linker_script[0] + user_link_flags.append("-Wl,-T," + linker_script.path) + additional_inputs.append(linker_script) + + # partially link if there are multiple object files + if len(objects_to_link.objects) + len(objects_to_link.pic_objects) > 1: + linking_output = cc_common.link( + name = ctx.label.name + ".o", + actions = ctx.actions, + feature_configuration = feature_configuration, + cc_toolchain = cc_toolchain, + user_link_flags = user_link_flags, + compilation_outputs = objects_to_link, + additional_inputs = additional_inputs, + ) + files = depset([linking_output.executable]) + else: + files = depset(objects_to_link.objects + objects_to_link.pic_objects) + + return [ + DefaultInfo(files = files), + CcInfo(compilation_context = compilation_context), + CcObjectInfo(objects = objects_to_link), + ] + +_cc_object = rule( + implementation = _cc_object_impl, + attrs = { + "srcs": attr.label_list(allow_files = constants.all_dot_exts), + "hdrs": attr.label_list(allow_files = constants.hdr_dot_exts), + "absolute_includes": attr.string_list(), + "local_includes": attr.string_list(), + "copts": attr.string_list(), + "asflags": attr.string_list(), + "deps": attr.label_list(providers = [CcInfo, CcObjectInfo]), + "includes_deps": attr.label_list(providers = [CcInfo]), + "linker_script": attr.label(allow_single_file = True), + "sdk_version": attr.string(), + "min_sdk_version": attr.string(), + "_android_product_variables": attr.label( + default = Label("//build/bazel/platforms:android_target_product_vars"), + providers = [platform_common.TemplateVariableInfo], + ), + }, + toolchains = ["//prebuilts/clang/host/linux-x86:nocrt_toolchain"], + fragments = ["cpp"], +) + +def cc_object( + name, + copts = [], + hdrs = [], + asflags = [], + srcs = [], + srcs_as = [], + deps = [], + native_bridge_supported = False, # TODO: not supported yet. + stl = "", + system_dynamic_deps = None, + sdk_version = "", + min_sdk_version = "", + **kwargs): + "Build macro to correspond with the cc_object Soong module." + + if system_dynamic_deps == None: + system_dynamic_deps = system_dynamic_deps_defaults + + _cc_object( + name = name, + hdrs = hdrs, + asflags = asflags, + copts = _CC_OBJECT_COPTS + copts, + srcs = srcs + srcs_as, + deps = deps, + includes_deps = static_stl_deps(stl) + system_dynamic_deps, + sdk_version = sdk_version, + min_sdk_version = min_sdk_version, + **kwargs + ) diff --git a/rules/cc/cc_proto.bzl b/rules/cc/cc_proto.bzl new file mode 100644 index 00000000..b72379d9 --- /dev/null +++ b/rules/cc/cc_proto.bzl @@ -0,0 +1,221 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load("//build/bazel/rules:proto_file_utils.bzl", "proto_file_utils") +load(":cc_library_common.bzl", "create_ccinfo_for_includes") +load(":cc_library_static.bzl", "cc_library_static") +load("@bazel_skylib//lib:paths.bzl", "paths") + +CcProtoGenInfo = provider(fields = ["headers", "sources"]) + +_SOURCES_KEY = "sources" +_HEADERS_KEY = "headers" + +def _cc_proto_sources_gen_rule_impl(ctx): + out_flags = [] + plugin_executable = None + out_arg = None + if ctx.attr.plugin: + plugin_executable = ctx.executable.plugin + else: + out_arg = "--cpp_out" + if ctx.attr.out_format: + out_flags.append(ctx.attr.out_format) + + + srcs = [] + hdrs = [] + includes = [] + for dep in ctx.attr.deps: + proto_info = dep[ProtoInfo] + if proto_info.proto_source_root == ".": + includes.append(paths.join(ctx.label.name, ctx.label.package)) + includes.append(ctx.label.name) + outs = _generate_cc_proto_action( + proto_info = proto_info, + protoc = ctx.executable._protoc, + ctx = ctx, + is_cc = True, + out_flags = out_flags, + plugin_executable = plugin_executable, + out_arg = out_arg, + ) + srcs.extend(outs[_SOURCES_KEY]) + hdrs.extend(outs[_HEADERS_KEY]) + + return [ + DefaultInfo(files = depset(direct = srcs + hdrs)), + create_ccinfo_for_includes(ctx, includes = includes), + CcProtoGenInfo( + headers = hdrs, + sources = srcs, + ), + ] + +_cc_proto_sources_gen = rule( + implementation = _cc_proto_sources_gen_rule_impl, + attrs = { + "deps": attr.label_list( + providers = [ProtoInfo], + doc = """ +proto_library or any other target exposing ProtoInfo provider with *.proto files +""", + mandatory = True, + ), + "_protoc": attr.label( + default = Label("//external/protobuf:aprotoc"), + executable = True, + cfg = "exec", + ), + "plugin": attr.label( + executable = True, + cfg = "exec", + ), + "out_format": attr.string( + doc = """ +Optional argument specifying the out format, e.g. lite. +If not provided, defaults to full protos. +""", + ), + }, + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], + provides = [CcInfo, CcProtoGenInfo], +) + +def _src_extension(is_cc): + if is_cc: + return "cc" + return "c" + +def _generate_cc_proto_action( + proto_info, + protoc, + ctx, + plugin_executable, + out_arg, + out_flags, + is_cc): + type_dictionary = { + _SOURCES_KEY: ".pb." + _src_extension(is_cc), + _HEADERS_KEY: ".pb.h", + } + return proto_file_utils.generate_proto_action( + proto_info, + protoc, + ctx, + type_dictionary, + out_flags, + plugin_executable = plugin_executable, + out_arg = out_arg, + mnemonic = "CcProtoGen", + ) + +def _cc_proto_sources_impl(ctx): + srcs = ctx.attr.src[CcProtoGenInfo].sources + return [ + DefaultInfo(files = depset(direct = srcs)), + ] + +_cc_proto_sources = rule( + implementation = _cc_proto_sources_impl, + attrs = { + "src": attr.label( + providers = [CcProtoGenInfo], + ), + }, +) + +def _cc_proto_headers_impl(ctx): + hdrs = ctx.attr.src[CcProtoGenInfo].headers + return [ + DefaultInfo(files = depset(direct = hdrs)), + ] + +_cc_proto_headers = rule( + implementation = _cc_proto_headers_impl, + attrs = { + "src": attr.label( + providers = [CcProtoGenInfo], + ), + }, +) + +def _cc_proto_library( + name, + deps = [], + plugin = None, + target_compatible_with = [], + out_format = None, + proto_dep = None): + proto_lib_name = name + "_proto_gen" + srcs_name = name + "_proto_sources" + hdrs_name = name + "_proto_headers" + + _cc_proto_sources_gen( + name = proto_lib_name, + deps = deps, + plugin = plugin, + out_format = out_format, + ) + + _cc_proto_sources( + name = srcs_name, + src = proto_lib_name, + ) + + _cc_proto_headers( + name = hdrs_name, + src = proto_lib_name, + ) + + cc_library_static( + name = name, + srcs = [":" + srcs_name], + hdrs = [":" + hdrs_name], + deps = [ + proto_lib_name, + proto_dep, + ], + local_includes = ["."], + target_compatible_with = target_compatible_with, + ) + +def cc_lite_proto_library( + name, + deps = [], + plugin = None, + target_compatible_with = []): + _cc_proto_library( + name, + deps = deps, + plugin = plugin, + target_compatible_with = target_compatible_with, + out_format = "lite", + proto_dep = "//external/protobuf:libprotobuf-cpp-lite", + ) + +def cc_proto_library( + name, + deps = [], + plugin = None, + target_compatible_with = []): + _cc_proto_library( + name, + deps = deps, + plugin = plugin, + target_compatible_with = target_compatible_with, + proto_dep = "//external/protobuf:libprotobuf-cpp-full", + ) diff --git a/rules/cc/cc_stub_library.bzl b/rules/cc/cc_stub_library.bzl new file mode 100644 index 00000000..be18de13 --- /dev/null +++ b/rules/cc/cc_stub_library.bzl @@ -0,0 +1,86 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_skylib//lib:dicts.bzl", "dicts") +load("//build/bazel/platforms:rule_utilities.bzl", "ARCH_CONSTRAINT_ATTRS", "get_arch") + +# This file contains the implementation for the cc_stub_library rule. +# +# TODO(b/207812332): +# - ndk_api_coverage_parser: https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/coverage.go;l=248-262;drc=master + +CcStubInfo = provider( + fields = { + "stub_map": "The .map file containing library symbols for the specific API version.", + "version": "The API version of this library.", + "abi_symbol_list": "A plain-text list of all symbols of this library for the specific API version." + } +) + +def _cc_stub_gen_impl(ctx): + # The name of this target. + name = ctx.attr.name + + # All declared outputs of ndkstubgen. + out_stub_c = ctx.actions.declare_file("/".join([name, "stub.c"])) + out_stub_map = ctx.actions.declare_file("/".join([name, "stub.map"])) + out_abi_symbol_list = ctx.actions.declare_file("/".join([name, "abi_symbol_list.txt"])) + + outputs = [out_stub_c, out_stub_map, out_abi_symbol_list] + + arch = get_arch(ctx) + + ndkstubgen_args = ctx.actions.args() + ndkstubgen_args.add_all(["--arch", arch]) + ndkstubgen_args.add_all(["--api", ctx.attr.version]) + ndkstubgen_args.add_all(["--api-map", ctx.file._api_levels_file]) + # TODO(b/207812332): This always parses and builds the stub library as a dependency of an APEX. Parameterize this + # for non-APEX use cases. + ndkstubgen_args.add_all(["--apex", ctx.file.symbol_file]) + ndkstubgen_args.add_all(outputs) + ctx.actions.run( + executable = ctx.executable._ndkstubgen, + inputs = [ + ctx.file.symbol_file, + ctx.file._api_levels_file, + ], + outputs = outputs, + arguments = [ndkstubgen_args], + ) + + return [ + # DefaultInfo.files contains the .stub.c file only so that this target + # can be used directly in the srcs of a cc_library. + DefaultInfo(files = depset([out_stub_c])), + CcStubInfo( + stub_map = out_stub_map, + abi_symbol_list = out_abi_symbol_list, + version = ctx.attr.version, + ), + ] + +cc_stub_gen = rule( + implementation = _cc_stub_gen_impl, + attrs = dicts.add({ + # Public attributes + "symbol_file": attr.label(mandatory = True, allow_single_file = [".map.txt"]), + "version": attr.string(mandatory = True, default = "current"), + # Private attributes + "_api_levels_file": attr.label(default = "@soong_injection//api_levels:api_levels.json", allow_single_file = True), + # TODO(b/199038020): Use //build/soong/cc/ndkstubgen when py_runtime is set up on CI for hermetic python usage. + # "_ndkstubgen": attr.label(default = "@make_injection//:host/linux-x86/bin/ndkstubgen", executable = True, cfg = "host", allow_single_file = True), + "_ndkstubgen": attr.label(default = "//build/soong/cc/ndkstubgen", executable = True, cfg = "host"), + }, ARCH_CONSTRAINT_ATTRS), +) + diff --git a/rules/cc/generate_toc.bzl b/rules/cc/generate_toc.bzl new file mode 100644 index 00000000..dc3d6ad3 --- /dev/null +++ b/rules/cc/generate_toc.bzl @@ -0,0 +1,80 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +"""A macro to generate table of contents files of symbols from a shared library.""" + +CcTocInfo = provider( + "Information about the table of contents of a shared library", + fields = { + "toc": "The single file for the table of contents", + }, +) + +def _shared_library_toc_impl(ctx): + so_name = "lib" + ctx.attr.name + ".so" + toc_name = so_name + ".toc" + out_file = ctx.actions.declare_file(toc_name) + d_file = ctx.actions.declare_file(toc_name + ".d") + ctx.actions.run( + env = { + "CLANG_BIN": ctx.executable._readelf.dirname, + }, + inputs = ctx.files.src, + tools = [ + ctx.executable._readelf, + ], + outputs = [out_file, d_file], + executable = ctx.executable._toc_script, + arguments = [ + # Only Linux shared libraries for now. + "--elf", + "-i", + ctx.files.src[0].path, + "-o", + out_file.path, + "-d", + d_file.path, + ], + ) + + return [ + CcTocInfo(toc = out_file), + DefaultInfo(files = depset([out_file])), + ] + +shared_library_toc = rule( + implementation = _shared_library_toc_impl, + attrs = { + "src": attr.label( + # TODO(b/217908237): reenable allow_single_file + # allow_single_file = True, + mandatory = True, + ), + "_toc_script": attr.label( + cfg = "host", + executable = True, + allow_single_file = True, + default = "//build/soong/scripts:toc.sh", + ), + "_readelf": attr.label( + cfg = "host", + executable = True, + allow_single_file = True, + default = "//prebuilts/clang/host/linux-x86:llvm-readelf", + ), + }, + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], +) diff --git a/rules/cc/prebuilt_library_shared.bzl b/rules/cc/prebuilt_library_shared.bzl new file mode 100644 index 00000000..c25ceff4 --- /dev/null +++ b/rules/cc/prebuilt_library_shared.bzl @@ -0,0 +1,36 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +def prebuilt_library_shared( + name, + shared_library, + alwayslink = None, + **kwargs): + "Bazel macro to correspond with the *_prebuilt_library_shared Soong module types" + + native.cc_import( + name = name, + shared_library = shared_library, + alwayslink = alwayslink, + **kwargs + ) + + native.cc_import( + name = name + "_alwayslink", + shared_library = shared_library, + alwayslink = True, + **kwargs + ) diff --git a/rules/cc/prebuilt_library_static.bzl b/rules/cc/prebuilt_library_static.bzl new file mode 100644 index 00000000..950ff163 --- /dev/null +++ b/rules/cc/prebuilt_library_static.bzl @@ -0,0 +1,40 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +def prebuilt_library_static( + name, + static_library, + alwayslink = None, + export_includes = [], + export_system_includes = [], + **kwargs): + "Bazel macro to correspond with the *_prebuilt_library_static Soong module types" + + # TODO: Handle includes similarly to cc_library_static + # e.g. includes = ["clang-r416183b/prebuilt_include/llvm/lib/Fuzzer"], + native.cc_import( + name = name, + static_library = static_library, + alwayslink = alwayslink, + **kwargs + ) + + native.cc_import( + name = name + "_alwayslink", + static_library = static_library, + alwayslink = True, + **kwargs + ) diff --git a/rules/static_libc.bzl b/rules/cc/static_libc.bzl index 0942ec9e..935125c8 100644 --- a/rules/static_libc.bzl +++ b/rules/cc/static_libc.bzl @@ -1,3 +1,19 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + # Rules and macros to define a cc toolchain with a static libc. # Used to bootstrap cc development using the bionic lib build by Soong. # Rule: _libc_config @@ -5,6 +21,7 @@ # Macro: static_libc # Creates the libc_config target and filegroups needed by cc_toolchain. LibcConfigInfo = provider(fields = ["include_dirs", "system_libraries"]) + def _libc_config_impl(ctx): include_dirs = ctx.attr.include_dirs system_libraries = [file.path for file in ctx.files.system_libraries] @@ -13,6 +30,7 @@ def _libc_config_impl(ctx): system_libraries = system_libraries, ) return [provider] + _libc_config = rule( implementation = _libc_config_impl, attrs = { @@ -20,6 +38,7 @@ _libc_config = rule( "system_libraries": attr.label_list(default = [], allow_files = True), }, ) + def static_libc( name, include_dirs = {}, @@ -42,6 +61,7 @@ def static_libc( name = "%s_system_libraries" % name, srcs = system_libraries, ) + # Create the libc config. include_paths = [path for path in include_dirs.keys()] _libc_config( @@ -49,6 +69,7 @@ def static_libc( include_dirs = include_paths, system_libraries = system_libraries, ) + # Also create cc_library target for direct dependencies. native.cc_library( name = "%s_library" % name, diff --git a/rules/cc/stl.bzl b/rules/cc/stl.bzl new file mode 100644 index 00000000..0c9e03db --- /dev/null +++ b/rules/cc/stl.bzl @@ -0,0 +1,71 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Helpers for stl property resolution. +# These mappings taken from build/soong/cc/stl.go + +load("//build/bazel/product_variables:constants.bzl", "constants") + +_libcpp_stl_names = { + "libc++": True, + "libc++_static": True, + "c++_shared": True, + "c++_static": True, + "": True, + "system": True, +} + +# https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/stl.go;l=157;drc=55d98d2ba142d6c35894b1092397e2b5a70bc2e8 +_common_static_deps = select({ + constants.ArchVariantToConstraints["android"]: ["//external/libcxxabi:libc++demangle"], + "//conditions:default": [], +}) + +# https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/stl.go;l=162;drc=cb0ac95bde896fa2aa59193a37ceb580758c322c +# this should vary based on vndk version +# skip libm and libc because then we would have duplicates due to system_shared_library +_libunwind = "//prebuilts/clang/host/linux-x86:libunwind" + +_static_binary_deps = select({ + constants.ArchVariantToConstraints["android"]: [_libunwind], + constants.ArchVariantToConstraints["linux_bionic"]: [_libunwind], + "//conditions:default": [], +}) + +def static_stl_deps(stl_name): + # TODO(b/201079053): Handle useSdk, windows, fuschia, preferably with selects. + if stl_name in _libcpp_stl_names: + return ["//external/libcxx:libc++_static"] + _common_static_deps + elif stl_name == "none": + return [] + else: + fail("Unhandled stl %s" % stl_name) + +def static_binary_stl_deps(stl_name): + base = static_stl_deps(stl_name) + if stl_name == "none": + return base + else: + return base + _static_binary_deps + +def shared_stl_deps(stl_name): + # TODO(b/201079053): Handle useSdk, windows, fuschia, preferably with selects. + if stl_name in _libcpp_stl_names: + return (_common_static_deps, ["//external/libcxx:libc++"]) + elif stl_name == "none": + return ([], []) + else: + fail("Unhandled stl %s" % stl_name) diff --git a/rules/cc/stripped_cc_common.bzl b/rules/cc/stripped_cc_common.bzl new file mode 100644 index 00000000..ad4f4b1f --- /dev/null +++ b/rules/cc/stripped_cc_common.bzl @@ -0,0 +1,193 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +"""A macro to handle shared library stripping.""" + +load("@rules_cc//examples:experimental_cc_shared_library.bzl", "CcSharedLibraryInfo", "cc_shared_library") +load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cpp_toolchain") + +# Keep this consistent with soong/cc/strip.go#NeedsStrip. +def needs_strip(attrs): + force_disable = attrs.none + force_enable = attrs.all or attrs.keep_symbols or attrs.keep_symbols_and_debug_frame + return force_enable and not force_disable + +# Keep this consistent with soong/cc/strip.go#strip and soong/cc/builder.go#transformStrip. +def get_strip_args(attrs): + strip_args = [] + keep_symbols_list = attrs.keep_symbols_list + keep_mini_debug_info = False + if attrs.keep_symbols: + strip_args += ["--keep-symbols"] + elif attrs.keep_symbols_and_debug_frame: + strip_args += ["--keep-symbols-and-debug-frame"] + elif len(keep_symbols_list) > 0: + strip_args += ["-k" + ",".join(keep_symbols_list)] + elif not attrs.all: + strip_args += ["--keep-mini-debug-info"] + keep_mini_debug_info = True + + if not keep_mini_debug_info: + strip_args += ["--add-gnu-debuglink"] + + return strip_args + +# https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/builder.go;l=131-146;drc=master +def _stripped_impl(ctx, prefix = "", extension = ""): + out_file = ctx.actions.declare_file(prefix + ctx.attr.name + extension) + if not needs_strip(ctx.attr): + ctx.actions.symlink( + output = out_file, + target_file = ctx.files.src[0], + ) + return out_file + cc_toolchain = find_cpp_toolchain(ctx) + d_file = ctx.actions.declare_file(ctx.attr.name + ".d") + ctx.actions.run( + env = { + "CREATE_MINIDEBUGINFO": ctx.executable._create_minidebuginfo.path, + "XZ": ctx.executable._xz.path, + "CLANG_BIN": ctx.executable._ar.dirname, + }, + inputs = ctx.files.src, + tools = [ + ctx.executable._ar, + ctx.executable._create_minidebuginfo, + ctx.executable._objcopy, + ctx.executable._readelf, + ctx.executable._strip, + ctx.executable._strip_script, + ctx.executable._xz, + ], + outputs = [out_file, d_file], + executable = ctx.executable._strip_script, + arguments = get_strip_args(ctx.attr) + [ + "-i", + ctx.files.src[0].path, + "-o", + out_file.path, + "-d", + d_file.path, + ], + ) + return out_file + +common_attrs = { + "keep_symbols": attr.bool(default = False), + "keep_symbols_and_debug_frame": attr.bool(default = False), + "all": attr.bool(default = False), + "none": attr.bool(default = False), + "keep_symbols_list": attr.string_list(default = []), + "_xz": attr.label( + cfg = "host", + executable = True, + allow_single_file = True, + default = "//prebuilts/build-tools:linux-x86/bin/xz", + ), + "_create_minidebuginfo": attr.label( + cfg = "host", + executable = True, + allow_single_file = True, + default = "//prebuilts/build-tools:linux-x86/bin/create_minidebuginfo", + ), + "_strip_script": attr.label( + cfg = "host", + executable = True, + allow_single_file = True, + default = "//build/soong/scripts:strip.sh", + ), + "_ar": attr.label( + cfg = "host", + executable = True, + allow_single_file = True, + default = "//prebuilts/clang/host/linux-x86:llvm-ar", + ), + "_strip": attr.label( + cfg = "host", + executable = True, + allow_single_file = True, + default = "//prebuilts/clang/host/linux-x86:llvm-strip", + ), + "_readelf": attr.label( + cfg = "host", + executable = True, + allow_single_file = True, + default = "//prebuilts/clang/host/linux-x86:llvm-readelf", + ), + "_objcopy": attr.label( + cfg = "host", + executable = True, + allow_single_file = True, + default = "//prebuilts/clang/host/linux-x86:llvm-objcopy", + ), + "_cc_toolchain": attr.label( + default = Label("@local_config_cc//:toolchain"), + providers = [cc_common.CcToolchainInfo], + ), +} + +def _stripped_shared_library_impl(ctx): + out_file = _stripped_impl(ctx, "lib", ".so") + + return [ + DefaultInfo(files = depset([out_file])), + ctx.attr.src[CcSharedLibraryInfo], + ] + +stripped_shared_library = rule( + implementation = _stripped_shared_library_impl, + attrs = dict( + common_attrs, + src = attr.label( + mandatory = True, + # TODO(b/217908237): reenable allow_single_file + # allow_single_file = True, + providers = [CcSharedLibraryInfo], + ), + ), + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], +) + +# A marker provider to distinguish a cc_binary from everything else that exports +# a CcInfo. +StrippedCcBinaryInfo = provider() + +def _stripped_binary_impl(ctx): + common_providers = [ + ctx.attr.src[CcInfo], + ctx.attr.src[InstrumentedFilesInfo], + ctx.attr.src[DebugPackageInfo], + ctx.attr.src[OutputGroupInfo], + StrippedCcBinaryInfo(), # a marker for dependents + ] + + out_file = _stripped_impl(ctx) + + return [ + DefaultInfo( + files = depset([out_file]), + executable = out_file, + ), + ] + common_providers + +stripped_binary = rule( + implementation = _stripped_binary_impl, + attrs = dict( + common_attrs, + src = attr.label(mandatory = True, allow_single_file = True, providers = [CcInfo]), + ), + toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], +) diff --git a/rules/cc/versioned_cc_common.bzl b/rules/cc/versioned_cc_common.bzl new file mode 100644 index 00000000..f91d151d --- /dev/null +++ b/rules/cc/versioned_cc_common.bzl @@ -0,0 +1,116 @@ +""" +Copyright (C) 2022 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +"""A macro to handle build number stamping.""" + +load(":stripped_cc_common.bzl", "StrippedCcBinaryInfo") +load("@rules_cc//examples:experimental_cc_shared_library.bzl", "CcSharedLibraryInfo") + +def stamp_build_number(ctx, prefix = "", extension = ""): + if len(ctx.files.src) != 1: + fail("Expected only one input file for build number stamping") + + out_file = ctx.actions.declare_file(prefix + ctx.attr.name + extension) + android_constraint = ctx.attr._android_constraint[platform_common.ConstraintValueInfo] + + # TODO(b/228461735): We need to dist the output for device target. + if ctx.target_platform_has_constraint(android_constraint) or not ctx.attr.stamp_build_number: + ctx.actions.symlink( + output = out_file, + target_file = ctx.files.src[0], + ) + return out_file + + ctx.actions.run_shell( + inputs = ctx.files.src + [ctx.version_file], + outputs = [out_file], + command = """ + build_number=$(cat {file} | grep "BUILD_NUMBER" | cut -f2 -d' '); + {build_number_stamper} -i {input} -o {output} -s soong_build_number -v $build_number + """.format( + file = ctx.version_file.path, + input = ctx.files.src[0].path, + output = out_file.path, + build_number_stamper = ctx.executable._build_number_stamper.path, + ), + tools = [ctx.executable._build_number_stamper], + mnemonic = "StampBuildNumber", + ) + + return out_file + +common_attrs = { + "stamp_build_number": attr.bool( + default = False, + doc = "Whether to stamp the build number", + ), + "_build_number_stamper": attr.label( + cfg = "exec", + doc = "The build number stamp tool.", + executable = True, + default = "//prebuilts/build-tools:linux-x86/bin/symbol_inject", + allow_single_file = True, + ), + "_android_constraint": attr.label( + default = Label("//build/bazel/platforms/os:android"), + ), +} + +def _versioned_binary_impl(ctx): + common_providers = [ + ctx.attr.src[CcInfo], + ctx.attr.src[InstrumentedFilesInfo], + ctx.attr.src[DebugPackageInfo], + ctx.attr.src[OutputGroupInfo], + ] + + out_file = stamp_build_number(ctx) + + return [ + DefaultInfo( + files = depset([out_file]), + executable = out_file, + ), + ] + common_providers + +versioned_binary = rule( + implementation = _versioned_binary_impl, + attrs = dict( + common_attrs, + src = attr.label(mandatory = True, allow_single_file = True, providers = [CcInfo]), + ), +) + +def _versioned_shared_library_impl(ctx): + out_file = stamp_build_number(ctx, "lib", ".so") + + return [ + DefaultInfo(files = depset([out_file])), + ctx.attr.src[CcSharedLibraryInfo], + ] + +versioned_shared_library = rule( + implementation = _versioned_shared_library_impl, + attrs = dict( + common_attrs, + src = attr.label( + mandatory = True, + # TODO(b/217908237): reenable allow_single_file + # allow_single_file = True, + providers = [CcSharedLibraryInfo], + ), + ), +) diff --git a/rules/cc_library_headers.bzl b/rules/cc_library_headers.bzl deleted file mode 100644 index db6c7cfa..00000000 --- a/rules/cc_library_headers.bzl +++ /dev/null @@ -1,20 +0,0 @@ -load("//build/bazel/rules:cc_library_static.bzl", "cc_library_static") - -def cc_library_headers( - name, - deps = [], - hdrs = [], - includes = [], - native_bridge_supported = False, # TODO: not supported yet. - **kwargs): - "Bazel macro to correspond with the cc_library_headers Soong module." - - cc_library_static( - name = name, - deps = deps, - hdrs = hdrs, - includes = includes, - native_bridge_supported = native_bridge_supported, - **kwargs - ) - diff --git a/rules/cc_library_static.bzl b/rules/cc_library_static.bzl deleted file mode 100644 index d9b3c743..00000000 --- a/rules/cc_library_static.bzl +++ /dev/null @@ -1,149 +0,0 @@ -load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cpp_toolchain") - -def cc_library_static( - name, - srcs = [], - deps = [], - hdrs = [], - copts = [], - includes = [], - native_bridge_supported = False, # TODO: not supported yet. - whole_archive_deps = [], - **kwargs): - "Bazel macro to correspond with the cc_library_static Soong module." - mainlib_name = "%s_mainlib" % name - - # Silently drop these attributes for now: - # - native_bridge_supported - native.cc_library( - name = mainlib_name, - srcs = srcs, - hdrs = hdrs, - # TODO(b/187533117): Handle whole_archive_deps differently from regular static deps. - deps = deps + whole_archive_deps, - copts = copts, - includes = includes, - **kwargs - ) - - # Safeguard target to handle the no-srcs no-deps case. - # With no-srcs no-deps, this returns a stub. Otherwise, it's a passthrough no-op. - _empty_library_safeguard( - name = name, - deps = [mainlib_name], - ) - -# Returns a cloned copy of the given CcInfo object, except that all linker inputs -# with owner `old_owner_label` are recreated and owned by the current target. -# -# This is useful in the "macro with proxy rule" pattern, as some rules upstream -# may expect they are depending directly on a target which generates linker inputs, -# as opposed to a proxy target which is a level of indirection to such a target. -def _claim_ownership(ctx, old_owner_label, ccinfo): - linker_inputs = [] - # This is not ideal, as it flattens a depset. - for old_linker_input in ccinfo.linking_context.linker_inputs.to_list(): - if old_linker_input.owner == old_owner_label: - new_linker_input = cc_common.create_linker_input( - owner = ctx.label, - libraries = depset(direct = old_linker_input.libraries)) - linker_inputs.append(new_linker_input) - else: - linker_inputs.append(old_linker_input) - - linking_context = cc_common.create_linking_context(linker_inputs = depset(direct = linker_inputs)) - return CcInfo(compilation_context = ccinfo.compilation_context, linking_context = linking_context) - -def _empty_library_safeguard_impl(ctx): - if len(ctx.attr.deps) != 1: - fail("the deps attribute should always contain exactly one label") - - main_target = ctx.attr.deps[0] - if len(ctx.files.deps) > 0: - # This safeguard is a no-op, as a library was generated by the main target. - new_cc_info = _claim_ownership(ctx, main_target.label, main_target[CcInfo]) - return [new_cc_info, main_target[DefaultInfo]] - - # The main library is empty; link a stub and propagate it to match Soong behavior. - cc_toolchain = find_cpp_toolchain(ctx) - CPP_LINK_STATIC_LIBRARY_ACTION_NAME = "c++-link-static-library" - feature_configuration = cc_common.configure_features( - ctx = ctx, - cc_toolchain = cc_toolchain, - requested_features = ctx.features, - unsupported_features = ctx.disabled_features + ["linker_flags"], - ) - - output_file = ctx.actions.declare_file(ctx.label.name + ".a") - linker_input = cc_common.create_linker_input( - owner = ctx.label, - libraries = depset(direct = [ - cc_common.create_library_to_link( - actions = ctx.actions, - feature_configuration = feature_configuration, - cc_toolchain = cc_toolchain, - static_library = output_file, - ), - ]), - ) - compilation_context = cc_common.create_compilation_context() - linking_context = cc_common.create_linking_context(linker_inputs = depset(direct = [linker_input])) - - archiver_path = cc_common.get_tool_for_action( - feature_configuration = feature_configuration, - action_name = CPP_LINK_STATIC_LIBRARY_ACTION_NAME, - ) - archiver_variables = cc_common.create_link_variables( - feature_configuration = feature_configuration, - cc_toolchain = cc_toolchain, - output_file = output_file.path, - is_using_linker = False, - ) - command_line = cc_common.get_memory_inefficient_command_line( - feature_configuration = feature_configuration, - action_name = CPP_LINK_STATIC_LIBRARY_ACTION_NAME, - variables = archiver_variables, - ) - args = ctx.actions.args() - args.add_all(command_line) - - ctx.actions.run( - executable = archiver_path, - arguments = [args], - inputs = depset( - transitive = [ - cc_toolchain.all_files, - ], - ), - outputs = [output_file], - ) - - cc_info = cc_common.merge_cc_infos(cc_infos = [ - main_target[CcInfo], - CcInfo(compilation_context = compilation_context, linking_context = linking_context), - ]) - return [ - DefaultInfo(files = depset([output_file])), - cc_info, - ] - -# A rule which depends on a single cc_library target. If the cc_library target -# has no outputs (indicating that it has no srcs or deps), then this safeguard -# rule creates a single stub .a file using llvm-ar. This mimics Soong's behavior -# in this regard. Otherwise, this safeguard is a simple passthrough for the providers -# of the cc_library. -_empty_library_safeguard = rule( - implementation = _empty_library_safeguard_impl, - attrs = { - # This should really be a label attribute since it always contains a - # single dependency, but cc_shared_library requires that C++ rules - # depend on each other through the "deps" attribute. - "deps": attr.label_list(providers = [CcInfo]), - "_cc_toolchain": attr.label( - default = Label("@local_config_cc//:toolchain"), - providers = [cc_common.CcToolchainInfo], - ), - }, - toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], - fragments = ["cpp"], -) diff --git a/rules/cc_object.bzl b/rules/cc_object.bzl deleted file mode 100644 index 84905c91..00000000 --- a/rules/cc_object.bzl +++ /dev/null @@ -1,131 +0,0 @@ -load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cpp_toolchain") -load(":cc_constants.bzl", "constants") - -# "cc_object" module copts, taken from build/soong/cc/object.go -_CC_OBJECT_COPTS = ["-fno-addrsig"] - -# partialLd module link opts, taken from build/soong/cc/builder.go -# https://cs.android.com/android/platform/superproject/+/master:build/soong/cc/builder.go;l=87;drc=f2be52c4dcc2e3d743318e106633e61de0ad2afd -_CC_OBJECT_LINKOPTS = [ - "-fuse-ld=lld", - "-nostdlib", - "-no-pie", - "-Wl,-r", -] - - -CcObjectInfo = provider(fields = [ - # The merged compilation outputs for this cc_object and its transitive - # dependencies. - "objects", -]) - -def split_srcs_hdrs(files): - headers = [] - non_headers = [] - for f in files: - if f.extension in constants.hdr_exts: - headers += [f] - else: - non_headers += [f] - return non_headers, headers - - -def _cc_object_impl(ctx): - cc_toolchain = find_cpp_toolchain(ctx) - - feature_configuration = cc_common.configure_features( - ctx = ctx, - cc_toolchain = cc_toolchain, - requested_features = ctx.features, - unsupported_features = ctx.disabled_features + ["linker_flags"], - ) - - compilation_contexts = [] - deps_objects = [] - for obj in ctx.attr.deps: - compilation_contexts.append(obj[CcInfo].compilation_context) - deps_objects.append(obj[CcObjectInfo].objects) - - product_variables = ctx.attr._android_product_variables[platform_common.TemplateVariableInfo] - asflags = [flag.format(**product_variables.variables) for flag in ctx.attr.asflags] - - srcs, private_hdrs = split_srcs_hdrs(ctx.files.srcs) - - (compilation_context, compilation_outputs) = cc_common.compile( - name = ctx.label.name, - actions = ctx.actions, - feature_configuration = feature_configuration, - cc_toolchain = cc_toolchain, - srcs = srcs, - includes = ctx.attr.includes, - public_hdrs = ctx.files.hdrs, - private_hdrs = private_hdrs, - user_compile_flags = ctx.attr.copts + asflags, - compilation_contexts = compilation_contexts, - ) - - objects_to_link = cc_common.merge_compilation_outputs(compilation_outputs=deps_objects + [compilation_outputs]) - - # partially link if there are multiple object files - if len(objects_to_link.objects) + len(objects_to_link.pic_objects) > 1: - linking_output = cc_common.link( - name = ctx.label.name + ".o", - actions = ctx.actions, - feature_configuration = feature_configuration, - cc_toolchain = cc_toolchain, - user_link_flags = _CC_OBJECT_LINKOPTS, - compilation_outputs = objects_to_link, - ) - files = depset([linking_output.executable]) - else: - files = depset(objects_to_link.objects + objects_to_link.pic_objects) - - return [ - DefaultInfo(files = files), - CcInfo(compilation_context = compilation_context), - CcObjectInfo(objects = objects_to_link), - ] - -_cc_object = rule( - implementation = _cc_object_impl, - attrs = { - "srcs": attr.label_list(allow_files = constants.all_dot_exts), - "hdrs": attr.label_list(allow_files = constants.hdr_dot_exts), - "includes": attr.string_list(), - "copts": attr.string_list(), - "asflags": attr.string_list(), - "deps": attr.label_list(providers=[CcInfo, CcObjectInfo]), - "_cc_toolchain": attr.label( - default = Label("@local_config_cc//:toolchain"), - providers = [cc_common.CcToolchainInfo], - ), - "_android_product_variables": attr.label( - default = Label("//build/bazel/product_variables:android_product_variables"), - providers = [platform_common.TemplateVariableInfo], - ), - }, - toolchains = ["@bazel_tools//tools/cpp:toolchain_type"], - fragments = ["cpp"], -) - -def cc_object( - name, - copts = [], - hdrs = [], - asflags = [], - srcs = [], - deps = [], - native_bridge_supported = False, # TODO: not supported yet. - **kwargs): - "Build macro to correspond with the cc_object Soong module." - - _cc_object( - name = name, - hdrs = hdrs, - asflags = asflags, - copts = _CC_OBJECT_COPTS + copts, - srcs = srcs, - deps = deps, - **kwargs - ) diff --git a/rules/coverage/remote_coverage_tools/BUILD b/rules/coverage/remote_coverage_tools/BUILD new file mode 100644 index 00000000..4f3d211a --- /dev/null +++ b/rules/coverage/remote_coverage_tools/BUILD @@ -0,0 +1,9 @@ +# This is a stub BUILD to override remote_coverage_tools. +# See b/201242197 for more information. + +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "coverage_report_generator", + srcs = ["coverage_report_generator.sh"], +) diff --git a/rules/coverage/remote_coverage_tools/WORKSPACE b/rules/coverage/remote_coverage_tools/WORKSPACE new file mode 100644 index 00000000..bd9e9137 --- /dev/null +++ b/rules/coverage/remote_coverage_tools/WORKSPACE @@ -0,0 +1,2 @@ +# This is a stub WORKSPACE to override remote_coverage_tools. +# See b/201242197 for more information. diff --git a/rules/filegroup.bzl b/rules/filegroup.bzl new file mode 100644 index 00000000..63ecd232 --- /dev/null +++ b/rules/filegroup.bzl @@ -0,0 +1,83 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load("//build/bazel/rules/cc:cc_constants.bzl", "constants") + +def extension(f): + return f.split(".")[-1] + +def group_files_by_ext(files): + cpp = [] + c = [] + asm = [] + + # This for-loop iterator works because filegroups in Android don't use + # configurable selects. + for f in files: + if extension(f) in constants.c_src_exts: + c += [f] + elif extension(f) in constants.cpp_src_exts: + cpp += [f] + elif extension(f) in constants.as_src_exts: + asm += [f] + else: + # not C based + continue + return cpp, c, asm + +# Filegroup is a macro because it needs to expand to language specific source +# files for cc_library's srcs_as, srcs_c and srcs attributes. +def filegroup(name, srcs = [], **kwargs): + native.filegroup( + name = name, + srcs = srcs, + **kwargs + ) + + # These genrule prevent empty filegroups being used as deps to cc libraries, + # avoiding the error: + # + # in srcs attribute of cc_library rule //foo/bar:baz: + # '//foo/bar/some_other:baz2' does not produce any cc_library srcs files. + native.genrule( + name = name + "_null_cc", + outs = [name + "_null.cc"], + cmd = "touch $@", + ) + native.genrule( + name = name + "_null_c", + outs = [name + "_null.c"], + cmd = "touch $@", + ) + native.genrule( + name = name + "_null_s", + outs = [name + "_null.S"], + cmd = "touch $@", + ) + + cpp_srcs, c_srcs, as_srcs = group_files_by_ext(srcs) + native.filegroup( + name = name + "_cpp_srcs", + srcs = [name + "_null.cc"] + cpp_srcs, + ) + native.filegroup( + name = name + "_c_srcs", + srcs = [name + "_null.c"] + c_srcs, + ) + native.filegroup( + name = name + "_as_srcs", + srcs = [name + "_null.S"] + as_srcs, + ) diff --git a/rules/full_cc_library.bzl b/rules/full_cc_library.bzl deleted file mode 100644 index 4a8e7b03..00000000 --- a/rules/full_cc_library.bzl +++ /dev/null @@ -1,99 +0,0 @@ -load(":cc_library_static.bzl", "cc_library_static") -load("@rules_cc//examples:experimental_cc_shared_library.bzl", "CcSharedLibraryInfo", "cc_shared_library") - -def cc_library( - name, - # attributes for both targets - srcs = [], - hdrs = [], - deps = [], - whole_archive_deps = [], - dynamic_deps = [], - copts = [], - includes = [], - linkopts = [], - # attributes for the shared target - dynamic_deps_for_shared = [], - shared_copts = [], - shared_srcs = [], - static_deps_for_shared = [], - whole_archive_deps_for_shared = [], - user_link_flags = [], - version_script = None, - # attributes for the static target - dynamic_deps_for_static = [], - static_copts = [], - static_srcs = [], - static_deps_for_static = [], - whole_archive_deps_for_static = [], - **kwargs): - static_name = name + "_bp2build_cc_library_static" - shared_name = name + "_bp2build_cc_library_shared" - shared_root_name = name + "_bp2build_cc_library_shared_root" - _cc_library_proxy( - name = name, - static = static_name, - shared = shared_name, - ) - - # The static version of the library. - cc_library_static( - name = static_name, - hdrs = hdrs, - srcs = srcs + static_srcs, - copts = copts + static_copts, - includes = includes, - linkopts = linkopts, - # TODO(b/187533117): Handle whole_archive_deps differently than other deps. - deps = deps + static_deps_for_static + whole_archive_deps + whole_archive_deps_for_static, - # TODO(b/187746106): Handle dynamic_deps_for_static. - ) - - # The static library at the root of the shared library. - # This may be distinct from the static library if, for example, - # the static-variant srcs are different than the shared-variant srcs. - cc_library_static( - name = shared_root_name, - hdrs = hdrs, - srcs = srcs + shared_srcs, - copts = copts + shared_copts, - includes = includes, - linkopts = linkopts, - deps = deps + static_deps_for_shared + whole_archive_deps + whole_archive_deps_for_shared, - ) - - cc_shared_library( - name = shared_name, - user_link_flags = user_link_flags, - # b/184806113: Note this is a pretty a workaround so users don't have to - # declare all transitive static deps used by this target. It'd be great - # if a shared library could declare a transitive exported static dep - # instead of needing to declare each target transitively. - static_deps = ["//:__subpackages__"] + [shared_root_name], - dynamic_deps = dynamic_deps + dynamic_deps_for_shared, - version_script = version_script, - roots = [shared_root_name] + whole_archive_deps + whole_archive_deps_for_shared, - ) - -def _cc_library_proxy_impl(ctx): - static_files = ctx.attr.static[DefaultInfo].files.to_list() - shared_files = ctx.attr.shared[DefaultInfo].files.to_list() - - files = static_files + shared_files - - return [ - ctx.attr.shared[CcSharedLibraryInfo], - ctx.attr.static[CcInfo], - DefaultInfo( - files = depset(direct = files), - runfiles = ctx.runfiles(files = files), - ), - ] - -_cc_library_proxy = rule( - implementation = _cc_library_proxy_impl, - attrs = { - "shared": attr.label(mandatory = True, providers = [CcSharedLibraryInfo]), - "static": attr.label(mandatory = True, providers = [CcInfo]), - }, -) diff --git a/rules/java/BUILD b/rules/java/BUILD new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/rules/java/BUILD diff --git a/rules/java/library.bzl b/rules/java/library.bzl new file mode 100644 index 00000000..f45cd51b --- /dev/null +++ b/rules/java/library.bzl @@ -0,0 +1,8 @@ +"""Macro wrapping the java_library for bp2build. """ + +def java_library(name = "", srcs = [], deps = [], javacopts = [], **kwargs): + # Disable the error prone check of HashtableContains by default. See https://errorprone.info/bugpattern/HashtableContains + # HashtableContains error is reported when compiling //external/bouncycastle:bouncycastle-bcpkix-unbundled + opts = ["-Xep:HashtableContains:OFF"] + javacopts + + native.java_library(name, srcs = srcs, deps = deps, javacopts = opts, **kwargs) diff --git a/rules/java/proto.bzl b/rules/java/proto.bzl new file mode 100644 index 00000000..fa2d5675 --- /dev/null +++ b/rules/java/proto.bzl @@ -0,0 +1,186 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load("//build/bazel/rules:proto_file_utils.bzl", "proto_file_utils") +load("@bazel_skylib//lib:paths.bzl", "paths") +load(":library.bzl", "java_library") + +def _java_proto_sources_gen_rule_impl(ctx): + out_flags = [] + plugin_executable = None + out_arg = None + if ctx.attr.plugin: + plugin_executable = ctx.executable.plugin + else: + out_arg = "--java_out" + if ctx.attr.out_format: + out_flags.append(ctx.attr.out_format) + + srcs = [] + for dep in ctx.attr.deps: + proto_info = dep[ProtoInfo] + out_jar = _generate_java_proto_action( + proto_info = proto_info, + protoc = ctx.executable._protoc, + ctx = ctx, + out_flags = out_flags, + plugin_executable = plugin_executable, + out_arg = out_arg, + ) + srcs.append(out_jar) + + return [ + DefaultInfo(files = depset(direct = srcs)), + ] + +_java_proto_sources_gen = rule( + implementation = _java_proto_sources_gen_rule_impl, + attrs = { + "deps": attr.label_list( + providers = [ProtoInfo], + doc = """ +proto_library or any other target exposing ProtoInfo provider with *.proto files +""", + mandatory = True, + ), + "_protoc": attr.label( + default = Label("//external/protobuf:aprotoc"), + executable = True, + cfg = "exec", + ), + "plugin": attr.label( + executable = True, + cfg = "exec", + ), + "out_format": attr.string( + doc = """ +Optional argument specifying the out format, e.g. lite. +If not provided, defaults to full protos. +""", + ), + }, + toolchains = ["@bazel_tools//tools/jdk:toolchain_type"], +) + +def _generate_java_proto_action( + proto_info, + protoc, + ctx, + plugin_executable, + out_arg, + out_flags): + return proto_file_utils.generate_jar_proto_action( + proto_info, + protoc, + ctx, + out_flags, + plugin_executable = plugin_executable, + out_arg = out_arg, + mnemonic = "JavaProtoGen", + ) + +def _java_proto_library( + name, + deps = [], + plugin = None, + target_compatible_with = [], + out_format = None, + proto_dep = None): + proto_sources_name = name + "_proto_gen" + + _java_proto_sources_gen( + name = proto_sources_name, + deps = deps, + plugin = plugin, + out_format = out_format, + ) + + if proto_dep: + deps = [proto_dep] + else: + deps = [] + + java_library( + name = name, + srcs = [proto_sources_name], + deps = deps, + target_compatible_with = target_compatible_with, + ) + +def java_nano_proto_library( + name, + deps = [], + plugin = "//external/protobuf:protoc-gen-javanano", + target_compatible_with = []): + _java_proto_library( + name, + deps = deps, + plugin = plugin, + target_compatible_with = target_compatible_with, + proto_dep = "//external/protobuf:libprotobuf-java-nano", + ) + +def java_micro_proto_library( + name, + deps = [], + plugin = "//external/protobuf:protoc-gen-javamicro", + target_compatible_with = []): + _java_proto_library( + name, + deps = deps, + plugin = plugin, + target_compatible_with = target_compatible_with, + proto_dep = "//external/protobuf:libprotobuf-java-micro", + ) + +def java_lite_proto_library( + name, + deps = [], + plugin = None, + target_compatible_with = []): + _java_proto_library( + name, + deps = deps, + plugin = plugin, + target_compatible_with = target_compatible_with, + out_format = "lite", + proto_dep = "//external/protobuf:libprotobuf-java-lite", + ) + +def java_stream_proto_library( + name, + deps = [], + plugin = "//frameworks/base/tools/streaming_proto:protoc-gen-javastream", + target_compatible_with = []): + _java_proto_library( + name, + deps = deps, + plugin = plugin, + target_compatible_with = target_compatible_with, + ) + +def java_proto_library( + name, + deps = [], + plugin = None, + target_compatible_with = []): + _java_proto_library( + name, + deps = deps, + plugin = plugin, + target_compatible_with = target_compatible_with, + proto_dep = "//external/protobuf:libprotobuf-java-full", + ) diff --git a/rules/lunch.bzl b/rules/lunch.bzl deleted file mode 100644 index 04152518..00000000 --- a/rules/lunch.bzl +++ /dev/null @@ -1,62 +0,0 @@ -_CAPTURED_ENV_VARS = [ - "PWD", - "TARGET_PRODUCT", - "TARGET_BUILD_VARIANT", - "COMBINED_NINJA", - "KATI_NINJA", - "PACKAGE_NINJA", - "SOONG_NINJA", -] - -_ALLOWED_SPECIAL_CHARACTERS = [ - "/", - "_", - "-", - "'", - ".", -] - -# Since we write the env var value literally into a .bzl file, ensure that the string -# does not contain special characters like '"', '\n' and '\'. Use an allowlist approach -# and check that the remaining string is alphanumeric. -def _validate_env_value(env_var, env_value): - if env_value == None: - fail("The env var " + env_var + " is not defined.") - - for allowed_char in _ALLOWED_SPECIAL_CHARACTERS: - env_value = env_value.replace(allowed_char, "") - if not env_value.isalnum(): - fail("The value of " + - env_var + - " can only consist of alphanumeric and " + - str(_ALLOWED_SPECIAL_CHARACTERS) + - " characters: " + - str(env_value)) - -def _lunch_impl(rctx): - env_vars = {} - for env_var in _CAPTURED_ENV_VARS: - env_value = rctx.os.environ.get(env_var) - _validate_env_value(env_var, env_value) - env_vars[env_var] = env_value - - rctx.file("BUILD.bazel", """ -exports_files(["env.bzl"]) -""") - - # Re-export captured environment variables in a .bzl file. - rctx.file("env.bzl", "\n".join([ - item[0] + " = \"" + str(item[1]) + "\"" - for item in env_vars.items() - ])) - -_lunch = repository_rule( - implementation = _lunch_impl, - configure = True, - environ = _CAPTURED_ENV_VARS, - doc = "A repository rule to capture environment variables based on the lunch choice.", -) - -def lunch(): - # Hardcode repository name to @lunch. - _lunch(name = "lunch") diff --git a/rules/make_injection.bzl b/rules/make_injection.bzl new file mode 100644 index 00000000..38bb9986 --- /dev/null +++ b/rules/make_injection.bzl @@ -0,0 +1,83 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# A repository rule to run soong_ui --make-mode to provide the Bazel standalone +# build with prebuilts from Make/Soong that Bazel can't build yet. +def _impl(rctx): + target_product = rctx.os.environ.get("TARGET_PRODUCT", "aosp_arm") + target_build_variant = rctx.os.environ.get("TARGET_BUILD_VARIANT", "eng") + + binaries = rctx.attr.binaries + target_modules = rctx.attr.target_module_files + + build_dir = rctx.path(Label("//:WORKSPACE")).dirname + soong_ui_bash = str(build_dir) + "/build/soong/soong_ui.bash" + args = [ + soong_ui_bash, + "--make-mode", + "--skip-soong-tests", + ] + all_modules = target_modules.keys() + binaries + args += all_modules + + rctx.report_progress("Building modules with Soong: %s" % str(all_modules)) + out_dir = str(build_dir.dirname) + "/make_injection" + exec_result = rctx.execute( + args, + environment = { + "OUT_DIR": out_dir, + "TARGET_PRODUCT": target_product, + "TARGET_BUILD_VARIANT": target_build_variant, + "TOP": str(build_dir.dirname.dirname.dirname), + }, + quiet = False, # stream stdout so it shows progress + ) + if exec_result.return_code != 0: + fail(exec_result.stderr) + + # Get the explicit list of host binary paths to be exported + rctx.symlink(out_dir + "/host/linux-x86", "host/linux-x86") + binary_path_prefix = "host/linux-x86/bin" + binary_paths = ['"%s/%s"' % (binary_path_prefix, binary) for binary in binaries] + + # Get the explicit list of target installed files to be exported + rctx.symlink(out_dir + "/target", "target") + target_path_prefix = "target/product/generic" + target_paths = [] + for paths in target_modules.values(): + target_paths.extend(['"%s/%s"' % (target_path_prefix, path) for path in paths]) + + exports_files = """exports_files([ + %s +]) +""" % ",\n ".join(binary_paths + target_paths) + rctx.file("BUILD", exports_files) + +make_injection_repository = repository_rule( + implementation = _impl, + doc = """This rule exposes Soong prebuilts for migrating the build to Bazel. + +This rule allows the Bazel build (i.e. b build //bionic/...) to depend on prebuilts from +Soong. A use case is to allow the Bazel build to use prebuilt host tools in the +Bazel rules toolchains without first converting them to Bazel.""", + attrs = { + "binaries": attr.string_list(default = [], doc = "A list of host binary modules built for linux-x86."), + "target_module_files": attr.string_list_dict(default = {}, doc = "A dict of modules to the target files that should be exported."), + # See b/210399979 + "watch_android_bp_files": attr.label_list(allow_files = [".bp"], default = [], doc = "A list of Android.bp files to watch for changes to invalidate this repository rule."), + }, + environ = ["TARGET_PRODUCT", "TARGET_BUILD_VARIANT"], +) diff --git a/rules/prebuilt_file.bzl b/rules/prebuilt_file.bzl new file mode 100644 index 00000000..12e69824 --- /dev/null +++ b/rules/prebuilt_file.bzl @@ -0,0 +1,89 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +PrebuiltFileInfo = provider( + "Info needed for prebuilt_file modules", + fields = { + "src": "Source file of this prebuilt", + "dir": "Directory into which to install", + "filename": "Optional name for the installed file", + "installable": "Whether this is directly installable into one of the partitions", + }, +) +_handled_dirs = ["etc", "usr/share"] + +def _prebuilt_file_rule_impl(ctx): + srcs = ctx.files.src + if len(srcs) != 1: + fail("src for", ctx.label.name, "is expected to be singular, but is of len", len(srcs), ":\n", srcs) + + # Is this an acceptable directory, or a subdir under one? + dir = ctx.attr.dir + acceptable = False + for d in _handled_dirs: + if dir == d or dir.startswith(d + "/"): + acceptable = True + break + if not acceptable: + fail("dir for", ctx.label.name, "is `", dir, "`, but we only handle these:\n", _handled_dirs) + + return [ + PrebuiltFileInfo( + src = srcs[0], + dir = dir, + filename = ctx.attr.filename, + installable = ctx.attr.installable, + ), + DefaultInfo( + files = depset(srcs), + ), + ] + +_prebuilt_file = rule( + implementation = _prebuilt_file_rule_impl, + attrs = { + "src": attr.label( + mandatory = True, + allow_files = True, + # TODO(b/217908237): reenable allow_single_file + # allow_single_file = True, + ), + "dir": attr.string(mandatory = True), + "filename": attr.string(), + "installable": attr.bool(default = True), + }, +) + +def prebuilt_file( + name, + src, + dir, + filename = None, + installable = True, + # TODO(b/207489266): Fully support; + # data is currently dropped to prevent breakages from e.g. prebuilt_etc + data = [], + **kwargs): + "Bazel macro to correspond with the e.g. prebuilt_etc and prebuilt_usr_share Soong modules." + + _prebuilt_file( + name = name, + src = src, + dir = dir, + filename = filename, + installable = installable, + **kwargs + ) diff --git a/rules/proto_file_utils.bzl b/rules/proto_file_utils.bzl new file mode 100644 index 00000000..2ce3dd58 --- /dev/null +++ b/rules/proto_file_utils.bzl @@ -0,0 +1,151 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load("@bazel_skylib//lib:paths.bzl", "paths") + +def _generate_and_declare_output_files( + ctx, + file_names, + type_dictionary): + ret = {} + for typ in type_dictionary: + ret[typ] = [] + + for name in file_names: + short_path = name.short_path + for typ, ext in type_dictionary.items(): + # prefix with label.name to prevent collisions between targets + # if proto compliation becomes an aspect, can prefix with output + # information instead to allow reuse, e.g. multiple cc `lite` + # libraries containing the same proto file + out_name = paths.join(ctx.label.name, paths.replace_extension(short_path, ext)) + declared = ctx.actions.declare_file(out_name) + ret[typ].append(declared) + + return ret + +def _generate_jar_proto_action( + proto_info, + protoc, + ctx, + out_flags = [], + plugin_executable = None, + out_arg = None, + mnemonic = "ProtoGen"): + + jar_basename = ctx.label.name + "-proto_gen" + jar_name = jar_basename + "-src.jar" + jar_file = ctx.actions.declare_file(jar_name) + + _generate_proto_action( + proto_info = proto_info, + protoc = protoc, + ctx = ctx, + out_flags = out_flags, + plugin_executable = plugin_executable, + out_arg = out_arg, + mnemonic = mnemonic, + output_file = jar_file, + ) + + srcjar_name = jar_basename + ".srcjar" + srcjar_file = ctx.actions.declare_file(srcjar_name) + ctx.actions.symlink( + output = srcjar_file, + target_file = jar_file, + ) + + return srcjar_file + +def _generate_proto_action( + proto_info, + protoc, + ctx, + type_dictionary = None, + out_flags = [], + plugin_executable = None, + out_arg = None, + mnemonic = "ProtoGen", + output_file = None): + """ Utility function for creating proto_compiler action. + + Args: + proto_info: ProtoInfo + protoc: proto compiler executable. + ctx: context, used for declaring new files only. + type_dictionary: a dictionary of types to output extensions + out_flags: protoc output flags + plugin_executable: plugin executable file + out_arg: as appropriate, if plugin_executable and out_arg are both supplied, plugin_executable is preferred + mnemonic: (optional) a string to describe the proto compilation action + output_file: (optional) File, used to specify a specific file for protoc output (typically a JAR file) + + Returns: + Dictionary with declared files grouped by type from the type_dictionary. + """ + proto_srcs = proto_info.direct_sources + transitive_proto_srcs = proto_info.transitive_imports + + protoc_out_name = paths.join(ctx.bin_dir.path, ctx.label.package) + if output_file: + protoc_out_name = paths.join(protoc_out_name, output_file.basename) + out_files = { + "out": [output_file] + } + else: + protoc_out_name = paths.join(protoc_out_name, ctx.label.name) + out_files = _generate_and_declare_output_files( + ctx, + proto_srcs, + type_dictionary, + ) + + tools = [] + args = ctx.actions.args() + if plugin_executable: + tools.append(plugin_executable) + args.add("--plugin=protoc-gen-PLUGIN=" + plugin_executable.path) + args.add("--PLUGIN_out=" + ",".join(out_flags) + ":" + protoc_out_name) + else: + args.add("{}={}:{}".format(out_arg, ",".join(out_flags), protoc_out_name)) + + args.add_all(["-I", proto_info.proto_source_root]) + args.add_all(["-I{0}={1}".format(f.short_path, f.path) for f in transitive_proto_srcs.to_list()]) + args.add_all([f.short_path for f in proto_srcs]) + + inputs = depset( + direct = proto_srcs, + transitive = [transitive_proto_srcs], + ) + + outputs = [] + for outs in out_files.values(): + outputs.extend(outs) + + ctx.actions.run( + inputs = inputs, + executable = protoc, + tools = tools, + outputs = outputs, + arguments = [args], + mnemonic = mnemonic, + ) + return out_files + +proto_file_utils = struct( + generate_proto_action = _generate_proto_action, + generate_jar_proto_action = _generate_jar_proto_action, +) diff --git a/rules/python/BUILD b/rules/python/BUILD new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/rules/python/BUILD diff --git a/rules/python/library.bzl b/rules/python/library.bzl new file mode 100644 index 00000000..3d2308e6 --- /dev/null +++ b/rules/python/library.bzl @@ -0,0 +1,15 @@ +"""Macro wrapping the py_library rule for Soong/Bazel convergence.""" + +def py_library(imports = [".."], **kwargs): + # b/208215661: Always propagate the parent directory of this target so that + # dependent targets can use `import <modulename>` without using absolute + # imports, which Bazel uses by default. The eventual effect of this in a + # py_binary is that all directories contain py_library deps are added to the + # PYTHONPATH of the py_binary stub script, enabling `import <modulename>`. + if ".." not in imports: + imports.append("..") + + native.py_library( + imports = imports, + **kwargs, + ) diff --git a/rules/sh_binary.bzl b/rules/sh_binary.bzl new file mode 100644 index 00000000..82e1510e --- /dev/null +++ b/rules/sh_binary.bzl @@ -0,0 +1,68 @@ +ShBinaryInfo = provider( + "Info needed for sh_binary modules", + fields = { + "sub_dir": "Optional subdirectory to install into", + "filename": "Optional name for the installed file", + }, +) + +def sh_binary( + name, + srcs, + sub_dir = None, + filename = None, + **kwargs): + "Bazel macro to correspond with the sh_binary Soong module." + + internal_name = name + "_internal" + native.sh_binary( + name = internal_name, + srcs = srcs, + **kwargs + ) + + # We need this wrapper rule around native.sh_binary in order to provide extra + # attributes such as filename and sub_dir that are useful when building apex. + _sh_binary_combiner( + name = name, + sub_dir = sub_dir, + filename = filename, + dep = internal_name, + ) + +def _sh_binary_combiner_impl(ctx): + dep = ctx.attr.dep[DefaultInfo] + output = ctx.outputs.executable + + ctx.actions.run_shell( + outputs = [output], + inputs = [dep.files_to_run.executable], + command = "cp %s %s" % (dep.files_to_run.executable.path, output.path), + mnemonic = "CopyNativeShBinary", + ) + + files = depset(direct = [output], transitive = [dep.files]) + + return [ + DefaultInfo( + files = files, + runfiles = ctx.runfiles().merge(dep.data_runfiles).merge(dep.default_runfiles), + executable = output, + ), + ShBinaryInfo( + sub_dir = ctx.attr.sub_dir, + filename = ctx.attr.filename, + ), + ] + +_sh_binary_combiner = rule( + implementation = _sh_binary_combiner_impl, + attrs = { + "sub_dir": attr.string(), + "filename": attr.string(), + "dep": attr.label(mandatory = True), + }, + provides = [ShBinaryInfo], + executable = True, + doc = "Wrapper rule around native.sh_binary to provide extra attributes", +) diff --git a/rules/soong_injection.bzl b/rules/soong_injection.bzl index 527ce52c..9e2090ef 100644 --- a/rules/soong_injection.bzl +++ b/rules/soong_injection.bzl @@ -1,9 +1,29 @@ +""" +Copyright (C) 2021 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + def _impl(rctx): rctx.file("WORKSPACE", "") build_dir = str(rctx.path(Label("//:BUILD")).dirname.dirname) soong_injection_dir = build_dir + "/soong_injection" rctx.symlink(soong_injection_dir + "/mixed_builds", "mixed_builds") rctx.symlink(soong_injection_dir + "/cc_toolchain", "cc_toolchain") + rctx.symlink(soong_injection_dir + "/java_toolchain", "java_toolchain") + rctx.symlink(soong_injection_dir + "/product_config", "product_config") + rctx.symlink(soong_injection_dir + "/api_levels", "api_levels") + rctx.symlink(soong_injection_dir + "/metrics", "metrics") soong_injection_repository = repository_rule( implementation = _impl, diff --git a/rules_cc/examples/experimental_cc_shared_library.bzl b/rules_cc/examples/experimental_cc_shared_library.bzl index 355ed0ec..45659b90 100644 --- a/rules_cc/examples/experimental_cc_shared_library.bzl +++ b/rules_cc/examples/experimental_cc_shared_library.bzl @@ -287,6 +287,12 @@ def _filter_inputs( fail("We can't link " + str(owner) + " either statically or dynamically") + # Divergence from rules_cc: Add all dynamic dependencies as linker inputs + # even if they do not contain transitive dependencies of the roots. + # TODO(cparsons): Push this as an option upstream.. + for dynamic_dep_input in transitive_exports.values(): + linker_inputs.append(dynamic_dep_input) + return (exports, linker_inputs, link_once_static_libs) def _same_package_or_above(label_a, label_b): @@ -417,13 +423,32 @@ def _cc_shared_library_impl(ctx): ), ] -def _graph_structure_aspect_impl(target, ctx): +def _collect_graph_structure_info_from_children(ctx, attr): children = [] - - if hasattr(ctx.rule.attr, "deps"): - for dep in ctx.rule.attr.deps: + deps = getattr(ctx.rule.attr, attr, []) + if type(deps) == "list": + for dep in deps: if GraphNodeInfo in dep: children.append(dep[GraphNodeInfo]) + elif deps != None and GraphNodeInfo in deps: + # Single dep. + children.append(deps[GraphNodeInfo]) + return children + + +def _graph_structure_aspect_impl(target, ctx): + children = [] + + # This is a deviation from HEAD rules_cc because full_cc_library.bzl uses + # static/shared (among others) attrs to combine multiple targets into one, + # and the aspect needs to be able to traverse them to correctly populate + # linker_inputs in the cc_shared_library impl. + children += _collect_graph_structure_info_from_children(ctx, "deps") + children += _collect_graph_structure_info_from_children(ctx, "whole_archive_deps") + children += _collect_graph_structure_info_from_children(ctx, "dynamic_deps") + children += _collect_graph_structure_info_from_children(ctx, "implementation_deps") + children += _collect_graph_structure_info_from_children(ctx, "static") + children += _collect_graph_structure_info_from_children(ctx, "shared") # TODO(bazel-team): Add flag to Bazel that can toggle the initialization of # linkable_more_than_once. diff --git a/scripts/bp2build-progress/BUILD.bazel b/scripts/bp2build-progress/BUILD.bazel new file mode 100644 index 00000000..460308a4 --- /dev/null +++ b/scripts/bp2build-progress/BUILD.bazel @@ -0,0 +1,33 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +py_library( + name = "dependency_analysis", + srcs = ["dependency_analysis.py"], + visibility = ["//visibility:public"], +) + +py_binary( + name = "bp2build-progress", + srcs = ["bp2build-progress.py"], + visibility = ["//visibility:public"], + deps = [":dependency_analysis"], +) + +py_binary( + name = "bp2build-module-dep-infos", + srcs = ["bp2build-module-dep-infos.py"], + visibility = ["//visibility:public"], + deps = [":dependency_analysis"], +) diff --git a/scripts/bp2build-progress/README.md b/scripts/bp2build-progress/README.md new file mode 100644 index 00000000..61ffef51 --- /dev/null +++ b/scripts/bp2build-progress/README.md @@ -0,0 +1,40 @@ +# bp2build progress graphs + +This directory contains tools to generate reports and .png graphs of the +bp2build conversion progress, for any module. + +This tool relies on `json-module-graph` and `bp2build` to be buildable targets +for this branch. + +## Prerequisites + +* `/usr/bin/dot`: turning dot graphviz files into .pngs +* Optional: `/usr/bin/jq`: running the query scripts over the json-module-graph. + +Tip: `--use_queryview=true` allows running `bp2build-progress.py` without `jq`. + +## Instructions + +# Generate the report for a module, e.g. adbd + +``` +./bp2build-progress.py report -m adbd +``` + +or: + +``` +./bp2build-progress.py report -m adbd --use_queryview=true +``` + +# Generate the report for a module, e.g. adbd + +``` +./bp2build-progress.py graph -m adbd > /tmp/graph.in && dot -Tpng -o /tmp/graph.png /tmp/graph.in +``` + +or: + +``` +./bp2build-progress.py graph -m adbd --use_queryview=true > /tmp/graph.in && dot -Tpng -o /tmp/graph.png /tmp/graph.in +``` diff --git a/scripts/bp2build-progress/bp2build-module-dep-infos.py b/scripts/bp2build-progress/bp2build-module-dep-infos.py new file mode 100755 index 00000000..95cc1b72 --- /dev/null +++ b/scripts/bp2build-progress/bp2build-module-dep-infos.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A script to produce a csv report of all modules of a given type. + +There is one output row per module of the input type, each column corresponds +to one of the fields of the _ModuleTypeInfo named tuple described below. +The script allows to ignore certain dependency edges based on the target module +name, or the dependency tag name. + +Usage: + ./bp2build-module-dep-infos.py -m <module type> + --ignore_by_name <modules to ignore> + --ignore_by_tag <dependency tags to ignore> + +""" + +import argparse +import collections +import csv +import dependency_analysis +import sys + +_ModuleTypeInfo = collections.namedtuple( + "_ModuleTypeInfo", + [ + # map of module type to the set of properties used by modules + # of the given type in the dependency tree. + "type_to_properties", + # [java modules only] list of source file extensions used by this module. + "java_source_extensions", + ]) + +_DependencyRelation = collections.namedtuple("_DependencyRelation", [ + "transitive_dependency", + "top_level_module", +]) + + +def _get_java_source_extensions(module): + out = set() + if "Module" not in module: + return out + if "Java" not in module["Module"]: + return out + if "SourceExtensions" not in module["Module"]["Java"]: + return out + if module["Module"]["Java"]["SourceExtensions"]: + out.update(module["Module"]["Java"]["SourceExtensions"]) + return out + + +def _get_set_properties(module): + set_properties = set() + if "Module" not in module: + return set_properties + if "Android" not in module["Module"]: + return set_properties + if "SetProperties" not in module["Module"]["Android"]: + return set_properties + for prop in module["Module"]["Android"]["SetProperties"]: + set_properties.add(prop["Name"]) + return set_properties + + +def _should_ignore(module, ignored_names): + return (dependency_analysis.is_windows_variation(module) or + module["Name"] in ignored_names or + dependency_analysis.ignore_kind(module["Type"])) + +def _update_infos(module_name, type_infos, module_graph_map, ignored_dep_names): + module = module_graph_map[module_name] + if _should_ignore(module, ignored_dep_names) or module_name in type_infos: + return + for dep in module["Deps"]: + dep_name = dep["Name"] + if dep_name == module_name: + continue + _update_infos(dep_name, type_infos, module_graph_map, ignored_dep_names) + + java_source_extensions = _get_java_source_extensions(module) + type_to_properties = collections.defaultdict(set) + if module["Type"]: + type_to_properties[module["Type"]].update(_get_set_properties(module)) + for dep in module["Deps"]: + dep_name = dep["Name"] + if _should_ignore(module_graph_map[dep_name], ignored_dep_names): + continue + if dep_name == module_name: + continue + for dep_type, dep_type_properties in type_infos[dep_name].type_to_properties.items(): + type_to_properties[dep_type].update(dep_type_properties) + java_source_extensions.update(type_infos[dep_name].java_source_extensions) + type_infos[module_name] = _ModuleTypeInfo( + type_to_properties=type_to_properties, + java_source_extensions=java_source_extensions) + + +def module_type_info_from_json(module_graph, module_type, ignored_dep_names): + """Builds a map of module name to _ModuleTypeInfo for each module of module_type. + + Dependency edges pointing to modules in ignored_dep_names are not followed. + """ + module_graph_map = dict() + module_stack = [] + for module in module_graph: + # Windows variants have incomplete dependency information in the json module graph. + if dependency_analysis.is_windows_variation(module): + continue + module_graph_map[module["Name"]] = module + if module["Type"] == module_type: + module_stack.append(module["Name"]) + # dictionary of module name to _ModuleTypeInfo. + type_infos = {} + for module_name in module_stack: + # post-order traversal of the dependency graph builds the type_infos + # dictionary from the leaves so that common dependencies are visited + # only once. + _update_infos(module_name, type_infos, module_graph_map, ignored_dep_names) + + return { + name: info + for name, info in type_infos.items() + if module_graph_map[name]["Type"] == module_type + } + + +def main(): + parser = argparse.ArgumentParser(description="") + parser.add_argument("--module_type", "-m", help="name of Soong module type.") + parser.add_argument( + "--ignore_by_name", + type=str, + default="", + required=False, + help="Comma-separated list. When building the tree of transitive dependencies, will not follow dependency edges pointing to module names listed by this flag." + ) + args = parser.parse_args() + + module_type = args.module_type + ignore_by_name = args.ignore_by_name + + module_graph = dependency_analysis.get_json_module_type_info(module_type) + type_infos = module_type_info_from_json(module_graph, module_type, + ignore_by_name.split(",")) + writer = csv.writer(sys.stdout) + writer.writerow([ + "module name", + "properties", + "java source extensions", + ]) + for module, module_type_info in type_infos.items(): + writer.writerow([ + module, + ("[\"%s\"]" % '"\n"'.join([ + "%s: %s" % (mtype, ",".join(properties)) for mtype, properties in + module_type_info.type_to_properties.items() + ]) if len(module_type_info.type_to_properties) else "[]"), + ("[\"%s\"]" % '", "'.join(module_type_info.java_source_extensions) + if len(module_type_info.java_source_extensions) else "[]"), + ]) + + +if __name__ == "__main__": + main() diff --git a/scripts/bp2build-progress/bp2build-progress.py b/scripts/bp2build-progress/bp2build-progress.py new file mode 100755 index 00000000..822d0720 --- /dev/null +++ b/scripts/bp2build-progress/bp2build-progress.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A json-module-graph postprocessing script to generate a bp2build progress tracker. + +Usage: + ./bp2build-progress.py [report|graph] -m <module name> + +Example: + + To generate a report on the `adbd` module, run: + ./bp2build-progress report -m adbd + + To generate a graph on the `adbd` module, run: + ./bp2build-progress graph -m adbd > graph.in && dot -Tpng -o graph.png + graph.in + +""" + +import argparse +import collections +import datetime +import dependency_analysis +import os.path +import queue +import subprocess +import sys + +_ModuleInfo = collections.namedtuple("_ModuleInfo", [ + "name", + "kind", + "dirname", +]) + +_ReportData = collections.namedtuple("_ReportData", [ + "input_module", + "all_unconverted_modules", + "blocked_modules", + "dirs_with_unconverted_modules", + "kind_of_unconverted_modules", + "converted", +]) + + +def combine_report_data(data): + ret = _ReportData( + input_module=set(), + all_unconverted_modules=collections.defaultdict(set), + blocked_modules=collections.defaultdict(set), + dirs_with_unconverted_modules=set(), + kind_of_unconverted_modules=set(), + converted=set(), + ) + for item in data: + ret.input_module.add(item.input_module) + for key, value in item.all_unconverted_modules.items(): + ret.all_unconverted_modules[key].update(value) + for key, value in item.blocked_modules.items(): + ret.blocked_modules[key].update(value) + ret.dirs_with_unconverted_modules.update(item.dirs_with_unconverted_modules) + ret.kind_of_unconverted_modules.update(item.kind_of_unconverted_modules) + if len(ret.converted) == 0: + ret.converted.update(item.converted) + return ret + + +# Generate a dot file containing the transitive closure of the module. +def generate_dot_file(modules, converted, module): + DOT_TEMPLATE = """ +digraph mygraph {{ + node [shape=box]; + + %s +}} +""" + + make_node = lambda module, color: \ + ('"{name}" [label="{name}\\n{kind}" color=black, style=filled, ' + "fillcolor={color}]").format(name=module.name, kind=module.kind, color=color) + make_edge = lambda module, dep: \ + '"%s" -> "%s"' % (module.name, dep) + + # Check that all modules in the argument are in the list of converted modules + all_converted = lambda modules: all(map(lambda m: m in converted, modules)) + + dot_entries = [] + + for module, deps in modules.items(): + if module.name in converted: + # Skip converted modules (nodes) + continue + elif module.name not in converted: + if all_converted(deps): + dot_entries.append(make_node(module, "yellow")) + else: + dot_entries.append(make_node(module, "tomato")) + + # Print all edges for this module + for dep in list(deps): + # Skip converted deps (edges) + if dep not in converted: + dot_entries.append(make_edge(module, dep)) + + print(DOT_TEMPLATE % "\n ".join(dot_entries)) + + +# Generate a report for each module in the transitive closure, and the blockers for each module +def generate_report_data(modules, converted, input_module): + # Map of [number of unconverted deps] to list of entries, + # with each entry being the string: "<module>: <comma separated list of unconverted modules>" + blocked_modules = collections.defaultdict(set) + + # Map of unconverted modules to the modules they're blocking + # (i.e. reverse deps) + all_unconverted_modules = collections.defaultdict(set) + + dirs_with_unconverted_modules = set() + kind_of_unconverted_modules = set() + + for module, deps in sorted(modules.items()): + unconverted_deps = set(dep for dep in deps if dep not in converted) + for dep in unconverted_deps: + all_unconverted_modules[dep].add(module) + + unconverted_count = len(unconverted_deps) + if module.name not in converted: + report_entry = "{name} [{kind}] [{dirname}]: {unconverted_deps}".format( + name=module.name, + kind=module.kind, + dirname=module.dirname, + unconverted_deps=", ".join(sorted(unconverted_deps))) + blocked_modules[unconverted_count].add(report_entry) + dirs_with_unconverted_modules.add(module.dirname) + kind_of_unconverted_modules.add(module.kind) + + return _ReportData( + input_module=input_module, + all_unconverted_modules=all_unconverted_modules, + blocked_modules=blocked_modules, + dirs_with_unconverted_modules=dirs_with_unconverted_modules, + kind_of_unconverted_modules=kind_of_unconverted_modules, + converted=converted, + ) + + +def generate_report(report_data): + report_lines = [] + input_modules = sorted(report_data.input_module) + + report_lines.append("# bp2build progress report for: %s\n" % input_modules) + report_lines.append("Ignored module types: %s\n" % + sorted(dependency_analysis.IGNORED_KINDS)) + report_lines.append("# Transitive dependency closure:") + + for count, modules in sorted(report_data.blocked_modules.items()): + report_lines.append("\n%d unconverted deps remaining:" % count) + for module_string in sorted(modules): + report_lines.append(" " + module_string) + + report_lines.append("\n") + report_lines.append("# Unconverted deps of {}:\n".format(input_modules)) + for count, dep in sorted( + ((len(unconverted), dep) + for dep, unconverted in report_data.all_unconverted_modules.items()), + reverse=True): + report_lines.append("%s: blocking %d modules" % (dep, count)) + + report_lines.append("\n") + report_lines.append("# Dirs with unconverted modules:\n\n{}".format("\n".join( + sorted(report_data.dirs_with_unconverted_modules)))) + + report_lines.append("\n") + report_lines.append("# Kinds with unconverted modules:\n\n{}".format( + "\n".join(sorted(report_data.kind_of_unconverted_modules)))) + + report_lines.append("\n") + report_lines.append("# Converted modules:\n\n%s" % + "\n".join(sorted(report_data.converted))) + + report_lines.append("\n") + report_lines.append( + "Generated by: https://cs.android.com/android/platform/superproject/+/master:build/bazel/scripts/bp2build-progress/bp2build-progress.py" + ) + report_lines.append("Generated at: %s" % + datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S %z")) + print("\n".join(report_lines)) + + +def adjacency_list_from_json(module_graph, ignore_by_name, top_level_module): + # The set of ignored modules. These modules (and their dependencies) are not shown + # in the graph or report. + ignored = set() + + # A map of module name to _ModuleInfo + name_to_info = dict() + module_graph_map = dict() + q = queue.Queue() + + # Do a single pass to find all top-level modules to be ignored + for module in module_graph: + name = module["Name"] + if dependency_analysis.is_windows_variation(module): + continue + if ignore_kind(module["Type"]) or name in ignore_by_name: + ignored.add(module["Name"]) + continue + name_to_info[name] = _ModuleInfo( + name=name, + kind=module["Type"], + dirname=os.path.dirname(module["Blueprint"])) + module_graph_map[module["Name"]] = module + if module["Name"] == top_level_module: + q.put(module["Name"]) + + # An adjacency list for all modules in the transitive closure, excluding ignored modules. + module_adjacency_list = {} + visited = set() + # Create the adjacency list. + while not q.empty(): + module_name = q.get() + module = module_graph_map[module_name] + visited.add(module_name) + if module_name in ignored: + continue + if dependency_analysis.is_windows_variation(module): + # ignore the windows variations of modules + continue + + module_info = name_to_info[module_name] + module_adjacency_list[module_info] = set() + for dep in module["Deps"]: + dep_name = dep["Name"] + if dep_name in ignored or dep_name == module_name: + continue + module_adjacency_list[module_info].add(dep_name) + if dep_name not in visited: + q.put(dep_name) + + return module_adjacency_list + + +def ignore_kind(kind): + return kind in dependency_analysis.IGNORED_KINDS or "defaults" in kind + + +def bazel_target_to_dir(full_target): + dirname, _ = full_target.split(":") + return dirname[2:] + + +def adjacency_list_from_queryview_xml(module_graph, ignore_by_name, + top_level_module): + # The set of ignored modules. These modules (and their dependencies) are + # not shown in the graph or report. + ignored = set() + + # A map of module name to ModuleInfo + name_to_info = dict() + + # queryview embeds variant in long name, keep a map of the name with vaiarnt + # to just name + name_with_variant_to_name = dict() + + module_graph_map = dict() + q = queue.Queue() + + for module in module_graph: + ignore = False + if module.tag != "rule": + continue + kind = module.attrib["class"] + name_with_variant = module.attrib["name"] + name = None + variant = "" + for attr in module: + attr_name = attr.attrib["name"] + if attr_name == "soong_module_name": + name = attr.attrib["value"] + elif attr_name == "soong_module_variant": + variant = attr.attrib["value"] + elif attr_name == "soong_module_type" and kind == "generic_soong_module": + kind = attr.attrib["value"] + # special handling for filegroup srcs, if a source has the same name as + # the module, we don't convert it + elif kind == "filegroup" and attr_name == "srcs": + for item in attr: + if item.attrib["value"] == name: + ignore = True + if name in ignore_by_name: + ignore = True + + if ignore_kind(kind) or variant.startswith("windows") or ignore: + ignored.add(name_with_variant) + else: + if name == top_level_module: + q.put(name_with_variant) + name_with_variant_to_name.setdefault(name_with_variant, name) + name_to_info.setdefault( + name, + _ModuleInfo( + name=name, + kind=kind, + dirname=bazel_target_to_dir(name_with_variant), + )) + module_graph_map[name_with_variant] = module + + # An adjacency list for all modules in the transitive closure, excluding ignored modules. + module_adjacency_list = dict() + visited = set() + while not q.empty(): + name_with_variant = q.get() + module = module_graph_map[name_with_variant] + if module.tag != "rule": + continue + visited.add(name_with_variant) + if name_with_variant in ignored: + continue + + name = name_with_variant_to_name[name_with_variant] + module_info = name_to_info[name] + module_adjacency_list[module_info] = set() + for attr in module: + if attr.tag != "rule-input": + continue + dep_name_with_variant = attr.attrib["name"] + if dep_name_with_variant in ignored: + continue + dep_name = name_with_variant_to_name[dep_name_with_variant] + if name == dep_name: + continue + if dep_name_with_variant not in visited: + q.put(dep_name_with_variant) + module_adjacency_list[module_info].add(dep_name) + + return module_adjacency_list + + +def get_module_adjacency_list(top_level_module, use_queryview, ignore_by_name): + # The main module graph containing _all_ modules in the Soong build, + # and the list of converted modules. + try: + module_graph = dependency_analysis.get_queryview_module_info( + top_level_module + ) if use_queryview else dependency_analysis.get_json_module_info( + top_level_module) + converted = dependency_analysis.get_bp2build_converted_modules() + except subprocess.CalledProcessError as err: + print("Error running: '%s':", " ".join(err.cmd)) + print("Output:\n%s" % err.output.decode("utf-8")) + print("Error:\n%s" % err.stderr.decode("utf-8")) + sys.exit(-1) + + module_adjacency_list = None + if use_queryview: + module_adjacency_list = adjacency_list_from_queryview_xml( + module_graph, ignore_by_name, top_level_module) + else: + module_adjacency_list = adjacency_list_from_json(module_graph, + ignore_by_name, + top_level_module) + + return module_adjacency_list, converted + + +def main(): + parser = argparse.ArgumentParser(description="") + parser.add_argument("mode", help="mode: graph or report") + parser.add_argument( + "--module", + "-m", + action="append", + help="name(s) of Soong module(s). Multiple modules only supported for report" + ) + parser.add_argument( + "--use_queryview", + type=bool, + default=False, + required=False, + help="whether to use queryview or module_info") + parser.add_argument( + "--ignore_by_name", + type=str, + default="", + required=False, + help="Comma-separated list. When building the tree of transitive dependencies, will not follow dependency edges pointing to module names listed by this flag." + ) + args = parser.parse_args() + + if len(args.module) > 1 and args.mode != "report": + print("Can only support one module with mode {}", args.mode) + + mode = args.mode + use_queryview = args.use_queryview + ignore_by_name = args.ignore_by_name + + report_infos = [] + for top_level_module in args.module: + module_adjacency_list, converted = get_module_adjacency_list( + top_level_module, use_queryview, ignore_by_name) + + if mode == "graph": + generate_dot_file(module_adjacency_list, converted, top_level_module) + elif mode == "report": + report_infos.append( + generate_report_data(module_adjacency_list, converted, + top_level_module)) + else: + raise RuntimeError("unknown mode: %s" % mode) + + if mode == "report": + combined_data = combine_report_data(report_infos) + generate_report(combined_data) + + +if __name__ == "__main__": + main() diff --git a/scripts/bp2build-progress/dependency_analysis.py b/scripts/bp2build-progress/dependency_analysis.py new file mode 100644 index 00000000..6987c10e --- /dev/null +++ b/scripts/bp2build-progress/dependency_analysis.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utility functions to produce module or module type dependency graphs using json-module-graph or queryview.""" + +import json +import os +import os.path +import subprocess +import xml.etree.ElementTree + +# This list of module types are omitted from the report and graph +# for brevity and simplicity. Presence in this list doesn't mean +# that they shouldn't be converted, but that they are not that useful +# to be recorded in the graph or report currently. +IGNORED_KINDS = set([ + "license_kind", + "license", + "cc_defaults", + "cc_prebuilt_object", + "cc_prebuilt_library_headers", + "cc_prebuilt_library_shared", + "cc_prebuilt_library_static", + "cc_prebuilt_library_static", + "cc_prebuilt_library", + "java_defaults", + "ndk_prebuilt_static_stl", + "ndk_library", +]) + +SRC_ROOT_DIR = os.path.abspath(__file__ + "/../../../../..") + + +def _build_with_soong(target): + subprocess.check_output( + [ + "build/soong/soong_ui.bash", + "--make-mode", + "--skip-soong-tests", + target, + ], + cwd=SRC_ROOT_DIR, + env={ + # Use aosp_arm as the canonical target product. + "TARGET_PRODUCT": "aosp_arm", + "TARGET_BUILD_VARIANT": "userdebug", + }, + ) + + +def get_queryview_module_info(module): + """Returns the list of transitive dependencies of input module as built by queryview.""" + _build_with_soong("queryview") + + result = subprocess.check_output( + [ + "tools/bazel", "query", "--config=ci", "--config=queryview", + "--output=xml", + 'deps(attr("soong_module_name", "^{}$", //...))'.format(module) + ], + cwd=SRC_ROOT_DIR, + ) + return xml.etree.ElementTree.fromstring(result) + + +def get_json_module_info(module): + """Returns the list of transitive dependencies of input module as provided by Soong's json module graph.""" + _build_with_soong("json-module-graph") + # Run query.sh on the module graph for the top level module + result = subprocess.check_output( + [ + "build/bazel/json_module_graph/query.sh", "fullTransitiveDeps", + "out/soong/module-graph.json", module + ], + cwd=SRC_ROOT_DIR, + ) + return json.loads(result) + + +def get_bp2build_converted_modules(): + """ Returns the list of modules that bp2build can currently convert. """ + _build_with_soong("bp2build") + # Parse the list of converted module names from bp2build + with open( + os.path.join( + SRC_ROOT_DIR, + "out/soong/soong_injection/metrics/converted_modules.txt")) as f: + # Read line by line, excluding comments. + # Each line is a module name. + ret = [line.strip() for line in f.readlines() if not line.startswith("#")] + return set(ret) + + +def get_json_module_type_info(module_type): + """Returns the combined transitive dependency closures of all modules of module_type.""" + _build_with_soong("json-module-graph") + # Run query.sh on the module graph for the top level module type + result = subprocess.check_output( + [ + "build/bazel/json_module_graph/query.sh", + "fullTransitiveModuleTypeDeps", "out/soong/module-graph.json", + module_type + ], + cwd=SRC_ROOT_DIR, + ) + return json.loads(result) + + +def is_windows_variation(module): + """Returns True if input module's variant is Windows. + + Args: + module: an entry parsed from Soong's json-module-graph + """ + dep_variations = module.get("Variations") + dep_variation_os = "" + if dep_variations != None: + dep_variation_os = dep_variations.get("os") + return dep_variation_os == "windows" + + +def ignore_kind(kind): + return kind in IGNORED_KINDS or "defaults" in kind diff --git a/scripts/difftool/.gitignore b/scripts/difftool/.gitignore new file mode 100644 index 00000000..03ba38d4 --- /dev/null +++ b/scripts/difftool/.gitignore @@ -0,0 +1,2 @@ +__pycache__ +*.iml diff --git a/scripts/difftool/BUILD.bazel b/scripts/difftool/BUILD.bazel new file mode 100644 index 00000000..719bc412 --- /dev/null +++ b/scripts/difftool/BUILD.bazel @@ -0,0 +1,38 @@ +filegroup ( + name = "collect_zip", + srcs = [":collect"], + output_group = "python_zip_file", +) + +py_binary( + name = "collect", + srcs = ["collect.py"], + python_version = "PY3", +) + +filegroup ( + name = "difftool_zip", + srcs = [":difftool"], + output_group = "python_zip_file", +) + +py_library( + name = "difftool_commands", + srcs = [ + "clangcompile.py", + "commands.py", + ], +) + +py_test( + name = "difftool_test", + srcs = ["difftool_test.py"], + deps = [":difftool", ":collect"], +) + +py_binary( + name = "difftool", + srcs = ["difftool.py"], + deps = [":difftool_commands", ":collect"], + python_version = "PY3", +) diff --git a/scripts/difftool/README.md b/scripts/difftool/README.md new file mode 100644 index 00000000..a06f725f --- /dev/null +++ b/scripts/difftool/README.md @@ -0,0 +1,42 @@ +# Difftool + +This directory contains tools to compare build artifacts from two separate +build invocations as a way of gauging build correctness and debugging +potential problems with build systems under development. + +# Usage + +Use of these tools requires a multistep process: + +1. Build using legacy build system: + ``` + $ m libc + ``` +2. Collect results to a tmp directory. + ``` + $ ./collect.py out/combined-aosp_flame.ninja \ + out/target/product/flame/obj/libc.so \ + /tmp/legacyFiles + ``` +3. Build using the new build system: + ``` + $ USE_BAZEL_ANALYSIS=1 m libc + ``` +4. Collect results to a tmp directory. + ``` + $ ./collect.py out/combined-aosp_flame.ninja \ + out/target/product/flame/obj/libc.so \ + /tmp/newFiles + ``` +5. Run comparative analysis on the two tmp directories. (See + documentation of difftool.py for exact usage.) + ``` + $ ./difftool.py /tmp/legacyFiles \ + out/target/product/flame/obj/libc.so \ + /tmp/newFiles \ + out/target/product/flame/obj/libc.so + ``` + +Use `./collect.py -h` or `./difftool.py -h` for full usage information of +these subtools. + diff --git a/scripts/difftool/clangcompile.py b/scripts/difftool/clangcompile.py new file mode 100644 index 00000000..8bae76e5 --- /dev/null +++ b/scripts/difftool/clangcompile.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.""" + +"""Helpers pertaining to clang compile actions.""" + +import collections +import difflib +import pathlib +import subprocess +from typing import Callable +from commands import CommandInfo +from commands import flag_repr +from commands import is_flag_starts_with +from commands import parse_flag_groups + + +class ClangCompileInfo(CommandInfo): + """Contains information about a clang compile action commandline.""" + + def __init__(self, tool, args): + CommandInfo.__init__(self, tool, args) + + flag_groups = parse_flag_groups(args, _custom_flag_group) + + misc = [] + i_includes = [] + iquote_includes = [] + isystem_includes = [] + defines = [] + warnings = [] + file_flags = [] + for g in flag_groups: + if is_flag_starts_with("D", g) or is_flag_starts_with("U", g): + defines += [g] + elif is_flag_starts_with("I", g): + i_includes += [g] + elif is_flag_starts_with("isystem", g): + isystem_includes += [g] + elif is_flag_starts_with("iquote", g): + iquote_includes += [g] + elif is_flag_starts_with("W", g) or is_flag_starts_with("w", g): + warnings += [g] + elif (is_flag_starts_with("MF", g) or is_flag_starts_with("o", g) or + _is_src_group(g)): + file_flags += [g] + else: + misc += [g] + self.misc_flags = sorted(misc, key=flag_repr) + self.i_includes = _process_includes(i_includes) + self.iquote_includes = _process_includes(iquote_includes) + self.isystem_includes = _process_includes(isystem_includes) + self.defines = _process_defines(defines) + self.warnings = warnings + self.file_flags = file_flags + + def _str_for_field(self, field_name, values): + s = " " + field_name + ":\n" + for x in values: + s += " " + flag_repr(x) + "\n" + return s + + def __str__(self): + s = "ClangCompileInfo:\n" + s += self._str_for_field("Includes (-I)", self.i_includes) + s += self._str_for_field("Includes (-iquote)", self.iquote_includes) + s += self._str_for_field("Includes (-isystem)", self.isystem_includes) + s += self._str_for_field("Defines", self.defines) + s += self._str_for_field("Warnings", self.warnings) + s += self._str_for_field("Files", self.file_flags) + s += self._str_for_field("Misc", self.misc_flags) + return s + + +def _is_src_group(x): + """Returns true if the given flag group describes a source file.""" + return isinstance(x, str) and x.endswith(".cpp") + + +def _custom_flag_group(x): + """Identifies single-arg flag groups for clang compiles. + + Returns a flag group if the given argument corresponds to a single-argument + flag group for clang compile. (For example, `-c` is a single-arg flag for + clang compiles, but may not be for other tools.) + + See commands.parse_flag_groups documentation for signature details.""" + if x.startswith("-I") and len(x) > 2: + return ("I", x[2:]) + if x.startswith("-W") and len(x) > 2: + return (x) + elif x == "-c": + return x + return None + + +def _process_defines(defs): + """Processes and returns deduplicated define flags from all define args.""" + # TODO(cparsons): Determine and return effective defines (returning the last + # set value). + defines_by_var = collections.defaultdict(list) + for x in defs: + if isinstance(x, tuple): + var_name = x[0][2:] + else: + var_name = x[2:] + defines_by_var[var_name].append(x) + result = [] + for k in sorted(defines_by_var): + d = defines_by_var[k] + for x in d: + result += [x] + return result + + +def _process_includes(includes): + # Drop genfiles directories; makes diffing easier. + result = [] + for x in includes: + if isinstance(x, tuple): + if not x[1].startswith("bazel-out"): + result += [x] + else: + result += [x] + return result + + +# given a file, give a list of "information" about it +ExtractInfo = Callable[[pathlib.Path], list[str]] + + +def _diff(left_path: pathlib.Path, right_path: pathlib.Path, tool_name: str, + tool: ExtractInfo) -> list[str]: + """Returns a list of strings describing differences in `.o` files. + Returns the empty list if these files are deemed "similar enough". + + The given files must exist and must be object (.o) files.""" + errors = [] + + left = tool(left_path) + right = tool(right_path) + comparator = difflib.context_diff(left, right) + difflines = list(comparator) + if difflines: + err = "\n".join(difflines) + errors.append( + f"{left_path}\ndiffers from\n{right_path}\nper {tool_name}:\n{err}") + return errors + + +def _external_tool(*args) -> ExtractInfo: + return lambda file: subprocess.run([*args, str(file)], + check=True, capture_output=True, + encoding="utf-8").stdout.splitlines() + + +# TODO(usta) use nm as a data dependency +def nm_differences(left_path: pathlib.Path, right_path: pathlib.Path) -> list[ + str]: + """Returns differences in symbol tables. + Returns the empty list if these files are deemed "similar enough". + + The given files must exist and must be object (.o) files.""" + return _diff(left_path, right_path, "symbol tables", _external_tool("nm")) + + +# TODO(usta) use readelf as a data dependency +def elf_differences(left_path: pathlib.Path, right_path: pathlib.Path) -> list[ + str]: + """Returns differences in elf headers. + Returns the empty list if these files are deemed "similar enough". + + The given files must exist and must be object (.o) files.""" + return _diff(left_path, right_path, "elf headers", + _external_tool("readelf", "-h")) diff --git a/scripts/difftool/collect.py b/scripts/difftool/collect.py new file mode 100755 index 00000000..b820b2b6 --- /dev/null +++ b/scripts/difftool/collect.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Copies ninja file information to another directory for processing. + +Usage: + ./collect.py [ninja_file] [dest_directory] + +This script should be used as in preparation for further analysis +using difftool.py. See directory-level README for details. +""" + +import argparse +import os +import pathlib +import shutil + + +COLLECTION_INFO_FILENAME = "collection_info" + + +def subninja_files(ninja_file_path): + result = [] + with ninja_file_path.open() as f: + for line in f: + if line.startswith("subninja "): + result += [line[len("subninja "):].strip()] + return result + + +def main(): + parser = argparse.ArgumentParser(description="") + parser.add_argument("ninja_file", + help="the path to the root ninja file of the build " + + "to be analyzed. Ex: out/combined-aosp_flame.ninja") + parser.add_argument("dest_directory", + help="directory to copy build-related information for " + + "later difftool comparison. Ex: /tmp/buildArtifacts") + # TODO(usta): enable multiple files or even a glob to be specified + parser.add_argument("--file", dest="output_file", default=None, + help="the path to the output artifact to be analyzed. " + + "Ex: out/path/to/foo.so") + args = parser.parse_args() + dest = args.dest_directory + + if not os.path.isdir(dest): + raise Exception("invalid destination directory " + dest) + + collection_info_filepath = "" + if args.output_file is not None: + output_file = pathlib.Path(args.output_file) + if not output_file.is_file(): + raise Exception("Expected file %s was not found. " % output_file) + output_file_dest = pathlib.Path(dest).joinpath(output_file) + output_file_dest.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(output_file, output_file_dest) + collection_info_filepath = str(output_file) + + ninja_file = pathlib.Path(args.ninja_file) + main_ninja_basename = ninja_file.name + shutil.copy2(args.ninja_file, os.path.join(dest, main_ninja_basename)) + + for subninja_file in subninja_files(ninja_file): + parent_dir = pathlib.Path(subninja_file).parent + dest_dir = os.path.join(dest, parent_dir) + pathlib.Path(dest_dir).mkdir(parents=True, exist_ok=True) + shutil.copy2(subninja_file, os.path.join(dest, subninja_file)) + + collection_info = main_ninja_basename + "\n" + collection_info_filepath + pathlib.Path(dest).joinpath(COLLECTION_INFO_FILENAME).write_text(collection_info) + + +if __name__ == "__main__": + main() diff --git a/scripts/difftool/commands.py b/scripts/difftool/commands.py new file mode 100644 index 00000000..2c05dad4 --- /dev/null +++ b/scripts/difftool/commands.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.""" + +"""Helper functions and types for command processing for difftool.""" + + +class CommandInfo: + """Contains information about an action commandline.""" + + def __init__(self, tool, args): + self.tool = tool + self.args = args + + def __str__(self): + s = "CommandInfo:\n" + s += " Tool:\n" + s += " " + self.tool + "\n" + s += " Args:\n" + for x in self.args: + s += " " + x + "\n" + return s + + +def parse_flag_groups(args, custom_flag_group=None): + """Returns a list of flag groups based on the given args. + + An arg group consists of one-arg flags, two-arg groups, or positional args. + + Positional arguments (for example `a.out`) are returned as strings in the + list. + One-arg groups consist of a flag with no argument (for example, `--verbose`), + and are returned as a tuple of size one in the list. + Two-arg groups consist of a flag with a single argument (for example, + `--file bar.txt` or `--mode=verbose`), + and are returned as a tuple of size two in the list. + + Also accepts an optional function `custom_flag_group` to determine if a + single arg comprises a group. (custom_flag_group(x) should return a flag + group abiding by the above convention, or None to use non-custom logic. + This may be required to accurately parse arg groups. For example, `-a b` may + be either a one-arg group `-a` followed by a positonal group `b`, or a two-arg + group `-a b`.""" + flag_groups = [] + + i = 0 + while i < len(args): + if custom_flag_group: + g = custom_flag_group(args[i]) + if g is not None: + flag_groups += [g] + i += 1 + continue + + g = one_arg_group(args[i]) + if g is not None: + flag_groups += [g] + i += 1 + continue + + # Look for a two-arg group if there are at least 2 elements left. + if i < len(args) - 1: + g = two_arg_group(args[i], args[i+1]) + if g is not None: + flag_groups += [g] + i += 2 + continue + + # Not a recognized one arg group or two arg group. + if args[i].startswith("-"): + flag_groups += [(args[i])] + else: + flag_groups += [args[i]] + i += 1 + + return flag_groups + + +def remove_hyphens(x): + """Returns the given string with leading '--' or '-' removed.""" + if x.startswith("--"): + return x[2:] + elif x.startswith("-"): + return x[1:] + else: + return x + + +def two_arg_group(a, b): + """Determines whether two consecutive args belong to a single flag group. + + Two arguments belong to a single flag group if the first arg contains + a hyphen and the second does not. For example: `-foo bar` is a flag, + but `foo bar` and `--foo --bar` are not. + + Returns: + A tuple of the two args without hyphens if they belong to a single + flag, or None if they do not. """ + if a.startswith("-") and (not b.startswith("-")): + return (remove_hyphens(a), b) + else: + return None + + +def one_arg_group(x): + """Determines whether an arg comprises a complete flag group. + + An argument comprises a single flag group if it is of the form of + `-key=value` or `--key=value`. + + Returns: + A tuple of `(key, value)` of the flag group, if the arg comprises a + complete flag group, or None if it does not.""" + tokens = x.split("=") + if len(tokens) == 2: + return (remove_hyphens(tokens[0]), tokens[1]) + else: + return None + + +def is_flag_starts_with(prefix, x): + if isinstance(x, tuple): + return x[0].startswith(prefix) + else: + return x.startswith("--" + prefix) or x.startswith("-" + prefix) + + +def flag_repr(x): + if isinstance(x, tuple): + return f"-{x[0]} {x[1]}" + else: + return x + diff --git a/scripts/difftool/difftool.py b/scripts/difftool/difftool.py new file mode 100755 index 00000000..3622f4a6 --- /dev/null +++ b/scripts/difftool/difftool.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provides useful diff information for build artifacts. + +Uses collected build artifacts from two separate build invocations to +compare output artifacts of these builds and/or the commands executed +to generate them. + +See the directory-level README for information about full usage, including +the collection step: a preparatory step required before invocation of this +tool. + +Use `difftool.py --help` for full usage information of this tool. + +Example Usage: + ./difftool.py [left_dir] [left_output_file] [right_dir] [right_output_file] + +Difftool will compare [left_dir]/[left_output_file] and +[right_dir]/[right_output_file] and provide its best insightful analysis on the +differences between these files. The content and depth of this analysis depends +on the types of these files, and also on Difftool"s verbosity mode. Difftool +may also use command data present in the left and right directories as part of +its analysis. +""" + +import argparse +import enum +import functools +import os +import pathlib +import re +import subprocess +import sys +from typing import Callable + +import clangcompile +import commands +from collect import COLLECTION_INFO_FILENAME + +DiffFunction = Callable[[pathlib.Path, pathlib.Path], list[str]] +"""Given two files, produces a list of differences.""" + + +@functools.total_ordering +class DiffLevel(enum.Enum): + """Defines the level of differences that should trigger a failure. + + E.g. when set to WARNING, differences deemed WARNING or SEVERE are taken into + account while other differences (INFO, FINE etc.) will be ignored. + """ + SEVERE = 1 + WARNING = 2 + INFO = 3 + FINE = 4 + + def __lt__(self, other): + if self.__class__ is other.__class__: + return self.value < other.value + return NotImplemented + + +class EnumAction(argparse.Action): + """Parses command line options into Enum types.""" + + def __init__(self, **kwargs): + enum_type = kwargs.pop("type", None) + kwargs.setdefault("choices", list(e.name for e in enum_type)) + super(EnumAction, self).__init__(**kwargs) + self._enum = enum_type + + def __call__(self, parser, namespace, values, option_string=None): + value = self._enum[values] + setattr(namespace, self.dest, value) + + +class ArtifactType(enum.Enum): + CC_OBJECT = 1 + CC_SHARED_LIBRARY = 2 + OTHER = 99 + + +def _artifact_type(file_path): + ext = file_path.suffix + if ext == ".o": + return ArtifactType.CC_OBJECT + elif ext == ".so": + return ArtifactType.CC_SHARED_LIBRARY + else: + return ArtifactType.OTHER + + +# TODO(usta) use libdiff +def literal_diff(left_path: pathlib.Path, right_path: pathlib.Path) -> list[ + str]: + return subprocess.run(["diff", str(left_path), str(right_path)], + check=False, capture_output=True, + encoding="utf-8").stdout.splitlines() + + +@functools.cache +def _diff_fns(artifact_type: ArtifactType, level: DiffLevel) -> list[ + DiffFunction]: + fns = [] + + if artifact_type == ArtifactType.CC_OBJECT: + fns.append(clangcompile.nm_differences) + if level >= DiffLevel.WARNING: + fns.append(clangcompile.elf_differences) + else: + fns.append(literal_diff) + + return fns + + +def collect_commands(ninja_file_path: pathlib.Path, + output_file_path: pathlib.Path) -> list[str]: + """Returns a list of all command lines required to build the file at given + output_file_path_string, as described by the ninja file present at + ninja_file_path_string.""" + + ninja_tool_path = pathlib.Path( + "prebuilts/build-tools/linux-x86/bin/ninja").resolve() + wd = os.getcwd() + os.chdir(ninja_file_path.parent.absolute()) + result = subprocess.check_output([str(ninja_tool_path), + "-f", ninja_file_path.name, + "-t", "commands", + str(output_file_path)]).decode("utf-8") + os.chdir(wd) + return result.splitlines() + + +def file_differences(left_path: pathlib.Path, right_path: pathlib.Path, + level=DiffLevel.SEVERE) -> list[str]: + """Returns differences between the two given files. + Returns the empty list if these files are deemed "similar enough".""" + + errors = [] + if not left_path.is_file(): + errors += ["%s does not exist" % left_path] + if not right_path.is_file(): + errors += ["%s does not exist" % right_path] + if errors: + return errors + + left_type = _artifact_type(left_path) + right_type = _artifact_type(right_path) + if left_type != right_type: + errors += ["file types differ: %s and %s" % (left_type, right_type)] + return errors + + for fn in _diff_fns(left_type, level): + errors += fn(left_path, right_path) + + return errors + + +def parse_collection_info(info_file_path: pathlib.Path): + """Parses the collection info file at the given path and returns details.""" + if not info_file_path.is_file(): + raise Exception("Expected file %s was not found. " % info_file_path + + "Did you run collect.py for this directory?") + + info_contents = info_file_path.read_text().splitlines() + ninja_path = pathlib.Path(info_contents[0]) + target_file = None + + if len(info_contents) > 1 and info_contents[1]: + target_file = info_contents[1] + + return ninja_path, target_file + + +# Pattern to parse out env-setting command prefix, for example: +# +# FOO=BAR KEY=VALUE {main_command_args} +env_set_prefix_pattern = re.compile("^(( )*([^ =]+=[^ =]+)( )*)+(.*)$") + +# Pattern to parse out command prefixes which cd into the execroot and +# then remove the old output. For example: +# +# cd path/to/execroot && rm old_output && {main_command} +cd_rm_prefix_pattern = re.compile("^cd [^&]* &&( )+rm [^&]* && (.*)$") + +# Pattern to parse out any trailing comment suffix. For example: +# +# {main_command} # This comment should be removed. +comment_suffix_pattern = re.compile("(.*) # .*") + + +def rich_command_info(raw_command): + """Returns a command info object describing the raw command string.""" + cmd = raw_command.strip() + # Remove things unrelated to the core command. + m = env_set_prefix_pattern.fullmatch(cmd) + if m is not None: + cmd = m.group(5) + m = cd_rm_prefix_pattern.fullmatch(cmd) + if m is not None: + cmd = m.group(2) + m = comment_suffix_pattern.fullmatch(cmd) + if m is not None: + cmd = m.group(1) + tokens = cmd.split() + tool = tokens[0] + args = tokens[1:] + + if tool.endswith("clang") or tool.endswith("clang++"): + # TODO(cparsons): Disambiguate between clang compile and other clang + # commands. + return clangcompile.ClangCompileInfo(tool=tool, args=args) + else: + return commands.CommandInfo(tool=tool, args=args) + + +def main(): + parser = argparse.ArgumentParser(description="") + parser.add_argument("--level", + action=EnumAction, + default=DiffLevel.SEVERE, + type=DiffLevel, + help="the level of differences to be considered." + + "Diffs below the specified level are ignored.") + parser.add_argument("--verbose", "-v", + action=argparse.BooleanOptionalAction, + default=False, + help="log verbosely.") + parser.add_argument("left_dir", + help="the 'left' directory to compare build outputs " + + "from. This must be the target of an invocation " + + "of collect.py.") + parser.add_argument("--left_file", "-l", dest="left_file", default=None, + help="the output file (relative to execution root) for " + + "the 'left' build invocation.") + parser.add_argument("right_dir", + help="the 'right' directory to compare build outputs " + + "from. This must be the target of an invocation " + + "of collect.py.") + parser.add_argument("--right_file", "-r", dest="right_file", default=None, + help="the output file (relative to execution root) " + + "for the 'right' build invocation.") + parser.add_argument("--allow_missing_file", + action=argparse.BooleanOptionalAction, + default=False, + help="allow a missing output file; this is useful to " + + "compare actions even in the absence of " + + "an output file.") + args = parser.parse_args() + + level = args.level + left_diffinfo = pathlib.Path(args.left_dir).joinpath(COLLECTION_INFO_FILENAME) + right_diffinfo = pathlib.Path(args.right_dir).joinpath( + COLLECTION_INFO_FILENAME) + + left_ninja_name, left_file = parse_collection_info(left_diffinfo) + right_ninja_name, right_file = parse_collection_info(right_diffinfo) + if args.left_file: + left_file = pathlib.Path(args.left_file) + if args.right_file: + right_file = pathlib.Path(args.right_file) + + if left_file is None: + raise Exception("No left file specified. Either run collect.py with a " + + "target file, or specify --left_file.") + if right_file is None: + raise Exception("No right file specified. Either run collect.py with a " + + "target file, or specify --right_file.") + + left_path = pathlib.Path(args.left_dir).joinpath(left_file) + right_path = pathlib.Path(args.right_dir).joinpath(right_file) + if not args.allow_missing_file: + if not left_path.is_file(): + raise RuntimeError("Expected file %s was not found. " % left_path) + if not right_path.is_file(): + raise RuntimeError("Expected file %s was not found. " % right_path) + + file_diff_errors = file_differences(left_path, right_path, level) + + if file_diff_errors: + for err in file_diff_errors: + print(err) + if args.verbose: + left_ninja_path = pathlib.Path(args.left_dir).joinpath(left_ninja_name) + left_commands = collect_commands(left_ninja_path, left_file) + left_command_info = rich_command_info(left_commands[-1]) + right_ninja_path = pathlib.Path(args.right_dir).joinpath(right_ninja_name) + right_commands = collect_commands(right_ninja_path, right_file) + right_command_info = rich_command_info(right_commands[-1]) + print("======== ACTION COMPARISON: ========") + print("=== LEFT:\n") + print(left_command_info) + print() + print("=== RIGHT:\n") + print(right_command_info) + print() + sys.exit(1) + else: + print(f"{left_file} matches\n{right_file}") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/difftool/difftool_test.py b/scripts/difftool/difftool_test.py new file mode 100644 index 00000000..fe3832a1 --- /dev/null +++ b/scripts/difftool/difftool_test.py @@ -0,0 +1,124 @@ +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for difftool.py.""" + +import os +import pathlib +import unittest +import clangcompile +import difftool + + +def get_path(name): + return os.path.join(os.getenv("TEST_TMPDIR"), name) + + +def create_file(name, content): + path = get_path(name) + with open(path, "w") as f: + f.write(content) + return pathlib.Path(path) + + +def _substring_in_list(s, slist): + for elem in slist: + if s in elem: + return True + return False + + +class DifftoolTest(unittest.TestCase): + + def assertNotInErrors(self, expected, errorlist): + if _substring_in_list(expected, errorlist): + self.fail("{!r} found in errors: {!r}".format(expected, errorlist)) + + def assertInErrors(self, expected, errorlist): + if not _substring_in_list(expected, errorlist): + self.fail("{!r} not found in errors: {!r}".format(expected, errorlist)) + + def test_file_differences_not_exist(self): + obj_file = create_file("foo.o", "object contents") + + diffs = difftool.file_differences(pathlib.Path("doesntexist.o"), + obj_file) + self.assertEqual(["doesntexist.o does not exist"], diffs) + + @unittest.skip("TODO(usta)") + def test_file_differences_different_types(self): + obj_file = create_file("foo.o", "object contents") + obj_file_two = create_file("foo2.o", "object contents two") + txt_file = create_file("foo3.txt", "other") + so_file = create_file("bar.so", "shared lib contents") + + diffs = difftool.file_differences(obj_file, so_file) + self.assertInErrors("file types differ", diffs) + + diffs = difftool.file_differences(obj_file, txt_file) + self.assertInErrors("file types differ", diffs) + + diffs = difftool.file_differences(so_file, obj_file) + self.assertInErrors("file types differ", diffs) + + diffs = difftool.file_differences(obj_file, obj_file_two) + self.assertNotInErrors("file types differ", diffs) + + @unittest.skip("TODO(usta)") + def test_object_contents_differ(self): + obj_file = create_file("foo.o", "object contents\none\n") + obj_file_two = create_file("foo2.o", "object contents\ntwo\n") + + diffs = difftool.file_differences(obj_file, obj_file_two) + self.assertNotInErrors("object_contents", diffs) + self.assertInErrors("one", diffs) + self.assertInErrors("two", diffs) + + def test_soong_clang_compile_info(self): + fake_cmd = ("PWD=/proc/self/cwd prebuilts/clang -c -Wall -Wno-unused " + + "foo.cpp -Iframeworks/av/include -Dsomedefine " + + "-misc_flag misc_arg " + + "-o foo.o # comment") + info = difftool.rich_command_info(fake_cmd) + self.assertIsInstance(info, clangcompile.ClangCompileInfo) + self.assertEqual([("I", "frameworks/av/include")], info.i_includes) + self.assertEqual(["-Dsomedefine"], info.defines) + self.assertEqual(["-Wall", "-Wno-unused"], info.warnings) + self.assertEqual(["-c", ("misc_flag", "misc_arg")], info.misc_flags) + self.assertEqual(["foo.cpp", ("o", "foo.o")], info.file_flags) + + def test_bazel_clang_compile_info(self): + fake_cmd = ("cd out/bazel/execroot && rm -f foo.o && " + + "prebuilts/clang -MD -MF bazel-out/foo.d " + + "-iquote . -iquote bazel-out/foo/bin " + + "-I frameworks/av/include " + + "-I bazel-out/frameworks/av/include/bin " + + " -Dsomedefine " + + "-misc_flag misc_arg " + + "-Werror=int-conversion " + + "-Wno-reserved-id-macro " + "-o foo.o # comment") + info = difftool.rich_command_info(fake_cmd) + self.assertIsInstance(info, clangcompile.ClangCompileInfo) + self.assertEqual([("iquote", ".")], info.iquote_includes) + self.assertEqual([("I", "frameworks/av/include")], info.i_includes) + self.assertEqual(["-Dsomedefine"], info.defines) + self.assertEqual(["-Werror=int-conversion", "-Wno-reserved-id-macro"], + info.warnings) + self.assertEqual(["-MD", ("misc_flag", "misc_arg")], info.misc_flags) + self.assertEqual([("MF", "bazel-out/foo.d"), ("o", "foo.o")], info.file_flags) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/gen_build_number.sh b/scripts/gen_build_number.sh new file mode 100755 index 00000000..80085acd --- /dev/null +++ b/scripts/gen_build_number.sh @@ -0,0 +1,33 @@ +#!/bin/bash -e +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Script used as --workspace_status_command. +# Must execute at the root of workspace. +# https://docs.bazel.build/versions/main/command-line-reference.html#flag--workspace_status_command + +if [[ ! -f "WORKSPACE" ]]; then + echo "ERROR: gen_build_number.sh must be executed at the root of Bazel workspace." >&2 + exit 1 +fi + +# TODO(b/228463719): figure out how to get the path properly. +BUILD_NUMBER_FILE=out/soong/build_number.txt +if [[ -f ${BUILD_NUMBER_FILE} ]]; then + BUILD_NUMBER=$(cat ${BUILD_NUMBER_FILE}) +else + BUILD_NUMBER=eng.${USER:0:6}.$(date '+%Y%m%d.%H%M%S') +fi + +echo "BUILD_NUMBER ${BUILD_NUMBER}"
\ No newline at end of file diff --git a/scripts/milestone-2/demo.sh b/scripts/milestone-2/demo.sh index 139f83ce..3f63841f 100755 --- a/scripts/milestone-2/demo.sh +++ b/scripts/milestone-2/demo.sh @@ -61,7 +61,7 @@ function bazel() { # Run the bp2build converter to generate BUILD files into out/soong/bp2build. function generate() { log "Running the bp2build converter.." - GENERATE_BAZEL_FILES=true "${AOSP_ROOT}/build/soong/soong_ui.bash" --make-mode nothing --skip-soong-tests + "${AOSP_ROOT}/build/soong/soong_ui.bash" --make-mode --skip-soong-tests bp2build log "Successfully generated BUILD files in out/soong/bp2build." } diff --git a/scripts/queryview-bottom-up.sh b/scripts/queryview-bottom-up.sh index 1870e8d0..0ad2a1dd 100755 --- a/scripts/queryview-bottom-up.sh +++ b/scripts/queryview-bottom-up.sh @@ -64,6 +64,13 @@ set -euo pipefail +# Convenience function to use the checked-in bazel binary +function bazel() { + # We're in <root>/build/bazel/scripts + AOSP_ROOT="$(dirname $0)/../../.." + "${AOSP_ROOT}/tools/bazel" "$@" +} + T=${1:-//bionic/libc:libc--android_arm_armv7-a-neon_shared} COMMON_BAZEL_OPTS="--noshow_loading_progress --color=no --curses=no" diff --git a/scripts/run_apex_tests.sh b/scripts/run_apex_tests.sh new file mode 100755 index 00000000..f6f5a4f3 --- /dev/null +++ b/scripts/run_apex_tests.sh @@ -0,0 +1,73 @@ +#!/bin/bash -eux +# +# Script to run some local APEX tests while APEX support is WIP and not easily testable on CI + +set -o pipefail + +# TODO: Refactor build/make/envsetup.sh to make gettop() available elsewhere +function gettop +{ + # Function uses potentially uninitialzied variables + set +u + + local TOPFILE=build/bazel/bazel.sh + if [ -n "$TOP" -a -f "$TOP/$TOPFILE" ] ; then + # The following circumlocution ensures we remove symlinks from TOP. + (cd "$TOP"; PWD= /bin/pwd) + else + if [ -f $TOPFILE ] ; then + # The following circumlocution (repeated below as well) ensures + # that we record the true directory name and not one that is + # faked up with symlink names. + PWD= /bin/pwd + else + local HERE=$PWD + local T= + while [ \( ! \( -f $TOPFILE \) \) -a \( "$PWD" != "/" \) ]; do + \cd .. + T=`PWD= /bin/pwd -P` + done + \cd "$HERE" + if [ -f "$T/$TOPFILE" ]; then + echo "$T" + fi + fi + fi + + set -u +} + +AOSP_ROOT=`gettop` + +# Generate BUILD files into out/soong/bp2build +"${AOSP_ROOT}/build/soong/soong_ui.bash" --make-mode BP2BUILD_VERBOSE=1 bp2build --skip-soong-tests + +BUILD_FLAGS_LIST=( + --color=no + --curses=no + --show_progress_rate_limit=5 + --config=bp2build +) +BUILD_FLAGS="${BUILD_FLAGS_LIST[@]}" + +TEST_FLAGS_LIST=( + --keep_going + --test_output=errors +) +TEST_FLAGS="${TEST_FLAGS_LIST[@]}" + +BUILD_TARGETS_LIST=( + //build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal + //system/timezone/apex:com.android.tzdata +) +BUILD_TARGETS="${BUILD_TARGETS_LIST[@]}" + +echo "Building APEXes with Bazel..." +${AOSP_ROOT}/tools/bazel --max_idle_secs=5 build ${BUILD_FLAGS} --platforms //build/bazel/platforms:android_x86 -k ${BUILD_TARGETS} +${AOSP_ROOT}/tools/bazel --max_idle_secs=5 build ${BUILD_FLAGS} --platforms //build/bazel/platforms:android_x86_64 -k ${BUILD_TARGETS} +${AOSP_ROOT}/tools/bazel --max_idle_secs=5 build ${BUILD_FLAGS} --platforms //build/bazel/platforms:android_arm -k ${BUILD_TARGETS} +${AOSP_ROOT}/tools/bazel --max_idle_secs=5 build ${BUILD_FLAGS} --platforms //build/bazel/platforms:android_arm64 -k ${BUILD_TARGETS} + +set +x +echo +echo "All tests passed, you are awesome!" diff --git a/scripts/run_presubmits.sh b/scripts/run_presubmits.sh index f999df04..d70d9c79 100755 --- a/scripts/run_presubmits.sh +++ b/scripts/run_presubmits.sh @@ -4,7 +4,7 @@ set -o pipefail if [[ ! -d "build/bazel/ci" ]]; then echo "Please run this script from TOP". - exit -1 + exit 1 fi echo "Running presubmit scripts..." diff --git a/tests/apex/BUILD b/tests/apex/BUILD new file mode 100644 index 00000000..560d4b85 --- /dev/null +++ b/tests/apex/BUILD @@ -0,0 +1,44 @@ +load(":apex_diff_test.bzl", "apex_diff_test") +load(":apex_test.bzl", "apex_compression_test") +load(":apex_aab_test.bzl", "apex_aab_test") + +apex_diff_test( + name = "com.android.tzdata", + apex1 = "//system/timezone/apex:com.android.tzdata.apex", + apex2 = "@make_injection//:target/product/generic/system/apex/com.android.tzdata.apex", +) + +apex_diff_test( + name = "build.bazel.examples.apex.minimal", + apex1 = "//build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal.apex", + apex2 = "@make_injection//:target/product/generic/system/product/apex/build.bazel.examples.apex.minimal.apex", +) + +apex_diff_test( + name = "com.android.adbd_uncompressed", + apex1 = "//packages/modules/adb/apex:com.android.adbd.apex", + apex2 = "@make_injection//:target/product/generic/system/apex/com.android.adbd.capex", +) + +apex_diff_test( + name = "com.android.adbd_compressed", + apex1 = "//packages/modules/adb/apex:com.android.adbd.capex", + apex2 = "@make_injection//:target/product/generic/system/apex/com.android.adbd.capex", +) + +apex_compression_test( + name = "build.bazel.examples.apex.minimal_apex", + apex = "//build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal.apex", + compressed = False, +) + +apex_compression_test( + name = "build.bazel.examples.apex.minimal_capex", + apex = "//build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal_compressed.capex", + compressed = True, +) + +apex_aab_test( + name = "build.bazel.examples.apex.minimal_mainline-module", + apex = "//build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal", +)
\ No newline at end of file diff --git a/tests/apex/apex_aab_test.bzl b/tests/apex/apex_aab_test.bzl new file mode 100644 index 00000000..566f4313 --- /dev/null +++ b/tests/apex/apex_aab_test.bzl @@ -0,0 +1,52 @@ +""" +Copyright (C) 2022 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +load("//build/bazel/rules/apex:mainline_modules.bzl", "apex_aab") + +def apex_aab_test(name, apex, **kwargs): + """Diff the .aab generated by Bazel and Soong""" + + aab_name = name + "_apex_aab" + apex_aab( + name = aab_name, + mainline_module = apex, + ) + + native.sh_library( + name = name + "_wrapper_sh_lib", + data = [ + ":" + aab_name, + "build.bazel.examples.apex.minimal.aab", + ], + ) + + args = [ + "$(location //build/bazel/tests/apex:" + aab_name + ")", + "$(location build.bazel.examples.apex.minimal.aab)", + ] + + native.sh_test( + name = name, + srcs = ["apex_aab_test.sh"], + deps = ["@bazel_tools//tools/bash/runfiles"], + data = [ + ":" + name + "_wrapper_sh_lib", + "@bazel_tools//tools/zip:zipper", + ":" + aab_name, + "build.bazel.examples.apex.minimal.aab", + ], + args = args, + ) diff --git a/tests/apex/apex_aab_test.sh b/tests/apex/apex_aab_test.sh new file mode 100755 index 00000000..d28cea4b --- /dev/null +++ b/tests/apex/apex_aab_test.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -xeuo pipefail + +readonly arg_aab_filepath=$1 +readonly arg_prebuilt_aab_filepath=$2 + +readonly ZIPPER=$(rlocation bazel_tools/tools/zip/zipper/zipper) +readonly -a AAB_FILES=( + "BundleConfig.pb" + "base/apex.pb" + "base/apex/arm64-v8a.build_info.pb" + "base/apex/arm64-v8a.img" + "base/apex/armeabi-v7a.build_info.pb" + "base/apex/armeabi-v7a.img" + "base/apex/x86.build_info.pb" + "base/apex/x86.img" + "base/apex/x86_64.build_info.pb" + "base/apex/x86_64.img" + "base/manifest/AndroidManifest.xml" + "base/root/apex_manifest.pb" +) +readonly -a EXCLUDE_FILES=( + # The following files are 1)not in bazel built abb file; 2)not same as the + # ones created by Soong, so exclude them in diff to make the test case pass. + #(TODO: b/190817312) assets/NOTICE.html.gz is not in bazel built aab file. + "assets" + "NOTICE.html.gz" + #(TODO: b/222587783) base/assets.pb is not in bazel built aab file + "assets.pb" + #(TODO: b/222588072) all .img files are different + "*.img" + #(TODO: b/222588241) all .build_info.pb files are different + "*.build_info.pb" + #(TODO: b/222588061) base/root/apex_manifest.pb + "apex_manifest.pb" + #(TODO: b/222587792) base/manifest/AndroidManifest.xml + # two bytes are different, prebuilt has 0x20, bazel built has 0x1f + "AndroidManifest.xml" +) + +# Check if .aab file contains specified files +function aab_contains_files() { + local aab_filepath=$1 + shift + local expected_files=("$@") + local aab_entries=$($ZIPPER v "$aab_filepath") + for file in "${expected_files[@]}"; do + if ! echo -e "$aab_entries" | grep "$file"; then + echo "Failed to find file $file in $aab_filepath" + exit 1 + fi + done +} + +# Test .aab file contains required files +function test_aab_contains_required_files() { + if [ "${arg_aab_filepath: -4}" != ".aab" ]; then + echo "@arg_aab_filepath does not have .aab as extension." + exit 1 + fi + aab_contains_files "$arg_aab_filepath" "${AAB_FILES[@]}" +} + +function test_aab_files_diff() { + local prebuilt_aab_file_dir=$(dirname "$arg_prebuilt_aab_filepath") + + local extracted_prebuilt_aab_dir=$(mktemp -d -p "$prebuilt_aab_file_dir" prebuilt_XXXXXX) + $ZIPPER x "$arg_prebuilt_aab_filepath" -d "$extracted_prebuilt_aab_dir" + + local extracted_aab_dir=$(mktemp -d -p "$prebuilt_aab_file_dir" aab_XXXXXX) + $ZIPPER x "$arg_aab_filepath" -d "$extracted_aab_dir" + + local diff_exclude= + for pattern in "${EXCLUDE_FILES[@]}"; do + diff_exclude="$diff_exclude -x $pattern" + done + + if ! diff -w $diff_exclude -r $extracted_prebuilt_aab_dir $extracted_aab_dir; then + echo ".aab file content is not same as the prebuilt one." + exit 1 + fi + + rm -rf "${extracted_prebuilt_aab_dir}" + rm -rf "${extracted_aab_dir}" +} + +test_aab_contains_required_files +test_aab_files_diff + +echo "Passed all test cases."
\ No newline at end of file diff --git a/tests/apex/apex_diff_test.bzl b/tests/apex/apex_diff_test.bzl new file mode 100644 index 00000000..e809f943 --- /dev/null +++ b/tests/apex/apex_diff_test.bzl @@ -0,0 +1,50 @@ +load("@bazel_skylib//rules:diff_test.bzl", "diff_test") + +def apex_diff_test(name, apex1, apex2, expected_diff=None, **kwargs): + """A test that compares the content list of two APEXes, determined by `deapexer`.""" + + native.genrule( + name = name + "_apex1_deapex", + tools = [ + "@make_injection//:host/linux-x86/bin/deapexer", + "//external/e2fsprogs/debugfs:debugfs", + ], + srcs = [apex1], + outs = [name + ".apex1.txt"], + cmd = "$(location @make_injection//:host/linux-x86/bin/deapexer) --debugfs_path=$(location //external/e2fsprogs/debugfs:debugfs) list $< > $@", + ) + + native.genrule( + name = name + "_apex2_deapex", + tools = [ + "@make_injection//:host/linux-x86/bin/deapexer", + "//external/e2fsprogs/debugfs:debugfs", + ], + srcs = [apex2], + outs = [name + ".apex2.txt"], + cmd = "$(location @make_injection//:host/linux-x86/bin/deapexer) --debugfs_path=$(location //external/e2fsprogs/debugfs:debugfs) list $< > $@", + ) + + if expected_diff == None: + diff_test( + name = name + "_content_diff_test", + file1 = name + ".apex1.txt", + file2 = name + ".apex2.txt", + ) + else: + # Make our own diff to compare against the expected one + native.genrule( + name = name + "_apex1_apex2_diff", + srcs = [ + name + ".apex1.txt", + name + ".apex2.txt", + ], + outs = [name + ".apex1.apex2.diff.txt"], + # Expected to generate a diff (and return a failing exit status) + cmd_bash = "diff $(SRCS) > $@ || true", + ) + diff_test( + name = name + "_content_diff_test", + file1 = name + ".apex1.apex2.diff.txt", + file2 = expected_diff, + ) diff --git a/tests/apex/apex_test.bzl b/tests/apex/apex_test.bzl new file mode 100644 index 00000000..a9181ec1 --- /dev/null +++ b/tests/apex/apex_test.bzl @@ -0,0 +1,42 @@ +""" +Copyright (C) 2022 The Android Open Source Project + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +def apex_compression_test(name, apex, compressed, **kwargs): + """This verifies APEX or compressed APEX file: + 1) has the correct file extension name + 2) contains the required files specified by the APEX file format + """ + + native.sh_library( + name = name + "_wrapper_sh_lib", + data = [apex], + ) + + args = ["$(location " + apex + ")"] + if compressed: + args.append("compressed") + + native.sh_test( + name = name, + srcs = ["apex_test.sh"], + deps = ["@bazel_tools//tools/bash/runfiles"], + data = [ + ":" + name + "_wrapper_sh_lib", + "@bazel_tools//tools/zip:zipper", + apex, + ], + args = args, + ) diff --git a/tests/apex/apex_test.sh b/tests/apex/apex_test.sh new file mode 100755 index 00000000..78f89eb6 --- /dev/null +++ b/tests/apex/apex_test.sh @@ -0,0 +1,88 @@ +#!/bin/bash + +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -xeuo pipefail + +readonly arg_apex_filepath=$1 +arg_compressed=false +[ $# -eq 2 ] && [ "$2" = "compressed" ] && arg_compressed=true + +readonly ZIPPER=$(rlocation bazel_tools/tools/zip/zipper/zipper) +readonly -a APEX_FILES=( + "apex_manifest.pb" + "AndroidManifest.xml" + "apex_payload.img" + "apex_pubkey" + "META-INF/CERT\.SF" + "META-INF/CERT\.RSA" + "META-INF/MANIFEST\.MF" +) +readonly -a CAPEX_FILES=( + "apex_manifest.pb" + "AndroidManifest.xml" + "original_apex" + "apex_pubkey" + "META-INF/CERT\.SF" + "META-INF/CERT\.RSA" + "META-INF/MANIFEST\.MF" +) + +# Check if apex file contains specified files +function apex_contains_files() { + local apex_filepath=$1 + shift + local expected_files=("$@") + local apex_entries=$($ZIPPER v "$apex_filepath") + for file in "${expected_files[@]}"; do + if ! echo -e "$apex_entries" | grep "$file"; then + echo "Failed to find file $file in $apex_filepath" + exit 1 + fi + done +} + +# Test compressed apex file required files. +function test_capex_contains_required_files() { + if [ "${arg_apex_filepath: -6}" != ".capex" ]; then + echo "@arg_apex_filepath does not have .capex as extension." + exit 1 + fi + apex_contains_files "$arg_apex_filepath" "${CAPEX_FILES[@]}" + + # Check files in original_apex extracted from the compressed apex file + local apex_file_dir=$(dirname "$arg_apex_filepath") + local extracted_capex=$(mktemp -d -p "$apex_file_dir") + $ZIPPER x "$arg_apex_filepath" -d "$extracted_capex" + apex_contains_files "$extracted_capex/original_apex" "${APEX_FILES[@]}" + rm -rf "${extracted_capex}" +} + +# Test apex file contains required files +function test_apex_contains_required_files() { + if [ "${arg_apex_filepath: -5}" != ".apex" ]; then + echo "@arg_apex_filepath does not have .apex as extension." + exit 1 + fi + apex_contains_files "$arg_apex_filepath" "${APEX_FILES[@]}" +} + +if [ $arg_compressed == true ]; then + test_capex_contains_required_files +else + test_apex_contains_required_files +fi + +echo "Passed all test cases."
\ No newline at end of file diff --git a/tests/apex/build.bazel.examples.apex.minimal.aab b/tests/apex/build.bazel.examples.apex.minimal.aab Binary files differnew file mode 100644 index 00000000..4edd3a20 --- /dev/null +++ b/tests/apex/build.bazel.examples.apex.minimal.aab diff --git a/tests/bionic/BUILD b/tests/bionic/BUILD index b2a04fd8..8c6e3236 100644 --- a/tests/bionic/BUILD +++ b/tests/bionic/BUILD @@ -1,10 +1,29 @@ +load("@soong_injection//cc_toolchain:constants.bzl", "constants") + # This test requires bp2build to run and the generated BUILD files in the source tree. sh_test( name = "verify_bionic_outputs", srcs = ["verify_bionic_outputs.sh"], data = [ - "//bionic/linker:ld-android", + "//bionic/libc", + "//bionic/libc:libc_bp2build_cc_library_static", "//bionic/libdl:libdl_android", + "//bionic/libdl:libdl_android_bp2build_cc_library_static", + "//bionic/linker:ld-android", + "//bionic/linker:ld-android_bp2build_cc_library_static", + "//prebuilts/clang/host/linux-x86:test_tools", + ], + env = {"CLANG_DEFAULT_VERSION": constants.CLANG_DEFAULT_VERSION}, + deps = ["@bazel_tools//tools/bash/runfiles"], +) + +sh_test( + name = "compare_libc_stripping", + srcs = ["compare_libc_stripping.sh"], + data = [ + "//bionic/libc", + "//bionic/libc:libc_unstripped", ], + env = {"CLANG_DEFAULT_VERSION": constants.CLANG_DEFAULT_VERSION}, deps = ["@bazel_tools//tools/bash/runfiles"], ) diff --git a/tests/bionic/compare_libc_stripping.sh b/tests/bionic/compare_libc_stripping.sh new file mode 100755 index 00000000..928d0eee --- /dev/null +++ b/tests/bionic/compare_libc_stripping.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# +# Copyright 2021 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash" + +# Smoke test to check that the stripped libc.so is smaller than the unstripped one. +function test_libc_stripping_basic() { + local readonly base="__main__/bionic/libc" + local readonly stripped_path="${base}/libc.so" + local readonly unstripped_path="${base}/liblibc_unstripped.so" + local stripped="$(rlocation $stripped_path)" + local unstripped="$(rlocation $unstripped_path)" + + if [ ! -e "$stripped" ]; then + >&2 echo "Missing stripped file; expected '$stripped_path'; got '$stripped'" + exit 2 + fi + if [ ! -e "$unstripped" ]; then + >&2 echo "Missing unstripped file; expected '$unstripped_path'; got '$unstripped'" + exit 2 + fi + + local stripped_size=$(stat -c %s "${stripped}") + local unstripped_size=$(stat -c %s "${unstripped}") + + # Check that the unstripped size is not greater or equal to the stripped size. + if [ "${stripped_size}" -ge "${unstripped_size}" ]; then + echo "Expected the size of stripped libc.so to be strictly smaller than the unstripped one." + exit 1 + fi +} + +test_libc_stripping_basic diff --git a/tests/bionic/verify_bionic_outputs.sh b/tests/bionic/verify_bionic_outputs.sh index cc743039..0001ff56 100755 --- a/tests/bionic/verify_bionic_outputs.sh +++ b/tests/bionic/verify_bionic_outputs.sh @@ -18,43 +18,51 @@ set -euo pipefail source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash" +READELF="$(rlocation __main__/prebuilts/clang/host/linux-x86/${CLANG_DEFAULT_VERSION}/bin/llvm-readelf)" +NM="$(rlocation __main__/prebuilts/clang/host/linux-x86/${CLANG_DEFAULT_VERSION}/bin/llvm-nm)" + # This should be abstracted to a unit-test library when it has more uses. function assert_contains_regex() { local needle="$1" local haystack="$2" - local message="${3:-Expected regexp "$needle" not found in "$haystack"}" + local message="${3:-Expected regexp "$needle" not found in\n"$haystack"}" echo "${haystack}" | grep "${needle}" && return 0 - echo "$message" + echo -e "$message" exit 1 } -# Test that a library is the expected filetype. -function test_filetype() { +# Test that a library is a static library. +function test_is_static_library() { local filepath="$(readlink -f $1)"; shift - local regex="$1"; shift - local file_output="$(file ${filepath})" - assert_contains_regex "${regex}" "${file_output}" + local metadata="$($READELF -h ${filepath})" + assert_contains_regex "Type:.*REL (Relocatable file)" "${metadata}" +} + +# Test that a library is a shared library. +function test_is_shared_library() { + local filepath="$(readlink -f $1)"; shift + local metadata="$($READELF -h ${filepath})" + assert_contains_regex "Type:.*DYN (Shared object file)" "${metadata}" } # Test that the shared library contains a symbol function test_shared_library_symbols() { local filepath="$(readlink -f $1)"; shift - local symbols="$1"; shift - local nm_output="$(nm -D "${filepath}")" - for symbol in "${symbols[@]}" - do + local symbols=("$@"); shift + local nm_output="$($NM -D "${filepath}")" + for symbol in "${symbols[@]}"; do assert_contains_regex "${symbol}" "${nm_output}" done } # Test file contents of //bionic/linker:ld-android function test_ld-android() { - local shared_library="$(rlocation __main__/bionic/linker/libld-android_bp2build_cc_library_shared.so)" - local static_library="$(rlocation __main__/bionic/linker/libld-android_bp2build_cc_library_static_mainlib.a)" + local shared_library="$(rlocation __main__/bionic/linker/ld-android.so)" + local static_library="$(rlocation __main__/bionic/linker/libld-android_bp2build_cc_library_static.a)" - test_filetype "${shared_library}" "shared object.*dynamically linked" - test_filetype "${static_library}" "current ar archive" + test_is_shared_library "${shared_library}" + test_is_static_library "${static_library}" symbols=( __loader_add_thread_local_dtor @@ -82,15 +90,15 @@ function test_ld-android() { _db_dlactivity ) - test_shared_library_symbols "${shared_library}" "${symbols}" + test_shared_library_symbols "${shared_library}" "${symbols[@]}" } function test_libdl_android() { - local shared_library="$(rlocation __main__/bionic/libdl/liblibdl_android_bp2build_cc_library_shared.so)" - local static_library="$(rlocation __main__/bionic/libdl/liblibdl_android_bp2build_cc_library_static_mainlib.a)" + local shared_library="$(rlocation __main__/bionic/libdl/libdl_android.so)" + local static_library="$(rlocation __main__/bionic/libdl/liblibdl_android_bp2build_cc_library_static.a)" - test_filetype "${shared_library}" "shared object.*dynamically linked" - test_filetype "${static_library}" "current ar archive" + test_is_shared_library "${shared_library}" + test_is_static_library "${static_library}" symbols=( android_create_namespace @@ -101,18 +109,52 @@ function test_libdl_android() { android_link_namespaces android_set_application_target_sdk_version android_update_LD_LIBRARY_PATH - __loader_android_create_namespace - __loader_android_dlwarning - __loader_android_get_exported_namespace - __loader_android_get_LD_LIBRARY_PATH - __loader_android_init_anonymous_namespace - __loader_android_link_namespaces - __loader_android_set_application_target_sdk_version - __loader_android_update_LD_LIBRARY_PATH ) - test_shared_library_symbols "${shared_library}" "${symbols}" + test_shared_library_symbols "${shared_library}" "${symbols[@]}" +} + +function test_libc() { + local shared_library="$(rlocation __main__/bionic/libc/libc.so)" + local static_library="$(rlocation __main__/bionic/libc/liblibc_bp2build_cc_library_static.a)" + + test_is_shared_library "${shared_library}" + test_is_static_library "${static_library}" + + symbols=( + __libc_get_static_tls_bounds + __libc_register_thread_exit_callback + __libc_iterate_dynamic_tls + __libc_register_dynamic_tls_listeners + android_reset_stack_guards + ffsl + ffsll + pidfd_getfd + pidfd_open + pidfd_send_signal + process_madvise + _Unwind_Backtrace # apex llndk + _Unwind_DeleteException # apex llndk + _Unwind_Find_FDE # apex llndk + _Unwind_FindEnclosingFunction # apex llndk + _Unwind_GetCFA # apex llndk + _Unwind_GetDataRelBase # apex llndk + _Unwind_GetGR # apex llndk + _Unwind_GetIP # apex llndk + _Unwind_GetIPInfo # apex llndk + _Unwind_GetLanguageSpecificData # apex llndk + _Unwind_GetRegionStart # apex llndk + _Unwind_GetTextRelBase # apex llndk + _Unwind_RaiseException # apex llndk + _Unwind_Resume # apex llndk + _Unwind_Resume_or_Rethrow # apex llndk + _Unwind_SetGR # apex llndk + _Unwind_SetIP # apex llndk + ) + + test_shared_library_symbols "${shared_library}" "${symbols[@]}" } test_ld-android test_libdl_android +test_libc diff --git a/tests/rules/BUILD b/tests/rules/BUILD new file mode 100644 index 00000000..28fd9c3b --- /dev/null +++ b/tests/rules/BUILD @@ -0,0 +1,33 @@ +load("//build/bazel/rules:sh_binary.bzl", "sh_binary") + +sh_library( + name = "lib1", + srcs = ["lib1.sh"], +) + +sh_library( + name = "lib2", + srcs = ["lib2.sh"], + deps = [":lib3"], +) + +sh_library( + name = "lib3", + srcs = ["lib3.sh"], +) + +sh_binary( + name = "bin_with_deps", + srcs = ["bin_with_deps.sh"], + deps = [ + "lib1", + "lib2", + ], +) + +genrule( + name = "test_bin_with_deps", + outs = ["out.txt"], + cmd = "$(location :bin_with_deps) > $@", + tools = [":bin_with_deps"], +) diff --git a/tests/rules/bin_with_deps.sh b/tests/rules/bin_with_deps.sh new file mode 100755 index 00000000..c906dd4f --- /dev/null +++ b/tests/rules/bin_with_deps.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +source $0.runfiles/__main__/build/bazel/tests/rules/lib1.sh +source $0.runfiles/__main__/build/bazel/tests/rules/lib2.sh +source $0.runfiles/__main__/build/bazel/tests/rules/lib3.sh + +lib1 +lib2 +lib3 diff --git a/tests/rules/lib1.sh b/tests/rules/lib1.sh new file mode 100755 index 00000000..d2fa335a --- /dev/null +++ b/tests/rules/lib1.sh @@ -0,0 +1,3 @@ +function lib1() { + echo lib1 +} diff --git a/tests/rules/lib2.sh b/tests/rules/lib2.sh new file mode 100755 index 00000000..39f2c7bb --- /dev/null +++ b/tests/rules/lib2.sh @@ -0,0 +1,3 @@ +function lib2() { + echo lib2 +} diff --git a/tests/rules/lib3.sh b/tests/rules/lib3.sh new file mode 100755 index 00000000..7ff0b776 --- /dev/null +++ b/tests/rules/lib3.sh @@ -0,0 +1,3 @@ +function lib3() { + echo lib3 +} diff --git a/vendor/google/BUILD b/vendor/google/BUILD new file mode 100644 index 00000000..ce0d6e8e --- /dev/null +++ b/vendor/google/BUILD @@ -0,0 +1,19 @@ +load("//build/bazel/rules/apex:mainline_modules.bzl", "apex_aab") + +modules = [ + "//build/bazel/examples/apex/minimal:build.bazel.examples.apex.minimal", +] +name_label_map = {module[module.index(":") + 1:]: module for module in modules} + +[ + apex_aab( + name = "%s_apex_aab" % name, + mainline_module = label, + ) + for name, label in name_label_map.items() +] + +filegroup( + name = "mainline_modules", + srcs = ["%s_apex_aab" % name for name, label in name_label_map.items()], +) diff --git a/vendor/google/build_mainline_modules.sh b/vendor/google/build_mainline_modules.sh new file mode 100755 index 00000000..3be4da02 --- /dev/null +++ b/vendor/google/build_mainline_modules.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BAZEL=tools/bazel + +function main() { + if [ ! -e "build/make/core/Makefile" ]; then + echo "$0 must be run from the top of the Android source tree." + exit 1 + fi + "build/soong/soong_ui.bash" --build-mode --all-modules --dir="$(pwd)" bp2build USE_BAZEL_ANALYSIS= + ${BAZEL} build //build/bazel/vendor/google:mainline_modules --config=bp2build +} + +main |