diff options
| author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 05:01:05 +0000 |
|---|---|---|
| committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 05:01:05 +0000 |
| commit | 17dfec23daa64ed4b98fbe4d9d31b6b7220c5b97 (patch) | |
| tree | 34d720e66745b8daf1632bb1fcd89fb9c0a40894 | |
| parent | 2495809a0455cb320b2699d2c04474d4ad696df4 (diff) | |
| parent | fe4d9ac26508ce8bf14d0bb56f97499664f561a4 (diff) | |
| download | bazelbuild-rules_android-android14-mainline-permission-release.tar.gz | |
Snap for 10453563 from fe4d9ac26508ce8bf14d0bb56f97499664f561a4 to mainline-permission-releaseaml_per_341614000aml_per_341510010aml_per_341410020aml_per_341311000aml_per_341110020aml_per_341110010aml_per_341011100aml_per_341011020aml_per_340916010android14-mainline-permission-release
Change-Id: I3cd4d28d541a3ced57df2ada87829e8aeb137426
225 files changed, 16746 insertions, 507 deletions
diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 0000000..a7a9f55 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,2 @@ +common --experimental_google_legacy_api +common --experimental_enable_android_migration_apis diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ac51a05 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bazel-* @@ -0,0 +1,55 @@ +load("@bazel_gazelle//:def.bzl", "gazelle") +load("@rules_license//rules:license.bzl", "license") + +package( + default_visibility = ["//visibility:public"], + default_applicable_licenses = [":license"], +) + +license( + name = "license", + package_name = "bazelbuild/rules_android", + copyright_notice = "Copyright © 2023 The Bazel Authors. All rights reserved.", + license_kinds = [ + "@rules_license//licenses/spdx:Apache-2.0", + ], + license_text = "LICENSE", +) + +# gazelle:prefix github.com/bazelbuild/rules_android +gazelle(name = "gazelle") + +# Common default platform definitions for use by Android projects. + +platform( + name = "x86", + constraint_values = [ + "@platforms//os:android", + "@platforms//cpu:x86_32", + ], +) + +platform( + name = "x86_64", + constraint_values = [ + "@platforms//os:android", + "@platforms//cpu:x86_64", + ], +) + +platform( + name = "armeabi-v7a", + constraint_values = [ + "@platforms//os:android", + "@platforms//cpu:armv7", + ], +) + +platform( + name = "arm64-v8a", + constraint_values = + [ + "@platforms//cpu:arm64", + "@platforms//os:android", + ], +) @@ -12,7 +12,7 @@ third_party { type: GIT value: "https://github.com/bazelbuild/rules_android" } - version: "ab13c86fafc79b965b7ad6e4d91c821760d869d3" - last_upgrade_date { year: 2021 month: 2 day: 12 } + version: "a51e0d5a49ebdc5051c1eca467272d794aaf6d42" + last_upgrade_date { year: 2023 month: 3 day: 20 } license_type: NOTICE } diff --git a/MODULE.bazel b/MODULE.bazel new file mode 100644 index 0000000..e5a32bd --- /dev/null +++ b/MODULE.bazel @@ -0,0 +1,48 @@ +module( + name = "bazel_build_rules_android", + version = "0.2.0", +) + +bazel_dep(name = "platforms", version = "0.0.5") +bazel_dep(name = "rules_java", version = "5.3.5") +rules_java_toolchains = use_extension("@rules_java//java:extensions.bzl", "toolchains") +use_repo(rules_java_toolchains, "remote_java_tools") + +bazel_dep(name = "protobuf", version = "3.19.0", repo_name = "com_google_protobuf") +bazel_dep(name = "rules_jvm_external", version = "4.5") +bazel_dep(name = "bazel_skylib", version = "1.0.3") + +register_toolchains("//toolchains/android:all") +register_toolchains("//toolchains/android_sdk:all") +register_toolchains("//toolchains/emulator:all") + +# go-related dependency setup +bazel_dep(name = "rules_go", version = "0.34.0", repo_name = "io_bazel_rules_go") +bazel_dep(name = "gazelle", version = "0.28.0") +go_sdk = use_extension("@io_bazel_rules_go//go:extensions.bzl", "go_sdk") +go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") +go_deps.from_file(go_mod = "//:go.mod") +use_repo( + go_deps, + "org_golang_google_protobuf", + "com_github_google_go_cmp", + "org_golang_x_sync", +) +maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven") +maven.install( + name = "rules_android_maven", + artifacts = [ + "com.android.tools.build:bundletool:1.6.1", + ], + repositories = [ + "https://maven.google.com", + "https://repo1.maven.org/maven2", + ], +) +use_repo( + maven, + "rules_android_maven" +) + +remote_android_extensions = use_extension("@bazel_tools//tools/android:android_extensions.bzl", "remote_android_tools_extensions") +use_repo(remote_android_extensions, "android_tools", "android_gmaven_r8") @@ -4,7 +4,8 @@ NOTE: This branch contains a development preview of the Starlark implementation of Android rules for Bazel. This code is incomplete and may not function as-is. -A version of Bazel built at or near head and the following flags are necessary to use these rules: +A version of Bazel built at or near head or a recent pre-release and the following flags are necessary to use these rules: + ``` --experimental_enable_android_migration_apis --experimental_google_legacy_api @@ -24,26 +25,36 @@ tree](https://source.bazel.build/bazel/+/master:src/main/java/com/google/devtool For the list of Android rules, see the Bazel [documentation](https://docs.bazel.build/versions/master/be/android.html). ## Getting Started -To use the new Bazel Android rules, add the following to your WORKSPACE file: +To use the Starlark Bazel Android rules, add the following to your WORKSPACE file: load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + + # Or a later commit + RULES_ANDROID_COMMIT= "0bf3093bd011acd35de3c479c8990dd630d552aa" + RULES_ANDROID_SHA = "b75a673a66c157138ab53f4d8612a6e655d38b69bb14207c1a6675f0e10afa61" http_archive( name = "build_bazel_rules_android", - urls = ["https://github.com/bazelbuild/rules_android/archive/refs/heads/pre-alpha.zip"], - strip_prefix = "rules_android-pre-alpha", + url = "https://github.com/bazelbuild/rules_android/archive/%s.zip" % RULES_ANDROID_COMMIT, + sha256 = RULES_ANDROID_SHA, + strip_prefix = "rules_android-%s" % RULES_ANDROID_COMMIT, ) + load("@build_bazel_rules_android//:prereqs.bzl", "rules_android_prereqs") + rules_android_prereqs() load("@build_bazel_rules_android//:defs.bzl", "rules_android_workspace") rules_android_workspace() - + register_toolchains( - "@build_bazel_rules_android//toolchains/android:android_default_toolchain", - "@build_bazel_rules_android//toolchains/android_sdk:android_sdk_tools", + "@build_bazel_rules_android//toolchains/android:android_default_toolchain", + "@build_bazel_rules_android//toolchains/android_sdk:android_sdk_tools", ) - Then, in your BUILD files, import and use the rules: - load("@build_bazel_rules_android//rules:rules.bzl", "android_library") + load("@build_bazel_rules_android//rules:rules.bzl", "android_binary", "android_library") + android_binary( + ... + ) + android_library( ... ) @@ -1,3 +1,25 @@ workspace(name = "build_bazel_rules_android") -register_toolchains("//android/toolchains/emulator:all") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +maybe( + android_sdk_repository, + name = "androidsdk", +) + +maybe( + android_ndk_repository, + name = "androidndk", +) + +load("prereqs.bzl", "rules_android_prereqs") +rules_android_prereqs() + +load("defs.bzl", "rules_android_workspace") + +rules_android_workspace() + +register_toolchains("//toolchains/android:all") +register_toolchains("//toolchains/android_sdk:all") +register_toolchains("//toolchains/emulator:all") diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod new file mode 100644 index 0000000..0aca21e --- /dev/null +++ b/WORKSPACE.bzlmod @@ -0,0 +1,14 @@ +workspace(name = "build_bazel_rules_android") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +maybe( + android_sdk_repository, + name = "androidsdk", +) + +maybe( + android_ndk_repository, + name = "androidndk", +) diff --git a/android/BUILD b/android/BUILD new file mode 100644 index 0000000..4db0c10 --- /dev/null +++ b/android/BUILD @@ -0,0 +1,18 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Package used for redirecting Starlark rules from //android/rules.bzl to //rules/rules.bzl. +Used for easier migration to a new branch due to directory differences. +""" diff --git a/android/rules.bzl b/android/rules.bzl new file mode 100644 index 0000000..556968d --- /dev/null +++ b/android/rules.bzl @@ -0,0 +1,64 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Redirecting starlark rules to //rules/rules.bzl for easier migration to a new branch.""" + +load( + "//rules/rules.bzl", + _aar_import = "aar_import", + _android_archive = "android_archive", + _android_binary = "android_binary", + _android_bundle_to_apks = "android_bundle_to_apks", + _android_device = "android_device", + _android_device_script_fixture = "android_device_script_fixture", + _android_host_service_fixture = "android_host_service_fixture", + _android_instrumentation_test = "android_instrumentation_test_macro", + _android_library = "android_library_macro", + _android_local_test = "android_local_test", + _android_ndk_repository = "android_ndk_repository", + _android_sdk = "android_sdk", + _android_sdk_repository = "android_sdk_repository", + _android_tools_defaults_jar = "android_tools_defaults_jar", + _apk_import = "apk_import", +) + +aar_import = _aar_import + +android_archive = _android_archive + +android_binary = _android_binary + +android_bundle_to_apks = _android_bundle_to_apks + +android_device = _android_device + +android_device_script_fixture = _android_device_script_fixture + +android_host_service_fixture = _android_host_service_fixture + +android_instrumentation_test = _android_instrumentation_test + +android_library = _android_library + +android_local_test = _android_local_test + +android_ndk_repository = _android_ndk_repository + +android_sdk = _android_sdk + +android_sdk_repository = _android_sdk_repository + +android_tools_defaults_jar = _android_tools_defaults_jar + +apk_import = _apk_import @@ -14,10 +14,15 @@ """Workspace setup macro for rules_android.""" +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") +load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") +load("@robolectric//bazel:robolectric.bzl", "robolectric_repositories") load("@rules_jvm_external//:defs.bzl", "maven_install") def rules_android_workspace(): """ Sets up workspace dependencies for rules_android.""" + protobuf_deps() maven_install( name = "rules_android_maven", @@ -29,3 +34,35 @@ def rules_android_workspace(): "https://repo1.maven.org/maven2", ], ) + + go_rules_dependencies() + + go_register_toolchains(version = "1.18.3") + + gazelle_dependencies() + # gazelle:repository go_repository name=org_golang_x_xerrors importpath=golang.org/x/xerrors + + go_repository( + name = "org_golang_google_protobuf", + importpath = "google.golang.org/protobuf", + sum = "h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=", + version = "v1.28.1", + ) + + go_repository( + name = "com_github_google_go_cmp", + importpath = "github.com/google/go-cmp", + sum = "h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=", + version = "v0.5.9", + ) + + go_repository( + name = "org_golang_x_sync", + importpath = "golang.org/x/sync", + sum = "h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=", + version = "v0.0.0-20210220032951-036812b2e83c", + ) + + robolectric_repositories() + + diff --git a/examples/basicapp/WORKSPACE b/examples/basicapp/WORKSPACE new file mode 100644 index 0000000..b7aa64a --- /dev/null +++ b/examples/basicapp/WORKSPACE @@ -0,0 +1,44 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +android_sdk_repository( + name = "androidsdk", +) + +android_ndk_repository( + name = "androidndk", +) + +maybe( + http_archive, + name = "rules_jvm_external", + strip_prefix = "rules_jvm_external-fa73b1a8e4846cee88240d0019b8f80d39feb1c3", + sha256 = "7e13e48b50f9505e8a99cc5a16c557cbe826e9b68d733050cd1e318d69f94bb5", + url = "https://github.com/bazelbuild/rules_jvm_external/archive/fa73b1a8e4846cee88240d0019b8f80d39feb1c3.zip", +) + +maybe( + http_archive, + name = "bazel_skylib", + urls = [ + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", + ], + sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c", +) +load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") +bazel_skylib_workspace() + +local_repository( + name = "rules_android", + path = "../..", # rules_android's WORKSPACE relative to this inner workspace +) +load("@rules_android//:prereqs.bzl", "rules_android_prereqs") +rules_android_prereqs() +load("@rules_android//:defs.bzl", "rules_android_workspace") +rules_android_workspace() +register_toolchains( + "@rules_android//toolchains/android:android_default_toolchain", + "@rules_android//toolchains/android_sdk:android_sdk_tools", +) + diff --git a/examples/basicapp/java/com/basicapp/AndroidManifest.xml b/examples/basicapp/java/com/basicapp/AndroidManifest.xml new file mode 100644 index 0000000..d9b0640 --- /dev/null +++ b/examples/basicapp/java/com/basicapp/AndroidManifest.xml @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.basic" > + + <uses-sdk + android:minSdkVersion="18" + android:targetSdkVersion="30" /> + + <application + android:allowBackup="true" + android:icon="@drawable/ic_launcher" + android:label="@string/app_name" > + <activity + android:name="com.basicapp.BasicActivity" + android:exported="true"> + <intent-filter> + <action android:name="android.intent.action.MAIN" /> + <category android:name="android.intent.category.LAUNCHER" /> + </intent-filter> + </activity> + </application> +</manifest> diff --git a/examples/basicapp/java/com/basicapp/BUILD b/examples/basicapp/java/com/basicapp/BUILD new file mode 100644 index 0000000..adcb05b --- /dev/null +++ b/examples/basicapp/java/com/basicapp/BUILD @@ -0,0 +1,14 @@ +load("@rules_android//rules:rules.bzl", "android_binary", "android_library") + +android_binary( + name = "basic_app", + manifest = "AndroidManifest.xml", + deps = [":basic_lib"], +) + +android_library( + name = "basic_lib", + srcs = ["BasicActivity.java"], + manifest = "AndroidManifest.xml", + resource_files = glob(["res/**"]), +) diff --git a/examples/basicapp/java/com/basicapp/BasicActivity.java b/examples/basicapp/java/com/basicapp/BasicActivity.java new file mode 100644 index 0000000..03c9aef --- /dev/null +++ b/examples/basicapp/java/com/basicapp/BasicActivity.java @@ -0,0 +1,59 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.basicapp; + +import android.app.Activity; +import android.os.Bundle; +import android.view.Menu; +import android.view.View; +import android.widget.Button; +import android.widget.TextView; + +/** + * The main activity of the Basic Sample App. + */ +public class BasicActivity extends Activity { + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.basic_activity); + + final Button buttons[] = { + findViewById(R.id.button_id_fizz), findViewById(R.id.button_id_buzz), + }; + + for (Button b : buttons) { + b.setOnClickListener( + new View.OnClickListener() { + public void onClick(View v) { + TextView tv = findViewById(R.id.text_hello); + if (v.getId() == R.id.button_id_fizz) { + tv.setText("fizz"); + } else if (v.getId() == R.id.button_id_buzz) { + tv.setText("buzz"); + } + } + }); + } + } + + @Override + public boolean onCreateOptionsMenu(Menu menu) { + // Inflate the menu; this adds items to the action bar if it is present. + getMenuInflater().inflate(R.menu.menu, menu); + return true; + } +} diff --git a/examples/basicapp/java/com/basicapp/res/drawable-hdpi/ic_launcher.png b/examples/basicapp/java/com/basicapp/res/drawable-hdpi/ic_launcher.png Binary files differnew file mode 100644 index 0000000..6ab2add --- /dev/null +++ b/examples/basicapp/java/com/basicapp/res/drawable-hdpi/ic_launcher.png diff --git a/examples/basicapp/java/com/basicapp/res/drawable-mdpi/ic_launcher.png b/examples/basicapp/java/com/basicapp/res/drawable-mdpi/ic_launcher.png Binary files differnew file mode 100644 index 0000000..c0a73c3 --- /dev/null +++ b/examples/basicapp/java/com/basicapp/res/drawable-mdpi/ic_launcher.png diff --git a/examples/basicapp/java/com/basicapp/res/drawable-xhdpi/ic_launcher.png b/examples/basicapp/java/com/basicapp/res/drawable-xhdpi/ic_launcher.png Binary files differnew file mode 100644 index 0000000..014b0f1 --- /dev/null +++ b/examples/basicapp/java/com/basicapp/res/drawable-xhdpi/ic_launcher.png diff --git a/examples/basicapp/java/com/basicapp/res/drawable-xxhdpi/ic_launcher.png b/examples/basicapp/java/com/basicapp/res/drawable-xxhdpi/ic_launcher.png Binary files differnew file mode 100644 index 0000000..20703a1 --- /dev/null +++ b/examples/basicapp/java/com/basicapp/res/drawable-xxhdpi/ic_launcher.png diff --git a/examples/basicapp/java/com/basicapp/res/layout/basic_activity.xml b/examples/basicapp/java/com/basicapp/res/layout/basic_activity.xml new file mode 100644 index 0000000..f84199c --- /dev/null +++ b/examples/basicapp/java/com/basicapp/res/layout/basic_activity.xml @@ -0,0 +1,23 @@ +<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android" + android:layout_width="match_parent" + android:layout_height="match_parent" + android:orientation="vertical" > + + <TextView + android:id="@+id/text_hello" + android:text="@string/hello_world" + android:layout_width="wrap_content" + android:layout_height="wrap_content" /> + + <Button + android:id="@+id/button_id_fizz" + android:layout_height="wrap_content" + android:layout_width="wrap_content" + android:text="fizz" /> + <Button + android:id="@+id/button_id_buzz" + android:layout_height="wrap_content" + android:layout_width="wrap_content" + android:text="buzz" /> + +</LinearLayout> diff --git a/examples/basicapp/java/com/basicapp/res/menu/menu.xml b/examples/basicapp/java/com/basicapp/res/menu/menu.xml new file mode 100644 index 0000000..a56bed6 --- /dev/null +++ b/examples/basicapp/java/com/basicapp/res/menu/menu.xml @@ -0,0 +1,8 @@ +<menu xmlns:android="http://schemas.android.com/apk/res/android" + xmlns:app="http://schemas.android.com/apk/res-auto" + xmlns:tools="http://schemas.android.com/tools" + tools:context="com.basicapp.BasicActivity" > + <item android:id="@+id/action_settings" + android:title="@string/action_settings" + android:orderInCategory="100" /> +</menu> diff --git a/examples/basicapp/java/com/basicapp/res/values/dimens.xml b/examples/basicapp/java/com/basicapp/res/values/dimens.xml new file mode 100644 index 0000000..47c8224 --- /dev/null +++ b/examples/basicapp/java/com/basicapp/res/values/dimens.xml @@ -0,0 +1,5 @@ +<resources> + <!-- Default screen margins, per the Android Design guidelines. --> + <dimen name="activity_horizontal_margin">16dp</dimen> + <dimen name="activity_vertical_margin">16dp</dimen> +</resources> diff --git a/examples/basicapp/java/com/basicapp/res/values/strings.xml b/examples/basicapp/java/com/basicapp/res/values/strings.xml new file mode 100644 index 0000000..565c987 --- /dev/null +++ b/examples/basicapp/java/com/basicapp/res/values/strings.xml @@ -0,0 +1,8 @@ +<?xml version="1.0" encoding="utf-8"?> +<resources> + + <string name="app_name" translatable="false">basicapp</string> + <string name="hello_world" translatable="false">Hello world!</string> + <string name="action_settings" translatable="false">Settings</string> + +</resources> @@ -0,0 +1,9 @@ +module github.com/bazelbuild/rules_android + +go 1.18 + +require ( + github.com/google/go-cmp v0.5.9 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + google.golang.org/protobuf v1.28.1 +) @@ -0,0 +1,10 @@ +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= @@ -0,0 +1,4 @@ +# UUID Group Name +# +mdb:copybara-git-writers mdb/copybara-git-writers +mdb:mobile-ninjas-releaser mdb/mobile-ninjas-releaser diff --git a/kokoro/presubmit/download_bazel.sh b/kokoro/presubmit/download_bazel.sh new file mode 100644 index 0000000..2c96b70 --- /dev/null +++ b/kokoro/presubmit/download_bazel.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +function DownloadBazel() { + # Utility function to download a specified version of bazel to a given + # installation directory. + # Positional arguments: + # ver: The version to install. Supports "latest" (major and minor releases), + # "latest-with-prereleases" (all versions from "latest" + prereleases), + # major/minor releases such as 5.2.0, and also prereleases such as + # 6.0.0-pre.20220720.3. Release candidates with "rc" in the name are NOT + # supported. + # platform: The platform to install. Currently only "linux" has been + # validated. + # arch: Architecture to install. Currently only "x86_64" has been validated. + # dest: Where to install Bazel. Must be a user-writeable directory, + # otherwise the root user must call this function through sudo. + # Returns: + # Echoes the installation directory at the end of installation. + ( + set -euxo pipefail + # Significantly cribbed from + # devtools/kokoro/vanadium/linux_scripts/usr/local/bin/use_bazel.sh + # Temporary workaround solution until use_bazel.sh can download prereleases. + + # Positional arguments + local ver="$1" + local platform="$2" + local arch="$3" + local dest="$4" + + # Function-local helper variables + local gcs_uri="" + local revision_identifier="" + if [[ "$ver" == "latest" || "$ver" == "latest-with-prereleases" ]]; then + # Query binary blob bucket to find the latest prerelease + if [[ "$ver" == "latest" ]]; then + # Filter out prereleases + ver=$(gsutil ls -l gs://bazel/**/*-installer-"${platform}"-"${arch}".sh | grep "gs://" | grep -v rc | grep -v pre | tail -n1 | awk '{print $NF}') + else + ver=$(gsutil ls -l gs://bazel/**/*-installer-"${platform}"-"${arch}".sh | grep "gs://" | grep -v rc | tail -n1 | awk '{print $NF}') + fi + ver=$(echo "$ver" | sed -n "s/.*bazel\-\(.*\)\-installer.*/\1/p") + fi + if [[ "$ver" =~ pre ]]; then + revision_identifier=$(echo "$ver" | awk -F"-" '{print $1}') + gcs_uri="gs://bazel/${revision_identifier}/rolling/${ver}/bazel-${ver}-installer-${platform}-${arch}.sh" + else + gcs_uri="gs://bazel/${ver}/release/bazel-${ver}-installer-${platform}-${arch}.sh" + fi + + # Download the installer from GCS + gsutil -q cp "$gcs_uri" "$dest"/bazel_installer.sh + mkdir -p "$dest"/install + # Run the installer + bash "$dest"/bazel_installer.sh --prefix="$dest"/install > /dev/null + ls -d "$dest"/install + ) +} + + diff --git a/kokoro/presubmit/kokoro_presubmit.sh b/kokoro/presubmit/kokoro_presubmit.sh new file mode 100644 index 0000000..4879800 --- /dev/null +++ b/kokoro/presubmit/kokoro_presubmit.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -x + +source "${KOKORO_GFILE_DIR}/download_bazel.sh" +echo "== installing bazel =========================================" +bazel_install_dir=$(mktemp -d) +BAZEL_VERSION="latest-with-prereleases" +DownloadBazel "$BAZEL_VERSION" linux x86_64 "$bazel_install_dir" +bazel="$bazel_install_dir/install/bin/bazel" +chmod +x "$bazel" +bazel_detected_version=$("$bazel" version | grep "Build label" | awk -F": " '{print $2}') +echo "=============================================================" + +function Cleanup() { + # Clean up all temporary directories: bazel install, sandbox, and + # android_tools. + rm -rf "$bazel_install_dir" +} +trap Cleanup EXIT + +# Kokoro is no longer updating toolchains in their images, so install newer +# android build tools, because the latest one installed (26.0.2) has some bug +# in APPT2 which causes the magic number to be incorrect for some files it +# outputs. +# +# Use "yes" to accept sdk licenses. +cd "$ANDROID_HOME" +yes | tools/bin/sdkmanager --install "build-tools;30.0.3" &>/dev/null +yes | tools/bin/sdkmanager --licenses &>/dev/null + +# ANDROID_HOME is already in the environment. +export ANDROID_NDK_HOME="/opt/android-ndk-r16b" + +# Create a tmpfs in the sandbox at "/tmp/hsperfdata_$USERNAME" to avoid the +# problems described in https://github.com/bazelbuild/bazel/issues/3236 +# Basically, the JVM creates a file at /tmp/hsperfdata_$USERNAME/$PID, but +# processes all get a PID of 2 in the sandbox, so concurrent Java build actions +# could crash because they're trying to modify the same file. So, tell the +# sandbox to mount a tmpfs at /tmp/hsperfdata_$(whoami) so that each JVM gets +# its own version of that directory. +hsperfdata_dir="/tmp/hsperfdata_$(whoami)_rules_android" +mkdir "$hsperfdata_dir" + +COMMON_ARGS=( + "--sandbox_tmpfs_path=$hsperfdata_dir" + "--verbose_failures" + "--experimental_google_legacy_api" + "--experimental_enable_android_migration_apis" +) + +# Go to rules_android workspace and run relevant tests. +cd "${KOKORO_ARTIFACTS_DIR}/git/rules_android" +"$bazel" test "${COMMON_ARGS[@]}" //src/common/golang/... \ + //src/tools/ak/{bucketize,compile,dex,extractaar,finalrjar,generatemanifest,link,liteparse,manifest,mindex,nativelib,res,rjar}/... + +# Go to basic app workspace in the source tree +cd "${KOKORO_ARTIFACTS_DIR}/git/rules_android/examples/basicapp" +"$bazel" build "${COMMON_ARGS[@]}" //java/com/basicapp:basic_app + diff --git a/kokoro/presubmit/presubmit.cfg b/kokoro/presubmit/presubmit.cfg new file mode 100644 index 0000000..8bf3662 --- /dev/null +++ b/kokoro/presubmit/presubmit.cfg @@ -0,0 +1,17 @@ + +# The version of bazel to use to test the Starlark Android Rules. +# Update this as newer versions of bazel are released. +build_params { + key: "bazel_version" + value: "5.0.0" +} + +env_vars { + key: "bazel_version" + value: "$[bazel_version]" +} + +gfile_resources: "/x20/teams/bazel/releases/bazel-$[bazel_version]-linux-x86_64" +gfile_resources: "/google_src/files/head/depot/google3/third_party/bazel_rules/rules_android/kokoro/presubmit/download_bazel.sh" + +build_file: "rules_android/kokoro/presubmit/kokoro_presubmit.sh" diff --git a/mobile_install/adapters/aar_import.bzl b/mobile_install/adapters/aar_import.bzl new file mode 100644 index 0000000..f1e5db9 --- /dev/null +++ b/mobile_install/adapters/aar_import.bzl @@ -0,0 +1,113 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for aar_import.""" + +load(":adapters/base.bzl", "make_adapter") +load( + ":providers.bzl", + "MIAndroidAarNativeLibsInfo", + "MIAndroidAssetsInfo", + "MIAndroidDexInfo", + "MIAndroidResourcesInfo", + "MIJavaResourcesInfo", + "providers", +) +load(":resources.bzl", "liteparse") +load(":transform.bzl", "dex") +load("//rules:java.bzl", _java = "java") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return ["deps", "exports"] + +def _adapt(target, ctx): + """Adapts the rule and target data. + + Args: + target: The target. + ctx: The context. + + Returns: + A list of providers. + """ + + assets = depset() + assets_dir = None + if AndroidAssetsInfo in target: + assets = target[AndroidAssetsInfo].assets + assets_dir = target[AndroidAssetsInfo].local_asset_dir + + label = None + resources = depset() + if AndroidResourcesInfo in target: + label = target[AndroidResourcesInfo].label + resources = target[AndroidResourcesInfo].direct_android_resources + + return [ + providers.make_mi_android_aar_native_libs_info( + native_libs = target[AndroidNativeLibsInfo].native_libs, + deps = providers.collect( + MIAndroidAarNativeLibsInfo, + ctx.rule.attr.deps, + ctx.rule.attr.exports, + ), + ), + providers.make_mi_android_assets_info( + assets = assets, + assets_dir = assets_dir, + deps = providers.collect( + MIAndroidAssetsInfo, + ctx.rule.attr.deps, + ctx.rule.attr.exports, + ), + ), + providers.make_mi_android_dex_info( + dex_shards = dex( + ctx, + target[JavaInfo].runtime_output_jars, + target[JavaInfo].transitive_deps, + ), + deps = providers.collect( + MIAndroidDexInfo, + ctx.rule.attr.deps, + ctx.rule.attr.exports, + ), + ), + providers.make_mi_android_resources_info( + # TODO(b/124229660): The package for an aar should be retrieved from + # the AndroidManifest.xml in the aar. Using the package is a short + # term work-around. + package = _java.resolve_package_from_label( + ctx.label, + ctx.rule.attr.package, + ), + label = label, + r_pb = liteparse(ctx), + resources = resources, + deps = providers.collect( + MIAndroidResourcesInfo, + ctx.rule.attr.deps, + ctx.rule.attr.exports, + ), + ), + providers.make_mi_java_resources_info( + deps = providers.collect( + MIJavaResourcesInfo, + ctx.rule.attr.deps, + ctx.rule.attr.exports, + ), + ), + ] + +aar_import = make_adapter(_aspect_attrs, _adapt) diff --git a/mobile_install/adapters/android_binary.bzl b/mobile_install/adapters/android_binary.bzl new file mode 100644 index 0000000..98641e1 --- /dev/null +++ b/mobile_install/adapters/android_binary.bzl @@ -0,0 +1,149 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for android_binary.""" + +load(":adapters/base.bzl", "make_adapter") +load(":launcher.bzl", "make_launcher") +load(":launcher_direct.bzl", "make_direct_launcher") +load(":process.bzl", "process") +load( + ":providers.bzl", + "MIAndroidAarNativeLibsInfo", + "MIAndroidAssetsInfo", + "MIAndroidDexInfo", + "MIAndroidResourcesInfo", + "MIJavaResourcesInfo", + "providers", +) +load(":resources.bzl", "get_assets_dir") +load(":transform.bzl", "dex", "filter_jars") +load(":utils.bzl", "utils") +load("//rules/flags:flags.bzl", "flags") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return ["_android_sdk", "deps", "resources", "instruments"] + +def extract(target, ctx): + # extract is made visibile for testing + """extract the rule and target data. + + Args: + target: The target. + ctx: The context. + + Returns: + Input for process method + """ + return dict( + debug_key = utils.only(ctx.rule.files.debug_key, allow_empty = True), + debug_signing_keys = ctx.rule.files.debug_signing_keys, + debug_signing_lineage_file = utils.only(ctx.rule.files.debug_signing_lineage_file, allow_empty = True), + key_rotation_min_sdk = ctx.rule.attr.key_rotation_min_sdk, + merged_manifest = target[AndroidIdeInfo].generated_manifest, + native_libs = target[AndroidIdeInfo].native_libs, + package = target[AndroidIdeInfo].java_package, + resource_apk = target[AndroidIdeInfo].resource_apk, + resource_src_jar = target[AndroidIdeInfo].resource_jar.source_jar, # This is the R with real ids. + aar_native_libs_info = providers.make_mi_android_aar_native_libs_info( + deps = providers.collect( + MIAndroidAarNativeLibsInfo, + ctx.rule.attr.deps, + ), + ), + android_assets_info = providers.make_mi_android_assets_info( + assets = depset(ctx.rule.files.assets), + assets_dir = get_assets_dir( + ctx.rule.files.assets[0], + ctx.rule.attr.assets_dir, + ) if ctx.rule.files.assets else None, + deps = providers.collect( + MIAndroidAssetsInfo, + ctx.rule.attr.deps, + ), + ), + android_dex_info = providers.make_mi_android_dex_info( + dex_shards = dex( + ctx, + filter_jars( + ctx.label.name + "_resources.jar", + target[JavaInfo].runtime_output_jars, + ) + + ( + ), + target[JavaInfo].transitive_deps, + ), + deps = providers.collect(MIAndroidDexInfo, ctx.rule.attr.deps), + ), + # TODO(djwhang): It wasteful to collect packages in + # android_resources_info, rather we should be looking to pull them + # from the resources_v3_info. + android_resources_info = providers.make_mi_android_resources_info( + package = target[AndroidIdeInfo].java_package, + deps = providers.collect( + MIAndroidResourcesInfo, + ctx.rule.attr.deps, + ), + ), + java_resources_info = providers.make_mi_java_resources_info( + deps = providers.collect( + MIJavaResourcesInfo, + ctx.rule.attr.deps, + ), + ), + android_jar = ctx.rule.attr._android_sdk[AndroidSdkInfo].android_jar, + instrumented_app = ctx.rule.attr.instruments, + apk = target.android.apk, + ) + +def adapt(target, ctx): + # adapt is made visibile for testing + """Adapts the android rule + + Args: + target: The target. + ctx: The context. + Returns: + A list of providers + """ + + # launcher is created here to be used as the sibling everywhere else. + launcher = utils.isolated_declare_file(ctx, ctx.label.name + "_mi/launcher") + mi_app_info = process(ctx, sibling = launcher, **extract(target, ctx)) + + if flags.get(ctx).use_direct_deploy: + mi_app_launch_info = make_direct_launcher( + ctx, + mi_app_info, + launcher, + use_adb_root = flags.get(ctx).use_adb_root, + ) + else: + mi_app_launch_info = make_launcher( + ctx, + mi_app_info, + launcher, + use_adb_root = flags.get(ctx).use_adb_root, + ) + + return [ + mi_app_info, + mi_app_launch_info, + OutputGroupInfo( + mobile_install_INTERNAL_ = depset(mi_app_launch_info.runfiles).to_list(), + mobile_install_launcher_INTERNAL_ = [mi_app_launch_info.launcher], + ), + ] + +android_binary = make_adapter(_aspect_attrs, adapt) diff --git a/mobile_install/adapters/android_instrumentation_test.bzl b/mobile_install/adapters/android_instrumentation_test.bzl new file mode 100644 index 0000000..ffd9c4d --- /dev/null +++ b/mobile_install/adapters/android_instrumentation_test.bzl @@ -0,0 +1,76 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for android_instrumentation_test.""" + +load(":adapters/base.bzl", "make_adapter") +load(":launcher.bzl", "make_launcher") +load(":launcher_direct.bzl", "make_direct_launcher") +load(":providers.bzl", "MIAppInfo") +load(":utils.bzl", "utils") +load("//rules/flags:flags.bzl", "flags") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return ["test_app", "support_apps"] + +def _adapt(target, ctx): + if not hasattr(ctx.attr, "_android_test_runner"): + fail("mobile-install does not support running tests on mac, check b/134172473 for more details") + + # TODO(b/): Tests have yet to be optimized so, this is an irrelevant error. + # if flags.get(ctx).enable_splits: + # fail("mobile-install does not support running tests for split apks, check b/139762843 for more details! To run tests with mobile-install without splits, pass --define=enable_splits=False") + + launcher = utils.isolated_declare_file(ctx, ctx.label.name + "_mi/launcher") + + test_app = ctx.rule.attr.test_app + + # TODO(manalinandan): Re-enable direct deploy for test. + # if _flags.get(ctx).use_direct_deploy: + if False: + mi_app_launch_info = make_direct_launcher( + ctx, + test_app[MIAppInfo], + launcher, + test_args = ctx.rule.attr.args, + test_support_apps = ctx.rule.attr.support_apps, + use_adb_root = flags.get(ctx).use_adb_root, + is_test = True, + ) + else: + googplayservices_container_app = None + test_support_apps = [] + for support_app in ctx.rule.attr.support_apps: + # Checks if the support_apps is an android_binary rule and 'GoogPlayServices' is present in the label + # This implies there is a GoogPlayServices container binary in the dependency + if MIAppInfo in support_app and "GoogPlayServices" in str(support_app.label): + googplayservices_container_app = support_app + elif MIAppInfo in support_app: + test_support_apps.append(support_app[MIAppInfo].apk) + mi_app_launch_info = make_launcher( + ctx, + test_app[MIAppInfo], + launcher, + test_args = ctx.rule.attr.args, + test_support_apks = test_support_apps, + googplayservices_container_app = googplayservices_container_app, + use_adb_root = flags.get(ctx).use_adb_root, + is_test = True, + ) + return [OutputGroupInfo( + mobile_install_INTERNAL_ = depset(mi_app_launch_info.runfiles).to_list(), + mobile_install_launcher_INTERNAL_ = [mi_app_launch_info.launcher], + )] + +android_instrumentation_test = make_adapter(_aspect_attrs, _adapt) diff --git a/mobile_install/adapters/android_library.bzl b/mobile_install/adapters/android_library.bzl new file mode 100644 index 0000000..698bf27 --- /dev/null +++ b/mobile_install/adapters/android_library.bzl @@ -0,0 +1,117 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for android_library.""" + +load(":adapters/base.bzl", "make_adapter") +load( + ":providers.bzl", + "MIAndroidAarNativeLibsInfo", + "MIAndroidAssetsInfo", + "MIAndroidDexInfo", + "MIAndroidResourcesInfo", + "MIAndroidSdkInfo", + "MIJavaResourcesInfo", + "providers", +) +load(":resources.bzl", "get_assets_dir") +load(":transform.bzl", "dex", "filter_jars") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return [ + "_android_sdk", + + # For the Google-internal kotlin rule to access the toolchain to + # get kotlin std and runtime libs. + "_toolchain", + "deps", + "exports", + ] + +def _adapt(target, ctx): + """Adapts the rule and target data. + + Args: + target: The target. + ctx: The context. + + Returns: + A list of providers. + """ + kt_toolchain = [ctx.rule.attr._toolchain] if hasattr(ctx.rule.attr, "_toolchain") else [] + if ctx.rule.attr.neverlink: + return [] + + if target[AndroidIdeInfo].idl_generated_java_files: + aidl_lib = [ctx.rule.attr._android_sdk[MIAndroidSdkInfo].aidl_lib] + else: + aidl_lib = [] + + return [ + providers.make_mi_android_aar_native_libs_info( + deps = providers.collect( + MIAndroidAarNativeLibsInfo, + ctx.rule.attr.deps, + ctx.rule.attr.exports, + ), + ), + providers.make_mi_android_assets_info( + assets = depset(ctx.rule.files.assets), + assets_dir = get_assets_dir( + ctx.rule.files.assets[0], + ctx.rule.attr.assets_dir, + ) if ctx.rule.files.assets else None, + deps = providers.collect( + MIAndroidAssetsInfo, + ctx.rule.attr.deps, + ctx.rule.attr.exports, + ), + ), + providers.make_mi_android_dex_info( + dex_shards = dex( + ctx, + filter_jars( + ctx.label.name + "_resources.jar", + target[JavaInfo].runtime_output_jars, + ), + target[JavaInfo].transitive_deps, + ), + deps = providers.collect( + MIAndroidDexInfo, + ctx.rule.attr.deps, + ctx.rule.attr.exports, + aidl_lib, + kt_toolchain, + ), + ), + providers.make_mi_android_resources_info( + package = target[AndroidIdeInfo].java_package, + deps = providers.collect( + MIAndroidResourcesInfo, + ctx.rule.attr.deps, + ctx.rule.attr.exports, + ), + ), + providers.make_mi_java_resources_info( + deps = providers.collect( + MIJavaResourcesInfo, + ctx.rule.attr.deps, + ctx.rule.attr.exports, + aidl_lib, + kt_toolchain, + ), + ), + ] + +android_library = make_adapter(_aspect_attrs, _adapt) diff --git a/mobile_install/adapters/android_sdk.bzl b/mobile_install/adapters/android_sdk.bzl new file mode 100644 index 0000000..44f8f19 --- /dev/null +++ b/mobile_install/adapters/android_sdk.bzl @@ -0,0 +1,39 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for android_sdk.""" + +load(":adapters/base.bzl", "make_adapter") +load(":providers.bzl", "MIAndroidSdkInfo") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return ["aidl_lib"] + +def _adapt(unused_target, ctx): + """Adapts the rule and target data. + + Args: + unused_target: The target. + ctx: The context. + + Returns: + A list of providers. + """ + return [ + MIAndroidSdkInfo( + aidl_lib = ctx.rule.attr.aidl_lib, + ), + ] + +android_sdk = make_adapter(_aspect_attrs, _adapt) diff --git a/mobile_install/adapters/apk_import.bzl b/mobile_install/adapters/apk_import.bzl new file mode 100644 index 0000000..425d230 --- /dev/null +++ b/mobile_install/adapters/apk_import.bzl @@ -0,0 +1,47 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for android_binary.""" + +load(":adapters/base.bzl", "make_adapter") +load(":providers.bzl", "MIAppInfo") +load(":utils.bzl", "utils") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return ["unsigned_apk"] + +def adapt(target, ctx): + # adapt is made visibile for testing + """Adapts the android rule + + Args: + target: The target. + ctx: The context. + Returns: + A list of providers + """ + apk = ctx.rule.file.unsigned_apk + + package_name_output_file = utils.isolated_declare_file(ctx, ctx.label.name + "/manifest_package_name.txt") + + utils.extract_package_name(ctx, apk, package_name_output_file) + + return [ + MIAppInfo( + apk = apk, + manifest_package_name = package_name_output_file, + ), + ] + +apk_import = make_adapter(_aspect_attrs, adapt) diff --git a/mobile_install/adapters/base.bzl b/mobile_install/adapters/base.bzl new file mode 100644 index 0000000..c969d9f --- /dev/null +++ b/mobile_install/adapters/base.bzl @@ -0,0 +1,33 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Provides the base adapter functions.""" + +def make_adapter(aspect_attrs, adapt): + """Creates an Adapter. + + Args: + aspect_attrs: A function that returns a list of attrs for the aspect. + adapt: A function that extracts and processes data from the target. + + Returns: + A struct that represents an adapter. + """ + if not aspect_attrs: + fail("aspect_attrs is None.") + if not adapt: + fail("adapt is None.") + return struct( + aspect_attrs = aspect_attrs, + adapt = adapt, + ) diff --git a/mobile_install/adapters/java_import.bzl b/mobile_install/adapters/java_import.bzl new file mode 100644 index 0000000..d4b9c92 --- /dev/null +++ b/mobile_install/adapters/java_import.bzl @@ -0,0 +1,71 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for java_import.""" + +load(":adapters/base.bzl", "make_adapter") +load( + ":providers.bzl", + "MIAndroidDexInfo", + "MIJavaResourcesInfo", + "providers", +) +load(":transform.bzl", "dex", "extract_jar_resources") +load(":utils.bzl", "utils") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return ["deps", "exports"] + +def _adapt(target, ctx): + """Adapts the rule and target data. + + Args: + target: The target. + ctx: The context. + + Returns: + A list of providers. + """ + if ctx.rule.attr.neverlink: + return [] + + return [ + providers.make_mi_android_dex_info( + dex_shards = dex( + ctx, + target[JavaInfo].runtime_output_jars, + target[JavaInfo].transitive_deps, + create_file = utils.declare_file, + ), + deps = providers.collect( + MIAndroidDexInfo, + ctx.rule.attr.deps, + ctx.rule.attr.exports, + ), + ), + providers.make_mi_java_resources_info( + java_resources = extract_jar_resources( + ctx, + target[JavaInfo].runtime_output_jars, + create_file = utils.declare_file, + ), + deps = providers.collect( + MIJavaResourcesInfo, + ctx.rule.attr.deps, + ctx.rule.attr.exports, + ), + ), + ] + +java_import = make_adapter(_aspect_attrs, _adapt) diff --git a/mobile_install/adapters/java_library.bzl b/mobile_install/adapters/java_library.bzl new file mode 100644 index 0000000..afeee64 --- /dev/null +++ b/mobile_install/adapters/java_library.bzl @@ -0,0 +1,70 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for java_library.""" + +load(":adapters/base.bzl", "make_adapter") +load( + ":providers.bzl", + "MIAndroidDexInfo", + "MIJavaResourcesInfo", + "providers", +) +load(":transform.bzl", "dex", "extract_jar_resources") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return ["deps", "exports", "runtime_deps"] + +def _adapt(target, ctx): + """Adapts the rule and target data. + + Args: + target: The target. + ctx: The context. + + Returns: + A list of providers. + """ + if ctx.rule.attr.neverlink: + return [] + + return [ + providers.make_mi_android_dex_info( + dex_shards = dex( + ctx, + target[JavaInfo].runtime_output_jars, + target[JavaInfo].transitive_deps, + ), + deps = providers.collect( + MIAndroidDexInfo, + ctx.rule.attr.deps, + ctx.rule.attr.runtime_deps, + ctx.rule.attr.exports, + ), + ), + providers.make_mi_java_resources_info( + java_resources = extract_jar_resources( + ctx, + target[JavaInfo].runtime_output_jars, + ), + deps = providers.collect( + MIJavaResourcesInfo, + ctx.rule.attr.deps, + ctx.rule.attr.runtime_deps, + ctx.rule.attr.exports, + ), + ), + ] + +java_library = make_adapter(_aspect_attrs, _adapt) diff --git a/mobile_install/adapters/java_lite_grpc_library.bzl b/mobile_install/adapters/java_lite_grpc_library.bzl new file mode 100644 index 0000000..5eff4a8 --- /dev/null +++ b/mobile_install/adapters/java_lite_grpc_library.bzl @@ -0,0 +1,59 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for _java_lite_grpc_library.""" + +load(":adapters/base.bzl", "make_adapter") +load(":providers.bzl", "MIAndroidDexInfo", "MIJavaResourcesInfo", "providers") +load(":transform.bzl", "dex", "extract_jar_resources") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return ["deps", "_toolchain"] + +def _adapt(target, ctx): + """Adapts the rule and target data. + + Args: + target: The target. + ctx: The context. + + Returns: + A list of providers. + """ + return [ + providers.make_mi_android_dex_info( + dex_shards = dex( + ctx, + target[JavaInfo].runtime_output_jars, + target[JavaInfo].transitive_deps, + ), + deps = providers.collect( + MIAndroidDexInfo, + ctx.rule.attr.deps, + [ctx.rule.attr._toolchain], + ), + ), + providers.make_mi_java_resources_info( + java_resources = extract_jar_resources( + ctx, + target[JavaInfo].runtime_output_jars, + ), + deps = providers.collect( + MIJavaResourcesInfo, + ctx.rule.attr.deps, + ), + ), + ] + +java_lite_grpc_library = make_adapter(_aspect_attrs, _adapt) diff --git a/mobile_install/adapters/java_lite_proto_library.bzl b/mobile_install/adapters/java_lite_proto_library.bzl new file mode 100644 index 0000000..b251b65 --- /dev/null +++ b/mobile_install/adapters/java_lite_proto_library.bzl @@ -0,0 +1,57 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for java_lite_proto_library. + +The java_lite_proto_library rule applies an aspect onto its proto dependencies. +Creates a "lite.jar" at every proto traversed. This adapter is used to just +propagate the deps, the proto_library rules. +""" + +load(":adapters/base.bzl", "make_adapter") +load(":providers.bzl", "MIAndroidDexInfo", "MIJavaResourcesInfo", "providers") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return ["deps", "_aspect_proto_toolchain_for_javalite"] + +def _adapt(target, ctx): + """Adapts the rule and target data. + + Args: + target: The target. + ctx: The context. + + Returns: + A list of providers. + """ + if not ctx.rule.attr.deps: + return [] + return [ + providers.make_mi_android_dex_info( + deps = providers.collect( + MIAndroidDexInfo, + ctx.rule.attr.deps, + [ctx.rule.attr._aspect_proto_toolchain_for_javalite], + ), + ), + providers.make_mi_java_resources_info( + deps = providers.collect( + MIJavaResourcesInfo, + ctx.rule.attr.deps, + [ctx.rule.attr._aspect_proto_toolchain_for_javalite], + ), + ), + ] + +java_lite_proto_library = make_adapter(_aspect_attrs, _adapt) diff --git a/mobile_install/adapters/java_rpc_toolchain.bzl b/mobile_install/adapters/java_rpc_toolchain.bzl new file mode 100644 index 0000000..76ee505 --- /dev/null +++ b/mobile_install/adapters/java_rpc_toolchain.bzl @@ -0,0 +1,39 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for java_rpc_toolchain.bzl.""" + +load(":adapters/base.bzl", "make_adapter") +load(":providers.bzl", "MIAndroidDexInfo", "providers") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return ["runtime"] # all potential implicit runtime deps + +def _adapt(unused_target, ctx): + """Adapts the rule and target data. + + Args: + unused_target: The target. + ctx: The context. + + Returns: + A list of providers. + """ + return [ + providers.make_mi_android_dex_info( + deps = providers.collect(MIAndroidDexInfo, ctx.rule.attr.runtime), + ), + ] + +java_rpc_toolchain = make_adapter(_aspect_attrs, _adapt) diff --git a/mobile_install/adapters/proto_lang_toolchain.bzl b/mobile_install/adapters/proto_lang_toolchain.bzl new file mode 100644 index 0000000..1e767e2 --- /dev/null +++ b/mobile_install/adapters/proto_lang_toolchain.bzl @@ -0,0 +1,50 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for proto_lang_toolchain.""" + +load(":adapters/base.bzl", "make_adapter") +load(":providers.bzl", "MIAndroidDexInfo", "MIJavaResourcesInfo", "providers") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return ["runtime"] + +def _adapt(unused_target, ctx): + """Adapts the rule and target data. + + Args: + unused_target: The target. + ctx: The context. + + Returns: + A list of providers. + """ + if not ctx.rule.attr.runtime: + return [] + return [ + providers.make_mi_android_dex_info( + deps = providers.collect( + MIAndroidDexInfo, + [ctx.rule.attr.runtime], + ), + ), + providers.make_mi_java_resources_info( + deps = providers.collect( + MIJavaResourcesInfo, + [ctx.rule.attr.runtime], + ), + ), + ] + +proto_lang_toolchain = make_adapter(_aspect_attrs, _adapt) diff --git a/mobile_install/adapters/proto_library.bzl b/mobile_install/adapters/proto_library.bzl new file mode 100644 index 0000000..9e5d8da --- /dev/null +++ b/mobile_install/adapters/proto_library.bzl @@ -0,0 +1,47 @@ +# Copyright 2018 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Rule adapter for proto_library.""" + +load(":adapters/base.bzl", "make_adapter") +load(":providers.bzl", "MIAndroidDexInfo", "providers") +load(":transform.bzl", "dex") + +def _aspect_attrs(): + """Attrs of the rule requiring traversal by the aspect.""" + return ["deps"] + +def _adapt(target, ctx): + """Adapts the rule and target data. + + Args: + target: The target. + ctx: The context. + + Returns: + A list of providers. + """ + if not JavaInfo in target: + return [] + return [ + providers.make_mi_android_dex_info( + dex_shards = dex( + ctx, + [j.class_jar for j in target[JavaInfo].outputs.jars], + target[JavaInfo].transitive_deps, + ), + deps = providers.collect(MIAndroidDexInfo, ctx.rule.attr.deps), + ), + ] + +proto_library = make_adapter(_aspect_attrs, _adapt) diff --git a/prereqs.bzl b/prereqs.bzl new file mode 100644 index 0000000..bf832d8 --- /dev/null +++ b/prereqs.bzl @@ -0,0 +1,97 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Sets up prerequisites for rules_android.""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + + +def rules_android_prereqs(): + """Downloads prerequisite repositories for rules_android.""" + maybe( + http_archive, + name = "rules_jvm_external", + strip_prefix = "rules_jvm_external-fa73b1a8e4846cee88240d0019b8f80d39feb1c3", + sha256 = "7e13e48b50f9505e8a99cc5a16c557cbe826e9b68d733050cd1e318d69f94bb5", + url = "https://github.com/bazelbuild/rules_jvm_external/archive/fa73b1a8e4846cee88240d0019b8f80d39feb1c3.zip", + ) + + maybe( + http_archive, + name = "com_google_protobuf", + sha256 = "87407cd28e7a9c95d9f61a098a53cf031109d451a7763e7dd1253abf8b4df422", + strip_prefix = "protobuf-3.19.1", + urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.1.tar.gz"], + ) + + maybe( + http_archive, + name = "remote_java_tools_for_rules_android", + sha256 = "8fb4d3138bd92a9d3324dae29c9f70d91ca2db18cd0bf1997446eed4657d19b3", + urls = [ + "https://mirror.bazel.build/bazel_java_tools/releases/java/v11.8/java_tools-v11.8.zip", + "https://github.com/bazelbuild/java_tools/releases/download/java_v11.8/java_tools-v11.8.zip", + ], + ) + + maybe( + http_archive, + name = "bazel_skylib", + sha256 = "1c531376ac7e5a180e0237938a2536de0c54d93f5c278634818e0efc952dd56c", + urls = [ + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.0.3/bazel-skylib-1.0.3.tar.gz", + ], + ) + + maybe( + http_archive, + name = "io_bazel_rules_go", + sha256 = "dd926a88a564a9246713a9c00b35315f54cbd46b31a26d5d8fb264c07045f05d", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.38.1/rules_go-v0.38.1.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.38.1/rules_go-v0.38.1.zip", + ], + ) + + maybe( + http_archive, + name = "bazel_gazelle", + sha256 = "5982e5463f171da99e3bdaeff8c0f48283a7a5f396ec5282910b9e8a49c0dd7e", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.25.0/bazel-gazelle-v0.25.0.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.25.0/bazel-gazelle-v0.25.0.tar.gz", + ], + ) + + maybe( + http_archive, + name = "robolectric", + urls = ["https://github.com/robolectric/robolectric-bazel/archive/4.9.2.tar.gz"], + strip_prefix = "robolectric-bazel-4.9.2", + sha256 = "7e007fcfdca7b7228cb4de72707e8b317026ea95000f963e91d5ae365be52d0d", + ) + + maybe( + http_archive, + name = "rules_license", + urls = [ + "https://github.com/bazelbuild/rules_license/releases/download/0.0.4/rules_license-0.0.4.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_license/releases/download/0.0.4/rules_license-0.0.4.tar.gz", + ], + sha256 = "6157e1e68378532d0241ecd15d3c45f6e5cfd98fc10846045509fb2a7cc9e381", + ) + + diff --git a/project.config b/project.config new file mode 100644 index 0000000..db40d49 --- /dev/null +++ b/project.config @@ -0,0 +1,7 @@ +[access] + inheritFrom = mobile-ninjas-releaser +[submit] + action = inherit +[access "refs/*"] + owner = group mdb/copybara-git-writers + owner = group mdb/mobile-ninjas-releaser diff --git a/rules/aar_import/BUILD b/rules/aar_import/BUILD index 50d41fe..b57f593 100644 --- a/rules/aar_import/BUILD +++ b/rules/aar_import/BUILD @@ -15,7 +15,7 @@ bzl_library( name = "bzl", srcs = glob(["*.bzl"]), deps = [ - "@rules_android//rules:common_bzl", - "@rules_android//rules/flags:bzl", + "//rules:common_bzl", + "//rules/flags:bzl", ], ) diff --git a/rules/aar_import/attrs.bzl b/rules/aar_import/attrs.bzl index 5ac9c7a..022231b 100644 --- a/rules/aar_import/attrs.bzl +++ b/rules/aar_import/attrs.bzl @@ -15,7 +15,7 @@ """Attributes.""" load( - "@rules_android//rules:attrs.bzl", + "//rules:attrs.bzl", _attrs = "attrs", ) @@ -24,15 +24,24 @@ ATTRS = _attrs.add( aar = attr.label( allow_single_file = [".aar"], mandatory = True, + doc = "The .aar file to process.", + ), + data = attr.label_list( + allow_files = True, + doc = "Files needed by this rule at runtime. May list file or rule " + + "targets. Generally allows any target.", ), - data = attr.label_list(allow_files = True), deps = attr.label_list( allow_files = False, providers = [JavaInfo], + doc = "The list of libraries to link against.", ), exports = attr.label_list( allow_files = False, - allow_rules = ["aar_import", "java_import"], + allow_rules = ["aar_import", "java_import", "kt_jvm_import"], + doc = "The closure of all rules reached via `exports` attributes are considered " + + "direct dependencies of any rule that directly depends on the target with " + + "`exports`. The `exports` are not direct deps of the rule they belong to.", ), has_lint_jar = attr.bool( default = False, @@ -45,20 +54,20 @@ ATTRS = _attrs.add( ), srcjar = attr.label( allow_single_file = [".srcjar"], - doc = - "A srcjar file that contains the source code for the JVM " + - "artifacts stored within the AAR.", + doc = "A srcjar file that contains the source code for the JVM " + + "artifacts stored within the AAR.", ), _flags = attr.label( - default = "@rules_android//rules/flags", + default = "//rules/flags", ), _java_toolchain = attr.label( default = Label("//tools/jdk:toolchain_android_only"), ), _host_javabase = attr.label( - cfg = "host", + cfg = "exec", default = Label("//tools/jdk:current_java_runtime"), ), ), _attrs.DATA_CONTEXT, + _attrs.ANDROID_TOOLCHAIN_ATTRS, ) diff --git a/rules/aar_import/impl.bzl b/rules/aar_import/impl.bzl index 280060e..46e1337 100644 --- a/rules/aar_import/impl.bzl +++ b/rules/aar_import/impl.bzl @@ -15,25 +15,25 @@ """Implementation.""" load( - "@rules_android//rules:acls.bzl", + "//rules:acls.bzl", _acls = "acls", ) load( - "@rules_android//rules:common.bzl", + "//rules:common.bzl", _common = "common", ) -load("@rules_android//rules:intellij.bzl", "intellij") +load("//rules:intellij.bzl", "intellij") load( - "@rules_android//rules:java.bzl", + "//rules:java.bzl", _java = "java", ) -load("@rules_android//rules:providers.bzl", "AndroidLintRulesInfo") +load("//rules:providers.bzl", "AndroidLintRulesInfo") load( - "@rules_android//rules:resources.bzl", + "//rules:resources.bzl", _resources = "resources", ) load( - "@rules_android//rules:utils.bzl", + "//rules:utils.bzl", _get_android_toolchain = "get_android_toolchain", _utils = "utils", ) @@ -41,38 +41,44 @@ load( RULE_PREFIX = "_aar" ANDROID_MANIFEST = "AndroidManifest.xml" LINT_JAR = "lint.jar" -_UNEXPECTED_LINT_JAR_ERROR = ( - "In target %s, has_lint_jar attribute is required when the aar contains " + - "a lint.jar file." -) # Resources context dict fields. _PROVIDERS = "providers" _VALIDATION_RESULTS = "validation_results" -def _create_aar_artifact(ctx, name): - return ctx.actions.declare_file("%s/%s/%s" % (RULE_PREFIX, ctx.label.name, name)) - def _create_aar_tree_artifact(ctx, name): return ctx.actions.declare_directory("%s/unzipped/%s/%s" % (RULE_PREFIX, name, ctx.label.name)) +def create_aar_artifact(ctx, name): + return ctx.actions.declare_file("%s/%s/%s" % (RULE_PREFIX, ctx.label.name, name)) + # Create an action to extract a file (specified by the parameter filename) from an AAR file. -def _extract_single_file( +# Will optionally create an empty output if the requested file does not exist.. +def extract_single_file( ctx, out_file, aar, filename, - unzip_tool): - args = ctx.actions.args() - args.add(aar) - args.add(filename) - args.add("-d", out_file.dirname) - - ctx.actions.run( - executable = unzip_tool, - arguments = [args], + unzip_tool, + create_empty_file = False): + ctx.actions.run_shell( + tools = [unzip_tool], inputs = [aar], outputs = [out_file], + command = + """ + if ! {create_empty_file} || {unzip_tool} -l {aar} | grep -q {file}; then + {unzip_tool} -q {aar} {file} -d {dirname}; + else + touch {dirname}/{file}; + fi + """.format( + unzip_tool = unzip_tool.executable.path, + aar = aar.path, + file = out_file.basename, + dirname = out_file.dirname, + create_empty_file = str(create_empty_file).lower(), + ), mnemonic = "AarFileExtractor", progress_message = "Extracting %s from %s" % (filename, aar.basename), ) @@ -220,7 +226,7 @@ def _extract_and_merge_jars( extracts and merges all Jars. """ jars_tree_artifact = _create_aar_tree_artifact(ctx, "jars") - jars_params_file = _create_aar_artifact(ctx, "jar_merging_params") + jars_params_file = create_aar_artifact(ctx, "jar_merging_params") _extract_jars( ctx, jars_tree_artifact, @@ -314,7 +320,7 @@ def _process_jars( ]) merged_java_info = java_common.merge(java_infos + r_java_info) - jdeps_artifact = _create_aar_artifact(ctx, "jdeps.proto") + jdeps_artifact = create_aar_artifact(ctx, "jdeps.proto") _create_import_deps_check( ctx, [out_jar], @@ -384,26 +390,31 @@ def _process_lint_rules( ctx, aar, unzip_tool): - providers = [] + transitive_lint_jars = [info.lint_jars for info in _utils.collect_providers( + AndroidLintRulesInfo, + ctx.attr.exports, + )] if ctx.attr.has_lint_jar: - lint_jar = _create_aar_artifact(ctx, LINT_JAR) - _extract_single_file( + lint_jar = create_aar_artifact(ctx, LINT_JAR) + extract_single_file( ctx, lint_jar, aar, LINT_JAR, unzip_tool, ) - providers.append(AndroidLintRulesInfo( - lint_jar = lint_jar, - )) - - providers.extend(_utils.collect_providers( - AndroidLintRulesInfo, - ctx.attr.exports, - )) - return providers + return [ + AndroidLintRulesInfo( + lint_jars = depset(direct = [lint_jar], transitive = transitive_lint_jars), + ), + ] + elif transitive_lint_jars: + return [ + AndroidLintRulesInfo(lint_jars = depset(transitive = transitive_lint_jars)), + ] + else: + return [] def _collect_proguard( ctx, @@ -443,8 +454,8 @@ def impl(ctx): package = _java.resolve_package_from_label(ctx.label, ctx.attr.package) # Extract the AndroidManifest.xml from the AAR. - android_manifest = _create_aar_artifact(ctx, ANDROID_MANIFEST) - _extract_single_file( + android_manifest = create_aar_artifact(ctx, ANDROID_MANIFEST) + extract_single_file( ctx, android_manifest, aar, @@ -452,11 +463,19 @@ def impl(ctx): unzip_tool, ) + # Bump min SDK to floor + manifest_ctx = _resources.bump_min_sdk( + ctx, + manifest = android_manifest, + floor = _resources.DEPOT_MIN_SDK_FLOOR if _acls.in_enforce_min_sdk_floor_rollout(str(ctx.label)) else 0, + enforce_min_sdk_floor_tool = _get_android_toolchain(ctx).enforce_min_sdk_floor_tool.files_to_run, + ) + resources_ctx = _process_resources( ctx, aar = aar, package = package, - manifest = android_manifest, + manifest = manifest_ctx.processed_manifest, deps = ctx.attr.deps, aar_resources_extractor_tool = _get_android_toolchain(ctx).aar_resources_extractor.files_to_run, @@ -464,7 +483,7 @@ def impl(ctx): ) providers.extend(resources_ctx.providers) - merged_jar = _create_aar_artifact(ctx, "classes_and_libs_merged.jar") + merged_jar = create_aar_artifact(ctx, "classes_and_libs_merged.jar") jvm_ctx = _process_jars( ctx, out_jar = merged_jar, @@ -493,7 +512,7 @@ def impl(ctx): providers.extend(jvm_ctx.providers) validation_outputs.extend(jvm_ctx.validation_results) - native_libs = _create_aar_artifact(ctx, "native_libs.zip") + native_libs = create_aar_artifact(ctx, "native_libs.zip") _extract_native_libs( ctx, native_libs, @@ -517,7 +536,7 @@ def impl(ctx): ) # Will be empty if there's no proguard.txt file in the aar - proguard_spec = _create_aar_artifact(ctx, "proguard.txt") + proguard_spec = create_aar_artifact(ctx, "proguard.txt") providers.append(_collect_proguard( ctx, proguard_spec, @@ -536,7 +555,7 @@ def impl(ctx): ctx, aar = aar, package = package, - manifest = android_manifest, + manifest = manifest_ctx.processed_manifest, checks = _get_android_toolchain(ctx).aar_import_checks.files_to_run, )) diff --git a/rules/aar_import/rule.bzl b/rules/aar_import/rule.bzl index 02a1d61..3274c5d 100644 --- a/rules/aar_import/rule.bzl +++ b/rules/aar_import/rule.bzl @@ -17,15 +17,31 @@ load(":attrs.bzl", _ATTRS = "ATTRS") load(":impl.bzl", _impl = "impl") +RULE_DOC = """ +#### Examples + +The following example shows how to use `aar_import`. +<pre><code>aar_import( + name = "hellobazellib", + aar = "lib.aar", + package = "bazel.hellobazellib", + deps = [ + "//java/bazel/hellobazellib/activities", + "//java/bazel/hellobazellib/common", + ], +)</code></pre> +""" + aar_import = rule( attrs = _ATTRS, fragments = ["android"], implementation = _impl, + doc = RULE_DOC, provides = [ AndroidIdeInfo, AndroidLibraryResourceClassJarProvider, AndroidNativeLibsInfo, JavaInfo, ], - toolchains = ["@rules_android//toolchains/android:toolchain_type"], + toolchains = ["//toolchains/android:toolchain_type"], ) diff --git a/rules/acls.bzl b/rules/acls.bzl index ffa1377..f9e74ee 100644 --- a/rules/acls.bzl +++ b/rules/acls.bzl @@ -27,236 +27,281 @@ To update a list: 1. Directly add/remove/edit targets in the appropriate .bzl file """ -load("@rules_android//rules/acls:aar_import_deps_checker.bzl", "AAR_IMPORT_DEPS_CHECKER_FALLBACK", "AAR_IMPORT_DEPS_CHECKER_ROLLOUT") -load("@rules_android//rules/acls:aar_import_explicit_exports_manifest.bzl", "AAR_IMPORT_EXPLICIT_EXPORTS_MANIFEST") -load("@rules_android//rules/acls:aar_import_exports_r_java.bzl", "AAR_IMPORT_EXPORTS_R_JAVA") -load("@rules_android//rules/acls:aar_propagate_resources.bzl", "AAR_PROPAGATE_RESOURCES_FALLBACK", "AAR_PROPAGATE_RESOURCES_ROLLOUT") -load("@rules_android//rules/acls:ait_install_snapshots.bzl", "APP_INSTALLATION_SNAPSHOT", "APP_INSTALLATION_SNAPSHOT_FALLBACK") -load("@rules_android//rules/acls:ait_virtual_device.bzl", "AIT_VIRTUAL_DEVICE_FALLBACK", "AIT_VIRTUAL_DEVICE_ROLLOUT") -load("@rules_android//rules/acls:allow_resource_conflicts.bzl", "ALLOW_RESOURCE_CONFLICTS") -load("@rules_android//rules/acls:android_archive_dogfood.bzl", "ANDROID_ARCHIVE_DOGFOOD") -load("@rules_android//rules/acls:android_archive_excluded_deps_denylist.bzl", "ANDROID_ARCHIVE_EXCLUDED_DEPS_DENYLIST") -load("@rules_android//rules/acls:android_test_lockdown.bzl", "ANDROID_TEST_LOCKDOWN_GENERATOR_FUNCTIONS", "ANDROID_TEST_LOCKDOWN_TARGETS") -load("@rules_android//rules/acls:android_device_plugin_rollout.bzl", "ANDROID_DEVICE_PLUGIN_FALLBACK", "ANDROID_DEVICE_PLUGIN_ROLLOUT") -load("@rules_android//rules/acls:android_instrumentation_binary_starlark_resources.bzl", "ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_FALLBACK", "ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_ROLLOUT") -load("@rules_android//rules/acls:android_feature_splits_dogfood.bzl", "ANDROID_FEATURE_SPLITS_DOGFOOD") -load("@rules_android//rules/acls:android_library_implicit_exports.bzl", "ANDROID_LIBRARY_IMPLICIT_EXPORTS", "ANDROID_LIBRARY_IMPLICIT_EXPORTS_GENERATOR_FUNCTIONS") -load("@rules_android//rules/acls:android_library_resources_without_srcs.bzl", "ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS", "ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS_GENERATOR_FUNCTIONS") -load("@rules_android//rules/acls:android_library_starlark_resource_outputs.bzl", "ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_FALLBACK", "ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_ROLLOUT") -load("@rules_android//rules/acls:android_lint_checks_rollout.bzl", "ANDROID_LINT_CHECKS_FALLBACK", "ANDROID_LINT_CHECKS_ROLLOUT") -load("@rules_android//rules/acls:android_lint_rollout.bzl", "ANDROID_LINT_FALLBACK", "ANDROID_LINT_ROLLOUT") -load("@rules_android//rules/acls:lint_registry_rollout.bzl", "LINT_REGISTRY_FALLBACK", "LINT_REGISTRY_ROLLOUT") -load("@rules_android//rules/acls:android_build_stamping_rollout.bzl", "ANDROID_BUILD_STAMPING_FALLBACK", "ANDROID_BUILD_STAMPING_ROLLOUT") -load("@rules_android//rules/acls:b122039567.bzl", "B122039567") -load("@rules_android//rules/acls:b123854163.bzl", "B123854163") -load("@rules_android//rules/acls:dex2oat_opts.bzl", "CAN_USE_DEX2OAT_OPTIONS") -load("@rules_android//rules/acls:fix_export_exporting_rollout.bzl", "FIX_EXPORT_EXPORTING_FALLBACK", "FIX_EXPORT_EXPORTING_ROLLOUT") -load("@rules_android//rules/acls:fix_resource_transitivity_rollout.bzl", "FIX_RESOURCE_TRANSITIVITY_FALLBACK", "FIX_RESOURCE_TRANSITIVITY_ROLLOUT") -load("@rules_android//rules/acls:host_dex2oat_rollout.bzl", "AIT_USE_HOST_DEX2OAT_ROLLOUT", "AIT_USE_HOST_DEX2OAT_ROLLOUT_FALLBACK") -load("@rules_android//rules/acls:install_apps_in_data.bzl", "INSTALL_APPS_IN_DATA") -load("@rules_android//rules/acls:local_test_multi_proto.bzl", "LOCAL_TEST_MULTI_PROTO_PKG") -load("@rules_android//rules/acls:local_test_rollout.bzl", "LOCAL_TEST_FALLBACK", "LOCAL_TEST_ROLLOUT") -load("@rules_android//rules/acls:local_test_starlark_resources.bzl", "LOCAL_TEST_STARLARK_RESOURCES_FALLBACK", "LOCAL_TEST_STARLARK_RESOURCES_ROLLOUT") -load("@rules_android//rules/acls:android_test_platform_rollout.bzl", "ANDROID_TEST_PLATFORM_FALLBACK", "ANDROID_TEST_PLATFORM_ROLLOUT") -load("@rules_android//rules/acls:sourceless_binary_rollout.bzl", "SOURCELESS_BINARY_FALLBACK", "SOURCELESS_BINARY_ROLLOUT") -load("@rules_android//rules/acls:test_to_instrument_test_rollout.bzl", "TEST_TO_INSTRUMENT_TEST_FALLBACK", "TEST_TO_INSTRUMENT_TEST_ROLLOUT") +load("//rules/acls:aar_import_deps_checker.bzl", "AAR_IMPORT_DEPS_CHECKER_FALLBACK", "AAR_IMPORT_DEPS_CHECKER_ROLLOUT") +load("//rules/acls:aar_import_explicit_exports_manifest.bzl", "AAR_IMPORT_EXPLICIT_EXPORTS_MANIFEST") +load("//rules/acls:aar_import_exports_r_java.bzl", "AAR_IMPORT_EXPORTS_R_JAVA") +load("//rules/acls:aar_propagate_resources.bzl", "AAR_PROPAGATE_RESOURCES_FALLBACK", "AAR_PROPAGATE_RESOURCES_ROLLOUT") +load("//rules/acls:ait_install_snapshots.bzl", "APP_INSTALLATION_SNAPSHOT", "APP_INSTALLATION_SNAPSHOT_FALLBACK") +load("//rules/acls:allow_resource_conflicts.bzl", "ALLOW_RESOURCE_CONFLICTS") +load("//rules/acls:android_archive_dogfood.bzl", "ANDROID_ARCHIVE_DOGFOOD") +load("//rules/acls:android_archive_duplicate_class_allowlist.bzl", "ANDROID_ARCHIVE_DUPLICATE_CLASS_ALLOWLIST") +load("//rules/acls:android_archive_excluded_deps_denylist.bzl", "ANDROID_ARCHIVE_EXCLUDED_DEPS_DENYLIST") +load("//rules/acls:android_archive_exposed_package_allowlist.bzl", "ANDROID_ARCHIVE_EXPOSED_PACKAGE_ALLOWLIST") +load("//rules/acls:android_test_lockdown.bzl", "ANDROID_TEST_LOCKDOWN_GENERATOR_FUNCTIONS", "ANDROID_TEST_LOCKDOWN_TARGETS") +load("//rules/acls:android_device_plugin_rollout.bzl", "ANDROID_DEVICE_PLUGIN_FALLBACK", "ANDROID_DEVICE_PLUGIN_ROLLOUT") +load("//rules/acls:android_instrumentation_binary_starlark_resources.bzl", "ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_FALLBACK", "ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_ROLLOUT") +load("//rules/acls:android_binary_starlark_javac.bzl", "ANDROID_BINARY_STARLARK_JAVAC_FALLBACK", "ANDROID_BINARY_STARLARK_JAVAC_ROLLOUT") +load("//rules/acls:android_binary_starlark_split_transition.bzl", "ANDROID_BINARY_STARLARK_SPLIT_TRANSITION_FALLBACK", "ANDROID_BINARY_STARLARK_SPLIT_TRANSITION_ROLLOUT") +load("//rules/acls:android_feature_splits_dogfood.bzl", "ANDROID_FEATURE_SPLITS_DOGFOOD") +load("//rules/acls:android_library_resources_without_srcs.bzl", "ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS", "ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS_GENERATOR_FUNCTIONS") +load("//rules/acls:android_library_starlark_resource_outputs.bzl", "ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_FALLBACK", "ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_ROLLOUT") +load("//rules/acls:android_library_use_aosp_aidl_compiler.bzl", "ANDROID_LIBRARY_USE_AOSP_AIDL_COMPILER_ALLOWLIST") +load("//rules/acls:android_lint_checks_rollout.bzl", "ANDROID_LINT_CHECKS_FALLBACK", "ANDROID_LINT_CHECKS_ROLLOUT") +load("//rules/acls:android_lint_rollout.bzl", "ANDROID_LINT_FALLBACK", "ANDROID_LINT_ROLLOUT") +load("//rules/acls:lint_registry_rollout.bzl", "LINT_REGISTRY_FALLBACK", "LINT_REGISTRY_ROLLOUT") +load("//rules/acls:android_build_stamping_rollout.bzl", "ANDROID_BUILD_STAMPING_FALLBACK", "ANDROID_BUILD_STAMPING_ROLLOUT") +load("//rules/acls:b122039567.bzl", "B122039567") +load("//rules/acls:b123854163.bzl", "B123854163") +load("//rules/acls:databinding.bzl", "DATABINDING_ALLOWED", "DATABINDING_DISALLOWED") +load("//rules/acls:dex2oat_opts.bzl", "CAN_USE_DEX2OAT_OPTIONS") +load("//rules/acls:fix_export_exporting_rollout.bzl", "FIX_EXPORT_EXPORTING_FALLBACK", "FIX_EXPORT_EXPORTING_ROLLOUT") +load("//rules/acls:fix_resource_transitivity_rollout.bzl", "FIX_RESOURCE_TRANSITIVITY_FALLBACK", "FIX_RESOURCE_TRANSITIVITY_ROLLOUT") +load("//rules/acls:host_dex2oat_rollout.bzl", "AIT_USE_HOST_DEX2OAT_ROLLOUT", "AIT_USE_HOST_DEX2OAT_ROLLOUT_FALLBACK") +load("//rules/acls:install_apps_in_data.bzl", "INSTALL_APPS_IN_DATA") +load("//rules/acls:local_test_multi_proto.bzl", "LOCAL_TEST_MULTI_PROTO_PKG") +load("//rules/acls:local_test_rollout.bzl", "LOCAL_TEST_FALLBACK", "LOCAL_TEST_ROLLOUT") +load("//rules/acls:local_test_starlark_resources.bzl", "LOCAL_TEST_STARLARK_RESOURCES_FALLBACK", "LOCAL_TEST_STARLARK_RESOURCES_ROLLOUT") +load("//rules/acls:android_test_platform_rollout.bzl", "ANDROID_TEST_PLATFORM_FALLBACK", "ANDROID_TEST_PLATFORM_ROLLOUT") +load("//rules/acls:sourceless_binary_rollout.bzl", "SOURCELESS_BINARY_FALLBACK", "SOURCELESS_BINARY_ROLLOUT") +load("//rules/acls:test_to_instrument_test_rollout.bzl", "TEST_TO_INSTRUMENT_TEST_FALLBACK", "TEST_TO_INSTRUMENT_TEST_ROLLOUT") load( - "@rules_android//rules/acls:partial_jetification_targets.bzl", + "//rules/acls:partial_jetification_targets.bzl", "PARTIAL_JETIFICATION_TARGETS_FALLBACK", "PARTIAL_JETIFICATION_TARGETS_ROLLOUT", ) -load("@rules_android//rules/acls:kt_android_library_rollout.bzl", "KT_ANDROID_LIBRARY_FALLBACK", "KT_ANDROID_LIBRARY_ROLLOUT") -load("@rules_android//rules/acls:android_instrumentation_test_manifest_check_rollout.bzl", "ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_FALLBACK", "ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_ROLLOUT") -load("@rules_android//rules/acls:android_instrumentation_derived_test_class_rollout.bzl", "ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_FALLBACK", "ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_ROLLOUT") +load("//rules/acls:kt_android_library_rollout.bzl", "KT_ANDROID_LIBRARY_FALLBACK", "KT_ANDROID_LIBRARY_ROLLOUT") +load("//rules/acls:android_instrumentation_test_manifest_check_rollout.bzl", "ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_FALLBACK", "ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_ROLLOUT") +load("//rules/acls:android_instrumentation_test_prebuilt_test_apk.bzl", "ANDROID_INSTRUMENTATION_TEST_PREBUILT_TEST_APK_FALLBACK", "ANDROID_INSTRUMENTATION_TEST_PREBUILT_TEST_APK_ROLLOUT") +load("//rules/acls:android_rules_with_kt_rollout.bzl", "ANDROID_RULES_WITH_KT_FALLBACK", "ANDROID_RULES_WITH_KT_ROLLOUT") +load("//rules/acls:baseline_profiles_rollout.bzl", "BASELINE_PROFILES_ROLLOUT") +load("//rules/acls:enforce_min_sdk_floor_rollout.bzl", "ENFORCE_MIN_SDK_FLOOR_FALLBACK", "ENFORCE_MIN_SDK_FLOOR_ROLLOUT") +load("//rules/acls:android_apk_to_bundle_features_lockdown.bzl", "ANDROID_APK_TO_BUNDLE_FEATURES") +load("//rules/acls:android_local_test_jdk_sts_rollout.bzl", "ANDROID_LOCAL_TEST_JDK_STS_FALLBACK", "ANDROID_LOCAL_TEST_JDK_STS_ROLLOUT") def _in_aar_import_deps_checker(fqn): - return not _matches(fqn, AAR_IMPORT_DEPS_CHECKER_FALLBACK_DICT) and _matches(fqn, AAR_IMPORT_DEPS_CHECKER_ROLLOUT_DICT) + return not matches(fqn, AAR_IMPORT_DEPS_CHECKER_FALLBACK_DICT) and matches(fqn, AAR_IMPORT_DEPS_CHECKER_ROLLOUT_DICT) def _in_aar_import_explicit_exports_manifest(fqn): - return _matches(fqn, AAR_IMPORT_EXPLICIT_EXPORTS_MANIFEST_DICT) + return matches(fqn, AAR_IMPORT_EXPLICIT_EXPORTS_MANIFEST_DICT) def _in_aar_import_exports_r_java(fqn): - return _matches(fqn, AAR_IMPORT_EXPORTS_R_JAVA_DICT) + return matches(fqn, AAR_IMPORT_EXPORTS_R_JAVA_DICT) def _in_aar_propagate_resources(fqn): - return not _matches(fqn, AAR_PROPAGATE_RESOURCES_FALLBACK_DICT) and _matches(fqn, AAR_PROPAGATE_RESOURCES_ROLLOUT_DICT) - -def _in_ait_virtual_device(fqn): - return not _matches(fqn, AIT_VIRTUAL_DEVICE_FALLBACK_DICT) and _matches(fqn, AIT_VIRTUAL_DEVICE_ROLLOUT_DICT) + return not matches(fqn, AAR_PROPAGATE_RESOURCES_FALLBACK_DICT) and matches(fqn, AAR_PROPAGATE_RESOURCES_ROLLOUT_DICT) def _in_android_archive_dogfood(fqn): - return _matches(fqn, ANDROID_ARCHIVE_DOGFOOD_DICT) + return matches(fqn, ANDROID_ARCHIVE_DOGFOOD_DICT) def _in_android_archive_excluded_deps_denylist(fqn): - return _matches(fqn, ANDROID_ARCHIVE_EXCLUDED_DEPS_DENYLIST_DICT) + return matches(fqn, ANDROID_ARCHIVE_EXCLUDED_DEPS_DENYLIST_DICT) def _in_android_device_plugin_rollout(fqn): - return not _matches(fqn, ANDROID_DEVICE_PLUGIN_FALLBACK_DICT) and _matches(fqn, ANDROID_DEVICE_PLUGIN_ROLLOUT_DICT) + return not matches(fqn, ANDROID_DEVICE_PLUGIN_FALLBACK_DICT) and matches(fqn, ANDROID_DEVICE_PLUGIN_ROLLOUT_DICT) def _in_android_instrumentation_binary_starlark_resources(fqn): - return not _matches(fqn, ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_FALLBACK_DICT) and _matches(fqn, ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_ROLLOUT_DICT) + return not matches(fqn, ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_FALLBACK_DICT) and matches(fqn, ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_ROLLOUT_DICT) + +def _in_android_binary_starlark_javac(fqn): + return not matches(fqn, ANDROID_BINARY_STARLARK_JAVAC_FALLBACK_DICT) and matches(fqn, ANDROID_BINARY_STARLARK_JAVAC_ROLLOUT_DICT) + +def _in_android_binary_starlark_split_transition(fqn): + return not matches(fqn, ANDROID_BINARY_STARLARK_SPLIT_TRANSITION_FALLBACK_DICT) and matches(fqn, ANDROID_BINARY_STARLARK_SPLIT_TRANSITION_ROLLOUT_DICT) def _in_android_feature_splits_dogfood(fqn): - return _matches(fqn, ANDROID_FEATURE_SPLITS_DOGFOOD_DICT) + return matches(fqn, ANDROID_FEATURE_SPLITS_DOGFOOD_DICT) def _in_android_lint_checks_rollout(fqn): - return not _matches(fqn, ANDROID_LINT_CHECKS_FALLBACK_DICT) and _matches(fqn, ANDROID_LINT_CHECKS_ROLLOUT_DICT) + return not matches(fqn, ANDROID_LINT_CHECKS_FALLBACK_DICT) and matches(fqn, ANDROID_LINT_CHECKS_ROLLOUT_DICT) def _in_android_lint_rollout(fqn): - return not _matches(fqn, ANDROID_LINT_FALLBACK_DICT) and _matches(fqn, ANDROID_LINT_ROLLOUT_DICT) + return not matches(fqn, ANDROID_LINT_FALLBACK_DICT) and matches(fqn, ANDROID_LINT_ROLLOUT_DICT) def _in_lint_registry_rollout(fqn): - return not _matches(fqn, LINT_REGISTRY_FALLBACK_DICT) and _matches(fqn, LINT_REGISTRY_ROLLOUT_DICT) + return not matches(fqn, LINT_REGISTRY_FALLBACK_DICT) and matches(fqn, LINT_REGISTRY_ROLLOUT_DICT) def _in_android_build_stamping_rollout(fqn): - return not _matches(fqn, ANDROID_BUILD_STAMPING_FALLBACK_DICT) and _matches(fqn, ANDROID_BUILD_STAMPING_ROLLOUT_DICT) + return not matches(fqn, ANDROID_BUILD_STAMPING_FALLBACK_DICT) and matches(fqn, ANDROID_BUILD_STAMPING_ROLLOUT_DICT) def _in_android_test_lockdown_allowlist(fqn, generator): if generator == "android_test": - return _matches(fqn, ANDROID_TEST_LOCKDOWN_TARGETS) + return matches(fqn, ANDROID_TEST_LOCKDOWN_TARGETS) return generator in ANDROID_TEST_LOCKDOWN_GENERATOR_FUNCTIONS_DICT def _in_b122039567(fqn): - return _matches(fqn, B122039567_DICT) + return matches(fqn, B122039567_DICT) def _in_b123854163(fqn): - return _matches(fqn, B123854163_DICT) - -def _in_android_library_implicit_exports(fqn): - return _matches(fqn, ANDROID_LIBRARY_IMPLICIT_EXPORTS_DICT) - -def _in_android_library_implicit_exports_generator_functions(gfn): - return gfn in ANDROID_LIBRARY_IMPLICIT_EXPORTS_GENERATOR_FUNCTIONS_DICT + return matches(fqn, B123854163_DICT) def _in_android_library_resources_without_srcs(fqn): - return _matches(fqn, ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS_DICT) + return matches(fqn, ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS_DICT) def _in_android_library_resources_without_srcs_generator_functions(gfn): return gfn in ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS_GENERATOR_FUNCTIONS_DICT def _in_android_library_starlark_resource_outputs_rollout(fqn): - return not _matches(fqn, ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_FALLBACK_DICT) and _matches(fqn, ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_ROLLOUT_DICT) + return not matches(fqn, ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_FALLBACK_DICT) and matches(fqn, ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_ROLLOUT_DICT) + +def _in_android_library_use_aosp_aidl_compiler_allowlist(fqn): + return matches(fqn, ANDROID_LIBRARY_USE_AOSP_AIDL_COMPILER_ALLOWLIST_DICT) def _in_app_installation_snapshot(fqn): - return not _matches(fqn, APP_INSTALLATION_SNAPSHOT_FALLBACK_DICT) and _matches(fqn, APP_INSTALLATION_SNAPSHOT_DICT) + return not matches(fqn, APP_INSTALLATION_SNAPSHOT_FALLBACK_DICT) and matches(fqn, APP_INSTALLATION_SNAPSHOT_DICT) + +def _in_databinding_allowed(fqn): + return not matches(fqn, DATABINDING_DISALLOWED_DICT) and matches(fqn, DATABINDING_ALLOWED_DICT) def _in_dex2oat_opts(fqn): - return _matches(fqn, CAN_USE_DEX2OAT_OPTIONS_DICT) + return matches(fqn, CAN_USE_DEX2OAT_OPTIONS_DICT) def _in_fix_export_exporting_rollout(fqn): - return not _matches(fqn, FIX_EXPORT_EXPORTING_FALLBACK_DICT) and _matches(fqn, FIX_EXPORT_EXPORTING_ROLLOUT_DICT) + return not matches(fqn, FIX_EXPORT_EXPORTING_FALLBACK_DICT) and matches(fqn, FIX_EXPORT_EXPORTING_ROLLOUT_DICT) def _in_fix_resource_transivity_rollout(fqn): - return not _matches(fqn, FIX_RESOURCE_TRANSIVITY_FALLBACK_DICT) and _matches(fqn, FIX_RESOURCE_TRANSIVITY_ROLLOUT_DICT) + return not matches(fqn, FIX_RESOURCE_TRANSIVITY_FALLBACK_DICT) and matches(fqn, FIX_RESOURCE_TRANSIVITY_ROLLOUT_DICT) def _in_host_dex2oat_rollout(fqn): - return not _matches(fqn, AIT_USE_HOST_DEX2OAT_ROLLOUT_FALLBACK_DICT) and _matches(fqn, AIT_USE_HOST_DEX2OAT_ROLLOUT_DICT) + return not matches(fqn, AIT_USE_HOST_DEX2OAT_ROLLOUT_FALLBACK_DICT) and matches(fqn, AIT_USE_HOST_DEX2OAT_ROLLOUT_DICT) def _in_install_apps_in_data(fqn): - return _matches(fqn, AIT_INSTALL_APPS_IN_DATA_DICT) + return matches(fqn, AIT_INSTALL_APPS_IN_DATA_DICT) def _in_local_test_multi_proto(fqn): - return _matches(fqn, LOCAL_TEST_MULTI_PROTO_PKG_DICT) + return matches(fqn, LOCAL_TEST_MULTI_PROTO_PKG_DICT) def _in_local_test_rollout(fqn): - return not _matches(fqn, LOCAL_TEST_FALLBACK_DICT) and _matches(fqn, LOCAL_TEST_ROLLOUT_DICT) + return not matches(fqn, LOCAL_TEST_FALLBACK_DICT) and matches(fqn, LOCAL_TEST_ROLLOUT_DICT) def _in_local_test_starlark_resources(fqn): - return not _matches(fqn, LOCAL_TEST_STARLARK_RESOURCES_FALLBACK_DICT) and _matches(fqn, LOCAL_TEST_STARLARK_RESOURCES_ROLLOUT_DICT) + return not matches(fqn, LOCAL_TEST_STARLARK_RESOURCES_FALLBACK_DICT) and matches(fqn, LOCAL_TEST_STARLARK_RESOURCES_ROLLOUT_DICT) def _in_android_test_platform_rollout(fqn): - return not _matches(fqn, ANDROID_TEST_PLATFORM_FALLBACK_DICT) and _matches(fqn, ANDROID_TEST_PLATFORM_ROLLOUT_DICT) + return not matches(fqn, ANDROID_TEST_PLATFORM_FALLBACK_DICT) and matches(fqn, ANDROID_TEST_PLATFORM_ROLLOUT_DICT) def _in_sourceless_binary_rollout(fqn): - return not _matches(fqn, SOURCELESS_BINARY_FALLBACK_DICT) and _matches(fqn, SOURCELESS_BINARY_ROLLOUT_DICT) + return not matches(fqn, SOURCELESS_BINARY_FALLBACK_DICT) and matches(fqn, SOURCELESS_BINARY_ROLLOUT_DICT) def _in_test_to_instrument_test_rollout(fqn): - return not _matches(fqn, TEST_TO_INSTRUMENT_TEST_FALLBACK_DICT) and _matches(fqn, TEST_TO_INSTRUMENT_TEST_ROLLOUT_DICT) + return not matches(fqn, TEST_TO_INSTRUMENT_TEST_FALLBACK_DICT) and matches(fqn, TEST_TO_INSTRUMENT_TEST_ROLLOUT_DICT) def _in_allow_resource_conflicts(fqn): - return _matches(fqn, ALLOW_RESOURCE_CONFLICTS_DICT) + return matches(fqn, ALLOW_RESOURCE_CONFLICTS_DICT) def _in_partial_jetification_targets(fqn): - return not _matches(fqn, PARTIAL_JETIFICATION_TARGETS_FALLBACK_DICT) and _matches(fqn, PARTIAL_JETIFICATION_TARGETS_ROLLOUT_DICT) + return not matches(fqn, PARTIAL_JETIFICATION_TARGETS_FALLBACK_DICT) and matches(fqn, PARTIAL_JETIFICATION_TARGETS_ROLLOUT_DICT) def _in_kt_android_library_rollout(fqn): - return not _matches(fqn, KT_ANDROID_LIBRARY_FALLBACK_DICT) and _matches(fqn, KT_ANDROID_LIBRARY_ROLLOUT_DICT) + return not matches(fqn, KT_ANDROID_LIBRARY_FALLBACK_DICT) and matches(fqn, KT_ANDROID_LIBRARY_ROLLOUT_DICT) def _in_android_instrumentation_test_manifest_check_rollout(fqn): - return not _matches(fqn, ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_FALLBACK_DICT) and _matches(fqn, ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_ROLLOUT_DICT) + return not matches(fqn, ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_FALLBACK_DICT) and matches(fqn, ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_ROLLOUT_DICT) + +def _in_android_instrumentation_test_prebuilt_test_apk(fqn): + return matches(fqn, ANDROID_INSTRUMENTATION_TEST_PREBUILT_TEST_APK_ROLLOUT_DICT) and not matches(fqn, ANDROID_INSTRUMENTATION_TEST_PREBUILT_TEST_APK_FALLBACK_DICT) + +def _in_android_rules_with_kt_rollout(fqn): + return not matches(fqn, ANDROID_RULES_WITH_KT_FALLBACK_DICT) and matches(fqn, ANDROID_RULES_WITH_KT_ROLLOUT_DICT) + +def _get_android_archive_exposed_package_allowlist(fqn): + return ANDROID_ARCHIVE_EXPOSED_PACKAGE_ALLOWLIST.get(fqn, []) + +def _in_baseline_profiles_rollout(fqn): + return matches(fqn, BASELINE_PROFILES_ROLLOUT) + +def _in_enforce_min_sdk_floor_rollout(fqn): + return not matches(fqn, ENFORCE_MIN_SDK_FLOOR_FALLBACK_DICT) and matches(fqn, ENFORCE_MIN_SDK_FLOOR_ROLLOUT_DICT) + +def _in_android_apk_to_bundle_features(fqn): + return matches(fqn, ANDROID_APK_TO_BUNDLE_FEATURES_DICT) + +def _get_android_archive_duplicate_class_allowlist(fqn): + return ANDROID_ARCHIVE_DUPLICATE_CLASS_ALLOWLIST.get(fqn, []) -def _in_android_instrumentation_test_derived_test_class_rollout(fqn): - return not _matches(fqn, ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_FALLBACK_DICT) and _matches(fqn, ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_ROLLOUT_DICT) +def _in_android_local_test_jdk_sts_rollout(fqn): + return not matches(fqn, ANDROID_LOCAL_TEST_JDK_STS_FALLBACK_DICT) and matches(fqn, ANDROID_LOCAL_TEST_JDK_STS_ROLLOUT_DICT) -def _make_dict(lst): - """Do not use this method outside of this file.""" +def make_dict(lst): + """Do not use this method outside of acls directory.""" return {t: True for t in lst} -AAR_IMPORT_DEPS_CHECKER_FALLBACK_DICT = _make_dict(AAR_IMPORT_DEPS_CHECKER_FALLBACK) -AAR_IMPORT_DEPS_CHECKER_ROLLOUT_DICT = _make_dict(AAR_IMPORT_DEPS_CHECKER_ROLLOUT) -AAR_IMPORT_EXPLICIT_EXPORTS_MANIFEST_DICT = _make_dict(AAR_IMPORT_EXPLICIT_EXPORTS_MANIFEST) -AAR_IMPORT_EXPORTS_R_JAVA_DICT = _make_dict(AAR_IMPORT_EXPORTS_R_JAVA) -AAR_PROPAGATE_RESOURCES_FALLBACK_DICT = _make_dict(AAR_PROPAGATE_RESOURCES_FALLBACK) -AAR_PROPAGATE_RESOURCES_ROLLOUT_DICT = _make_dict(AAR_PROPAGATE_RESOURCES_ROLLOUT) -AIT_VIRTUAL_DEVICE_FALLBACK_DICT = _make_dict(AIT_VIRTUAL_DEVICE_FALLBACK) -AIT_VIRTUAL_DEVICE_ROLLOUT_DICT = _make_dict(AIT_VIRTUAL_DEVICE_ROLLOUT) -ANDROID_ARCHIVE_DOGFOOD_DICT = _make_dict(ANDROID_ARCHIVE_DOGFOOD) -ANDROID_ARCHIVE_EXCLUDED_DEPS_DENYLIST_DICT = _make_dict(ANDROID_ARCHIVE_EXCLUDED_DEPS_DENYLIST) -ANDROID_DEVICE_PLUGIN_ROLLOUT_DICT = _make_dict(ANDROID_DEVICE_PLUGIN_ROLLOUT) -ANDROID_DEVICE_PLUGIN_FALLBACK_DICT = _make_dict(ANDROID_DEVICE_PLUGIN_FALLBACK) -ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_ROLLOUT_DICT = _make_dict(ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_ROLLOUT) -ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_FALLBACK_DICT = _make_dict(ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_FALLBACK) -ANDROID_FEATURE_SPLITS_DOGFOOD_DICT = _make_dict(ANDROID_FEATURE_SPLITS_DOGFOOD) -ANDROID_LIBRARY_IMPLICIT_EXPORTS_DICT = _make_dict(ANDROID_LIBRARY_IMPLICIT_EXPORTS) -ANDROID_LIBRARY_IMPLICIT_EXPORTS_GENERATOR_FUNCTIONS_DICT = _make_dict(ANDROID_LIBRARY_IMPLICIT_EXPORTS_GENERATOR_FUNCTIONS) -ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS_DICT = _make_dict(ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS) -ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS_GENERATOR_FUNCTIONS_DICT = _make_dict(ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS_GENERATOR_FUNCTIONS) -ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_FALLBACK_DICT = _make_dict(ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_FALLBACK) -ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_ROLLOUT_DICT = _make_dict(ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_ROLLOUT) -ANDROID_LINT_CHECKS_FALLBACK_DICT = _make_dict(ANDROID_LINT_CHECKS_FALLBACK) -ANDROID_LINT_CHECKS_ROLLOUT_DICT = _make_dict(ANDROID_LINT_CHECKS_ROLLOUT) -ANDROID_LINT_FALLBACK_DICT = _make_dict(ANDROID_LINT_FALLBACK) -ANDROID_LINT_ROLLOUT_DICT = _make_dict(ANDROID_LINT_ROLLOUT) -LINT_REGISTRY_FALLBACK_DICT = _make_dict(LINT_REGISTRY_FALLBACK) -LINT_REGISTRY_ROLLOUT_DICT = _make_dict(LINT_REGISTRY_ROLLOUT) -ANDROID_BUILD_STAMPING_ROLLOUT_DICT = _make_dict(ANDROID_BUILD_STAMPING_ROLLOUT) -ANDROID_BUILD_STAMPING_FALLBACK_DICT = _make_dict(ANDROID_BUILD_STAMPING_FALLBACK) -ANDROID_TEST_LOCKDOWN_GENERATOR_FUNCTIONS_DICT = _make_dict(ANDROID_TEST_LOCKDOWN_GENERATOR_FUNCTIONS) -ANDROID_TEST_LOCKDOWN_TARGETS_DICT = _make_dict(ANDROID_TEST_LOCKDOWN_TARGETS) -APP_INSTALLATION_SNAPSHOT_DICT = _make_dict(APP_INSTALLATION_SNAPSHOT) -APP_INSTALLATION_SNAPSHOT_FALLBACK_DICT = _make_dict(APP_INSTALLATION_SNAPSHOT_FALLBACK) -B122039567_DICT = _make_dict(B122039567) -B123854163_DICT = _make_dict(B123854163) -CAN_USE_DEX2OAT_OPTIONS_DICT = _make_dict(CAN_USE_DEX2OAT_OPTIONS) -FIX_RESOURCE_TRANSIVITY_FALLBACK_DICT = _make_dict(FIX_RESOURCE_TRANSITIVITY_FALLBACK) -FIX_RESOURCE_TRANSIVITY_ROLLOUT_DICT = _make_dict(FIX_RESOURCE_TRANSITIVITY_ROLLOUT) -FIX_EXPORT_EXPORTING_FALLBACK_DICT = _make_dict(FIX_EXPORT_EXPORTING_FALLBACK) -FIX_EXPORT_EXPORTING_ROLLOUT_DICT = _make_dict(FIX_EXPORT_EXPORTING_ROLLOUT) -AIT_USE_HOST_DEX2OAT_ROLLOUT_DICT = _make_dict(AIT_USE_HOST_DEX2OAT_ROLLOUT) -AIT_USE_HOST_DEX2OAT_ROLLOUT_FALLBACK_DICT = _make_dict(AIT_USE_HOST_DEX2OAT_ROLLOUT_FALLBACK) -AIT_INSTALL_APPS_IN_DATA_DICT = _make_dict(INSTALL_APPS_IN_DATA) -LOCAL_TEST_MULTI_PROTO_PKG_DICT = _make_dict(LOCAL_TEST_MULTI_PROTO_PKG) -LOCAL_TEST_FALLBACK_DICT = _make_dict(LOCAL_TEST_FALLBACK) -LOCAL_TEST_ROLLOUT_DICT = _make_dict(LOCAL_TEST_ROLLOUT) -LOCAL_TEST_STARLARK_RESOURCES_FALLBACK_DICT = _make_dict(LOCAL_TEST_STARLARK_RESOURCES_FALLBACK) -LOCAL_TEST_STARLARK_RESOURCES_ROLLOUT_DICT = _make_dict(LOCAL_TEST_STARLARK_RESOURCES_ROLLOUT) -ANDROID_TEST_PLATFORM_FALLBACK_DICT = _make_dict(ANDROID_TEST_PLATFORM_FALLBACK) -ANDROID_TEST_PLATFORM_ROLLOUT_DICT = _make_dict(ANDROID_TEST_PLATFORM_ROLLOUT) -SOURCELESS_BINARY_FALLBACK_DICT = _make_dict(SOURCELESS_BINARY_FALLBACK) -SOURCELESS_BINARY_ROLLOUT_DICT = _make_dict(SOURCELESS_BINARY_ROLLOUT) -TEST_TO_INSTRUMENT_TEST_FALLBACK_DICT = _make_dict(TEST_TO_INSTRUMENT_TEST_FALLBACK) -TEST_TO_INSTRUMENT_TEST_ROLLOUT_DICT = _make_dict(TEST_TO_INSTRUMENT_TEST_ROLLOUT) -ALLOW_RESOURCE_CONFLICTS_DICT = _make_dict(ALLOW_RESOURCE_CONFLICTS) -PARTIAL_JETIFICATION_TARGETS_ROLLOUT_DICT = _make_dict(PARTIAL_JETIFICATION_TARGETS_ROLLOUT) -PARTIAL_JETIFICATION_TARGETS_FALLBACK_DICT = _make_dict(PARTIAL_JETIFICATION_TARGETS_FALLBACK) -KT_ANDROID_LIBRARY_ROLLOUT_DICT = _make_dict(KT_ANDROID_LIBRARY_ROLLOUT) -KT_ANDROID_LIBRARY_FALLBACK_DICT = _make_dict(KT_ANDROID_LIBRARY_FALLBACK) -ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_ROLLOUT_DICT = _make_dict(ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_ROLLOUT) -ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_FALLBACK_DICT = _make_dict(ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_FALLBACK) -ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_ROLLOUT_DICT = _make_dict(ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_ROLLOUT) -ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_FALLBACK_DICT = _make_dict(ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_FALLBACK) - -def _matches(fqn, dct): +AAR_IMPORT_DEPS_CHECKER_FALLBACK_DICT = make_dict(AAR_IMPORT_DEPS_CHECKER_FALLBACK) +AAR_IMPORT_DEPS_CHECKER_ROLLOUT_DICT = make_dict(AAR_IMPORT_DEPS_CHECKER_ROLLOUT) +AAR_IMPORT_EXPLICIT_EXPORTS_MANIFEST_DICT = make_dict(AAR_IMPORT_EXPLICIT_EXPORTS_MANIFEST) +AAR_IMPORT_EXPORTS_R_JAVA_DICT = make_dict(AAR_IMPORT_EXPORTS_R_JAVA) +AAR_PROPAGATE_RESOURCES_FALLBACK_DICT = make_dict(AAR_PROPAGATE_RESOURCES_FALLBACK) +AAR_PROPAGATE_RESOURCES_ROLLOUT_DICT = make_dict(AAR_PROPAGATE_RESOURCES_ROLLOUT) +ANDROID_ARCHIVE_DOGFOOD_DICT = make_dict(ANDROID_ARCHIVE_DOGFOOD) +ANDROID_ARCHIVE_EXCLUDED_DEPS_DENYLIST_DICT = make_dict(ANDROID_ARCHIVE_EXCLUDED_DEPS_DENYLIST) +ANDROID_DEVICE_PLUGIN_ROLLOUT_DICT = make_dict(ANDROID_DEVICE_PLUGIN_ROLLOUT) +ANDROID_DEVICE_PLUGIN_FALLBACK_DICT = make_dict(ANDROID_DEVICE_PLUGIN_FALLBACK) +ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_ROLLOUT_DICT = make_dict(ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_ROLLOUT) +ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_FALLBACK_DICT = make_dict(ANDROID_INSTRUMENTATION_BINARY_STARLARK_RESOURCES_FALLBACK) +ANDROID_BINARY_STARLARK_JAVAC_ROLLOUT_DICT = make_dict(ANDROID_BINARY_STARLARK_JAVAC_ROLLOUT) +ANDROID_BINARY_STARLARK_JAVAC_FALLBACK_DICT = make_dict(ANDROID_BINARY_STARLARK_JAVAC_FALLBACK) +ANDROID_BINARY_STARLARK_SPLIT_TRANSITION_ROLLOUT_DICT = make_dict(ANDROID_BINARY_STARLARK_SPLIT_TRANSITION_ROLLOUT) +ANDROID_BINARY_STARLARK_SPLIT_TRANSITION_FALLBACK_DICT = make_dict(ANDROID_BINARY_STARLARK_SPLIT_TRANSITION_FALLBACK) +ANDROID_FEATURE_SPLITS_DOGFOOD_DICT = make_dict(ANDROID_FEATURE_SPLITS_DOGFOOD) +ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS_DICT = make_dict(ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS) +ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS_GENERATOR_FUNCTIONS_DICT = make_dict(ANDROID_LIBRARY_RESOURCES_WITHOUT_SRCS_GENERATOR_FUNCTIONS) +ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_FALLBACK_DICT = make_dict(ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_FALLBACK) +ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_ROLLOUT_DICT = make_dict(ANDROID_LIBRARY_STARLARK_RESOURCE_OUTPUTS_ROLLOUT) +ANDROID_LINT_CHECKS_FALLBACK_DICT = make_dict(ANDROID_LINT_CHECKS_FALLBACK) +ANDROID_LINT_CHECKS_ROLLOUT_DICT = make_dict(ANDROID_LINT_CHECKS_ROLLOUT) +ANDROID_LINT_FALLBACK_DICT = make_dict(ANDROID_LINT_FALLBACK) +ANDROID_LINT_ROLLOUT_DICT = make_dict(ANDROID_LINT_ROLLOUT) +ANDROID_RULES_WITH_KT_ROLLOUT_DICT = make_dict(ANDROID_RULES_WITH_KT_ROLLOUT) +ANDROID_RULES_WITH_KT_FALLBACK_DICT = make_dict(ANDROID_RULES_WITH_KT_FALLBACK) + +LINT_REGISTRY_FALLBACK_DICT = make_dict(LINT_REGISTRY_FALLBACK) +LINT_REGISTRY_ROLLOUT_DICT = make_dict(LINT_REGISTRY_ROLLOUT) +ANDROID_BUILD_STAMPING_ROLLOUT_DICT = make_dict(ANDROID_BUILD_STAMPING_ROLLOUT) +ANDROID_BUILD_STAMPING_FALLBACK_DICT = make_dict(ANDROID_BUILD_STAMPING_FALLBACK) +ANDROID_TEST_LOCKDOWN_GENERATOR_FUNCTIONS_DICT = make_dict(ANDROID_TEST_LOCKDOWN_GENERATOR_FUNCTIONS) +ANDROID_TEST_LOCKDOWN_TARGETS_DICT = make_dict(ANDROID_TEST_LOCKDOWN_TARGETS) +APP_INSTALLATION_SNAPSHOT_DICT = make_dict(APP_INSTALLATION_SNAPSHOT) +APP_INSTALLATION_SNAPSHOT_FALLBACK_DICT = make_dict(APP_INSTALLATION_SNAPSHOT_FALLBACK) +B122039567_DICT = make_dict(B122039567) +B123854163_DICT = make_dict(B123854163) +CAN_USE_DEX2OAT_OPTIONS_DICT = make_dict(CAN_USE_DEX2OAT_OPTIONS) +FIX_RESOURCE_TRANSIVITY_FALLBACK_DICT = make_dict(FIX_RESOURCE_TRANSITIVITY_FALLBACK) +FIX_RESOURCE_TRANSIVITY_ROLLOUT_DICT = make_dict(FIX_RESOURCE_TRANSITIVITY_ROLLOUT) +FIX_EXPORT_EXPORTING_FALLBACK_DICT = make_dict(FIX_EXPORT_EXPORTING_FALLBACK) +FIX_EXPORT_EXPORTING_ROLLOUT_DICT = make_dict(FIX_EXPORT_EXPORTING_ROLLOUT) +AIT_USE_HOST_DEX2OAT_ROLLOUT_DICT = make_dict(AIT_USE_HOST_DEX2OAT_ROLLOUT) +AIT_USE_HOST_DEX2OAT_ROLLOUT_FALLBACK_DICT = make_dict(AIT_USE_HOST_DEX2OAT_ROLLOUT_FALLBACK) +AIT_INSTALL_APPS_IN_DATA_DICT = make_dict(INSTALL_APPS_IN_DATA) +LOCAL_TEST_MULTI_PROTO_PKG_DICT = make_dict(LOCAL_TEST_MULTI_PROTO_PKG) +LOCAL_TEST_FALLBACK_DICT = make_dict(LOCAL_TEST_FALLBACK) +LOCAL_TEST_ROLLOUT_DICT = make_dict(LOCAL_TEST_ROLLOUT) +LOCAL_TEST_STARLARK_RESOURCES_FALLBACK_DICT = make_dict(LOCAL_TEST_STARLARK_RESOURCES_FALLBACK) +LOCAL_TEST_STARLARK_RESOURCES_ROLLOUT_DICT = make_dict(LOCAL_TEST_STARLARK_RESOURCES_ROLLOUT) +ANDROID_TEST_PLATFORM_FALLBACK_DICT = make_dict(ANDROID_TEST_PLATFORM_FALLBACK) +ANDROID_TEST_PLATFORM_ROLLOUT_DICT = make_dict(ANDROID_TEST_PLATFORM_ROLLOUT) +SOURCELESS_BINARY_FALLBACK_DICT = make_dict(SOURCELESS_BINARY_FALLBACK) +SOURCELESS_BINARY_ROLLOUT_DICT = make_dict(SOURCELESS_BINARY_ROLLOUT) +TEST_TO_INSTRUMENT_TEST_FALLBACK_DICT = make_dict(TEST_TO_INSTRUMENT_TEST_FALLBACK) +TEST_TO_INSTRUMENT_TEST_ROLLOUT_DICT = make_dict(TEST_TO_INSTRUMENT_TEST_ROLLOUT) +ALLOW_RESOURCE_CONFLICTS_DICT = make_dict(ALLOW_RESOURCE_CONFLICTS) +PARTIAL_JETIFICATION_TARGETS_ROLLOUT_DICT = make_dict(PARTIAL_JETIFICATION_TARGETS_ROLLOUT) +PARTIAL_JETIFICATION_TARGETS_FALLBACK_DICT = make_dict(PARTIAL_JETIFICATION_TARGETS_FALLBACK) +KT_ANDROID_LIBRARY_ROLLOUT_DICT = make_dict(KT_ANDROID_LIBRARY_ROLLOUT) +KT_ANDROID_LIBRARY_FALLBACK_DICT = make_dict(KT_ANDROID_LIBRARY_FALLBACK) +ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_ROLLOUT_DICT = make_dict(ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_ROLLOUT) +ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_FALLBACK_DICT = make_dict(ANDROID_INSTRUMENTATION_TEST_MANIFEST_CHECK_FALLBACK) +ANDROID_INSTRUMENTATION_TEST_PREBUILT_TEST_APK_ROLLOUT_DICT = make_dict(ANDROID_INSTRUMENTATION_TEST_PREBUILT_TEST_APK_ROLLOUT) +ANDROID_INSTRUMENTATION_TEST_PREBUILT_TEST_APK_FALLBACK_DICT = make_dict(ANDROID_INSTRUMENTATION_TEST_PREBUILT_TEST_APK_FALLBACK) +BASELINE_PROFILES_ROLLOUT_DICT = make_dict(BASELINE_PROFILES_ROLLOUT) +ENFORCE_MIN_SDK_FLOOR_ROLLOUT_DICT = make_dict(ENFORCE_MIN_SDK_FLOOR_ROLLOUT) +ENFORCE_MIN_SDK_FLOOR_FALLBACK_DICT = make_dict(ENFORCE_MIN_SDK_FLOOR_FALLBACK) +ANDROID_APK_TO_BUNDLE_FEATURES_DICT = make_dict(ANDROID_APK_TO_BUNDLE_FEATURES) +ANDROID_LIBRARY_USE_AOSP_AIDL_COMPILER_ALLOWLIST_DICT = make_dict(ANDROID_LIBRARY_USE_AOSP_AIDL_COMPILER_ALLOWLIST) +ANDROID_LOCAL_TEST_JDK_STS_FALLBACK_DICT = make_dict(ANDROID_LOCAL_TEST_JDK_STS_FALLBACK) +ANDROID_LOCAL_TEST_JDK_STS_ROLLOUT_DICT = make_dict(ANDROID_LOCAL_TEST_JDK_STS_ROLLOUT) +DATABINDING_ALLOWED_DICT = make_dict(DATABINDING_ALLOWED) +DATABINDING_DISALLOWED_DICT = make_dict(DATABINDING_DISALLOWED) + +def matches(fqn, dct): # Labels with workspace names ("@workspace//pkg:target") are not supported. if fqn.startswith("@"): return False @@ -292,29 +337,32 @@ def _matches(fqn, dct): return False acls = struct( + get_android_archive_duplicate_class_allowlist = _get_android_archive_duplicate_class_allowlist, + get_android_archive_exposed_package_allowlist = _get_android_archive_exposed_package_allowlist, in_aar_import_deps_checker = _in_aar_import_deps_checker, in_aar_import_explicit_exports_manifest = _in_aar_import_explicit_exports_manifest, in_aar_import_exports_r_java = _in_aar_import_exports_r_java, in_aar_propagate_resources = _in_aar_propagate_resources, - in_ait_virtual_device = _in_ait_virtual_device, in_b122039567 = _in_b122039567, in_b123854163 = _in_b123854163, in_android_archive_dogfood = _in_android_archive_dogfood, in_android_archive_excluded_deps_denylist = _in_android_archive_excluded_deps_denylist, in_android_device_plugin_rollout = _in_android_device_plugin_rollout, in_android_instrumentation_binary_starlark_resources = _in_android_instrumentation_binary_starlark_resources, + in_android_binary_starlark_javac = _in_android_binary_starlark_javac, + in_android_binary_starlark_split_transition = _in_android_binary_starlark_split_transition, in_android_feature_splits_dogfood = _in_android_feature_splits_dogfood, - in_android_library_implicit_exports = _in_android_library_implicit_exports, - in_android_library_implicit_exports_generator_functions = _in_android_library_implicit_exports_generator_functions, in_android_library_starlark_resource_outputs_rollout = _in_android_library_starlark_resource_outputs_rollout, in_android_library_resources_without_srcs = _in_android_library_resources_without_srcs, in_android_library_resources_without_srcs_generator_functions = _in_android_library_resources_without_srcs_generator_functions, + in_android_library_use_aosp_aidl_compiler_allowlist = _in_android_library_use_aosp_aidl_compiler_allowlist, in_android_lint_checks_rollout = _in_android_lint_checks_rollout, in_android_lint_rollout = _in_android_lint_rollout, in_lint_registry_rollout = _in_lint_registry_rollout, in_android_build_stamping_rollout = _in_android_build_stamping_rollout, in_android_test_lockdown_allowlist = _in_android_test_lockdown_allowlist, in_app_installation_snapshot = _in_app_installation_snapshot, + in_databinding_allowed = _in_databinding_allowed, in_dex2oat_opts = _in_dex2oat_opts, in_fix_export_exporting_rollout = _in_fix_export_exporting_rollout, in_fix_resource_transivity_rollout = _in_fix_resource_transivity_rollout, @@ -330,11 +378,16 @@ acls = struct( in_partial_jetification_targets = _in_partial_jetification_targets, in_kt_android_library_rollout = _in_kt_android_library_rollout, in_android_instrumentation_test_manifest_check_rollout = _in_android_instrumentation_test_manifest_check_rollout, - in_android_instrumentation_test_derived_test_class_rollout = _in_android_instrumentation_test_derived_test_class_rollout, + in_android_instrumentation_test_prebuilt_test_apk = _in_android_instrumentation_test_prebuilt_test_apk, + in_android_rules_with_kt_rollout = _in_android_rules_with_kt_rollout, + in_baseline_profiles_rollout = _in_baseline_profiles_rollout, + in_enforce_min_sdk_floor_rollout = _in_enforce_min_sdk_floor_rollout, + in_android_apk_to_bundle_features = _in_android_apk_to_bundle_features, + in_android_local_test_jdk_sts_rollout = _in_android_local_test_jdk_sts_rollout, ) # Visible for testing testing = struct( - matches = _matches, - make_dict = _make_dict, + matches = matches, + make_dict = make_dict, ) diff --git a/rules/acls/android_apk_to_bundle_features_lockdown.bzl b/rules/acls/android_apk_to_bundle_features_lockdown.bzl new file mode 100644 index 0000000..28bacb6 --- /dev/null +++ b/rules/acls/android_apk_to_bundle_features_lockdown.bzl @@ -0,0 +1,18 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Allowlist for specifying `features` on the android_apk_to_bundle rule.""" + +ANDROID_APK_TO_BUNDLE_FEATURES = [ +] diff --git a/rules/acls/android_archive_duplicate_class_allowlist.bzl b/rules/acls/android_archive_duplicate_class_allowlist.bzl new file mode 100644 index 0000000..a170d83 --- /dev/null +++ b/rules/acls/android_archive_duplicate_class_allowlist.bzl @@ -0,0 +1,21 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Allowlist for android_archive targets to skip duplicate class validation.""" + +# Map of {"//some:target": ["list.class", "of.class", "classes.class"]} which will be excluded from +# duplicate class validation. +# keep sorted +ANDROID_ARCHIVE_DUPLICATE_CLASS_ALLOWLIST = { +} diff --git a/rules/acls/android_archive_excluded_deps_denylist.bzl b/rules/acls/android_archive_excluded_deps_denylist.bzl index 2a46e68..ba9fe07 100644 --- a/rules/acls/android_archive_excluded_deps_denylist.bzl +++ b/rules/acls/android_archive_excluded_deps_denylist.bzl @@ -17,5 +17,5 @@ # keep sorted ANDROID_ARCHIVE_EXCLUDED_DEPS_DENYLIST = [ # Failure test support. - "@rules_android//test/rules/android_archive/java/com/testdata/denied:__pkg__", + "//test/rules/android_archive/java/com/testdata/denied:__pkg__", ] diff --git a/rules/acls/android_archive_exposed_package_allowlist.bzl b/rules/acls/android_archive_exposed_package_allowlist.bzl new file mode 100644 index 0000000..0803134 --- /dev/null +++ b/rules/acls/android_archive_exposed_package_allowlist.bzl @@ -0,0 +1,22 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Allowlist for android_archive targets to expose packages that would otherwise be restricted.""" + +# Map of {"target": ["list", "of", "packages"]} which will be excluded from +# exposed package checks. +# keep sorted +ANDROID_ARCHIVE_EXPOSED_PACKAGE_ALLOWLIST = { + "//test/rules/android_archive/java/com/testdata:archive_denied_package_allowlisted": ["androidx.test"], +} diff --git a/rules/acls/android_library_implicit_exports.bzl b/rules/acls/android_binary_starlark_javac.bzl index 521c2ac..54f6058 100644 --- a/rules/acls/android_library_implicit_exports.bzl +++ b/rules/acls/android_binary_starlark_javac.bzl @@ -1,4 +1,4 @@ -# Copyright 2020 The Bazel Authors. All rights reserved. +# Copyright 2021 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Allow list for b/144163743 - deprecated implicit exports.""" +"""Allow list for rollout of Starlark java compilation in android_binary_internal.""" -# These macros can create android_library targets with deps and no srcs with no way for -# their users to control this. -ANDROID_LIBRARY_IMPLICIT_EXPORTS_GENERATOR_FUNCTIONS = [ +# keep sorted +ANDROID_BINARY_STARLARK_JAVAC_ROLLOUT = [ ] -ANDROID_LIBRARY_IMPLICIT_EXPORTS = [ +# keep sorted +ANDROID_BINARY_STARLARK_JAVAC_FALLBACK = [ ] diff --git a/rules/acls/android_binary_starlark_split_transition.bzl b/rules/acls/android_binary_starlark_split_transition.bzl new file mode 100644 index 0000000..0cc737b --- /dev/null +++ b/rules/acls/android_binary_starlark_split_transition.bzl @@ -0,0 +1,22 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Allow and fallback lists for using the Starlark implementation of the android split transition""" + +ANDROID_BINARY_STARLARK_SPLIT_TRANSITION_ROLLOUT = [ + "//tools/build_defs/android/test/dev/android_binary_internal/java/com/nativelibs:__pkg__", +] + +ANDROID_BINARY_STARLARK_SPLIT_TRANSITION_FALLBACK = [ +] diff --git a/rules/acls/android_instrumentation_derived_test_class_rollout.bzl b/rules/acls/android_instrumentation_derived_test_class_rollout.bzl index 782beb8..d00129d 100644 --- a/rules/acls/android_instrumentation_derived_test_class_rollout.bzl +++ b/rules/acls/android_instrumentation_derived_test_class_rollout.bzl @@ -14,10 +14,18 @@ """Rollout list for enabling test class derivation in android_instrumentation_test,""" -ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_ROLLOUT = [ +load("//rules:acls.bzl", "make_dict", "matches") + +_ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_ROLLOUT = [ "//:__subpackages__", ] -ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_FALLBACK = [ +_ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_FALLBACK = [ "//javatests/notinacl:__subpackages__", ] + +_ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_ROLLOUT_DICT = make_dict(_ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_ROLLOUT) +_ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_FALLBACK_DICT = make_dict(_ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_FALLBACK) + +def acls_in_android_instrumentation_test_derived_test_class_rollout(fqn): + return not matches(fqn, _ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_FALLBACK_DICT) and matches(fqn, _ANDROID_INSTRUMENTATION_TEST_DERIVED_TEST_CLASS_ROLLOUT_DICT) diff --git a/rules/acls/android_instrumentation_test_prebuilt_test_apk.bzl b/rules/acls/android_instrumentation_test_prebuilt_test_apk.bzl new file mode 100644 index 0000000..fb3b431 --- /dev/null +++ b/rules/acls/android_instrumentation_test_prebuilt_test_apk.bzl @@ -0,0 +1,22 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Allow list of a_i_t targets allowed to use a prebuilt test apk.""" + +# keep sorted +ANDROID_INSTRUMENTATION_TEST_PREBUILT_TEST_APK_ROLLOUT = [ +] + +ANDROID_INSTRUMENTATION_TEST_PREBUILT_TEST_APK_FALLBACK = [ +] diff --git a/rules/acls/android_library_use_aosp_aidl_compiler.bzl b/rules/acls/android_library_use_aosp_aidl_compiler.bzl new file mode 100644 index 0000000..3825458 --- /dev/null +++ b/rules/acls/android_library_use_aosp_aidl_compiler.bzl @@ -0,0 +1,18 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 3.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Allow list for the `idl_use_aosp_compiler` attribute in the `android_library` rule.""" + +ANDROID_LIBRARY_USE_AOSP_AIDL_COMPILER_ALLOWLIST = [ +] diff --git a/rules/acls/android_lint_checks_rollout.bzl b/rules/acls/android_lint_checks_rollout.bzl index ee44280..8fee3ff 100644 --- a/rules/acls/android_lint_checks_rollout.bzl +++ b/rules/acls/android_lint_checks_rollout.bzl @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Allow list for enabling Android Lint checks in the Android Rules.""" +"""Allow list for using the "fully enabled" lint check config.""" # keep sorted ANDROID_LINT_CHECKS_ROLLOUT = [ diff --git a/rules/acls/android_local_test_jdk_sts_rollout.bzl b/rules/acls/android_local_test_jdk_sts_rollout.bzl new file mode 100644 index 0000000..22055fa --- /dev/null +++ b/rules/acls/android_local_test_jdk_sts_rollout.bzl @@ -0,0 +1,24 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Allow and fallback lists for using the latest JDK runtime in android_local_test.""" + +# keep sorted +ANDROID_LOCAL_TEST_JDK_STS_ROLLOUT = [ +] + +# keep sorted +ANDROID_LOCAL_TEST_JDK_STS_FALLBACK = [ + "//:__subpackages__", +] diff --git a/rules/acls/android_rules_with_kt_rollout.bzl b/rules/acls/android_rules_with_kt_rollout.bzl new file mode 100644 index 0000000..7240b92 --- /dev/null +++ b/rules/acls/android_rules_with_kt_rollout.bzl @@ -0,0 +1,22 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Allow and fallback lists for Kotlin Compilation in the Android Rules.""" + +# keep sorted +ANDROID_RULES_WITH_KT_ROLLOUT = [ +] + +ANDROID_RULES_WITH_KT_FALLBACK = [ +] diff --git a/rules/acls/baseline_profiles_rollout.bzl b/rules/acls/baseline_profiles_rollout.bzl new file mode 100644 index 0000000..e4677bc --- /dev/null +++ b/rules/acls/baseline_profiles_rollout.bzl @@ -0,0 +1,19 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Allowlist for targets allowed to set baseline_profiles.""" + +# keep sorted +BASELINE_PROFILES_ROLLOUT = [ +] diff --git a/rules/acls/databinding.bzl b/rules/acls/databinding.bzl new file mode 100644 index 0000000..98415c7 --- /dev/null +++ b/rules/acls/databinding.bzl @@ -0,0 +1,24 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Allowlist for databinding""" + +# keep sorted +DATABINDING_ALLOWED = [ + "//:__subpackages__", +] + +# keep sorted +DATABINDING_DISALLOWED = [ +] diff --git a/rules/acls/enforce_min_sdk_floor_rollout.bzl b/rules/acls/enforce_min_sdk_floor_rollout.bzl new file mode 100644 index 0000000..94643eb --- /dev/null +++ b/rules/acls/enforce_min_sdk_floor_rollout.bzl @@ -0,0 +1,22 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rollout list for enabling enforce min SDK floor.""" + +ENFORCE_MIN_SDK_FLOOR_ROLLOUT = [ + "//:__subpackages__", +] + +ENFORCE_MIN_SDK_FLOOR_FALLBACK = [ +] diff --git a/rules/android_application/BUILD b/rules/android_application/BUILD index 31cf119..45f31fe 100644 --- a/rules/android_application/BUILD +++ b/rules/android_application/BUILD @@ -21,7 +21,7 @@ bzl_library( name = "bzl", srcs = glob(["*.bzl"]), deps = [ - "@rules_android//rules:common_bzl", - "@rules_android//rules/flags:bzl", + "//rules:common_bzl", + "//rules/flags:bzl", ], ) diff --git a/rules/android_application/android_application.bzl b/rules/android_application/android_application.bzl index f7e1710..4c666f1 100644 --- a/rules/android_application/android_application.bzl +++ b/rules/android_application/android_application.bzl @@ -18,7 +18,7 @@ This file exists to inject the correct version of android_binary. """ load(":android_application_rule.bzl", _android_application_macro = "android_application_macro") -load("@rules_android//rules:android_binary.bzl", _android_binary = "android_binary") +load("//rules:android_binary.bzl", _android_binary = "android_binary") def android_application(**attrs): """Rule to build an Android Application (app bundle). diff --git a/rules/android_application/android_application_rule.bzl b/rules/android_application/android_application_rule.bzl index ab26456..ce3f321 100644 --- a/rules/android_application/android_application_rule.bzl +++ b/rules/android_application/android_application_rule.bzl @@ -17,33 +17,33 @@ load(":android_feature_module_rule.bzl", "get_feature_module_paths") load(":attrs.bzl", "ANDROID_APPLICATION_ATTRS") load( - "@rules_android//rules:aapt.bzl", + "//rules:aapt.bzl", _aapt = "aapt", ) load( - "@rules_android//rules:bundletool.bzl", + "//rules:bundletool.bzl", _bundletool = "bundletool", ) load( - "@rules_android//rules:busybox.bzl", + "//rules:busybox.bzl", _busybox = "busybox", ) load( - "@rules_android//rules:common.bzl", + "//rules:common.bzl", _common = "common", ) load( - "@rules_android//rules:java.bzl", + "//rules:java.bzl", _java = "java", ) load( - "@rules_android//rules:providers.bzl", + "//rules:providers.bzl", "AndroidBundleInfo", "AndroidFeatureModuleInfo", "StarlarkAndroidResourcesInfo", ) load( - "@rules_android//rules:utils.bzl", + "//rules:utils.bzl", "get_android_toolchain", _log = "log", ) @@ -57,7 +57,7 @@ def _verify_attrs(attrs, fqn): if hasattr(attrs, attr): _log.error("Unsupported attr: %s in android_application" % attr) - if not attrs.get("manifest_values", default = {}).get("applicationId"): + if not attrs.get("manifest_values", {}).get("applicationId"): _log.error("%s missing required applicationId in manifest_values" % fqn) for attr in ["deps"]: @@ -140,7 +140,6 @@ def _process_feature_module( ctx, inputs = [filtered_res, native_libs], output = out, - exclude_build_data = True, java_toolchain = _common.get_java_toolchain(ctx), ) @@ -204,15 +203,16 @@ def _create_feature_manifest( progress_message = "Generating Priority AndroidManifest.xml for " + feature_target.label.name, ) - _busybox.merge_manifests( - ctx, - out_file = manifest, - manifest = priority_manifest, - mergee_manifests = depset([info.manifest]), - java_package = java_package, - busybox = android_resources_busybox.files_to_run, - host_javabase = host_javabase, - manifest_values = {"MODULE_TITLE": "@string/" + info.title_id}, + args = ctx.actions.args() + args.add("--main_manifest", priority_manifest.path) + args.add("--feature_manifest", info.manifest.path) + args.add("--feature_title", "@string/" + info.title_id) + args.add("--out", manifest.path) + ctx.actions.run( + executable = ctx.attr._merge_manifests.files_to_run, + inputs = [priority_manifest, info.manifest], + outputs = [manifest], + arguments = [args], ) return manifest @@ -284,11 +284,25 @@ def _impl(ctx): ) # Create `blaze run` script + base_apk_info = ctx.attr.base_module[ApkInfo] + deploy_script_files = [base_apk_info.signing_keys[-1]] subs = { "%bundletool_path%": get_android_toolchain(ctx).bundletool.files_to_run.executable.short_path, "%aab%": ctx.outputs.unsigned_aab.short_path, - "%key%": ctx.attr.base_module[ApkInfo].signing_keys[0].short_path, + "%newest_key%": base_apk_info.signing_keys[-1].short_path, } + if base_apk_info.signing_lineage: + signer_properties = _common.create_signer_properties(ctx, base_apk_info.signing_keys[0]) + subs["%oldest_signer_properties%"] = signer_properties.short_path + subs["%lineage%"] = base_apk_info.signing_lineage.short_path + subs["%min_rotation_api%"] = base_apk_info.signing_min_v3_rotation_api_version + deploy_script_files.extend( + [signer_properties, base_apk_info.signing_lineage, base_apk_info.signing_keys[0]], + ) + else: + subs["%oldest_signer_properties%"] = "" + subs["%lineage%"] = "" + subs["%min_rotation_api%"] = "" ctx.actions.expand_template( template = ctx.file._bundle_deploy, output = ctx.outputs.deploy_script, @@ -304,14 +318,14 @@ def _impl(ctx): executable = ctx.outputs.deploy_script, runfiles = ctx.runfiles([ ctx.outputs.unsigned_aab, - ctx.attr.base_module[ApkInfo].signing_keys[0], get_android_toolchain(ctx).bundletool.files_to_run.executable, - ]), + ] + deploy_script_files), ), ] android_application = rule( attrs = ANDROID_APPLICATION_ATTRS, + cfg = android_common.android_platforms_transition, fragments = [ "android", "java", @@ -322,7 +336,7 @@ android_application = rule( "deploy_script": "%{name}.sh", "unsigned_aab": "%{name}_unsigned.aab", }, - toolchains = ["@rules_android//toolchains/android:toolchain_type"], + toolchains = ["//toolchains/android:toolchain_type"], _skylark_testable = True, ) @@ -337,8 +351,8 @@ def android_application_macro(_android_binary, **attrs): fqn = "//%s:%s" % (native.package_name(), attrs["name"]) # Must pop these because android_binary does not have these attributes. - app_integrity_config = attrs.pop("app_integrity_config", default = None) - rotation_config = attrs.pop("rotation_config", default = None) + app_integrity_config = attrs.pop("app_integrity_config", None) + rotation_config = attrs.pop("rotation_config", None) # Simply fall back to android_binary if no feature splits or bundle_config if not attrs.get("feature_modules", None) and not (attrs.get("bundle_config", None) or attrs.get("bundle_config_file", None)): @@ -352,9 +366,9 @@ def android_application_macro(_android_binary, **attrs): base_split_name = "%s_base" % name # default to [] if feature_modules = None is passed - feature_modules = attrs.pop("feature_modules", default = []) or [] - bundle_config = attrs.pop("bundle_config", default = None) - bundle_config_file = attrs.pop("bundle_config_file", default = None) + feature_modules = attrs.pop("feature_modules", []) or [] + bundle_config = attrs.pop("bundle_config", None) + bundle_config_file = attrs.pop("bundle_config_file", None) # bundle_config is deprecated in favor of bundle_config_file # In the future bundle_config will accept a build rule rather than a raw file. @@ -382,4 +396,5 @@ def android_application_macro(_android_binary, **attrs): transitive_configs = attrs.get("transitive_configs", []), feature_modules = feature_modules, application_id = attrs["manifest_values"]["applicationId"], + visibility = attrs.get("visibility", None), ) diff --git a/rules/android_application/android_feature_module.bzl b/rules/android_application/android_feature_module.bzl index 840d961..6b8732a 100644 --- a/rules/android_application/android_feature_module.bzl +++ b/rules/android_application/android_feature_module.bzl @@ -22,11 +22,11 @@ load( _android_feature_module_macro = "android_feature_module_macro", ) load( - "@rules_android//rules:android_binary.bzl", + "//rules:android_binary.bzl", _android_binary = "android_binary", ) load( - "@rules_android//rules/android_library:rule.bzl", + "//rules/android_library:rule.bzl", _android_library_macro = "android_library_macro", ) diff --git a/rules/android_application/android_feature_module_rule.bzl b/rules/android_application/android_feature_module_rule.bzl index 3041656..a0f7d2b 100644 --- a/rules/android_application/android_feature_module_rule.bzl +++ b/rules/android_application/android_feature_module_rule.bzl @@ -15,15 +15,16 @@ """android_feature_module rule.""" load(":attrs.bzl", "ANDROID_FEATURE_MODULE_ATTRS") -load("@rules_android//rules:java.bzl", _java = "java") +load("//rules:java.bzl", _java = "java") load( - "@rules_android//rules:providers.bzl", + "//rules:providers.bzl", "AndroidFeatureModuleInfo", ) -load("@rules_android//rules:acls.bzl", "acls") +load("//rules:acls.bzl", "acls") load( - "@rules_android//rules:utils.bzl", + "//rules:utils.bzl", "get_android_toolchain", + "utils", ) def _impl(ctx): @@ -39,7 +40,7 @@ def _impl(ctx): args.add(ctx.attr.binary[ApkInfo].unsigned_apk.path) args.add(ctx.configuration.coverage_enabled) args.add(ctx.fragments.android.desugar_java8_libs) - args.add(ctx.attr.library.label) + args.add(utils.dedupe_split_attr(ctx.split_attr.library).label) args.add(get_android_toolchain(ctx).xmllint_tool.files_to_run.executable) args.add(get_android_toolchain(ctx).unzip_tool.files_to_run.executable) @@ -59,7 +60,7 @@ def _impl(ctx): return [ AndroidFeatureModuleInfo( binary = ctx.attr.binary, - library = ctx.attr.library, + library = utils.dedupe_split_attr(ctx.split_attr.library), title_id = ctx.attr.title_id, title_lib = ctx.attr.title_lib, feature_name = ctx.attr.feature_name, @@ -77,7 +78,7 @@ android_feature_module = rule( ], implementation = _impl, provides = [AndroidFeatureModuleInfo], - toolchains = ["@rules_android//toolchains/android:toolchain_type"], + toolchains = ["//toolchains/android:toolchain_type"], _skylark_testable = True, ) @@ -140,7 +141,7 @@ EOF ) # Create AndroidManifest.xml - min_sdk_version = getattr(attrs, "min_sdk_version", "14") or "14" + min_sdk_version = getattr(attrs, "min_sdk_version", "21") or "21" package = _java.resolve_package_from_label(Label(fqn), getattr(attrs, "custom_package", None)) native.genrule( name = targets.manifest_lib.name, diff --git a/rules/android_application/attrs.bzl b/rules/android_application/attrs.bzl index 33e43fb..55864c2 100644 --- a/rules/android_application/attrs.bzl +++ b/rules/android_application/attrs.bzl @@ -15,7 +15,7 @@ """Attributes for android_application.""" load( - "@rules_android//rules:attrs.bzl", + "//rules:attrs.bzl", _attrs = "attrs", ) @@ -46,23 +46,33 @@ ANDROID_APPLICATION_ATTRS = _attrs.add( allow_single_file = True, default = ":bundle_deploy.sh_template", ), + _bundle_keystore_properties = attr.label( + allow_single_file = True, + default = "//rules:bundle_keystore_properties.tmpl", + ), _feature_manifest_script = attr.label( allow_single_file = True, - cfg = "host", + cfg = "exec", executable = True, default = ":gen_android_feature_manifest.sh", ), _java_toolchain = attr.label( default = Label("//tools/jdk:toolchain_android_only"), ), + _merge_manifests = attr.label( + default = ":merge_feature_manifests.par", + allow_single_file = True, + cfg = "exec", + executable = True, + ), _priority_feature_manifest_script = attr.label( allow_single_file = True, - cfg = "host", + cfg = "exec", executable = True, default = ":gen_priority_android_feature_manifest.sh", ), _host_javabase = attr.label( - cfg = "host", + cfg = "exec", default = Label("//tools/jdk:current_java_runtime"), ), ), @@ -74,6 +84,7 @@ ANDROID_FEATURE_MODULE_ATTRS = dict( feature_name = attr.string(), library = attr.label( allow_rules = ["android_library"], + cfg = android_common.multi_cpu_configuration, mandatory = True, doc = "android_library target to include as a feature split.", ), @@ -82,7 +93,7 @@ ANDROID_FEATURE_MODULE_ATTRS = dict( title_lib = attr.string(), _feature_module_validation_script = attr.label( allow_single_file = True, - cfg = "host", + cfg = "exec", executable = True, default = ":feature_module_validation.sh", ), diff --git a/rules/android_application/bundle_deploy.sh_template b/rules/android_application/bundle_deploy.sh_template index 37f6d4d..bcc1f8f 100644 --- a/rules/android_application/bundle_deploy.sh_template +++ b/rules/android_application/bundle_deploy.sh_template @@ -2,7 +2,10 @@ bundletool="%bundletool_path%" aab="%aab%" -key="%key%" +oldest_signer_properties="%oldest_signer_properties%" +newest_key="%newest_key%" +lineage="%lineage%" +min_rotation_api="%min_rotation_api%" tmp="$(mktemp /tmp/XXXXbundle.apks)" function cleanup { @@ -10,15 +13,30 @@ function cleanup { } trap cleanup EXIT -java -jar "$bundletool" build-apks \ - --bundle="$aab" \ - --output="$tmp" \ - --overwrite \ - --local-testing \ - --ks="$key" \ - --ks-pass=pass:android \ - --ks-key-alias=androiddebugkey \ - --key-pass=pass:android || exit +args=( + --bundle="$aab" + --output="$tmp" + --overwrite + --local-testing + --ks="$newest_key" + --ks-pass=pass:android + --ks-key-alias=androiddebugkey + --key-pass=pass:android +) + +if [[ ! -z "$lineage" ]]; then + args+=(--lineage="$lineage") +fi + +if [[ ! -z "$oldest_signer_properties" ]]; then + args+=(--oldest-signer="$oldest_signer_properties") +fi + +if [[ ! -z "$min_rotation_api" ]]; then + args+=(--rotation-min-sdk-version="$min_rotation_api") +fi + +java -jar "$bundletool" build-apks "${args[@]}" || exit java -jar "$bundletool" install-apks \ --adb="$(which adb)" \ diff --git a/rules/android_binary.bzl b/rules/android_binary.bzl index 5d33512..16c7a40 100644 --- a/rules/android_binary.bzl +++ b/rules/android_binary.bzl @@ -14,8 +14,9 @@ """Bazel rule for building an APK.""" +load(":common.bzl", "common") load(":migration_tag_DONOTUSE.bzl", "add_migration_tag") -load("@rules_android//rules/android_binary_internal:rule.bzl", "android_binary_internal_macro") +load("//rules/android_binary_internal:rule.bzl", "android_binary_internal_macro") def android_binary(**attrs): """Bazel android_binary rule. @@ -25,7 +26,7 @@ def android_binary(**attrs): Args: **attrs: Rule attributes """ - android_binary_internal_name = ":%s_RESOURCES_DO_NOT_USE" % attrs["name"] + android_binary_internal_name = ":" + attrs["name"] + common.PACKAGED_RESOURCES_SUFFIX android_binary_internal_macro( **dict( attrs, diff --git a/rules/android_binary_internal/BUILD b/rules/android_binary_internal/BUILD index 2440016..aa1e0ef 100644 --- a/rules/android_binary_internal/BUILD +++ b/rules/android_binary_internal/BUILD @@ -13,6 +13,6 @@ bzl_library( name = "bzl", srcs = glob(["*.bzl"]), deps = [ - "@rules_android//rules:common_bzl", + "//rules:common_bzl", ], ) diff --git a/rules/android_binary_internal/attrs.bzl b/rules/android_binary_internal/attrs.bzl index 5a7bca7..5d5fe49 100644 --- a/rules/android_binary_internal/attrs.bzl +++ b/rules/android_binary_internal/attrs.bzl @@ -15,13 +15,21 @@ """Attributes.""" load( - "@rules_android//rules:attrs.bzl", + "//rules:attrs.bzl", _attrs = "attrs", ) +load( + "//rules:native_deps.bzl", + "split_config_aspect", +) ATTRS = _attrs.replace( _attrs.add( dict( + srcs = attr.label_list( + # TODO(timpeut): Set PropertyFlag direct_compile_time_input + allow_files = [".java", ".srcjar"], + ), deps = attr.label_list( allow_files = True, allow_rules = [ @@ -50,17 +58,33 @@ ATTRS = _attrs.replace( allow_files = False, allow_rules = ["android_binary", "android_test"], ), + proguard_specs = attr.label_list(allow_empty = True, allow_files = True), resource_configuration_filters = attr.string_list(), densities = attr.string_list(), nocompress_extensions = attr.string_list(), shrink_resources = _attrs.tristate.create( default = _attrs.tristate.auto, ), + _java_toolchain = attr.label( + default = Label("//tools/jdk:toolchain_android_only"), + ), _defined_resource_files = attr.bool(default = False), _enable_manifest_merging = attr.bool(default = True), + _cc_toolchain_split = attr.label( + cfg = android_common.multi_cpu_configuration, + default = "@bazel_tools//tools/cpp:current_cc_toolchain", + aspects = [split_config_aspect], + ), + _grep_includes = attr.label( + allow_single_file = True, + executable = True, + cfg = "exec", + default = Label("@@bazel_tools//tools/cpp:grep-includes"), + ), ), _attrs.COMPILATION, _attrs.DATA_CONTEXT, + _attrs.ANDROID_TOOLCHAIN_ATTRS, ), # TODO(b/167599192): don't override manifest attr to remove .xml file restriction. manifest = attr.label( diff --git a/rules/android_binary_internal/impl.bzl b/rules/android_binary_internal/impl.bzl index 6838a81..2ea8ac8 100644 --- a/rules/android_binary_internal/impl.bzl +++ b/rules/android_binary_internal/impl.bzl @@ -14,23 +14,42 @@ """Implementation.""" -load("@rules_android//rules:acls.bzl", "acls") -load("@rules_android//rules:java.bzl", "java") +load("//rules:acls.bzl", "acls") +load("//rules:common.bzl", "common") +load("//rules:data_binding.bzl", "data_binding") +load("//rules:java.bzl", "java") load( - "@rules_android//rules:processing_pipeline.bzl", + "//rules:processing_pipeline.bzl", "ProviderInfo", "processing_pipeline", ) -load("@rules_android//rules:resources.bzl", _resources = "resources") -load("@rules_android//rules:utils.bzl", "compilation_mode", "get_android_toolchain", "utils") +load("//rules:resources.bzl", _resources = "resources") +load("//rules:utils.bzl", "compilation_mode", "get_android_toolchain", "utils") +load( + "//rules:native_deps.bzl", + _process_native_deps = "process", +) + +def _process_manifest(ctx, **unused_ctxs): + manifest_ctx = _resources.bump_min_sdk( + ctx, + manifest = ctx.file.manifest, + floor = _resources.DEPOT_MIN_SDK_FLOOR if (_is_test_binary(ctx) and acls.in_enforce_min_sdk_floor_rollout(str(ctx.label))) else 0, + enforce_min_sdk_floor_tool = get_android_toolchain(ctx).enforce_min_sdk_floor_tool.files_to_run, + ) -def _process_resources(ctx, java_package, **unused_ctxs): + return ProviderInfo( + name = "manifest_ctx", + value = manifest_ctx, + ) + +def _process_resources(ctx, manifest_ctx, java_package, **unused_ctxs): packaged_resources_ctx = _resources.package( ctx, assets = ctx.files.assets, assets_dir = ctx.attr.assets_dir, resource_files = ctx.files.resource_files, - manifest = ctx.file.manifest, + manifest = manifest_ctx.processed_manifest, manifest_values = utils.expand_make_vars(ctx, ctx.attr.manifest_values), resource_configs = ctx.attr.resource_configuration_filters, densities = ctx.attr.densities, @@ -59,6 +78,103 @@ def _process_resources(ctx, java_package, **unused_ctxs): value = packaged_resources_ctx, ) +def _validate_manifest(ctx, packaged_resources_ctx, **unused_ctxs): + manifest_validation_ctx = _resources.validate_min_sdk( + ctx, + manifest = packaged_resources_ctx.processed_manifest, + floor = _resources.DEPOT_MIN_SDK_FLOOR if acls.in_enforce_min_sdk_floor_rollout(str(ctx.label)) else 0, + enforce_min_sdk_floor_tool = get_android_toolchain(ctx).enforce_min_sdk_floor_tool.files_to_run, + ) + + return ProviderInfo( + name = "manifest_validation_ctx", + value = manifest_validation_ctx, + ) + +def _process_native_libs(ctx, **_unusued_ctxs): + providers = [] + if acls.in_android_binary_starlark_split_transition(str(ctx.label)): + providers.append(_process_native_deps( + ctx, + filename = "nativedeps", + )) + return ProviderInfo( + name = "native_libs_ctx", + value = struct(providers = providers), + ) + +def _process_build_stamp(_unused_ctx, **_unused_ctxs): + return ProviderInfo( + name = "stamp_ctx", + value = struct( + resource_files = [], + deps = [], + java_info = None, + providers = [], + ), + ) + +def _process_data_binding(ctx, java_package, packaged_resources_ctx, **_unused_ctxs): + if ctx.attr.enable_data_binding and not acls.in_databinding_allowed(str(ctx.label)): + fail("This target is not allowed to use databinding and enable_data_binding is True.") + return ProviderInfo( + name = "db_ctx", + value = data_binding.process( + ctx, + defines_resources = True, + enable_data_binding = ctx.attr.enable_data_binding, + java_package = java_package, + layout_info = packaged_resources_ctx.data_binding_layout_info, + artifact_type = "APPLICATION", + deps = utils.collect_providers(DataBindingV2Info, utils.dedupe_split_attr(ctx.split_attr.deps)), + data_binding_exec = get_android_toolchain(ctx).data_binding_exec.files_to_run, + data_binding_annotation_processor = + get_android_toolchain(ctx).data_binding_annotation_processor[JavaPluginInfo], + data_binding_annotation_template = + utils.only(get_android_toolchain(ctx).data_binding_annotation_template.files.to_list()), + ), + ) + +def _process_jvm(ctx, db_ctx, packaged_resources_ctx, stamp_ctx, **_unused_ctxs): + native_name = ctx.label.name.removesuffix(common.PACKAGED_RESOURCES_SUFFIX) + java_info = java.compile_android( + ctx, + # Use the same format as the class jar from native android_binary. + # Some macros expect the class jar to be named like this. + ctx.actions.declare_file("%s/lib%s.jar" % (ctx.label.name, native_name)), + ctx.actions.declare_file(ctx.label.name + "-src.jar"), + srcs = ctx.files.srcs + db_ctx.java_srcs, + javac_opts = ctx.attr.javacopts + db_ctx.javac_opts, + r_java = packaged_resources_ctx.r_java, + enable_deps_without_srcs = True, + deps = utils.collect_providers(JavaInfo, utils.dedupe_split_attr(ctx.split_attr.deps) + stamp_ctx.deps), + plugins = + utils.collect_providers(JavaPluginInfo, ctx.attr.plugins) + + db_ctx.java_plugins, + annotation_processor_additional_outputs = + db_ctx.java_annotation_processor_additional_outputs, + annotation_processor_additional_inputs = + db_ctx.java_annotation_processor_additional_inputs, + strict_deps = "DEFAULT", + java_toolchain = common.get_java_toolchain(ctx), + ) + java_info = java_common.add_constraints( + java_info, + constraints = ["android"], + ) + + providers = [] + if acls.in_android_binary_starlark_javac(str(ctx.label)): + providers.append(java_info) + + return ProviderInfo( + name = "jvm_ctx", + value = struct( + java_info = java_info, + providers = providers, + ), + ) + def use_legacy_manifest_merger(ctx): """Whether legacy manifest merging is enabled. @@ -86,11 +202,28 @@ def finalize(ctx, providers, validation_outputs, **unused_ctxs): ) return providers +def _is_test_binary(ctx): + """Whether this android_binary target is a test binary. + + Args: + ctx: The context. + + Returns: + Boolean indicating whether the target is a test target. + """ + return ctx.attr.testonly or ctx.attr.instruments or str(ctx.label).find("/javatests/") >= 0 + # Order dependent, as providers will not be available to downstream processors # that may depend on the provider. Iteration order for a dictionary is based on # insertion. PROCESSORS = dict( + ManifestProcessor = _process_manifest, + StampProcessor = _process_build_stamp, ResourceProcessor = _process_resources, + ValidateManifestProcessor = _validate_manifest, + NativeLibsProcessor = _process_native_libs, + DataBindingProcessor = _process_data_binding, + JvmProcessor = _process_jvm, ) _PROCESSING_PIPELINE = processing_pipeline.make_processing_pipeline( diff --git a/rules/android_binary_internal/rule.bzl b/rules/android_binary_internal/rule.bzl index 3070688..b08a39e 100644 --- a/rules/android_binary_internal/rule.bzl +++ b/rules/android_binary_internal/rule.bzl @@ -17,7 +17,7 @@ load(":attrs.bzl", "ATTRS") load(":impl.bzl", "impl") load( - "@rules_android//rules:attrs.bzl", + "//rules:attrs.bzl", _attrs = "attrs", ) @@ -43,11 +43,12 @@ def make_rule( attrs = attrs, implementation = implementation, provides = provides, - toolchains = ["@rules_android//toolchains/android:toolchain_type"], + toolchains = ["//toolchains/android:toolchain_type"], _skylark_testable = True, fragments = [ "android", "java", + "cpp", ], ) @@ -67,7 +68,7 @@ def sanitize_attrs(attrs, allowed_attrs = ATTRS.keys()): Returns: A dictionary containing valid attributes. """ - for attr_name in attrs.keys(): + for attr_name in list(attrs.keys()): if attr_name not in allowed_attrs and attr_name not in _DEFAULT_ALLOWED_ATTRS: attrs.pop(attr_name, None) diff --git a/rules/android_common/BUILD b/rules/android_common/BUILD new file mode 100644 index 0000000..241cf20 --- /dev/null +++ b/rules/android_common/BUILD @@ -0,0 +1,18 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") + +package( + default_visibility = + ["//:__subpackages__"], +) + +licenses(["notice"]) + +filegroup( + name = "all_files", + srcs = glob(["**"]), +) + +bzl_library( + name = "bzl", + srcs = glob(["*.bzl"]), +) diff --git a/rules/android_common/reexport_android_common.bzl b/rules/android_common/reexport_android_common.bzl new file mode 100644 index 0000000..22399ea --- /dev/null +++ b/rules/android_common/reexport_android_common.bzl @@ -0,0 +1,21 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A workaround to expose android_common native module in android/common.bzl. + +Redefine native symbols with a new name as a workaround for +exporting them in `rules:common.bzl` with their original name. +""" + +native_android_common = android_common diff --git a/rules/android_library/BUILD b/rules/android_library/BUILD index 09ca5b8..4eaeaf3 100644 --- a/rules/android_library/BUILD +++ b/rules/android_library/BUILD @@ -15,7 +15,7 @@ bzl_library( name = "bzl", srcs = glob(["*.bzl"]), deps = [ - "@rules_android//rules:common_bzl", - "@rules_android//rules/flags:bzl", + "//rules:common_bzl", + "//rules/flags:bzl", ], ) diff --git a/rules/android_library/attrs.bzl b/rules/android_library/attrs.bzl index 8e38b01..6b7380b 100644 --- a/rules/android_library/attrs.bzl +++ b/rules/android_library/attrs.bzl @@ -15,7 +15,7 @@ """Attributes.""" load( - "@rules_android//rules:attrs.bzl", + "//rules:attrs.bzl", _attrs = "attrs", ) @@ -26,30 +26,172 @@ ATTRS = _attrs.add( [CcInfo], [JavaInfo], ], + doc = ( + "The list of other libraries to link against. Permitted library types " + + "are: `android_library`, `java_library` with `android` constraint and " + + "`cc_library` wrapping or producing `.so` native libraries for the " + + "Android target platform." + ), + ), + enable_data_binding = attr.bool( + default = False, + doc = ( + "If true, this rule processes [data binding]" + + "(https://developer.android.com/topic/libraries/data-binding) " + + "expressions in layout resources included through the [resource_files]" + + "(https://docs.bazel.build/versions/main/be/android.html#android_binary.resource_files) " + + "attribute. Without this setting, data binding expressions produce build " + + "failures. To build an Android app with data binding, you must also do the following:" + + "\n\n1. Set this attribute for all Android rules that transitively depend on " + + "this one. This is because dependers inherit the rule's data binding " + + "expressions through resource merging. So they also need to build with " + + "data binding to parse those expressions." + + "\n\n2. Add a `deps =` entry for the data binding runtime library to all targets " + + "that set this attribute. The location of this library depends on your depot setup." + ), ), - enable_data_binding = attr.bool(default = False), exported_plugins = attr.label_list( providers = [ [JavaPluginInfo], ], - cfg = "host", + cfg = "exec", + doc = ( + "The list of [java_plugin](https://docs.bazel.build/versions/main/be/java.html#java_plugin)s " + + "(e.g. annotation processors) to export to libraries that directly depend on this library. " + + "The specified list of `java_plugin`s will be applied to any library which directly depends on " + + "this library, just as if that library had explicitly declared these labels in " + + "[plugins](#android_library-plugins)." + ), ), exports = attr.label_list( providers = [ [CcInfo], [JavaInfo], ], + doc = ( + "The closure of all rules reached via `exports` attributes are considered " + + "direct dependencies of any rule that directly depends on the target with " + + "`exports`. The `exports` are not direct deps of the rule they belong to." + ), + ), + exports_manifest = _attrs.tristate.create( + default = _attrs.tristate.no, + doc = ( + "Whether to export manifest entries to `android_binary` targets that " + + "depend on this target. `uses-permissions` attributes are never exported." + ), + ), + idl_import_root = attr.string( + doc = ( + "Package-relative path to the root of the java package tree containing idl " + + "sources included in this library. This path will be used as the import root " + + "when processing idl sources that depend on this library." + + "\n\n" + + "When `idl_import_root` is specified, both `idl_parcelables` and `idl_srcs` must " + + "be at the path specified by the java package of the object they represent " + + "under `idl_import_root`. When `idl_import_root` is not specified, both " + + "`idl_parcelables` and `idl_srcs` must be at the path specified by their " + + "package under a Java root. " + + "See [examples](#examples)" + ), + ), + idl_parcelables = attr.label_list( + allow_files = [".aidl"], + doc = ( + "List of Android IDL definitions to supply as imports. These files will " + + "be made available as imports for any `android_library` target that depends " + + "on this library, directly or via its transitive closure, but will not be " + + "translated to Java or compiled. Only `.aidl` files that correspond directly " + + "to `.java` sources in this library should be included (e.g., custom " + + "implementations of Parcelable), otherwise `idl_srcs` should be used." + + "\n\n" + + "These files must be placed appropriately for the aidl compiler to find " + + "them. See the description of [idl_import_root](#android_library-idl_import_root) " + + "for information about what this means." + ), + ), + idl_preprocessed = attr.label_list( + allow_files = [".aidl"], + doc = ( + "List of preprocessed Android IDL definitions to supply as imports. These " + + "files will be made available as imports for any `android_library` target " + + "that depends on this library, directly or via its transitive closure, but " + + "will not be translated to Java or compiled. Only preprocessed `.aidl` " + + "files that correspond directly to `.java` sources in this library should " + + "be included (e.g., custom implementations of Parcelable), otherwise use " + + "`idl_srcs` for Android IDL definitions that need to be translated to Java " + + "interfaces and use `idl_parcelable` for non-preprocessed AIDL files." + ), + ), + idl_srcs = attr.label_list( + allow_files = [".aidl"], + doc = ( + "List of Android IDL definitions to translate to Java interfaces. After " + + "the Java interfaces are generated, they will be compiled together with " + + "the contents of `srcs`. These files will be made available as imports " + + "for any `android_library` target that depends on this library, directly " + + "or via its transitive closure." + + "\n\n" + + "These files must be placed appropriately for the aidl compiler to find " + + "them. See the description of [idl_import_root](#android_library-idl_import_root) " + + "for information about what this means." + ), + ), + idl_uses_aosp_compiler = attr.bool( + default = False, + doc = ( + "Use the upstream AOSP compiler to generate Java files out of `idl_srcs`." + + "The upstream AOSP compiler provides several new language features that the " + + "Google3-only compiler doesn't provide. For example: structured parcelables, " + + "unions, enums, nested type declarations, constant expressions, annotations, " + + "and more. " + + "See [AIDL Doc](https://source.android.com/docs/core/architecture/aidl/overview) " + + "for more details. " + + "Note: the use of the AOSP compiler in google3 is restricted due to performance " + + "considerations. This should not be broadly used unless these features are " + + "strictly required." + ), + ), + idlopts = attr.string_list( + mandatory = False, + allow_empty = True, + default = [], + doc = ( + "Add these flags to the AIDL compiler command." + ), + ), + neverlink = attr.bool( + default = False, + doc = ( + "Only use this library for compilation and not at runtime. The outputs " + + "of a rule marked as neverlink will not be used in `.apk` creation. " + + "Useful if the library will be provided by the runtime environment during execution." + ), + ), + proguard_specs = attr.label_list( + allow_files = True, + doc = ( + "Files to be used as Proguard specification. These will describe the set " + + "of specifications to be used by Proguard. If specified, they will be " + + "added to any `android_binary` target depending on this library. The " + + "files included here must only have idempotent rules, namely -dontnote, " + + "-dontwarn, assumenosideeffects, and rules that start with -keep. Other " + + "options can only appear in `android_binary`'s proguard_specs, to " + + "ensure non-tautological merges." + ), ), - exports_manifest = - _attrs.tristate.create(default = _attrs.tristate.no), - idl_import_root = attr.string(), - idl_parcelables = attr.label_list(allow_files = [".aidl"]), - idl_preprocessed = attr.label_list(allow_files = [".aidl"]), - idl_srcs = attr.label_list(allow_files = [".aidl"]), - neverlink = attr.bool(default = False), - proguard_specs = attr.label_list(allow_files = True), srcs = attr.label_list( allow_files = [".java", ".srcjar"], + doc = ( + "The list of `.java` or `.srcjar` files that are processed to create the " + + "target. `srcs` files of type `.java` are compiled. For *readability's " + + "sake*, it is not good to put the name of a generated `.java` source " + + "file into the `srcs`. Instead, put the depended-on rule name in the `srcs`, " + + "as described below." + + "\n\n" + + "`srcs` files of type `.srcjar` are unpacked and compiled. (This is useful " + + "if you need to generate a set of `.java` files with a genrule or build extension.)" + ), ), # TODO(b/127517031): Remove these entries once fixed. _defined_assets = attr.bool(default = False), @@ -64,10 +206,11 @@ ATTRS = _attrs.add( # TODO(str): Remove when fully migrated to android_instrumentation_test _android_test_migration = attr.bool(default = False), _flags = attr.label( - default = "@rules_android//rules/flags", + default = "//rules/flags", ), _package_name = attr.string(), # for sending the package name to the outputs callback ), _attrs.COMPILATION, _attrs.DATA_CONTEXT, + _attrs.ANDROID_TOOLCHAIN_ATTRS, ) diff --git a/rules/android_library/impl.bzl b/rules/android_library/impl.bzl index 286d7b5..5d4b2ed 100644 --- a/rules/android_library/impl.bzl +++ b/rules/android_library/impl.bzl @@ -14,27 +14,26 @@ """Implementation.""" -load("@rules_android//rules:acls.bzl", "acls") -load("@rules_android//rules:attrs.bzl", _attrs = "attrs") -load("@rules_android//rules:common.bzl", _common = "common") -load("@rules_android//rules:data_binding.bzl", _data_binding = "data_binding") -load("@rules_android//rules:idl.bzl", _idl = "idl") -load("@rules_android//rules:intellij.bzl", _intellij = "intellij") -load("@rules_android//rules:java.bzl", _java = "java") +load("//rules:acls.bzl", "acls") +load("//rules:attrs.bzl", _attrs = "attrs") +load("//rules:common.bzl", _common = "common") +load("//rules:data_binding.bzl", _data_binding = "data_binding") +load("//rules:idl.bzl", _idl = "idl") +load("//rules:intellij.bzl", _intellij = "intellij") +load("//rules:java.bzl", _java = "java") load( - "@rules_android//rules:processing_pipeline.bzl", + "//rules:processing_pipeline.bzl", "ProviderInfo", "processing_pipeline", ) -load("@rules_android//rules:proguard.bzl", _proguard = "proguard") -load("@rules_android//rules:resources.bzl", _resources = "resources") -load("@rules_android//rules:utils.bzl", "get_android_sdk", "get_android_toolchain", "log", "utils") -load("@rules_android//rules/flags:flags.bzl", _flags = "flags") +load("//rules:proguard.bzl", _proguard = "proguard") +load("//rules:providers.bzl", "AndroidLintRulesInfo") +load("//rules:resources.bzl", _resources = "resources") +load("//rules:utils.bzl", "get_android_sdk", "get_android_toolchain", "log", "utils") +load("//rules/flags:flags.bzl", _flags = "flags") _USES_DEPRECATED_IMPLICIT_EXPORT_ERROR = ( - "The android_library rule will be deprecating the use of deps to export " + - "targets implicitly. " + - "Please use android_library.exports to explicitly specify the exported " + + "Use android_library.exports to explicitly specify the exported " + "targets of %s." ) @@ -53,6 +52,14 @@ _IDL_SRC_FROM_DIFFERENT_PACKAGE_ERROR = ( "package or depend on an appropriate rule there." ) +_IDL_USES_AOSP_COMPILER_ERROR = ( + "Use of `idl_uses_aosp_compiler` is not allowed for %s." +) + +_IDL_IDLOPTS_UNSUPPORTERD_ERROR = ( + "`idlopts` is supported only if `idl_uses_aosp_compiler` is set to true." +) + # Android library AAR context attributes. _PROVIDERS = "providers" _VALIDATION_OUTPUTS = "validation_outputs" @@ -65,35 +72,26 @@ _AARContextInfo = provider( }, ) +def _has_srcs(ctx): + return ctx.files.srcs or ctx.files.idl_srcs or getattr(ctx.files, "common_srcs", False) + def _uses_deprecated_implicit_export(ctx): - if not ctx.attr.deps: - return False - return not (ctx.files.srcs or - ctx.files.idl_srcs or - ctx.attr._defined_assets or - ctx.files.resource_files or - ctx.attr.manifest) + return (ctx.attr.deps and not (_has_srcs(ctx) or + ctx.attr._defined_assets or + ctx.files.resource_files or + ctx.attr.manifest)) def _uses_resources_and_deps_without_srcs(ctx): - if not ctx.attr.deps: - return False - if not (ctx.attr._defined_assets or - ctx.files.resource_files or - ctx.attr.manifest): - return False - return not (ctx.files.srcs or ctx.files.idl_srcs) + return (ctx.attr.deps and + (ctx.attr._defined_assets or ctx.files.resource_files or ctx.attr.manifest) and + not _has_srcs(ctx)) def _check_deps_without_java_srcs(ctx): - if not ctx.attr.deps or ctx.files.srcs or ctx.files.idl_srcs: + if not ctx.attr.deps or _has_srcs(ctx): return False gfn = getattr(ctx.attr, "generator_function", "") if _uses_deprecated_implicit_export(ctx): - if (acls.in_android_library_implicit_exports_generator_functions(gfn) or - acls.in_android_library_implicit_exports(str(ctx.label))): - return True - else: - # TODO(b/144163743): add a test for this. - log.error(_USES_DEPRECATED_IMPLICIT_EXPORT_ERROR % ctx.label) + log.error(_USES_DEPRECATED_IMPLICIT_EXPORT_ERROR % ctx.label) if _uses_resources_and_deps_without_srcs(ctx): if (acls.in_android_library_resources_without_srcs_generator_functions(gfn) or acls.in_android_library_resources_without_srcs(str(ctx.label))): @@ -111,6 +109,15 @@ def _validate_rule_context(ctx): if ctx.label.package != idl_src.label.package: log.error(_IDL_SRC_FROM_DIFFERENT_PACKAGE_ERROR % idl_src.label) + # Ensure that the AOSP AIDL compiler is used only in allowlisted packages + if (ctx.attr.idl_uses_aosp_compiler and + not acls.in_android_library_use_aosp_aidl_compiler_allowlist(str(ctx.label))): + log.error(_IDL_USES_AOSP_COMPILER_ERROR % ctx.label) + + # Check if idlopts is with idl_uses_aosp_compiler + if ctx.attr.idlopts and not ctx.attr.idl_uses_aosp_compiler: + log.error(_IDL_IDLOPTS_UNSUPPORTERD_ERROR) + return struct( enable_deps_without_srcs = _check_deps_without_java_srcs(ctx), ) @@ -121,7 +128,20 @@ def _exceptions_processor(ctx, **unused_ctxs): value = _validate_rule_context(ctx), ) -def _process_resources(ctx, java_package, **unused_ctxs): +def _process_manifest(ctx, **unused_ctxs): + manifest_ctx = _resources.bump_min_sdk( + ctx, + manifest = ctx.file.manifest, + floor = _resources.DEPOT_MIN_SDK_FLOOR if acls.in_enforce_min_sdk_floor_rollout(str(ctx.label)) else 0, + enforce_min_sdk_floor_tool = get_android_toolchain(ctx).enforce_min_sdk_floor_tool.files_to_run, + ) + + return ProviderInfo( + name = "manifest_ctx", + value = manifest_ctx, + ) + +def _process_resources(ctx, java_package, manifest_ctx, **unused_ctxs): # exports_manifest can be overridden by a bazel flag. if ctx.attr.exports_manifest == _attrs.tristate.auto: exports_manifest = ctx.fragments.android.get_exports_manifest_default @@ -131,7 +151,7 @@ def _process_resources(ctx, java_package, **unused_ctxs): # Process Android Resources resources_ctx = _resources.process( ctx, - manifest = ctx.file.manifest, + manifest = manifest_ctx.processed_manifest, resource_files = ctx.attr.resource_files, defined_assets = ctx.attr._defined_assets, assets = ctx.attr.assets, @@ -151,7 +171,6 @@ def _process_resources(ctx, java_package, **unused_ctxs): # misbehavior on the Java side. fix_resource_transitivity = bool(ctx.attr.srcs), fix_export_exporting = acls.in_fix_export_exporting_rollout(str(ctx.label)), - propagate_resources = not ctx.attr._android_test_migration, # Tool and Processing related inputs aapt = get_android_toolchain(ctx).aapt2.files_to_run, @@ -200,10 +219,14 @@ def _process_idl(ctx, **unused_sub_ctxs): aidl = get_android_sdk(ctx).aidl, aidl_lib = get_android_sdk(ctx).aidl_lib, aidl_framework = get_android_sdk(ctx).framework_aidl, + uses_aosp_compiler = ctx.attr.idl_uses_aosp_compiler, + idlopts = ctx.attr.idlopts, ), ) def _process_data_binding(ctx, java_package, resources_ctx, **unused_sub_ctxs): + if ctx.attr.enable_data_binding and not acls.in_databinding_allowed(str(ctx.label)): + fail("This target is not allowed to use databinding and enable_data_binding is True.") return ProviderInfo( name = "db_ctx", value = _data_binding.process( @@ -216,7 +239,7 @@ def _process_data_binding(ctx, java_package, resources_ctx, **unused_sub_ctxs): exports = utils.collect_providers(DataBindingV2Info, ctx.attr.exports), data_binding_exec = get_android_toolchain(ctx).data_binding_exec.files_to_run, data_binding_annotation_processor = - get_android_toolchain(ctx).data_binding_annotation_processor[JavaPluginInfo], + get_android_toolchain(ctx).data_binding_annotation_processor, data_binding_annotation_template = utils.only(get_android_toolchain(ctx).data_binding_annotation_template.files.to_list()), ), @@ -252,10 +275,7 @@ def _process_jvm(ctx, exceptions_ctx, resources_ctx, idl_ctx, db_ctx, **unused_s deps = utils.collect_providers(JavaInfo, ctx.attr.deps, idl_ctx.idl_deps), exports = utils.collect_providers(JavaInfo, ctx.attr.exports), - plugins = ( - utils.collect_providers(JavaPluginInfo, ctx.attr.plugins) + - db_ctx.java_plugins - ), + plugins = utils.collect_providers(JavaPluginInfo, ctx.attr.plugins, db_ctx.java_plugins), exported_plugins = utils.collect_providers( JavaPluginInfo, ctx.attr.exported_plugins, @@ -272,11 +292,25 @@ def _process_jvm(ctx, exceptions_ctx, resources_ctx, idl_ctx, db_ctx, **unused_s java_toolchain = _common.get_java_toolchain(ctx), ) + providers = [java_info] + + # Propagate Lint rule Jars from any exported AARs (b/229993446) + android_lint_rules = [info.lint_jars for info in utils.collect_providers( + AndroidLintRulesInfo, + ctx.attr.exports, + )] + if android_lint_rules: + providers.append( + AndroidLintRulesInfo( + lint_jars = depset(transitive = android_lint_rules), + ), + ) + return ProviderInfo( name = "jvm_ctx", value = struct( java_info = java_info, - providers = [java_info], + providers = providers, ), ) @@ -369,11 +403,11 @@ def _process_native(ctx, idl_ctx, **unused_ctx): ), ) -def _process_intellij(ctx, java_package, resources_ctx, idl_ctx, jvm_ctx, **unused_sub_ctxs): +def _process_intellij(ctx, java_package, manifest_ctx, resources_ctx, idl_ctx, jvm_ctx, **unused_sub_ctxs): android_ide_info = _intellij.make_android_ide_info( ctx, java_package = java_package, - manifest = ctx.file.manifest, + manifest = manifest_ctx.processed_manifest, defines_resources = resources_ctx.defines_resources, merged_manifest = resources_ctx.merged_manifest, resources_apk = resources_ctx.resources_apk, @@ -404,6 +438,7 @@ def _process_coverage(ctx, **unused_ctx): providers = [ coverage_common.instrumented_files_info( ctx, + source_attributes = ["srcs"], dependency_attributes = ["assets", "deps", "exports"], ), ], @@ -415,6 +450,7 @@ def _process_coverage(ctx, **unused_ctx): # insertion. PROCESSORS = dict( ExceptionsProcessor = _exceptions_processor, + ManifestProcessor = _process_manifest, ResourceProcessor = _process_resources, IdlProcessor = _process_idl, DataBindingProcessor = _process_data_binding, diff --git a/rules/android_library/rule.bzl b/rules/android_library/rule.bzl index 02cc1fc..30b928a 100644 --- a/rules/android_library/rule.bzl +++ b/rules/android_library/rule.bzl @@ -14,14 +14,93 @@ """android_library rule.""" -load("@rules_android//rules:acls.bzl", "acls") +load("//rules:acls.bzl", "acls") load(":attrs.bzl", _ATTRS = "ATTRS") load(":impl.bzl", _impl = "impl") load( - "@rules_android//rules:attrs.bzl", + "//rules:attrs.bzl", _attrs = "attrs", ) +_RULE_DOC = """ +#### Examples + +The following example shows how to use android libraries with resources. +<pre><code>android_library( + name = "hellobazellib", + srcs = glob(["*.java"]), + resource_files = glob(["res/**/*"]), + manifest = "AndroidManifest.xml", + deps = [ + "//java/bazel/hellobazellib/activities", + "//java/bazel/hellobazellib/common", + "//java/bazel/hellobazellib/math", + "//java/bazel/hellobazellib/service", + ], +)</code></pre> + +The following example shows how to set `idl_import_root`. Let //java/bazel/helloandroid/BUILD contain: +<pre><code>android_library( + name = "parcelable", + srcs = ["MyParcelable.java"], # bazel.helloandroid.MyParcelable + # MyParcelable.aidl will be used as import for other .aidl + # files that depend on it, but will not be compiled. + idl_parcelables = ["MyParcelable.aidl"] # bazel.helloandroid.MyParcelable + # We don't need to specify idl_import_root since the aidl file + # which declares bazel.helloandroid.MyParcelable + # is present at java/bazel/helloandroid/MyParcelable.aidl + # underneath a java root (java/). +) +android_library( + name = "foreign_parcelable", + srcs = ["src/android/helloandroid/OtherParcelable.java"], # android.helloandroid.OtherParcelable + idl_parcelables = [ + "src/android/helloandroid/OtherParcelable.aidl" # android.helloandroid.OtherParcelable + ], + # We need to specify idl_import_root because the aidl file which + # declares android.helloandroid.OtherParcelable is not positioned + # at android/helloandroid/OtherParcelable.aidl under a normal java root. + # Setting idl_import_root to "src" in //java/bazel/helloandroid + # adds java/bazel/helloandroid/src to the list of roots + # the aidl compiler will search for imported types. + idl_import_root = "src", +) +\\# Here, OtherInterface.aidl has an "import android.helloandroid.CallbackInterface;" statement. +android_library( + name = "foreign_interface", + idl_srcs = [ + "src/android/helloandroid/OtherInterface.aidl" # android.helloandroid.OtherInterface + "src/android/helloandroid/CallbackInterface.aidl" # android.helloandroid.CallbackInterface + ], + # As above, idl_srcs which are not correctly positioned under a java root + # must have idl_import_root set. Otherwise, OtherInterface (or any other + # interface in a library which depends on this one) will not be able + # to find CallbackInterface when it is imported. + idl_import_root = "src", +) +\\# MyParcelable.aidl is imported by MyInterface.aidl, so the generated +\\# MyInterface.java requires MyParcelable.class at compile time. +\\# Depending on :parcelable ensures that aidl compilation of MyInterface.aidl +\\# specifies the correct import roots and can access MyParcelable.aidl, and +\\# makes MyParcelable.class available to Java compilation of MyInterface.java +\\# as usual. +android_library( + name = "idl", + idl_srcs = ["MyInterface.aidl"], + deps = [":parcelable"], +) +\\# Here, ServiceParcelable uses and thus depends on ParcelableService, +\\# when it's compiled, but ParcelableService also uses ServiceParcelable, +\\# which creates a circular dependency. +\\# As a result, these files must be compiled together, in the same android_library. +android_library( + name = "circular_dependencies", + srcs = ["ServiceParcelable.java"], + idl_srcs = ["ParcelableService.aidl"], + idl_parcelables = ["ServiceParcelable.aidl"], +)</code></pre> +""" + def _outputs(name, _package_name, _defined_local_resources): outputs = dict( lib_jar = "lib%{name}.jar", @@ -70,6 +149,7 @@ def make_rule( "java", ], implementation = implementation, + doc = _RULE_DOC, provides = [ AndroidCcLinkParamsInfo, AndroidIdeInfo, @@ -80,8 +160,8 @@ def make_rule( ], outputs = outputs, toolchains = [ - "@rules_android//toolchains/android:toolchain_type", - "@rules_android//toolchains/android_sdk:toolchain_type", + "//toolchains/android:toolchain_type", + "//toolchains/android_sdk:toolchain_type", ] + additional_toolchains, _skylark_testable = True, ) diff --git a/rules/android_tools_defaults_jar.bzl b/rules/android_tools_defaults_jar.bzl index b226882..df937e6 100644 --- a/rules/android_tools_defaults_jar.bzl +++ b/rules/android_tools_defaults_jar.bzl @@ -28,5 +28,5 @@ android_tools_defaults_jar = rule( attrs = ANDROID_TOOLS_DEFAULTS_JAR_ATTRS, implementation = _impl, fragments = ["android"], - toolchains = ["@rules_android//toolchains/android_sdk:toolchain_type"], + toolchains = ["//toolchains/android_sdk:toolchain_type"], ) diff --git a/rules/attrs.bzl b/rules/attrs.bzl index af389ff..b93c2e8 100644 --- a/rules/attrs.bzl +++ b/rules/attrs.bzl @@ -15,6 +15,7 @@ """Common attributes for Android rules.""" load(":utils.bzl", "log") +load(":native_toolchain_attrs.bzl", "ANDROID_SDK_TOOLCHAIN_TYPE_DEFAULT") def _add(attrs, *others): new = {} @@ -78,8 +79,8 @@ _tristate = struct( _JAVA_RUNTIME = dict( _host_javabase = attr.label( - cfg = "host", - default = Label("@rules_android//tools/jdk:current_java_runtime"), + cfg = "exec", + default = Label("//tools/jdk:current_java_runtime"), ), ) @@ -102,23 +103,72 @@ _COMPILATION = _add( assets = attr.label_list( allow_files = True, cfg = "target", + doc = ("The list of assets to be packaged. This is typically a glob of " + + "all files under the assets directory. You can also reference " + + "other rules (any rule that produces files) or exported files in " + + "the other packages, as long as all those files are under the " + + "assets_dir directory in the corresponding package."), + ), + assets_dir = attr.string( + doc = ("The string giving the path to the files in assets. " + + "The pair assets and assets_dir describe packaged assets and either both " + + "attributes should be provided or none of them."), + ), + custom_package = attr.string( + doc = ("Java package for which java sources will be generated. " + + "By default the package is inferred from the directory where the BUILD file " + + "containing the rule is. You can specify a different package but this is " + + "highly discouraged since it can introduce classpath conflicts with other " + + "libraries that will only be detected at runtime."), ), - assets_dir = attr.string(), - custom_package = attr.string(), manifest = attr.label( allow_single_file = [".xml"], + doc = ("The name of the Android manifest file, normally " + + "AndroidManifest.xml. Must be defined if resource_files or assets are defined."), ), resource_files = attr.label_list( allow_files = True, + doc = ("The list of resources to be packaged. This " + + "is typically a glob of all files under the res directory. Generated files " + + "(from genrules) can be referenced by Label here as well. The only " + + "restriction is that the generated outputs must be under the same \"res\" " + + "directory as any other resource files that are included."), ), data = attr.label_list( allow_files = True, + doc = ( + "Files needed by this rule at runtime. May list file or rule targets. Generally allows any target.\n\n" + + "The default outputs and runfiles of targets in the `data` attribute should appear in the `*.runfiles` area of" + + "any executable which is output by or has a runtime dependency on this target. " + + "This may include data files or binaries used when this target's " + + "[srcs](https://docs.bazel.build/versions/main/be/common-definitions.html#typical.srcs) are executed. " + + "See the [data dependencies](https://docs.bazel.build/versions/main/build-ref.html#data) section " + + "for more information about how to depend on and use data files.\n\n" + + "New rules should define a `data` attribute if they process inputs which might use other inputs at runtime. " + + "Rules' implementation functions must also " + + "[populate the target's runfiles](https://docs.bazel.build/versions/main/skylark/rules.html#runfiles) " + + "from the outputs and runfiles of any `data` attribute, as well as runfiles from any dependency attribute " + + "which provides either source code or runtime dependencies." + ), ), plugins = attr.label_list( providers = [JavaPluginInfo], - cfg = "host", + cfg = "exec", + doc = ( + "Java compiler plugins to run at compile-time. " + + "Every `java_plugin` specified in the plugins attribute will be run whenever this rule is built. " + + "A library may also inherit plugins from dependencies that use [exported_plugins](https://docs.bazel.build/versions/main/be/java.html#java_library.exported_plugins). " + + "Resources generated by the plugin will be included in the resulting jar of this rule." + ), + ), + javacopts = attr.string_list( + doc = ( + "Extra compiler options for this library. " + + "Subject to \"[Make variable](https://docs.bazel.build/versions/main/be/make-variables.html)\" substitution and " + + "[Bourne shell tokenization](https://docs.bazel.build/versions/main/be/common-definitions.html#sh-tokenization).\n" + + "These compiler options are passed to javac after the global compiler options." + ), ), - javacopts = attr.string_list(), # TODO: Expose getPlugins() in JavaConfiguration.java # com/google/devtools/build/lib/rules/java/JavaConfiguration.java # com/google/devtools/build/lib/rules/java/JavaOptions.java @@ -139,23 +189,23 @@ _DATA_CONTEXT = _add( dict( # Additional attrs needed for AndroidDataContext _add_g3itr_xslt = attr.label( - cfg = "host", + cfg = "exec", default = Label("//tools/android/xslt:add_g3itr.xslt"), allow_single_file = True, ), _android_manifest_merge_tool = attr.label( - cfg = "host", + cfg = "exec", default = Label("//tools/android:merge_manifests"), executable = True, ), # TODO(b/145617058) Switching back to head RPBB until the Android rules release process is improved _android_resources_busybox = attr.label( - cfg = "host", - default = Label("@rules_android//rules:ResourceProcessorBusyBox"), + cfg = "exec", + default = Label("//rules:ResourceProcessorBusyBox"), executable = True, ), _xsltproc_tool = attr.label( - cfg = "host", + cfg = "exec", default = Label("//tools/android/xslt:xslt"), allow_files = True, ), @@ -172,18 +222,18 @@ _DATA_CONTEXT = _add( ANDROID_SDK_ATTRS = dict( aapt = attr.label( allow_single_file = True, - cfg = "host", + cfg = "exec", executable = True, mandatory = True, ), aapt2 = attr.label( allow_single_file = True, - cfg = "host", + cfg = "exec", executable = True, ), aidl = attr.label( allow_files = True, - cfg = "host", + cfg = "exec", executable = True, mandatory = True, ), @@ -192,80 +242,80 @@ ANDROID_SDK_ATTRS = dict( ), android_jar = attr.label( allow_single_file = [".jar"], - cfg = "host", + cfg = "exec", mandatory = True, ), annotations_jar = attr.label( allow_single_file = [".jar"], - cfg = "host", + cfg = "exec", ), apkbuilder = attr.label( allow_files = True, - cfg = "host", + cfg = "exec", executable = True, ), apksigner = attr.label( allow_files = True, - cfg = "host", + cfg = "exec", executable = True, mandatory = True, ), adb = attr.label( allow_single_file = True, - cfg = "host", + cfg = "exec", executable = True, mandatory = True, ), build_tools_version = attr.string(), dx = attr.label( allow_files = True, - cfg = "host", + cfg = "exec", executable = True, mandatory = True, ), framework_aidl = attr.label( allow_single_file = True, - cfg = "host", + cfg = "exec", mandatory = True, ), legacy_main_dex_list_generator = attr.label( allow_files = True, - cfg = "host", + cfg = "exec", executable = True, ), main_dex_classes = attr.label( allow_single_file = True, - cfg = "host", + cfg = "exec", mandatory = True, ), main_dex_list_creator = attr.label( allow_files = True, - cfg = "host", + cfg = "exec", executable = True, mandatory = True, ), proguard = attr.label( allow_files = True, - cfg = "host", + cfg = "exec", executable = True, mandatory = True, ), shrinked_android_jar = attr.label( allow_single_file = True, - cfg = "host", + cfg = "exec", ), source_properties = attr.label( allow_single_file = True, - cfg = "host", + cfg = "exec", ), zipalign = attr.label( allow_single_file = True, - cfg = "host", + cfg = "exec", executable = True, mandatory = True, ), _proguard = attr.label( - cfg = "host", + cfg = "exec", default = configuration_field( fragment = "java", name = "proguard_top", @@ -276,14 +326,23 @@ ANDROID_SDK_ATTRS = dict( ), ) -ANDROID_TOOLS_DEFAULTS_JAR_ATTRS = _add(_ANDROID_SDK) +# Attributes for resolving platform-based toolchains. Only needed by the native +# DexArchiveAspect. +_ANDROID_TOOLCHAIN_ATTRS = dict( + _android_sdk_toolchain_type = attr.label( + allow_rules = ["toolchain_type"], + default = ANDROID_SDK_TOOLCHAIN_TYPE_DEFAULT, + ), +) +ANDROID_TOOLS_DEFAULTS_JAR_ATTRS = _add(_ANDROID_SDK) attrs = struct( ANDROID_SDK = _ANDROID_SDK, COMPILATION = _COMPILATION, DATA_CONTEXT = _DATA_CONTEXT, JAVA_RUNTIME = _JAVA_RUNTIME, + ANDROID_TOOLCHAIN_ATTRS = _ANDROID_TOOLCHAIN_ATTRS, tristate = _tristate, add = _add, replace = _replace, diff --git a/rules/bundletool.bzl b/rules/bundletool.bzl index 161882a..cfc3e48 100644 --- a/rules/bundletool.bzl +++ b/rules/bundletool.bzl @@ -14,6 +14,7 @@ """Bazel Bundletool Commands.""" +load(":common.bzl", _common = "common") load(":java.bzl", _java = "java") _density_mapping = { @@ -196,9 +197,13 @@ def _bundle_to_apks( ctx, out = None, bundle = None, - universal = False, + mode = None, + system_apk_options = None, device_spec = None, keystore = None, + oldest_signer = None, + lineage = None, + rotation_min_sdk = None, modules = None, aapt2 = None, bundletool = None, @@ -210,8 +215,13 @@ def _bundle_to_apks( args.add("--bundle", bundle) args.add("--aapt2", aapt2.executable.path) - if universal: - args.add("--mode=universal") + if mode: + args.add("--mode", mode) + + if system_apk_options: + if mode != "SYSTEM": + fail("Unexpected system_apk_options specified, requires SYSTEM mode but got %s" % mode) + args.add_joined("--system-apk-options", system_apk_options, join_with = ",") if keystore: args.add("--ks", keystore.path) @@ -219,6 +229,19 @@ def _bundle_to_apks( args.add("--ks-key-alias", "AndroidDebugKey") inputs.append(keystore) + if lineage: + if not oldest_signer: + fail("Key rotation requires oldest_signer in %s" % ctx.label) + oldest_signer_properties = _common.create_signer_properties(ctx, oldest_signer) + args.add("--oldest-signer", oldest_signer_properties.path) + args.add("--lineage", lineage.short_path) + inputs.append(oldest_signer_properties) + inputs.append(oldest_signer) + inputs.append(lineage) + + if rotation_min_sdk: + args.add("--rotation-min-sdk-version", rotation_min_sdk) + if device_spec: args.add("--device-spec", device_spec) inputs.append(device_spec) diff --git a/rules/busybox.bzl b/rules/busybox.bzl index a92fe68..195d80a 100644 --- a/rules/busybox.bzl +++ b/rules/busybox.bzl @@ -203,6 +203,7 @@ def _package( additional_apks_to_link_against = [], nocompress_extensions = [], proto_format = False, + shrink_resource_cycles = False, version_name = None, version_code = None, android_jar = None, @@ -257,6 +258,8 @@ def _package( nocompress_extensions: A list of strings. File extension to leave uncompressed in the apk. proto_format: Boolean, whether to generate the resource table in proto format. + shrink_resource_cycles: Boolean, flag that enables more shrinking of + code and resources by instructing AAPT2 to emit conditional Proguard keep rules. version_name: A string. The version name to stamp the generated manifest with. Optional. version_code: A string. The version code to stamp the generated manifest with. Optional. android_jar: A File. The Android Jar. @@ -376,6 +379,8 @@ def _package( args.add_joined("--uncompressedExtensions", nocompress_extensions, join_with = ",") if proto_format: args.add("--resourceTableAsProto") + if shrink_resource_cycles: + args.add("--conditionalKeepRules=yes") if version_name: args.add("--versionName", version_name) if version_code: diff --git a/rules/common.bzl b/rules/common.bzl index 6e84559..508413f 100644 --- a/rules/common.bzl +++ b/rules/common.bzl @@ -14,13 +14,11 @@ """Bazel common library for the Android rules.""" -load(":java.bzl", _java = "java") -load(":utils.bzl", "get_android_sdk", "get_android_toolchain", _log = "log") +load(":utils.bzl", "get_android_toolchain", _log = "log") +load("//rules/android_common:reexport_android_common.bzl", _native_android_common = "native_android_common") -# TODO(ostonge): Remove once kotlin/jvm_library.internal.bzl -# is updated and released to use the java.resolve_package function -def _java_package(label, custom_package): - return _java.resolve_package_from_label(label, custom_package) +# Suffix attached to the Starlark portion of android_binary target +_PACKAGED_RESOURCES_SUFFIX = "_RESOURCES_DO_NOT_USE" # Validates that the packages listed under "deps" all have the given constraint. If a package # does not have this attribute, an error is generated. @@ -45,44 +43,6 @@ def _get_host_javabase(ctx): _log.error("Missing _host_javabase attr") return ctx.attr._host_javabase -def _sign_apk(ctx, unsigned_apk, signed_apk, keystore = None, signing_keys = [], signing_lineage = None): - """Signs an apk. Usage of keystore is deprecated. Prefer using signing_keys.""" - inputs = [unsigned_apk] - signer_args = ctx.actions.args() - signer_args.add("sign") - - if signing_keys: - inputs.extend(signing_keys) - for i, key in enumerate(signing_keys): - if i > 0: - signer_args.add("--next-signer") - signer_args.add("--ks") - signer_args.add(key.path) - signer_args.add("--ks-pass") - signer_args.add("pass:android") - if signing_lineage: - inputs.append(signing_lineage) - signer_args.add("--lineage", signing_lineage.path) - elif keystore: - inputs.append(keystore) - signer_args.add("--ks", keystore.path) - signer_args.add("--ks-pass", "pass:android") - - signer_args.add("--v1-signing-enabled", ctx.fragments.android.apk_signing_method_v1) - signer_args.add("--v1-signer-name", "CERT") - signer_args.add("--v2-signing-enabled", ctx.fragments.android.apk_signing_method_v2) - signer_args.add("--out", signed_apk.path) - signer_args.add(unsigned_apk.path) - ctx.actions.run( - executable = get_android_sdk(ctx).apk_signer, - inputs = inputs, - outputs = [signed_apk], - arguments = [signer_args], - mnemonic = "ApkSignerTool", - progress_message = "Signing APK for %s" % unsigned_apk.path, - ) - return signed_apk - def _filter_zip(ctx, in_zip, out_zip, filters = []): """Creates a copy of a zip file with files that match filters.""" args = ctx.actions.args() @@ -101,11 +61,22 @@ def _filter_zip(ctx, in_zip, out_zip, filters = []): progress_message = "Filtering %s" % in_zip.short_path, ) +def _create_signer_properties(ctx, oldest_key): + properties = ctx.actions.declare_file("%s/keystore.properties" % ctx.label.name) + ctx.actions.expand_template( + template = ctx.file._bundle_keystore_properties, + output = properties, + substitutions = {"%oldest_key%": oldest_key.short_path}, + ) + return properties + common = struct( + PACKAGED_RESOURCES_SUFFIX = _PACKAGED_RESOURCES_SUFFIX, check_rule = _check_rule, + create_signer_properties = _create_signer_properties, get_host_javabase = _get_host_javabase, get_java_toolchain = _get_java_toolchain, filter_zip = _filter_zip, - java_package = _java_package, - sign_apk = _sign_apk, ) + +android_common = _native_android_common diff --git a/rules/data_binding.bzl b/rules/data_binding.bzl index 9243673..e853058 100644 --- a/rules/data_binding.bzl +++ b/rules/data_binding.bzl @@ -130,6 +130,7 @@ def _setup_dependent_lib_artifacts(ctx, output_dir, deps): def _get_javac_opts( ctx, java_package, + artifact_type, dependency_artifacts_dir, aar_out_dir, class_info_path, @@ -145,7 +146,7 @@ def _get_javac_opts( dependency_artifacts_dir) javac_opts.append("-Aandroid.databinding.aarOutDir=" + aar_out_dir) javac_opts.append("-Aandroid.databinding.sdkDir=/not/used") - javac_opts.append("-Aandroid.databinding.artifactType=LIBRARY") + javac_opts.append("-Aandroid.databinding.artifactType=" + artifact_type) javac_opts.append("-Aandroid.databinding.exportClassListOutFile=" + "/tmp/exported_classes") javac_opts.append("-Aandroid.databinding.modulePackage=" + java_package) @@ -169,6 +170,7 @@ def _process( enable_data_binding = False, java_package = None, layout_info = None, + artifact_type = "LIBRARY", deps = [], exports = [], data_binding_exec = None, @@ -183,10 +185,11 @@ def _process( enable_data_binding: boolean. Determines whether Data Binding should be enabled. java_package: String. The Java package. + layout_info: A file. The layout-info zip file. + artifact_type: String. Either LIBRARY or APPLICATION. deps: sequence of DataBindingV2Info providers. A list of deps. Optional. exports: sequence of DataBindingV2Info providers. A list of exports. Optional. - layout_info: A file. The layout-info zip file. data_binding_exec: The DataBinding executable. data_binding_annotation_processor: JavaInfo. The JavaInfo for the annotation processor. @@ -197,6 +200,9 @@ def _process( A DataBindingContextInfo provider. """ + if artifact_type not in ["LIBRARY", "APPLICATION"]: + fail("Unexpected artifact type: " + artifact_type) + # TODO(b/154513292): Clean up bad usages of context objects. if resources_ctx: defines_resources = resources_ctx.defines_resources @@ -267,6 +273,7 @@ def _process( db_info[_JAVAC_OPTS] = _get_javac_opts( ctx, java_package, + artifact_type, ( br_out.path.rpartition(br_out.short_path)[0] + ctx.label.package + diff --git a/rules/flags/BUILD b/rules/flags/BUILD index 4895b09..a9a585c 100644 --- a/rules/flags/BUILD +++ b/rules/flags/BUILD @@ -1,7 +1,7 @@ # Flags for Android rules and mobile-install -load("@rules_android//rules/flags:flags.bzl", "flags") -load("@rules_android//rules/flags:flag_defs.bzl", "define_flags") +load("//rules/flags:flags.bzl", "flags") +load("//rules/flags:flag_defs.bzl", "define_flags") load("@bazel_skylib//:bzl_library.bzl", "bzl_library") diff --git a/rules/flags/flag_defs.bzl b/rules/flags/flag_defs.bzl index 08eee2d..61f2cf7 100644 --- a/rules/flags/flag_defs.bzl +++ b/rules/flags/flag_defs.bzl @@ -14,7 +14,7 @@ """Flag definitions.""" -load("@rules_android//rules/flags:flags.bzl", "flags") +load("//rules/flags:flags.bzl", "flags") def define_flags(): flags.DEFINE_bool( @@ -76,7 +76,7 @@ def define_flags(): flags.DEFINE_bool( name = "mi_desugar_java8_libs", - default = False, + default = True, description = "Set True with --config=android_java8_libs", ) @@ -91,3 +91,9 @@ def define_flags(): name = "stamp", description = "Accesses the native --stamp CLI flag", ) + + flags.DEFINE_bool( + name = "use_studio_deployer", + default = True, + description = "Use Studio Deployer to install apks", + ) diff --git a/rules/flags/flags.bzl b/rules/flags/flags.bzl index e6e25a6..5756592 100644 --- a/rules/flags/flags.bzl +++ b/rules/flags/flags.bzl @@ -14,7 +14,7 @@ """Bazel Flags.""" -load("@rules_android//rules:utils.bzl", "utils") +load("//rules:utils.bzl", "utils") _BoolFlagInfo = provider( doc = "Provides information about a boolean flag", diff --git a/rules/idl.bzl b/rules/idl.bzl index 8dc52d3..c77b137 100644 --- a/rules/idl.bzl +++ b/rules/idl.bzl @@ -47,22 +47,34 @@ def _gen_java_from_idl( transitive_idl_preprocessed = [], aidl = None, aidl_lib = None, - aidl_framework = None): + aidl_framework = None, + uses_aosp_compiler = False, + idlopts = []): args = ctx.actions.args() - args.add("-b") + + # Note: at the moment (2022/11/07), the flags that the AOSP compiler accepts is a superset of + # the Google3 compiler, but that might not be true in the future. + if uses_aosp_compiler: + args.add("--use-aosp-compiler") + + for opt in idlopts: + args.add(opt) + + args.add("-b") # fail on parcelable args.add_all(transitive_idl_import_roots, format_each = "-I%s") args.add(aidl_framework, format = "-p%s") args.add_all(transitive_idl_preprocessed, format_each = "-p%s") args.add(idl_src) args.add(out_idl_java_src) + aidl_lib_files = [aidl_lib.files] if aidl_lib else [] + ctx.actions.run( executable = aidl, arguments = [args], inputs = depset( [aidl_framework], - transitive = [ - aidl_lib.files, + transitive = aidl_lib_files + [ transitive_idl_imports, transitive_idl_preprocessed, ], @@ -129,7 +141,9 @@ def _process( exports = [], aidl = None, aidl_lib = None, - aidl_framework = None): + aidl_framework = None, + uses_aosp_compiler = False, + idlopts = []): """Processes Android IDL. Args: @@ -176,14 +190,22 @@ def _process( are supplied. aidl_lib: Target. A target pointing to the aidl_lib library required during Java compilation when Java code is generated from idl sources. - Optional, unless idl_srcs are supplied. + Optional. aidl_framework: Target. A target pointing to the aidl framework. Optional, unless idl_srcs are supplied. + uses_aosp_compiler: boolean. If True, the upstream AOSP AIDL compiler is + used instead of the Google3-only AIDL compiler. This allows wider range + of AIDL language features including the structured parcelable, enum, + union, and many more. On the other hand, using this may cause noticeable + regression in terms of code size and performance as the compiler doesn't + implement several optimization techniques that the Google3 compiler has. + idlopts: list of string. Additional flags to add to the AOSP AIDL compiler + invocation. Returns: A IDLContextInfo provider. """ - if idl_srcs and not (aidl and aidl_lib and aidl_framework): + if idl_srcs and not (aidl and aidl_framework): _log.error(_AIDL_TOOLCHAIN_MISSING_ERROR) transitive_idl_import_roots = [] @@ -224,13 +246,15 @@ def _process( aidl = aidl, aidl_lib = aidl_lib, aidl_framework = aidl_framework, + uses_aosp_compiler = uses_aosp_compiler, + idlopts = idlopts, ) return IDLContextInfo( idl_srcs = idl_srcs, idl_import_root = idl_import_root, idl_java_srcs = idl_java_srcs, - idl_deps = [aidl_lib] if idl_java_srcs else [], + idl_deps = [aidl_lib] if (idl_java_srcs and aidl_lib) else [], providers = [ # TODO(b/146216105): Make this a Starlark provider. AndroidIdlInfo( diff --git a/rules/java.bzl b/rules/java.bzl index 2e2fc58..25c7090 100644 --- a/rules/java.bzl +++ b/rules/java.bzl @@ -359,14 +359,14 @@ def _singlejar( output, mnemonic = "SingleJar", progress_message = "Merge into a single jar.", - exclude_build_data = False, + include_build_data = False, java_toolchain = None): args = ctx.actions.args() args.add("--output") args.add(output) args.add("--compression") args.add("--normalize") - if exclude_build_data: + if not include_build_data: args.add("--exclude_build_data") args.add("--warn_duplicate_resources") if inputs: diff --git a/rules/native_deps.bzl b/rules/native_deps.bzl new file mode 100644 index 0000000..fb4c728 --- /dev/null +++ b/rules/native_deps.bzl @@ -0,0 +1,334 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Defines the native libs processing and an aspect to collect build configuration +of split deps +""" + +load("//rules:common.bzl", "common") + +SplitConfigInfo = provider( + doc = "Provides information about configuration for a split config dep", + fields = dict( + build_config = "The build configuration of the dep.", + ), +) + +def _split_config_aspect_impl(__, ctx): + return SplitConfigInfo(build_config = ctx.configuration) + +split_config_aspect = aspect( + implementation = _split_config_aspect_impl, +) + +def process(ctx, filename): + """ Links native deps into a shared library + + Args: + ctx: The context. + filename: String. The name of the artifact containing the name of the + linked shared library + + Returns: + Tuple of (libs, libs_name) where libs is a depset of all native deps + and libs_name is a File containing the basename of the linked shared + library + """ + actual_target_name = ctx.label.name.removesuffix(common.PACKAGED_RESOURCES_SUFFIX) + native_libs_basename = None + libs_name = None + libs = dict() + for key, deps in ctx.split_attr.deps.items(): + cc_toolchain_dep = ctx.split_attr._cc_toolchain_split[key] + cc_toolchain = cc_toolchain_dep[cc_common.CcToolchainInfo] + build_config = cc_toolchain_dep[SplitConfigInfo].build_config + linker_input = cc_common.create_linker_input( + owner = ctx.label, + user_link_flags = ["-Wl,-soname=lib" + actual_target_name], + ) + cc_info = cc_common.merge_cc_infos( + cc_infos = _concat( + [CcInfo(linking_context = cc_common.create_linking_context( + linker_inputs = depset([linker_input]), + ))], + [dep[JavaInfo].cc_link_params_info for dep in deps if JavaInfo in dep], + [dep[AndroidCcLinkParamsInfo].link_params for dep in deps if AndroidCcLinkParamsInfo in dep], + [dep[CcInfo] for dep in deps if CcInfo in dep], + ), + ) + libraries = [] + + native_deps_lib = _link_native_deps_if_present(ctx, cc_info, cc_toolchain, build_config, actual_target_name) + if native_deps_lib: + libraries.append(native_deps_lib) + native_libs_basename = native_deps_lib.basename + + libraries.extend(_filter_unique_shared_libs(native_deps_lib, cc_info)) + + if libraries: + libs[key] = depset(libraries) + + if libs and native_libs_basename: + libs_name = ctx.actions.declare_file("nativedeps_filename/" + actual_target_name + "/" + filename) + ctx.actions.write(output = libs_name, content = native_libs_basename) + + transitive_native_libs = _get_transitive_native_libs(ctx) + return AndroidBinaryNativeLibsInfo(libs, libs_name, transitive_native_libs) + +# Collect all native shared libraries across split transitions. Some AARs +# contain shared libraries across multiple architectures, e.g. x86 and +# armeabi-v7a, and need to be packed into the APK. +def _get_transitive_native_libs(ctx): + return depset( + transitive = [ + dep[AndroidNativeLibsInfo].native_libs + for dep in ctx.attr.deps + if AndroidNativeLibsInfo in dep + ], + ) + +def _all_inputs(cc_info): + return [ + lib + for input in cc_info.linking_context.linker_inputs.to_list() + for lib in input.libraries + ] + +def _filter_unique_shared_libs(linked_lib, cc_info): + basenames = {} + artifacts = {} + if linked_lib: + basenames[linked_lib.basename] = linked_lib + for input in _all_inputs(cc_info): + if input.pic_static_library or input.static_library: + # This is not a shared library and will not be loaded by Android, so skip it. + continue + + artifact = None + if input.interface_library: + if input.resolved_symlink_interface_library: + artifact = input.resolved_symlink_interface_library + else: + artifact = input.interface_library + elif input.resolved_symlink_dynamic_library: + artifact = input.resolved_symlink_dynamic_library + else: + artifact = input.dynamic_library + + if not artifact: + fail("Should never happen: did not find artifact for link!") + + if artifact in artifacts: + # We have already reached this library, e.g., through a different solib symlink. + continue + artifacts[artifact] = None + basename = artifact.basename + if basename in basenames: + old_artifact = basenames[basename] + fail( + "Each library in the transitive closure must have a " + + "unique basename to avoid name collisions when packaged into " + + "an apk, but two libraries have the basename '" + basename + + "': " + artifact + " and " + old_artifact + ( + " (the library compiled for this target)" if old_artifact == linked_lib else "" + ), + ) + else: + basenames[basename] = artifact + + return artifacts.keys() + +def _contains_code_to_link(input): + if not input.static_library and not input.pic_static_library: + # this is a shared library so we're going to have to copy it + return False + if input.objects: + object_files = input.objects + elif input.pic_objects: + object_files = input.pic_objects + elif _is_any_source_file(input.static_library, input.pic_static_library): + # this is an opaque library so we're going to have to link it + return True + else: + # if we reach here, this is a cc_library without sources generating an + # empty archive which does not need to be linked + # TODO(hvd): replace all such cc_library with exporting_cc_library + return False + for obj in object_files: + if not _is_shared_library(obj): + # this library was built with a non-shared-library object so we should link it + return True + return False + +def _is_any_source_file(*files): + for file in files: + if file and file.is_source: + return True + return False + +def _is_shared_library(lib_artifact): + if (lib_artifact.extension in ["so", "dll", "dylib"]): + return True + + lib_name = lib_artifact.basename + + # validate against the regex "^.+\\.((so)|(dylib))(\\.\\d\\w*)+$", + # must match VERSIONED_SHARED_LIBRARY. + for ext in (".so.", ".dylib."): + name, _, version = lib_name.rpartition(ext) + if name and version: + version_parts = version.split(".") + for part in version_parts: + if not part[0].isdigit(): + return False + for c in part[1:].elems(): + if not (c.isalnum() or c == "_"): + return False + return True + return False + +def _get_build_info(ctx): + return cc_common.get_build_info(ctx) + +def _get_shared_native_deps_path( + linker_inputs, + link_opts, + linkstamps, + build_info_artifacts, + features, + is_test_target_partially_disabled_thin_lto): + fp = [] + for artifact in linker_inputs: + fp.append(artifact.short_path) + fp.append(str(len(link_opts))) + for opt in link_opts: + fp.append(opt) + for artifact in linkstamps: + fp.append(artifact.short_path) + for artifact in build_info_artifacts: + fp.append(artifact.short_path) + for feature in features: + fp.append(feature) + + fp.append("1" if is_test_target_partially_disabled_thin_lto else "0") + + fingerprint = "%x" % hash("".join(fp)) + return "_nativedeps/" + fingerprint + +def _get_static_mode_params_for_dynamic_library_libraries(libs): + linker_inputs = [] + for lib in libs: + if lib.pic_static_library: + linker_inputs.append(lib.pic_static_library) + elif lib.static_library: + linker_inputs.append(lib.static_library) + elif lib.interface_library: + linker_inputs.append(lib.interface_library) + else: + linker_inputs.append(lib.dynamic_library) + return linker_inputs + +def _link_native_deps_if_present(ctx, cc_info, cc_toolchain, build_config, actual_target_name, is_test_rule_class = False): + needs_linking = False + for input in _all_inputs(cc_info): + needs_linking = needs_linking or _contains_code_to_link(input) + + if not needs_linking: + return None + + # This does not need to be shareable, but we use this API to specify the + # custom file root (matching the configuration) + output_lib = ctx.actions.declare_shareable_artifact( + ctx.label.package + "/nativedeps/" + actual_target_name + "/lib" + actual_target_name + ".so", + build_config.bin_dir, + ) + + link_opts = cc_info.linking_context.user_link_flags + + linkstamps = [] + for input in cc_info.linking_context.linker_inputs.to_list(): + linkstamps.extend(input.linkstamps) + linkstamps_dict = {linkstamp: None for linkstamp in linkstamps} + + build_info_artifacts = _get_build_info(ctx) if linkstamps_dict else [] + requested_features = ["static_linking_mode", "native_deps_link"] + requested_features.extend(ctx.features) + if not "legacy_whole_archive" in ctx.disabled_features: + requested_features.append("legacy_whole_archive") + requested_features = sorted(requested_features) + feature_config = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + requested_features = requested_features, + unsupported_features = ctx.disabled_features, + ) + partially_disabled_thin_lto = ( + cc_common.is_enabled( + feature_name = "thin_lto_linkstatic_tests_use_shared_nonlto_backends", + feature_configuration = feature_config, + ) and not cc_common.is_enabled( + feature_name = "thin_lto_all_linkstatic_use_shared_nonlto_backends", + feature_configuration = feature_config, + ) + ) + test_only_target = ctx.attr.testonly or is_test_rule_class + share_native_deps = ctx.fragments.cpp.share_native_deps() + + linker_inputs = _get_static_mode_params_for_dynamic_library_libraries(cc_info.linking_context.libraries_to_link) + + if share_native_deps: + shared_path = _get_shared_native_deps_path( + linker_inputs, + link_opts, + [linkstamp.file() for linkstamp in linkstamps_dict], + build_info_artifacts, + requested_features, + test_only_target and partially_disabled_thin_lto, + ) + linked_lib = ctx.actions.declare_shareable_artifact(shared_path + ".so", build_config.bin_dir) + else: + linked_lib = output_lib + + cc_common.link( + name = ctx.label.name, + actions = ctx.actions, + linking_contexts = [cc_info.linking_context], + output_type = "dynamic_library", + never_link = True, + native_deps = True, + feature_configuration = feature_config, + cc_toolchain = cc_toolchain, + test_only_target = test_only_target, + stamp = ctx.attr.stamp, + grep_includes = ctx.file._grep_includes, + main_output = linked_lib, + use_shareable_artifact_factory = True, + build_config = build_config, + ) + + if (share_native_deps): + ctx.actions.symlink( + output = output_lib, + target_file = linked_lib, + ) + return output_lib + else: + return linked_lib + +def _concat(*list_of_lists): + res = [] + for list in list_of_lists: + res.extend(list) + return res diff --git a/rules/native_toolchain_attrs.bzl b/rules/native_toolchain_attrs.bzl new file mode 100644 index 0000000..c1317b8 --- /dev/null +++ b/rules/native_toolchain_attrs.bzl @@ -0,0 +1,16 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Default label for native android sdk toolchain type.""" +ANDROID_SDK_TOOLCHAIN_TYPE_DEFAULT = Label("@bazel_tools//tools/android:sdk_toolchain_type") diff --git a/rules/proguard.bzl b/rules/proguard.bzl index 837d622..5b1a16c 100644 --- a/rules/proguard.bzl +++ b/rules/proguard.bzl @@ -14,6 +14,8 @@ """Bazel Android Proguard library for the Android rules.""" +load(":utils.bzl", "utils") + _ProguardContextInfo = provider( doc = "Contains data from processing Proguard specs.", fields = dict( @@ -100,11 +102,50 @@ def _process( ], ) +def _collect_transitive_proguard_specs( + specs_to_include, + local_proguard_specs, + proguard_deps): + if len(local_proguard_specs) == 0: + return [] + + proguard_specs = local_proguard_specs + specs_to_include + for dep in proguard_deps: + proguard_specs.extend(dep.specs.to_list()) + + return sorted(proguard_specs) + +def _get_proguard_specs( + ctx, + resource_proguard_config, + proguard_specs_for_manifest = []): + proguard_deps = utils.collect_providers(ProguardSpecProvider, utils.dedupe_split_attr(ctx.split_attr.deps)) + if ctx.configuration.coverage_enabled and hasattr(ctx.attr, "_jacoco_runtime"): + proguard_deps.append(ctx.attr._jacoco_runtime[ProguardSpecProvider]) + + local_proguard_specs = [] + if ctx.files.proguard_specs: + local_proguard_specs = ctx.files.proguard_specs + proguard_specs = _collect_transitive_proguard_specs( + [resource_proguard_config], + local_proguard_specs, + proguard_deps, + ) + + if len(proguard_specs) > 0 and ctx.fragments.android.assume_min_sdk_version: + # NB: Order here is important. We're including generated Proguard specs before the user's + # specs so that they can override values. + proguard_specs = proguard_specs_for_manifest + proguard_specs + + return proguard_specs + proguard = struct( process = _process, + get_proguard_specs = _get_proguard_specs, ) testing = struct( validate_proguard_spec = _validate_proguard_spec, + collect_transitive_proguard_specs = _collect_transitive_proguard_specs, ProguardContextInfo = _ProguardContextInfo, ) diff --git a/rules/providers.bzl b/rules/providers.bzl index e7db209..4460af1 100644 --- a/rules/providers.bzl +++ b/rules/providers.bzl @@ -99,7 +99,7 @@ StarlarkAndroidResourcesInfo = provider( AndroidLintRulesInfo = provider( doc = "Provides extra lint rules to use with AndroidLint.", fields = dict( - lint_jar = "A file, a lint jar found in an aar.", + lint_jars = "A depset of lint rule jars found in AARs and exported by a target.", ), ) diff --git a/rules/resources.bzl b/rules/resources.bzl index 2ad89c6..45db2bb 100644 --- a/rules/resources.bzl +++ b/rules/resources.bzl @@ -31,6 +31,9 @@ load( _log = "log", ) +# Depot-wide min SDK floor +_DEPOT_MIN_SDK_FLOOR = 14 + _RESOURCE_FOLDER_TYPES = [ "anim", "animator", @@ -51,8 +54,8 @@ _RESOURCE_FOLDER_TYPES = [ _RESOURCE_QUALIFIER_SEP = "-" _MANIFEST_MISSING_ERROR = ( - "In target %s, manifest attribute is required when resource_files or " + - "assets are defined." + "In target %s, manifest attribute is required when resource_files, " + + "assets, or exports_manifest are specified." ) _ASSET_DEFINITION_ERROR = ( @@ -113,6 +116,7 @@ _PACKAGED_FINAL_MANIFEST = "processed_manifest" _PACKAGED_RESOURCE_APK = "resources_apk" _PACKAGED_CLASS_JAR = "class_jar" _PACKAGED_VALIDATION_RESULT = "validation_result" +_RESOURCE_PROGUARD_CONFIG = "resource_proguard_config" _ResourcesPackageContextInfo = provider( "Packaged resources context object", @@ -123,10 +127,28 @@ _ResourcesPackageContextInfo = provider( _PACKAGED_VALIDATION_RESULT: "Validation result.", _R_JAVA: "JavaInfo for R.jar", _DATA_BINDING_LAYOUT_INFO: "Databinding layout info file.", + _RESOURCE_PROGUARD_CONFIG: "Resource proguard config", _PROVIDERS: "The list of all providers to propagate.", }, ) +# Manifest context attributes +_PROCESSED_MANIFEST = "processed_manifest" + +_ManifestContextInfo = provider( + "Manifest context object", + fields = { + _PROCESSED_MANIFEST: "The manifest after the min SDK has been changed as necessary.", + }, +) + +_ManifestValidationContextInfo = provider( + "Manifest validation context object", + fields = { + _VALIDATION_OUTPUTS: "List of outputs given to OutputGroupInfo _validation group.", + }, +) + def _generate_dummy_manifest( ctx, out_manifest = None, @@ -373,10 +395,11 @@ def _is_resource_shrinking_enabled( def _should_shrink_resource_cycles( use_android_resource_cycle_shrinking, - resource_shrinking_enabled): - if use_android_resource_cycle_shrinking and not resource_shrinking_enabled: - fail("resource cycle shrinking can only be enabled when resource shrinking is enabled") - return use_android_resource_cycle_shrinking + resource_shrinking_enabled, + has_local_proguard_specs): + return (use_android_resource_cycle_shrinking and + resource_shrinking_enabled and + has_local_proguard_specs) def _filter_multi_cpu_configuration_targets( targets): @@ -425,7 +448,9 @@ def _package( should_throw_on_conflict = True, enable_data_binding = False, enable_manifest_merging = True, + should_compile_java_srcs = True, aapt = None, + has_local_proguard_specs = False, android_jar = None, legacy_merger = None, xsltproc = None, @@ -478,6 +503,7 @@ def _package( parameter is enabled. Without this setting, data binding expressions produce build failures. enable_manifest_merging: boolean. If true, manifest merging will be performed. + should_compile_java_srcs: boolean. If native android_binary should perform java compilation. aapt: FilesToRunProvider. The aapt executable or FilesToRunProvider. android_jar: File. The Android jar. legacy_merger: FilesToRunProvider. The legacy manifest merger executable. @@ -597,6 +623,16 @@ def _package( host_javabase = host_javabase, ) + resource_shrinking_enabled = _is_resource_shrinking_enabled( + shrink_resources, + use_android_resource_shrinking, + ) + shrink_resource_cycles = _should_shrink_resource_cycles( + use_android_resource_cycle_shrinking, + resource_shrinking_enabled, + has_local_proguard_specs, + ) + resource_apk = ctx.actions.declare_file(ctx.label.name + "_migrated/.ap_") r_java = ctx.actions.declare_file("_migrated/" + ctx.label.name + ".srcjar") r_txt = ctx.actions.declare_file(ctx.label.name + "_migrated/_symbols/R.txt") @@ -641,6 +677,7 @@ def _package( densities = densities, nocompress_extensions = nocompress_extensions, java_package = java_package, + shrink_resource_cycles = shrink_resource_cycles, version_name = manifest_values[_VERSION_NAME] if _VERSION_NAME in manifest_values else None, version_code = manifest_values[_VERSION_CODE] if _VERSION_CODE in manifest_values else None, android_jar = android_jar, @@ -653,22 +690,13 @@ def _package( packaged_resources_ctx[_PACKAGED_FINAL_MANIFEST] = processed_manifest packaged_resources_ctx[_PACKAGED_RESOURCE_APK] = resource_apk packaged_resources_ctx[_PACKAGED_VALIDATION_RESULT] = resource_files_zip - - resource_shrinking_enabled = _is_resource_shrinking_enabled( - shrink_resources, - use_android_resource_shrinking, - ) - shrink_resource_cycles = _should_shrink_resource_cycles( - use_android_resource_cycle_shrinking, - resource_shrinking_enabled, - ) + packaged_resources_ctx[_RESOURCE_PROGUARD_CONFIG] = proguard_cfg # Fix class jar name because some tests depend on {label_name}_resources.jar being the suffix of - # the path, with _RESOURCES_DO_NOT_USE removed from the label name. - _RESOURCES_SUFFIX = "_RESOURCES_DO_NOT_USE" + # the path, with _common.PACKAGED_RESOURCES_SUFFIX removed from the label name. class_jar_name = ctx.label.name + "_migrated/_resources.jar" - if ctx.label.name.endswith(_RESOURCES_SUFFIX): - label_name = ctx.label.name[:-len(_RESOURCES_SUFFIX)] + if ctx.label.name.endswith(_common.PACKAGED_RESOURCES_SUFFIX): + label_name = ctx.label.name.removesuffix(_common.PACKAGED_RESOURCES_SUFFIX) class_jar_name = ctx.label.name + "_migrated/" + label_name + "_resources.jar" class_jar = ctx.actions.declare_file(class_jar_name) @@ -728,6 +756,7 @@ def _package( r_txt = r_txt, resources_zip = resource_files_zip, databinding_info = data_binding_layout_info, + should_compile_java_srcs = should_compile_java_srcs, )) return _ResourcesPackageContextInfo(**packaged_resources_ctx) @@ -969,6 +998,149 @@ def _validate_resources(resource_files = None): if res_type not in _RESOURCE_FOLDER_TYPES: fail(_INCORRECT_RESOURCE_LAYOUT_ERROR % resource_file) +def _bump_min_sdk( + ctx, + manifest, + floor, + enforce_min_sdk_floor_tool): + """Bumps the min SDK attribute of AndroidManifest to the floor. + + Args: + ctx: The rules context. + manifest: File. The AndroidManifest.xml file. + floor: int. The min SDK floor. Manifest is unchanged if floor <= 0. + enforce_min_sdk_floor_tool: FilesToRunProvider. The enforce_min_sdk_tool executable or + FilesToRunprovider + + Returns: + A dict containing _ManifestContextInfo provider fields. + """ + manifest_ctx = {} + if not manifest or floor <= 0: + manifest_ctx[_PROCESSED_MANIFEST] = manifest + return _ManifestContextInfo(**manifest_ctx) + + args = ctx.actions.args() + args.add("-action", "bump") + args.add("-manifest", manifest) + args.add("-min_sdk_floor", floor) + + out_dir = "_migrated/_min_sdk_bumped/" + ctx.label.name + "/" + log = ctx.actions.declare_file( + out_dir + "log.txt", + ) + args.add("-log", log.path) + + out_manifest = ctx.actions.declare_file( + out_dir + "AndroidManifest.xml", + ) + args.add("-output", out_manifest.path) + ctx.actions.run( + executable = enforce_min_sdk_floor_tool, + inputs = [manifest], + outputs = [out_manifest, log], + arguments = [args], + mnemonic = "BumpMinSdkFloor", + progress_message = "Bumping up AndroidManifest min SDK %s" % str(ctx.label), + ) + manifest_ctx[_PROCESSED_MANIFEST] = out_manifest + + return _ManifestContextInfo(**manifest_ctx) + +def _set_default_min_sdk( + ctx, + manifest, + default, + enforce_min_sdk_floor_tool): + """ Sets the min SDK attribute of AndroidManifest to default if it is not already set. + + Args: + ctx: The rules context. + manifest: File. The AndroidManifest.xml file. + default: string. The default value for min SDK. The manifest is unchanged if it already + specifies a min SDK. + enforce_min_sdk_floor_tool: FilesToRunProvider. The enforce_min_sdk_tool executable or + FilesToRunprovider + + Returns: + A dict containing _ManifestContextInfo provider fields. + """ + manifest_ctx = {} + if not manifest or not default: + manifest_ctx[_PROCESSED_MANIFEST] = manifest + return _ManifestContextInfo(**manifest_ctx) + + args = ctx.actions.args() + args.add("-action", "set_default") + args.add("-manifest", manifest) + args.add("-default_min_sdk", default) + + out_dir = "_migrated/_min_sdk_default_set/" + ctx.label.name + "/" + log = ctx.actions.declare_file( + out_dir + "log.txt", + ) + args.add("-log", log.path) + + out_manifest = ctx.actions.declare_file( + out_dir + "AndroidManifest.xml", + ) + args.add("-output", out_manifest.path) + ctx.actions.run( + executable = enforce_min_sdk_floor_tool, + inputs = [manifest], + outputs = [out_manifest, log], + arguments = [args], + mnemonic = "SetDefaultMinSdkFloor", + progress_message = "Setting AndroidManifest min SDK to default %s" % str(ctx.label), + ) + manifest_ctx[_PROCESSED_MANIFEST] = out_manifest + + return _ManifestContextInfo(**manifest_ctx) + +def _validate_min_sdk( + ctx, + manifest, + floor, + enforce_min_sdk_floor_tool): + """Validates that the min SDK attribute of AndroidManifest is at least at the floor. + + Args: + ctx: The rules context. + manifest: File. The AndroidManifest.xml file. + floor: int. The min SDK floor. No validation is done if floor <= 0. + enforce_min_sdk_floor_tool: FilesToRunProvider. The enforce_min_sdk_tool executable or + FilesToRunprovider + + Returns: + A dict containing _ManifestValidationContextInfo provider fields. + """ + manifest_validation_ctx = {_VALIDATION_OUTPUTS: []} + if not manifest or floor <= 0: + return _ManifestValidationContextInfo(**manifest_validation_ctx) + + args = ctx.actions.args() + args.add("-action", "validate") + args.add("-manifest", manifest) + args.add("-min_sdk_floor", floor) + + out_dir = "_migrated/_min_sdk_validated/" + ctx.label.name + "/" + log = ctx.actions.declare_file( + out_dir + "log.txt", + ) + args.add("-log", log.path) + + ctx.actions.run( + executable = enforce_min_sdk_floor_tool, + inputs = [manifest], + outputs = [log], + arguments = [args], + mnemonic = "ValidateMinSdkFloor", + progress_message = "Validating AndroidManifest min SDK %s" % str(ctx.label), + ) + manifest_validation_ctx[_VALIDATION_OUTPUTS].append(log) + + return _ManifestValidationContextInfo(**manifest_validation_ctx) + def _process_starlark( ctx, java_package = None, @@ -1668,6 +1840,18 @@ resources = struct( # Exposed for android_local_test and android_library generate_dummy_manifest = _generate_dummy_manifest, + + # Exposed for android_library, aar_import, and android_binary + bump_min_sdk = _bump_min_sdk, + + # Exposed for use in AOSP + set_default_min_sdk = _set_default_min_sdk, + + # Exposed for android_binary + validate_min_sdk = _validate_min_sdk, + + # Exposed for android_library, aar_import, and android_binary + DEPOT_MIN_SDK_FLOOR = _DEPOT_MIN_SDK_FLOOR, ) testing = struct( diff --git a/rules/toolchains/emulator/toolchain.bzl b/rules/toolchains/emulator/toolchain.bzl index 61bfb77..6bf2e0f 100644 --- a/rules/toolchains/emulator/toolchain.bzl +++ b/rules/toolchains/emulator/toolchain.bzl @@ -38,20 +38,20 @@ emulator_toolchain = rule( attrs = { "emulator": attr.label( allow_files = True, - cfg = "host", + cfg = "exec", mandatory = True, ), "emulator_deps": attr.label_list( allow_files = True, - cfg = "host", + cfg = "exec", ), "emulator_head": attr.label( allow_files = True, - cfg = "host", + cfg = "exec", ), "emulator_head_deps": attr.label_list( allow_files = True, - cfg = "host", + cfg = "exec", ), "emulator_suffix": attr.string(default = ""), "emulator_head_suffix": attr.string(default = ""), diff --git a/rules/utils.bzl b/rules/utils.bzl index d796371..e9a28cc 100644 --- a/rules/utils.bzl +++ b/rules/utils.bzl @@ -414,11 +414,11 @@ VALIDATION_OUT={validation_out} ) def get_android_toolchain(ctx): - return ctx.toolchains["@rules_android//toolchains/android:toolchain_type"] + return ctx.toolchains["//toolchains/android:toolchain_type"] def get_android_sdk(ctx): if hasattr(ctx.fragments.android, "incompatible_use_toolchain_resolution") and ctx.fragments.android.incompatible_use_toolchain_resolution: - return ctx.toolchains["@rules_android//toolchains/android_sdk:toolchain_type"].android_sdk_info + return ctx.toolchains["//toolchains/android_sdk:toolchain_type"].android_sdk_info else: return ctx.attr._android_sdk[AndroidSdkInfo] diff --git a/src/common/golang/BUILD b/src/common/golang/BUILD new file mode 100644 index 0000000..4c1bf0f --- /dev/null +++ b/src/common/golang/BUILD @@ -0,0 +1,116 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description: +# Common libraries and utilities. +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_library( + name = "xml2", + srcs = ["marshal.go"], + importpath = "src/common/golang/xml2", +) + +go_test( + name = "xml2_test", + size = "small", + srcs = [ + "marshal_test.go", + ], + embed = [":xml2"], +) + +go_library( + name = "shard", + srcs = ["shard.go"], + importpath = "src/common/golang/shard", +) + +go_test( + name = "shard_test", + size = "small", + srcs = [ + "shard_test.go", + "zipshard_test.go", + ], + embed = [":shard"], +) + +go_library( + name = "walk", + srcs = ["walk.go"], + importpath = "src/common/golang/walk", +) + +go_library( + name = "ziputils", + srcs = ["ziputils.go"], + importpath = "src/common/golang/ziputils", + deps = ["@org_golang_x_sync//errgroup"], +) + +go_library( + name = "fileutils", + srcs = ["fileutils.go"], + importpath = "src/common/golang/fileutils", +) + +go_library( + name = "flags", + srcs = ["flags.go"], + importpath = "src/common/golang/flags", +) + +go_test( + name = "flagfile_test", + size = "small", + srcs = ["flagfile_test.go"], + embed = [":flagfile"], +) + +go_library( + name = "ini", + srcs = ["ini.go"], + importpath = "src/common/golang/ini", +) + +go_test( + name = "ini_test", + size = "small", + srcs = ["ini_test.go"], + embed = [":ini"], +) + +go_library( + name = "pprint", + srcs = ["pprint.go"], + importpath = "src/common/golang/pprint", +) + +go_library( + name = "flagfile", + srcs = ["flagfile.go"], + importpath = "src/common/golang/flagfile", +) + +genrule( + name = "a_txt", + outs = ["a.txt"], + cmd = "echo hello world > $@", +) + +go_library( + name = "runfilelocation", + srcs = ["runfilelocation.go"], + importpath = "src/common/golang/runfilelocation", + deps = ["@io_bazel_rules_go//go/runfiles"], +) + +go_test( + name = "runfilelocation_test", + srcs = ["runfilelocation_test.go"], + data = [":a_txt"], + embed = [":runfilelocation"], +) diff --git a/src/common/golang/fileutils.go b/src/common/golang/fileutils.go new file mode 100644 index 0000000..31ecc24 --- /dev/null +++ b/src/common/golang/fileutils.go @@ -0,0 +1,35 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fileutils provides utility functions to work with files. +package fileutils + +import ( + "io" + "os" +) + +// Copy will copy a file. +func Copy(in, out string) error { + inF, err := os.Open(in) + if err != nil { + return err + } + outF, err := os.Create(out) + if err != nil { + return err + } + _, err = io.Copy(outF, inF) + return err +} diff --git a/src/common/golang/flagfile.go b/src/common/golang/flagfile.go new file mode 100644 index 0000000..805068e --- /dev/null +++ b/src/common/golang/flagfile.go @@ -0,0 +1,118 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package flagfile installs a -flagfile command line flag. +// This package is only imported for the side effect of installing the flag +package flagfile + +import ( + "bufio" + "flag" + "fmt" + "io" + "os" + "strings" +) + +type flagFile string + +func (f *flagFile) String() string { + return string(*f) +} + +func (f *flagFile) Get() interface{} { + return string(*f) +} + +func (f *flagFile) Set(fn string) error { + file, err := os.Open(fn) + if err != nil { + return fmt.Errorf("error parsing flagfile %s: %v", fn, err) + } + defer file.Close() + + fMap, err := parseFlags(bufio.NewReader(file)) + if err != nil { + return err + } + for k, v := range fMap { + flag.Set(k, v) + } + return nil +} + +// parseFlags parses the contents is a naive flag file parser. +func parseFlags(r *bufio.Reader) (map[string]string, error) { + fMap := make(map[string]string) + eof := false + for !eof { + line, err := r.ReadString('\n') + if err != nil && err != io.EOF { + return nil, err + } + if err == io.EOF { + eof = true + } + line = strings.TrimSpace(line) + if line == "" { + continue + } + // When Bazel is used to create flag files, it may create entries that are wrapped within + // quotations '--a=b'. Verify that it is balanced and strip first and last quotation. + if strings.HasPrefix(line, "'") || strings.HasPrefix(line, "\"") { + if !strings.HasSuffix(line, line[:1]) { + return nil, fmt.Errorf("error parsing flags, found unbalanced quotation marks around flag entry: %s", line) + } + line = line[1 : len(line)-1] + } + // Check that the flag has at least 1 "-" but no more than 2 ("-a" or "--a"). + if !strings.HasPrefix(line, "-") || strings.HasPrefix(line, "---") { + return nil, fmt.Errorf("error parsing flags, expected flag start definition ('-' or '--') but, got: %s", line) + } + split := strings.SplitN(strings.TrimLeft(line, "-"), "=", 2) + k := split[0] + if len(split) == 2 { + fMap[k] = split[1] + continue + } + v, err := parseFlagValue(r) + if err != nil { + return nil, fmt.Errorf("error parsing flag value, got: %v", err) + } + fMap[k] = v + } + return fMap, nil +} + +func parseFlagValue(r *bufio.Reader) (string, error) { + pBytes, err := r.Peek(2) + if err != nil && err != io.EOF { + return "", err + } + peeked := string(pBytes) + // If the next line starts with "-", "'-" or '"-' assume it is the beginning of a new flag definition. + if strings.HasPrefix(peeked, "-") || peeked == "'-" || peeked == "\"-" { + return "", nil + } + // Next line contains the flag value. + line, err := r.ReadString('\n') + if err != nil && err != io.EOF { + return "", err + } + return strings.TrimSpace(line), nil +} + +func init() { + flag.Var(new(flagFile), "flagfile", "Path to flagfile containing flag values, --key=val on each line") +} diff --git a/src/common/golang/flagfile_test.go b/src/common/golang/flagfile_test.go new file mode 100644 index 0000000..7f87c57 --- /dev/null +++ b/src/common/golang/flagfile_test.go @@ -0,0 +1,128 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Unit test for the flagfile module. +package flagfile + +import ( + "bufio" + "reflect" + "strings" + "testing" +) + +func TestParseFlags(t *testing.T) { + tcs := []struct { + name string + in string + want map[string]string + wantErr string + }{ + { + name: "SingleLineFlagDefinitions", + in: ` +--a=b +'--1=2' +-foo=bar +--enable +"--baz=qux=quux" +`, + want: map[string]string{ + "a": "b", + "1": "2", + "foo": "bar", + "enable": "", + "baz": "qux=quux", + }, + }, + { + name: "MultiLineFlagDefinitions", + in: ` +--a +b +--1 +2 +-foo +bar +--enable +--baz +qux=quux +`, + want: map[string]string{ + "a": "b", + "1": "2", + "foo": "bar", + "enable": "", + "baz": "qux=quux", + }, + }, + { + name: "MixedMultiSingleLineFlagDefinitions", + in: ` +--a +b +"-1=2" +-foo +bar +--enable +'--baz=--qux=quux' +`, + want: map[string]string{ + "a": "b", + "1": "2", + "foo": "bar", + "enable": "", + "baz": "--qux=quux", + }, + }, + { + name: "NoFlags", + in: "", + want: map[string]string{}, + }, + { + name: "MalformedFlagMissingDash", + in: "a=b", + wantErr: "expected flag start definition ('-' or '--')", + }, + { + name: "MalformedFlagTooManyDashes", + in: "---a=b", + wantErr: "expected flag start definition ('-' or '--')", + }, + { + name: "UnbalancedQuotationsAroundFlag", + in: "'--a=b", + wantErr: "found unbalanced quotation marks around flag entry", + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + got, err := parseFlags(bufio.NewReader(strings.NewReader(tc.in))) + if err != nil { + if tc.wantErr != "" { + if !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("error got error: %s wanted to contain: %s", err, tc.wantErr) + } + return + } + t.Errorf("got unexpected error: %s", err) + return + } + if eq := reflect.DeepEqual(got, tc.want); !eq { + t.Errorf("error got: %v wanted: %v", got, tc.want) + } + }) + } +} diff --git a/src/common/golang/flags.go b/src/common/golang/flags.go new file mode 100644 index 0000000..be78af6 --- /dev/null +++ b/src/common/golang/flags.go @@ -0,0 +1,42 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package flags provides extensions to the built-in flag module. +package flags + +import ( + "flag" + "strings" +) + +// StringList provides a flag type that parses a,comma,separated,string into a []string. +type StringList []string + +func (i *StringList) String() string { + return strings.Join([]string(*i), ",") +} + +// Set sets the flag value. +func (i *StringList) Set(v string) error { + *i = strings.Split(v, ",") + return nil +} + +// NewStringList creates a new StringList flag +// var someFlag = flags.NewStringList("some_name", "some desc") +func NewStringList(name, help string) *StringList { + var r StringList + flag.Var(&r, name, help) + return &r +} diff --git a/src/common/golang/ini.go b/src/common/golang/ini.go new file mode 100644 index 0000000..92cffa5 --- /dev/null +++ b/src/common/golang/ini.go @@ -0,0 +1,86 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ini provides utility functions to read and write ini files. +package ini + +import ( + "fmt" + "io" + "io/ioutil" + "log" + "os" + "sort" + "strings" +) + +func parse(in string) map[string]string { + m := make(map[string]string) + lines := strings.Split(in, "\n") + for i, l := range lines { + l = strings.TrimSpace(l) + if len(l) == 0 { + // Skip empty line + continue + } + if strings.HasPrefix(l, ";") || strings.HasPrefix(l, "#") { + // Skip comment + continue + } + kv := strings.SplitN(l, "=", 2) + if len(kv) < 2 { + log.Printf("Invalid line in ini file at line:%v %q\n", i, l) + // Skip invalid line + continue + } + k := strings.TrimSpace(kv[0]) + v := strings.TrimSpace(kv[1]) + if ov, ok := m[k]; ok { + log.Printf("Overwrite \"%s=%s\", duplicate found at line:%v %q\n", k, ov, i, l) + } + m[k] = v + } + return m +} + +func write(f io.Writer, m map[string]string) { + var keys []string + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + fmt.Fprintf(f, "%s=%s\n", k, m[k]) + } +} + +// Read reads an ini file. +func Read(n string) (map[string]string, error) { + c, err := ioutil.ReadFile(n) + if err != nil { + return nil, err + } + return parse(string(c)), nil +} + +// Write writes an ini file. +func Write(n string, m map[string]string) error { + f, err := os.Create(n) + if err != nil { + return err + } + defer f.Close() + write(f, m) + return nil +} diff --git a/src/common/golang/ini_test.go b/src/common/golang/ini_test.go new file mode 100644 index 0000000..a712e90 --- /dev/null +++ b/src/common/golang/ini_test.go @@ -0,0 +1,113 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package ini + +import ( + "bytes" + "reflect" + "strings" + "testing" +) + +func TestParseFunc(t *testing.T) { + tests := []struct { + name string + in string + want map[string]string + }{ + { + name: "ini_single_line", + in: "test=abc", + want: map[string]string{"test": "abc"}, + }, + { + name: "ini_multi_line", + in: `key=data +key2=more data`, + want: map[string]string{"key": "data", "key2": "more data"}, + }, + { + name: "ini_with_comment", + in: `key=data +;key2=irrelevant data +#key3=more irrelevant data`, + want: map[string]string{"key": "data"}, + }, + { + name: "ini_with_whitespace", + in: `key = data +another_key = The data +yet_another_key = more data`, + want: map[string]string{"key": "data", "another_key": "The data", "yet_another_key": "more data"}, + }, + { + name: "ini_with_empty_data", + in: `key=data +key2= +key3=more data`, + want: map[string]string{"key": "data", "key2": "", "key3": "more data"}, + }, + { + name: "invalid_ini", + in: `key=data +invalid line +key2=The data`, + want: map[string]string{"key": "data", "key2": "The data"}, + }, + { + name: "ini_with_duplicate", + in: `key=data +key=duplicate`, + want: map[string]string{"key": "duplicate"}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + iniOut := parse(test.in) + if eq := reflect.DeepEqual(iniOut, test.want); !eq { + t.Errorf("Parsing ini failed for: %q got: %v wanted: %v", test.in, iniOut, test.want) + } + }) + } +} + +func TestWriteFunc(t *testing.T) { + tests := []struct { + name string + in map[string]string + want string + }{ + { + name: "ini_single_line", + in: map[string]string{"test": "abc"}, + want: "test=abc\n", + }, + { + name: "ini_multi_line", + in: map[string]string{"key": "data", "key2": "more data"}, + want: `key=data +key2=more data +`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + b := new(bytes.Buffer) + write(b, test.in) + if strings.Compare(b.String(), test.want) != 0 { + t.Errorf("Writing ini failed for: %q got: %v wanted: %v", test.in, b.String(), test.want) + } + }) + } +} diff --git a/src/common/golang/marshal.go b/src/common/golang/marshal.go new file mode 100644 index 0000000..76b0114 --- /dev/null +++ b/src/common/golang/marshal.go @@ -0,0 +1,322 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package xml2 provides drop-in replacement functionality for encoding/xml. +// +// There are existing issues with the encoding/xml package that affect AK tools. +// +// xml2.Encoder: +// +// The current encoding/xml Encoder has several issues around xml namespacing +// that makes the output produced by it incompatible with AAPT. +// +// * Tracked here: https://golang.org/issue/7535 +// +// The xml2.Encoder.EncodeToken verifies the validity of namespaces and encodes +// them. For everything else, xml2.Encoder will fallback to the xml.Encoder. +package xml2 + +import ( + "bytes" + "encoding/xml" + "fmt" + "io" + "log" +) + +const xmlNS = "xmlns" + +// Encoder is an xml encoder which behaves much like the encoding/xml Encoder. +type Encoder struct { + *xml.Encoder + p printer + prefixURI map[string]string + state []state + uriPrefix *uriPrefixMap +} + +// ChildEncoder returns an encoder whose state is copied the given parent Encoder and writes to w. +func ChildEncoder(w io.Writer, parent *Encoder) *Encoder { + e := NewEncoder(w) + for k, v := range parent.prefixURI { + e.prefixURI[k] = v + } + for k, v := range parent.uriPrefix.up { + e.uriPrefix.up[k] = make([]string, len(v)) + copy(e.uriPrefix.up[k], v) + } + return e +} + +// NewEncoder returns a new encoder that writes to w. +func NewEncoder(w io.Writer) *Encoder { + e := &Encoder{ + Encoder: xml.NewEncoder(w), + p: printer{Writer: w}, + prefixURI: make(map[string]string), + uriPrefix: &uriPrefixMap{up: make(map[string][]string)}, + } + return e +} + +// EncodeToken behaves almost the same as encoding/xml.Encoder.EncodeToken +// but deals with StartElement and EndElement differently. +func (enc *Encoder) EncodeToken(t xml.Token) error { + switch t := t.(type) { + case xml.StartElement: + enc.Encoder.Flush() // Need to flush the wrapped encoder before we write. + if err := enc.writeStart(&t); err != nil { + return err + } + case xml.EndElement: + enc.Encoder.Flush() // Need to flush the wrapped encoder before we write. + if err := enc.writeEnd(t.Name); err != nil { + return err + } + default: + // Delegate to the embedded encoder for everything else. + return enc.Encoder.EncodeToken(t) + } + return nil +} + +func (enc *Encoder) writeStart(start *xml.StartElement) error { + if start.Name.Local == "" { + return fmt.Errorf("start tag with no name") + } + enc.setUpState(start) + + // Begin creating the start tag. + var st bytes.Buffer + st.WriteByte('<') + n, err := enc.translateName(start.Name) + if err != nil { + return fmt.Errorf("translating start tag name %q failed, got: %v", start.Name.Local, err) + } + st.Write(n) + for _, attr := range start.Attr { + name := attr.Name + if name.Local == "" { + continue + } + st.WriteByte(' ') + n, err := enc.translateName(attr.Name) + if err != nil { + return fmt.Errorf("translating attribute name %q failed, got: %v", start.Name.Local, err) + } + st.Write(n) + st.WriteString(`="`) + xml.EscapeText(&st, []byte(attr.Value)) + st.WriteByte('"') + } + st.WriteByte('>') + + enc.p.writeIndent(1) + enc.p.Write(st.Bytes()) + return nil +} + +func (enc *Encoder) writeEnd(name xml.Name) error { + if name.Local == "" { + return fmt.Errorf("end tag with no name") + } + n, err := enc.translateName(name) + if err != nil { + return fmt.Errorf("translating end tag name %q failed, got: %v", name.Local, err) + } + sn := enc.tearDownState() + if sn == nil || name.Local != sn.Local && name.Space != sn.Space { + return fmt.Errorf("tags are unbalanced, got: %v, wanted: %v", name, sn) + } + + // Begin creating the end tag + var et bytes.Buffer + et.WriteString("</") + et.Write(n) + et.WriteByte('>') + + enc.p.writeIndent(-1) + enc.p.Write(et.Bytes()) + return nil +} + +func (enc *Encoder) setUpState(start *xml.StartElement) { + enc.state = append(enc.state, element{n: &start.Name}) // Store start element to verify balanced close tags. + // Track attrs that affect the state of the xml (e.g. xmlns, xmlns:foo). + for _, attr := range start.Attr { + // push any xmlns type attrs as xml namespaces are valid within the tag they are declared in, and onward. + if attr.Name.Space == "xmlns" || attr.Name.Local == "xmlns" { + prefix := attr.Name.Local + if attr.Name.Local == "xmlns" { + prefix = "" // Default xml namespace is being set. + } + // Store the previous state, to be restored when exiting the tag. + enc.state = append(enc.state, xmlns{prefix: prefix, uri: enc.prefixURI[prefix]}) + enc.prefixURI[prefix] = attr.Value + enc.uriPrefix.put(attr.Value, prefix) + } + } +} + +func (enc *Encoder) tearDownState() *xml.Name { + // Unwind the state setup on start element. + for len(enc.state) > 0 { + s := enc.state[len(enc.state)-1] + enc.state = enc.state[:len(enc.state)-1] + switch s := s.(type) { + case element: + // Stop unwinding As soon as an element type is seen and verify that the + // tags are balanced + return s.n + case xmlns: + if p, ok := enc.uriPrefix.removeLast(enc.prefixURI[s.prefix]); !ok || p != s.prefix { + // Unexpected error, internal state is corrupt. + if !ok { + log.Fatalf("xmlns attribute state corrupt, uri %q does not exist", enc.prefixURI[s.prefix]) + } + log.Fatalf("xmlns attributes state corrupt, got: %q, wanted: %q", s.prefix, p) + } + if s.uri == "" { + delete(enc.prefixURI, s.prefix) + } else { + enc.prefixURI[s.prefix] = s.uri + } + } + } + return nil +} + +func (enc *Encoder) translateName(name xml.Name) ([]byte, error) { + var n bytes.Buffer + if name.Space != "" { + prefix := "" + if name.Space == xmlNS { + prefix = xmlNS + } else if ns, ok := enc.uriPrefix.getLast(name.Space); ok { + // URI Space is defined in current context, use the namespace. + prefix = ns + } else if _, ok := enc.prefixURI[name.Space]; ok { + // If URI Space is not defined in current context, there is a possibility + // that the Space is in fact a namespace prefix. If present use it. + prefix = name.Space + } else { + return nil, fmt.Errorf("unknown namespace: %s", name.Space) + } + if prefix != "" { + n.WriteString(prefix) + n.WriteByte(':') + } + } + n.WriteString(name.Local) + return n.Bytes(), nil +} + +type printer struct { + io.Writer + indent string + prefix string + depth int + indentedIn bool + putNewline bool +} + +// writeIndent is directly cribbed from encoding/xml/marshal.go to keep indentation behavior the same. +func (p *printer) writeIndent(depthDelta int) { + if len(p.prefix) == 0 && len(p.indent) == 0 { + return + } + if depthDelta < 0 { + p.depth-- + if p.indentedIn { + p.indentedIn = false + return + } + p.indentedIn = false + } + if p.putNewline { + p.Write([]byte("\n")) + } else { + p.putNewline = true + } + if len(p.prefix) > 0 { + p.Write([]byte(p.prefix)) + } + if len(p.indent) > 0 { + for i := 0; i < p.depth; i++ { + p.Write([]byte(p.indent)) + } + } + if depthDelta > 0 { + p.depth++ + p.indentedIn = true + } + +} + +// uriPrefixMap is a multimap, mapping a uri to many xml namespace prefixes. The +// difference with this and a a traditional multimap is that, you can only get +// or remove the last prefixed added. This is mainly due to the way xml decoding +// is implemented by the encoding/xml Decoder. +type uriPrefixMap struct { + up map[string][]string +} + +// getLast returns a boolean which signifies if the entry exists and the last +// prefix stored for the given uri. +func (u *uriPrefixMap) getLast(uri string) (string, bool) { + ps, ok := u.up[uri] + if !ok { + return "", ok + } + return ps[len(ps)-1], ok +} + +func (u *uriPrefixMap) put(uri, prefix string) { + if _, ok := u.up[uri]; !ok { + // Though the mapping of url-to-prefix is implemented for a multimap, in practice, + // there should never be more than a single prefix defined for any given uri within + // at any point in time in an xml file. + u.up[uri] = make([]string, 1) + } + u.up[uri] = append(u.up[uri], prefix) +} + +// removeLast a boolean which signifies if the entry exists and returns the last +// prefix removed for the given uri. If the last entry is removed the key is +// also deleted. +func (u *uriPrefixMap) removeLast(uri string) (string, bool) { + p, ok := u.getLast(uri) + if ok { + if len(u.up[uri]) > 1 { + u.up[uri] = u.up[uri][:len(u.up[uri])-1] + } else { + delete(u.up, uri) + } + } + return p, ok +} + +// state stores the state of the xml when a new start element is seen. +type state interface{} + +// xml element state entry. +type element struct { + n *xml.Name +} + +// xmlns attribute state entry. +type xmlns struct { + prefix string + uri string +} diff --git a/src/common/golang/marshal_test.go b/src/common/golang/marshal_test.go new file mode 100644 index 0000000..7e4d04b --- /dev/null +++ b/src/common/golang/marshal_test.go @@ -0,0 +1,149 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package xml2 + +import ( + "bufio" + "bytes" + "encoding/xml" + "io" + "strings" + "testing" +) + +func TestEncoderEncodeToken(t *testing.T) { + tests := []struct { + name string + in string + want string + wantErr string + }{ + { + name: "xmlnsPrefixForElement", + in: "<foo:bar xmlns:foo=\"baz\"></foo:bar>", + want: "<foo:bar xmlns:foo=\"baz\"></foo:bar>", + }, + { + name: "xmlnsPrefixForAttribute", + in: "<foo bar:baz=\"qux\" xmlns:bar=\"quux\"></foo>", + want: "<foo bar:baz=\"qux\" xmlns:bar=\"quux\"></foo>", + }, + { + name: "defaultXmlnsAttribute", + in: "<foo xmlns=\"bar\"></foo>", + want: "<foo xmlns=\"bar\"></foo>", + }, + { + // The return value of Decoder.Token() makes it + // impossible for a decode then encode of an xml file + // be isomorphic. This is mainly due to the fact that + // xml.Name.Space contains the uri, and xml.Name does + // not store the prefix. Instead, make sure that the + // behavior remains consistent. + // + // That is, the last prefix defined for the space is the + // one applied when encoding the token. + name: "multipleDefsXmlnsPrefixesSameUri", + in: ` +<foo xmlns:bar="bar"> + <bar:baz xmlns:qux="bar"> + <qux:quux></qux:quux> + </bar:baz> +</foo>`, + want: ` +<foo xmlns:bar="bar"> + <qux:baz xmlns:qux="bar"> + <qux:quux></qux:quux> + </qux:baz> +</foo>`, + }, + { + name: "xmlnsPrefixUsedOnElementButNotDefined", + in: "<foo:bar></foo:bar>", + wantErr: "unknown namespace: foo", + }, + { + name: "xmlnsPrefixUsedOnAttrButNotDefined", + in: "<foo bar:baz=\"qux\"></foo>", + wantErr: "unknown namespace: bar", + }, + { + name: "xmlnsPrefixUsedOutsideOfDefiningTag", + in: ` +<foo xmlns:bar="baz" bar:qux="quux">corge</foo> +<grault bar:garply="waldo"></grault>`, + wantErr: "unknown namespace: bar", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var b bytes.Buffer + e := NewEncoder(bufio.NewWriter(&b)) + d := xml.NewDecoder(strings.NewReader(test.in)) + for { + tkn, err := d.Token() + if err != nil { + if err == io.EOF { + break + } + t.Fatalf("Unexpected error got: %v while reading: %s", err, test.in) + } + if err := e.EncodeToken(tkn); err != nil { + if test.wantErr != "" && strings.Contains(err.Error(), test.wantErr) { + // Do nothing, error is expected. + } else { + t.Errorf("Unexpected error during encode: %v", err) + } + return + } + } + e.Flush() + if b.String() != test.want { + t.Errorf("got: <%s> expected: <%s>", b.String(), test.want) + } + }) + } +} + +func TestChildEncoder(t *testing.T) { + // Setup the parent Encoder with the namespace "bar". + d := xml.NewDecoder(strings.NewReader("<foo xmlns:bar=\"bar\"><bar:baz>Hello World</bar:baz></foo>")) + tkn, err := d.Token() + if err != nil { + t.Fatalf("Error occurred during decoding, got: %v", err) + } + parentEnc := NewEncoder(&bytes.Buffer{}) + if err := parentEnc.EncodeToken(tkn); err != nil { + t.Fatalf("Error occurred while the parent encoder was encoding token %q got: %v", tkn, err) + } + + // Without instantiating the Encoder as a child, the "bar" namespace will be unknown and cause an + // error to occur when trying to encode the "bar" namespaced element "<bar:baz>". + tkn, err = d.Token() + if err != nil { + t.Fatalf("Error occurred during decoding, got: %v", err) + } + b := &bytes.Buffer{} + childEnc := ChildEncoder(b, parentEnc) + if err := childEnc.EncodeToken(tkn); err != nil { + t.Fatalf("Error occurred while the child encoder was encoding token %q got: %v", tkn, err) + } + childEnc.Flush() + + // Verify that the token is not mangled. + if want := "<bar:baz>"; b.String() != want { + t.Errorf("Error, got %q, wanted %q", b.String(), want) + } +} diff --git a/src/common/golang/pprint.go b/src/common/golang/pprint.go new file mode 100644 index 0000000..221ff0f --- /dev/null +++ b/src/common/golang/pprint.go @@ -0,0 +1,48 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package pprint provides colored "pretty print" output helper methods +package pprint + +import ( + "fmt" + "os" +) + +const ( + errorString = "\033[1m\033[31mERROR:\033[0m %s\n" + warningString = "\033[35mWARNING:\033[0m %s\n" + infoString = "\033[32mINFO:\033[0m %s\n" + clearLine = "\033[A\033[K" +) + +// Error prints an error message in bazel style colors +func Error(errorMsg string, args ...interface{}) { + fmt.Fprintf(os.Stderr, errorString, fmt.Sprintf(errorMsg, args...)) +} + +// Warning prints a warning message in bazel style colors +func Warning(warningMsg string, args ...interface{}) { + fmt.Fprintf(os.Stderr, warningString, fmt.Sprintf(warningMsg, args...)) +} + +// Info prints an info message in bazel style colors +func Info(infoMsg string, args ...interface{}) { + fmt.Fprintf(os.Stderr, infoString, fmt.Sprintf(infoMsg, args...)) +} + +// ClearLine deletes the line above the cursor's current position. +func ClearLine() { + fmt.Printf(clearLine) +} diff --git a/src/common/golang/runfilelocation.go b/src/common/golang/runfilelocation.go new file mode 100644 index 0000000..9116c18 --- /dev/null +++ b/src/common/golang/runfilelocation.go @@ -0,0 +1,35 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package runfilelocation provides utility functions to deal with runfiles + +package runfilelocation + +import ( + "os" + "path" + + "github.com/bazelbuild/rules_go/go/runfiles" +) + +// Find determines the absolute path to a given runfile +func Find(runfilePath string) (string, error) { + runfileLocation, err := runfiles.Rlocation(path.Join(os.Getenv("TEST_WORKSPACE"), runfilePath)) + + if err != nil { + return "", err + } + + return runfileLocation, err +} diff --git a/src/common/golang/runfilelocation_test.go b/src/common/golang/runfilelocation_test.go new file mode 100644 index 0000000..507196b --- /dev/null +++ b/src/common/golang/runfilelocation_test.go @@ -0,0 +1,58 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runfilelocation + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestValidRunfileLocation(t *testing.T) { + // Check that Find() returns a valid path to a runfile + runfilePath := "src/common/golang/a.txt" + + absRunFilePath, err := Find(runfilePath) + if err != nil { + t.Errorf("Runfile path through Runfilelocation() failed: %v", err) + } + + // Check that the path actually exists + contents, err := ioutil.ReadFile(absRunFilePath) + text := string(contents) + if err != nil { + t.Errorf("Could not read file: %v", err) + } + + if text != "hello world\n" { + t.Errorf("Expected 'hello world' in file, got %v instead.", text) + } +} + +func TestInvalidRunfileLocation(t *testing.T) { + invalidRunfilePath := "src/common/golang/b.txt" + + runfileLocationShouldNotExist, err := Find(invalidRunfilePath) + if err != nil { + // Even if the path is invalid, runfilelocation.Find() should return the path to where it _thinks_ + // the runfile should exist. + t.Errorf("Unexpected error: %v should have returned a runfile path. Instead got %v", invalidRunfilePath, err) + } + + // Check that the invalid runfile path actually does not exist. + if _, err := os.Stat(runfileLocationShouldNotExist); err == nil { + t.Errorf("Expected error, file should not have been found: %v", runfileLocationShouldNotExist) + } +} diff --git a/src/common/golang/shard.go b/src/common/golang/shard.go new file mode 100644 index 0000000..a805bdc --- /dev/null +++ b/src/common/golang/shard.go @@ -0,0 +1,90 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package shard provides functions to help sharding your data. +package shard + +import ( + "archive/zip" + "errors" + "fmt" + "hash/fnv" + "io" + "strings" +) + +// Func converts a name and a number of shards into a particular shard index. +type Func func(name string, shardCount int) int + +// FNV uses the FNV hash algo on the provided string and mods its result by shardCount. +func FNV(name string, shardCount int) int { + h := fnv.New32() + h.Write([]byte(name)) + return int(h.Sum32()) % shardCount +} + +// MakeSepFunc creates a shard function that takes a substring from 0 to the last occurrence of +// separator from the name to be sharded, and passes that onto the provided shard function. +func MakeSepFunc(sep string, s Func) Func { + return func(name string, shardCount int) int { + idx := strings.LastIndex(name, sep) + if idx == -1 { + return s(name, shardCount) + } + return s(name[:idx], shardCount) + } +} + +// ZipShard takes a given zip reader, and shards its content across the provided io.Writers +// utilizing the provided SharderFunc. +func ZipShard(r *zip.Reader, zws []*zip.Writer, fn Func) error { + sc := len(zws) + if sc == 0 { + return errors.New("no output writers") + } + + for _, f := range r.File { + if !f.Mode().IsRegular() { + continue + } + si := fn(f.Name, sc) + if si < 0 || si > sc { + return fmt.Errorf("s.Shard(%s, %d) yields invalid shard index: %d", f.Name, sc, si) + } + zw := zws[si] + var rc io.ReadCloser + rc, err := f.Open() + if err != nil { + return fmt.Errorf("%s: could not open: %v", f.Name, err) + } + var zo io.Writer + zo, err = zw.CreateHeader(&zip.FileHeader{ + Name: f.Name, + Method: zip.Store, + }) + if err != nil { + return fmt.Errorf("%s: could not create output entry: %v", f.Name, err) + } + if err := copyAndClose(zo, rc); err != nil { + return fmt.Errorf("%s: copy to output failed: %v", f.Name, err) + } + } + return nil +} + +func copyAndClose(w io.Writer, rc io.ReadCloser) error { + defer rc.Close() + _, err := io.Copy(w, rc) + return err +} diff --git a/src/common/golang/shard_test.go b/src/common/golang/shard_test.go new file mode 100644 index 0000000..5b60361 --- /dev/null +++ b/src/common/golang/shard_test.go @@ -0,0 +1,98 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shard + +import ( + "testing" +) + +func TestFNV(t *testing.T) { + tests := []struct { + name string + in []string + sc int + want []int + }{ + { + name: "shardCount2", + in: []string{"foo", "bar", "baz"}, + sc: 2, + want: []int{1, 0, 0}, + }, + { + name: "shardCount5", + in: []string{"foo", "bar", "baz"}, + sc: 5, + want: []int{0, 2, 0}, + }, + { + name: "shardCount9", + in: []string{"foo", "bar", "baz"}, + sc: 9, + want: []int{2, 7, 6}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for idx, in := range test.in { + if shard := FNV(in, test.sc); shard != test.want[idx] { + t.Errorf("FNV applied for: %q got: %v wanted: %v", in, shard, test.want[idx]) + } + } + }) + } +} + +func TestMakeSepFunc(t *testing.T) { + tests := []struct { + name string + sep string + in []string + sc int + want []int + }{ + { + name: "makeSepFunc", + sep: "@", + in: []string{"foo@postfix", "bar@postfix", "baz@postfix"}, + sc: 9, + want: []int{2, 7, 6}, + }, + { + name: "makeSepFuncWithNoSep", + sep: "", + in: []string{"foo", "bar", "baz"}, + sc: 9, + want: []int{2, 7, 6}, + }, + { + name: "makeSepFuncWithWrongSep", + sep: "*", + in: []string{"foo", "bar", "baz"}, + sc: 9, + want: []int{2, 7, 6}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + for idx, in := range test.in { + shardFn := MakeSepFunc(test.sep, FNV) + if shard := shardFn(in, test.sc); shard != test.want[idx] { + t.Errorf("MakeSepFunc applied for: %q got: %v wanted: %v", in, shard, test.want[idx]) + } + } + }) + } +} diff --git a/src/common/golang/walk.go b/src/common/golang/walk.go new file mode 100644 index 0000000..7b73484 --- /dev/null +++ b/src/common/golang/walk.go @@ -0,0 +1,51 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package walk provides an utility function to walk a directory tree collecting and deduping files. +package walk + +import ( + "fmt" + "os" + "path/filepath" +) + +// Files traverses a list of paths and returns a list of all the seen files. +func Files(paths []string) ([]string, error) { + var files []string + seen := make(map[string]bool) + visitFunc := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if seen[path] { + return nil + } + seen[path] = true + switch fType := info.Mode(); { + case fType.IsDir(): + // Do nothing. + default: + files = append(files, path) + } + return nil + } + for _, p := range paths { + err := filepath.Walk(p, visitFunc) + if err != nil { + return nil, fmt.Errorf("got error while walking %s got: %v", p, err) + } + } + return files, nil +} diff --git a/src/common/golang/zipshard_test.go b/src/common/golang/zipshard_test.go new file mode 100644 index 0000000..de6dedc --- /dev/null +++ b/src/common/golang/zipshard_test.go @@ -0,0 +1,193 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package shard + +import ( + "archive/zip" + "bytes" + "errors" + "fmt" + "reflect" + "sort" + "strings" + "testing" +) + +func TestSepSharder(t *testing.T) { + tcs := []struct { + name string + sep string + wantName string + }{ + { + name: "Hello", + sep: "/", + wantName: "Hello", + }, + { + name: "foo/bar/baz", + sep: "/", + wantName: "foo/bar", + }, + { + name: "com@google@Foo.dex", + sep: "@", + wantName: "com@google", + }, + } + + for _, tc := range tcs { + checkShard := func(name string, sc int) int { + if name != tc.wantName { + t.Errorf("makeSepSharder(%s).Shard(%s, 1): got name: %s wanted: %s", tc.sep, tc.name, name, tc.wantName) + } + return 0 + } + + s := MakeSepFunc(tc.sep, Func(checkShard)) + s(tc.name, 1) + } + +} + +func TestBadSharder(t *testing.T) { + srcZip, err := makeZip(map[string]string{"hello": "world"}) + if err != nil { + t.Fatalf("Could not make initial zip: %v", err) + } + + for _, shardVal := range []int{-1, -244, 123} { + zr, err := zip.NewReader(bytes.NewReader(srcZip), int64(len(srcZip))) + if err != nil { + t.Fatalf("could not read initial zip: %v", err) + } + zws := []*zip.Writer{zip.NewWriter(&bytes.Buffer{})} + + s := Func(func(name string, sc int) int { + return shardVal + }) + err = ZipShard(zr, zws, s) + if err == nil || !strings.Contains(err.Error(), "invalid shard index") { + t.Errorf("Returning shard value: %d gave: %v wanted an error with invalid shard index", shardVal, err) + } + } +} + +func TestZipShard(t *testing.T) { + tcs := []struct { + name string + contents map[string]string + shardCount int + want map[int][]string + zipShardErr error + }{ + { + name: "Vanilla", + contents: map[string]string{ + "foo/hello": "world", + "bar/something": "stuff", + "blah/nothing": "here", + "blah/everything": "nowhere", + "hello/everything": "nowhere", + }, + shardCount: 5, + want: map[int][]string{ + 0: {"hello/everything"}, + 3: {"foo/hello", "bar/something"}, + 4: {"blah/nothing", "blah/everything"}, + }, + }, + { + name: "no output shards", + contents: map[string]string{"something": "something"}, + shardCount: 0, + zipShardErr: errors.New("no output writers"), + }, + { + name: "empty input zip", + contents: map[string]string{}, + shardCount: 5, + want: map[int][]string{}, + }, + } + + for _, tc := range tcs { + srcZip, err := makeZip(tc.contents) + if err != nil { + t.Errorf("%s: could not create initial zip: %v", tc.name, err) + } + zr, err := zip.NewReader(bytes.NewReader(srcZip), int64(len(srcZip))) + + if err != nil { + t.Errorf("%s: could not read initial zip: %v", tc.name, err) + continue + } + bufs := make([]*bytes.Buffer, tc.shardCount) + zws := make([]*zip.Writer, tc.shardCount) + for i := range zws { + bufs[i] = new(bytes.Buffer) + zws[i] = zip.NewWriter(bufs[i]) + } + s := MakeSepFunc("/", Func(func(name string, sc int) int { + return len(name) % sc + })) + err = ZipShard(zr, zws, s) + if !reflect.DeepEqual(err, tc.zipShardErr) { + t.Errorf("%s: got zipshard error: %v wanted: %v", tc.name, err, tc.zipShardErr) + continue + } + for i, s := range bufs { + if err := zws[i].Close(); err != nil { + t.Errorf("%s: shard: %d cannot close zip writer: %v", tc.name, tc.shardCount, err) + continue + } + z, err := zip.NewReader(bytes.NewReader(s.Bytes()), int64(s.Len())) + if err != nil { + t.Errorf("%s: shard: %d cannot create zip reader: %v", tc.name, tc.shardCount, err) + continue + } + var fileNames []string + for _, f := range z.File { + fileNames = append(fileNames, f.Name) + } + sort.Strings(fileNames) + want, _ := tc.want[i] + sort.Strings(want) + if !reflect.DeepEqual(want, fileNames) { + t.Errorf("%s: shard: %d got: %s wanted: %s", tc.name, i, fileNames, want) + } + } + } +} + +func makeZip(contents map[string]string) ([]byte, error) { + var zin bytes.Buffer + zw := zip.NewWriter(&zin) + for name, body := range contents { + f, err := zw.Create(name) + if err != nil { + return nil, fmt.Errorf("%s: could not create: %v", name, err) + } + _, err = f.Write([]byte(body)) + if err != nil { + return nil, fmt.Errorf("%s: could not write: %s due to %v: ", name, body, err) + } + + } + if err := zw.Close(); err != nil { + return nil, err + } + return zin.Bytes(), nil +} diff --git a/src/common/golang/ziputils.go b/src/common/golang/ziputils.go new file mode 100644 index 0000000..e1a7af9 --- /dev/null +++ b/src/common/golang/ziputils.go @@ -0,0 +1,193 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package ziputils provides utility functions to work with zip files. +package ziputils + +import ( + "archive/zip" + "bytes" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "golang.org/x/sync/errgroup" +) + +// Empty file contains only the End of central directory record. 0x06054b50 +// https://en.wikipedia.org/wiki/Zip_(file_format) +var ( + emptyzip = append([]byte{0x50, 0x4b, 0x05, 0x06}, make([]byte, 18)...) + dirPerm os.FileMode = 0755 +) + +// EmptyZipReader wraps an reader whose contents are the empty zip. +type EmptyZipReader struct { + *bytes.Reader +} + +// NewEmptyZipReader creates and returns an EmptyZipReader struct. +func NewEmptyZipReader() *EmptyZipReader { + return &EmptyZipReader{bytes.NewReader(emptyzip)} +} + +// EmptyZip creates empty zip archive. +func EmptyZip(dst string) error { + zipfile, err := os.Create(dst) + if err != nil { + return err + } + defer zipfile.Close() + _, err = io.Copy(zipfile, NewEmptyZipReader()) + return err +} + +// Zip archives src into dst without compression. +func Zip(src, dst string) error { + fi, err := os.Stat(src) + if err != nil { + return err + } + + zipfile, err := os.Create(dst) + if err != nil { + return err + } + defer zipfile.Close() + + archive := zip.NewWriter(zipfile) + defer archive.Close() + + if !fi.Mode().IsDir() { + return WriteFile(archive, src, filepath.Base(src)) + } + + return filepath.Walk(src, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + return WriteFile(archive, path, strings.TrimPrefix(path, src+string(filepath.Separator))) + }) +} + +// WriteFile writes filename to the out zip writer. +func WriteFile(out *zip.Writer, filename, zipFilename string) error { + // It's important to set timestamps to zero, otherwise we would break caching for unchanged files + f, err := out.CreateHeader(&zip.FileHeader{Name: zipFilename, Method: zip.Store, Modified: time.Unix(0, 0)}) + if err != nil { + return err + } + contents, err := ioutil.ReadFile(filename) + if err != nil { + return err + } + _, err = f.Write(contents) + return err +} + +// WriteReader writes a reader to the out zip writer. +func WriteReader(out *zip.Writer, in io.Reader, filename string) error { + // It's important to set timestamps to zero, otherwise we would break caching for unchanged files + f, err := out.CreateHeader(&zip.FileHeader{Name: filename, Method: zip.Store, Modified: time.Unix(0, 0)}) + if err != nil { + return err + } + contents, err := ioutil.ReadAll(in) + if err != nil { + return err + } + _, err = f.Write(contents) + return err +} + +// Unzip expands srcZip in dst directory +func Unzip(srcZip, dst string) error { + reader, err := zip.OpenReader(srcZip) + if err != nil { + return err + } + defer reader.Close() + + _, err = os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return err + } + if os.IsNotExist(err) { + if err := os.MkdirAll(dst, dirPerm); err != nil { + return err + } + } + + for _, file := range reader.File { + path := filepath.Join(dst, file.Name) + + if file.FileInfo().IsDir() { + if err := os.MkdirAll(path, dirPerm); err != nil { + return err + } + continue + } + + dir := filepath.Dir(path) + _, err := os.Stat(dir) + if err != nil && !os.IsNotExist(err) { + return err + } + if os.IsNotExist(err) { + if err := os.MkdirAll(dir, dirPerm); err != nil { + return err + } + } + + if err := write(file, path); err != nil { + return err + } + } + + return nil +} + +// UnzipParallel expands zip archives in parallel. +// TODO(b/137549283) Update UnzipParallel and add test +func UnzipParallel(srcZipDestMap map[string]string) error { + var eg errgroup.Group + for z, d := range srcZipDestMap { + zip, dest := z, d + eg.Go(func() error { return Unzip(zip, dest) }) + } + return eg.Wait() +} + +func write(zf *zip.File, path string) error { + rc, err := zf.Open() + if err != nil { + return err + } + defer rc.Close() + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(f, rc) + return err +} diff --git a/src/java/com/example/sampleapp/AndroidManifest.xml b/src/java/com/example/sampleapp/AndroidManifest.xml new file mode 100644 index 0000000..b476bf3 --- /dev/null +++ b/src/java/com/example/sampleapp/AndroidManifest.xml @@ -0,0 +1,18 @@ +<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.example.sampleapp" + android:versionCode="1" + android:versionName="1.0"> + <uses-sdk android:minSdkVersion="21" /> + <application android:label="@string/app_name" + android:debuggable="true"> + <activity android:name=".SampleApp" + android:label="@string/app_name" + android:exported="true"> + <intent-filter> + <action android:name="android.intent.action.MAIN" /> + <category android:name="android.intent.category.LAUNCHER" /> + </intent-filter> + </activity> + </application> +</manifest> diff --git a/src/java/com/example/sampleapp/BUILD b/src/java/com/example/sampleapp/BUILD new file mode 100644 index 0000000..8530ec2 --- /dev/null +++ b/src/java/com/example/sampleapp/BUILD @@ -0,0 +1,31 @@ +# Sample app to demonstrate proper rule structuring and mobile-install usage. + +load("//rules:rules.bzl", "android_binary", "android_library") + +package(default_visibility = ["//src:__subpackages__"]) + +android_binary( + name = "sampleapp", + manifest = "AndroidManifest.xml", + multidex = "native", + deps = [ + ":lib", + ], +) + +android_library( + name = "lib", + srcs = glob(["*.java"]), + manifest = "AndroidManifest.xml", + resource_files = glob(["res/**"]), + deps = [ + ":native", + "@androidsdk//com.android.support:appcompat-v7-25.0.0", + "@androidsdk//com.android.support:support-v4-25.2.0", + ], +) + +cc_library( + name = "native", + srcs = ["native.c"], +) diff --git a/src/java/com/example/sampleapp/SampleApp.java b/src/java/com/example/sampleapp/SampleApp.java new file mode 100644 index 0000000..fe5f6fe --- /dev/null +++ b/src/java/com/example/sampleapp/SampleApp.java @@ -0,0 +1,39 @@ +/* + * Copyright 2018 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.sampleapp; + +import android.app.Activity; +import android.os.Bundle; +import android.widget.TextView; + +/** + * Minimal sample app to demonstrate mobile-install and rule best practices. + */ +public class SampleApp extends Activity { + + @Override + public void onCreate(Bundle state) { + super.onCreate(state); + + setContentView(R.layout.basic_activity); + } + + public native String getString(); + + static { + System.loadLibrary("sample"); + } +} diff --git a/src/java/com/example/sampleapp/native.c b/src/java/com/example/sampleapp/native.c new file mode 100644 index 0000000..021bfa5 --- /dev/null +++ b/src/java/com/example/sampleapp/native.c @@ -0,0 +1,22 @@ +/* + * Copyright 2022 The Bazel Authors. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include <string.h> +#include <jni.h> + +jstring +Java_com_example_sampleapp_SampleApp_getString(JNIEnv* env, jobject thiz) { + return (*env)->NewStringUTF(env, "Native String!"); +} diff --git a/src/java/com/example/sampleapp/res/layout/basic_activity.xml b/src/java/com/example/sampleapp/res/layout/basic_activity.xml new file mode 100644 index 0000000..61fb73e --- /dev/null +++ b/src/java/com/example/sampleapp/res/layout/basic_activity.xml @@ -0,0 +1,12 @@ +<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android" + android:layout_width="match_parent" + android:layout_height="match_parent" + android:orientation="vertical" > + + <TextView + android:id="@+id/text_hello" + android:text="@string/hello_world" + android:layout_width="wrap_content" + android:layout_height="wrap_content" /> + +</LinearLayout> diff --git a/src/java/com/example/sampleapp/res/values/strings.xml b/src/java/com/example/sampleapp/res/values/strings.xml new file mode 100644 index 0000000..2065009 --- /dev/null +++ b/src/java/com/example/sampleapp/res/values/strings.xml @@ -0,0 +1,5 @@ +<?xml version="1.0" encoding="utf-8"?> +<resources> + <string name="app_name">SampleApp</string> + <string name="hello_world" translatable="false">Hello world!</string> +</resources> diff --git a/src/tools/ak/BUILD b/src/tools/ak/BUILD new file mode 100644 index 0000000..51d524f --- /dev/null +++ b/src/tools/ak/BUILD @@ -0,0 +1,65 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description: +# Top level package for ak, a "busybox" for various minor build-related tools. +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_binary( + name = "ak", + srcs = [ + "ak.go", + ], + deps = [ + ":akcommands", + ":types", + "//src/common/golang:flagfile", + ], +) + +go_library( + name = "types", + srcs = ["types.go"], + importpath = "src/tools/ak/types", +) + +go_library( + name = "akhelper", + srcs = ["akhelper.go"], + importpath = "src/tools/ak/akhelper", +) + +go_library( + name = "manifestutils", + srcs = ["manifestutils.go"], + importpath = "src/tools/ak/manifestutils", + deps = [ + "//src/common/golang:xml2", + ], +) + +go_library( + name = "akcommands", + srcs = ["akcommands.go"], + importpath = "src/tools/ak/akcommands", + deps = [ + ":types", + "//src/tools/ak/bucketize", + "//src/tools/ak/compile", + "//src/tools/ak/dex", + "//src/tools/ak/extractaar", + "//src/tools/ak/finalrjar", + "//src/tools/ak/generatemanifest", + "//src/tools/ak/link", + "//src/tools/ak/liteparse", + "//src/tools/ak/manifest", + "//src/tools/ak/mindex", + "//src/tools/ak/nativelib", + "//src/tools/ak/patch", + "//src/tools/ak/repack", + "//src/tools/ak/rjar", + "//src/tools/ak/shellapk", + ], +) diff --git a/src/tools/ak/akcommands.go b/src/tools/ak/akcommands.go new file mode 100644 index 0000000..7a100ca --- /dev/null +++ b/src/tools/ak/akcommands.go @@ -0,0 +1,56 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package akcommands provides a map of all AK commands to their respective binaries. +package akcommands + +import ( + "src/tools/ak/bucketize/bucketize" + "src/tools/ak/compile/compile" + "src/tools/ak/dex/dex" + "src/tools/ak/extractaar/extractaar" + "src/tools/ak/finalrjar/finalrjar" + "src/tools/ak/generatemanifest/generatemanifest" + "src/tools/ak/link/link" + "src/tools/ak/liteparse/liteparse" + "src/tools/ak/manifest/manifest" + "src/tools/ak/mindex/mindex" + "src/tools/ak/nativelib/nativelib" + "src/tools/ak/patch/patch" + "src/tools/ak/repack/repack" + "src/tools/ak/rjar/rjar" + "src/tools/ak/shellapk/shellapk" + "src/tools/ak/types" +) + +var ( + // Cmds map AK commands to their respective binaries + Cmds = map[string]types.Command{ + "bucketize": bucketize.Cmd, + "compile": compile.Cmd, + "dex": dex.Cmd, + "extractaar": extractaar.Cmd, + "link": link.Cmd, + "liteparse": liteparse.Cmd, + "generatemanifest": generatemanifest.Cmd, + "manifest": manifest.Cmd, + "mindex": mindex.Cmd, + "nativelib": nativelib.Cmd, + "patch": patch.Cmd, + "repack": repack.Cmd, + "rjar": rjar.Cmd, + "finalrjar": finalrjar.Cmd, + "shellapk": shellapk.Cmd, + } +) diff --git a/src/tools/ak/akhelper.go b/src/tools/ak/akhelper.go new file mode 100644 index 0000000..5a91098 --- /dev/null +++ b/src/tools/ak/akhelper.go @@ -0,0 +1,27 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package akhelper provides globally used functions. +package akhelper + +import "strings" + +const ( + lnBreak = "\n " +) + +// FormatDesc returns an indented string with line breaks for each element in given string array. +func FormatDesc(desc []string) string { + return strings.Join(desc, lnBreak) +} diff --git a/src/tools/ak/bucketize/BUILD b/src/tools/ak/bucketize/BUILD new file mode 100644 index 0000000..750400a --- /dev/null +++ b/src/tools/ak/bucketize/BUILD @@ -0,0 +1,58 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description: +# Package for bucketize module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_binary( + name = "bucketize_bin", + srcs = ["bucketize_bin.go"], + deps = [ + ":bucketize", + "//src/common/golang:flagfile", + ], +) + +go_library( + name = "bucketize", + srcs = [ + "bucketize.go", + "partitioner.go", + "pipe.go", + ], + importpath = "src/tools/ak/bucketize/bucketize", + deps = [ + "//src/common/golang:flags", + "//src/common/golang:shard", + "//src/common/golang:walk", + "//src/common/golang:xml2", + "//src/tools/ak:akhelper", + "//src/tools/ak:types", + "//src/tools/ak/res", + ], +) + +go_test( + name = "bucketize_test", + size = "small", + srcs = [ + "bucketize_test.go", + "partitioner_test.go", + ], + embed = [":bucketize"], + deps = [ + "//src/common/golang:shard", + "//src/common/golang:walk", + "//src/tools/ak/res", + ], +) + +go_test( + name = "pipe_test", + size = "small", + srcs = ["pipe_test.go"], + embed = [":bucketize"], +) diff --git a/src/tools/ak/bucketize/bucketize.go b/src/tools/ak/bucketize/bucketize.go new file mode 100644 index 0000000..4e72b0f --- /dev/null +++ b/src/tools/ak/bucketize/bucketize.go @@ -0,0 +1,451 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package bucketize provides functionality to bucketize Android resources. +package bucketize + +import ( + "bytes" + "context" + "encoding/xml" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path" + "strings" + "sync" + + "src/common/golang/flags" + "src/common/golang/shard" + "src/common/golang/walk" + "src/common/golang/xml2" + "src/tools/ak/akhelper" + "src/tools/ak/res/res" + "src/tools/ak/types" +) + +const ( + numParsers = 25 +) + +// Archiver process the provided resource files and directories stores the data +type Archiver struct { + ResFiles []*res.PathInfo + Partitioner Partitioner +} + +// ResourcesAttribute correlates the attribute of a resources xml tag and the file where it originates +type ResourcesAttribute struct { + Attribute xml.Attr + ResFile *res.PathInfo +} + +var ( + // Cmd defines the command to run repack + Cmd = types.Command{ + Init: Init, + Run: Run, + Desc: desc, + Flags: []string{ + "res_paths", + "typed_outputs", + }, + } + + resPaths flags.StringList + typedOutputs flags.StringList + + initOnce sync.Once +) + +// Init initializes repack. +func Init() { + initOnce.Do(func() { + flag.Var(&resPaths, "res_paths", "List of res paths (a file or directory).") + flag.Var(&typedOutputs, "typed_outputs", akhelper.FormatDesc([]string{ + "A list of output file paths, each path prefixed with the res type it supports.", + "<res_type>:<file_path> i.e. string:/foo/bar/res-string-0.zip,string:/foo/bar/res-string-1.zip,...", + "The number of files per res type will determine shards."})) + }) +} + +func desc() string { + return "Bucketize Android resources." +} + +// MakeArchiver creates an Archiver +func makeArchiver(resFiles []string, p Partitioner) (*Archiver, error) { + pis, err := res.MakePathInfos(resFiles) + if err != nil { + return nil, fmt.Errorf("converting res path failed: %v", err) + } + return &Archiver{ResFiles: pis, Partitioner: p}, nil +} + +// Archive process the res directories and files of the archiver +func (a *Archiver) Archive(ctx context.Context) error { + ctx, cancel := context.WithCancel(prefixErr(ctx, "archive: ")) + defer cancel() + vPIC, nvPIC := separatePathInfosByValues(ctx, a.ResFiles) + vrCs := make([]<-chan *res.ValuesResource, 0, numParsers) + raCs := make([]<-chan *ResourcesAttribute, 0, numParsers) + errCs := make([]<-chan error, 0, numParsers) + for i := 0; i < numParsers; i++ { + vrC, raC, vErrC := handleValuesPathInfos(ctx, vPIC) + vrCs = append(vrCs, vrC) + raCs = append(raCs, raC) + errCs = append(errCs, vErrC) + } + mVRC := mergeValuesResourceStreams(ctx, vrCs) + mRAC := mergeResourcesAttributeStreams(ctx, raCs) + mErrC := mergeErrStreams(ctx, errCs) + return a.archive(ctx, nvPIC, mVRC, mRAC, mErrC) +} + +// archive takes PathInfo, ValuesResource and error channels and process the values given +func (a *Archiver) archive(ctx context.Context, piC <-chan *res.PathInfo, vrC <-chan *res.ValuesResource, raC <-chan *ResourcesAttribute, errC <-chan error) error { + var errs []error +Loop: + for piC != nil || vrC != nil || errC != nil || raC != nil { + select { + case e, ok := <-errC: + if !ok { + errC = nil + continue + } + errs = append(errs, e) + break Loop + case ra, ok := <-raC: + if !ok { + raC = nil + continue + } + a.Partitioner.CollectResourcesAttribute(ra) + case pi, ok := <-piC: + if !ok { + piC = nil + continue + } + a.Partitioner.CollectPathResource(*pi) + case vr, ok := <-vrC: + if !ok { + vrC = nil + continue + } + if err := a.Partitioner.CollectValues(vr); err != nil { + return fmt.Errorf("got error collecting values: %v", err) + } + } + } + + if len(errs) != 0 { + return errorf(ctx, "errors encountered: %v", errs) + } + if err := a.Partitioner.Close(); err != nil { + return fmt.Errorf("got error closing partitioner: %v", err) + } + return nil +} + +func handleValuesPathInfos(ctx context.Context, piC <-chan *res.PathInfo) (<-chan *res.ValuesResource, <-chan *ResourcesAttribute, <-chan error) { + vrC := make(chan *res.ValuesResource) + raC := make(chan *ResourcesAttribute) + errC := make(chan error) + go func() { + defer close(vrC) + defer close(raC) + defer close(errC) + for pi := range piC { + if !syncParse(prefixErr(ctx, fmt.Sprintf("%s values-parse: ", pi.Path)), pi, vrC, raC, errC) { + return + } + } + }() + return vrC, raC, errC +} + +func syncParse(ctx context.Context, pi *res.PathInfo, vrC chan<- *res.ValuesResource, raC chan<- *ResourcesAttribute, errC chan<- error) bool { + f, err := os.Open(pi.Path) + if err != nil { + return sendErr(ctx, errC, errorf(ctx, "open failed: %v", err)) + } + defer f.Close() + return syncParseReader(ctx, pi, xml.NewDecoder(f), vrC, raC, errC) +} + +func syncParseReader(ctx context.Context, pi *res.PathInfo, dec *xml.Decoder, vrC chan<- *res.ValuesResource, raC chan<- *ResourcesAttribute, errC chan<- error) bool { + // Shadow Encoder is used to track xml state, such as namespaces. The state will be inherited by child encoders. + parentEnc := xml2.NewEncoder(ioutil.Discard) + for { + t, err := dec.Token() + if err == io.EOF { + return true + } + if err != nil { + return sendErr(ctx, errC, errorf(ctx, "token failed: %v", err)) + } + if err := parentEnc.EncodeToken(t); err != nil { + return sendErr(ctx, errC, errorf(ctx, "encoding token token %s failed: %v", t, err)) + } + if se, ok := t.(xml.StartElement); ok && se.Name == res.ResourcesTagName { + for _, xmlAttr := range se.Attr { + raC <- &ResourcesAttribute{ResFile: pi, Attribute: xmlAttr} + } + // AAPT2 does not support a multiple resources sections in a single file and silently ignores + // subsequent resources sections. The parser will only parse the first resources tag and exit. + return parseRes(ctx, parentEnc, pi, dec, vrC, errC) + } + } +} + +func skipTag(se xml.StartElement) bool { + _, ok := res.ResourcesChildToSkip[se.Name] + return ok +} + +func parseRes(ctx context.Context, parentEnc *xml2.Encoder, pi *res.PathInfo, dec *xml.Decoder, vrC chan<- *res.ValuesResource, errC chan<- error) bool { + for { + t, err := dec.Token() + if err != nil { + return sendErr(ctx, errC, errorf(ctx, "extract token failed: %v", err)) + } + // Encode all tokens to the shadow Encoder at the top-level loop to keep track of any required xml state. + if err := parentEnc.EncodeToken(t); err != nil { + return sendErr(ctx, errC, errorf(ctx, "encoding token token %s failed: %v", t, err)) + } + switch t.(type) { + case xml.StartElement: + se := t.(xml.StartElement) + if skipTag(se) { + dec.Skip() + break + } + + fqn, err := extractFQN(se) + if err != nil { + return sendErr(ctx, errC, errorf(ctx, "extract name and type failed: %v", err)) + } + + b, err := extractElement(parentEnc, dec, se) + if err != nil { + return sendErr(ctx, errC, errorf(ctx, "extracting element failed: %v", err)) + } + + if !sendVR(ctx, vrC, &res.ValuesResource{pi, fqn, b.Bytes()}) { + return false + } + + if fqn.Type == res.Styleable { + // with a declare-styleable tag, parse its childen and treat them as direct children of resources + dsDec := xml.NewDecoder(b) + dsDec.Token() // we've already processed the first token (the declare-styleable start element) + if !parseRes(ctx, parentEnc, pi, dsDec, vrC, errC) { + return false + } + } + case xml.EndElement: + return true + } + } +} + +func extractFQN(se xml.StartElement) (res.FullyQualifiedName, error) { + if matches(se.Name, res.ItemTagName) { + nameAttr, resType, err := extractNameAndType(se) + if err != nil { + return res.FullyQualifiedName{}, err + } + return res.ParseName(nameAttr, resType) + } + + nameAttr, err := extractName(se) + if err != nil { + return res.FullyQualifiedName{}, err + } + if resType, ok := res.ResourcesTagToType[se.Name.Local]; ok { + return res.ParseName(nameAttr, resType) + } + return res.FullyQualifiedName{}, fmt.Errorf("%s: is an unhandled tag", se.Name.Local) + +} + +func extractName(se xml.StartElement) (nameAttr string, err error) { + for _, a := range se.Attr { + if matches(res.NameAttrName, a.Name) { + nameAttr = a.Value + break + } + } + if nameAttr == "" { + err = fmt.Errorf("%s: tag is missing %q attribute or is empty", se.Name.Local, res.NameAttrName.Local) + } + return +} + +func extractNameAndType(se xml.StartElement) (nameAttr string, resType res.Type, err error) { + var typeAttr string + for _, a := range se.Attr { + if matches(res.NameAttrName, a.Name) { + nameAttr = a.Value + } + if matches(res.TypeAttrName, a.Name) { + typeAttr = a.Value + } + } + if nameAttr == "" { + err = fmt.Errorf("%s: tag is missing %q attribute or is empty", se.Name.Local, res.NameAttrName.Local) + return + } + if typeAttr == "" { + err = fmt.Errorf("%s: tag is missing %q attribute or is empty", se.Name.Local, res.TypeAttrName.Local) + return + } + resType, err = res.ParseType(typeAttr) + return +} + +func matches(n1, n2 xml.Name) bool { + // Ignores xml.Name Space attributes unless both names specify Space. + if n1.Space == "" || n2.Space == "" { + return n1.Local == n2.Local + } + return n1.Local == n2.Local && n1.Space == n2.Space +} + +func extractElement(parentEnc *xml2.Encoder, dec *xml.Decoder, se xml.Token) (*bytes.Buffer, error) { + // copy tag contents to a buffer + b := &bytes.Buffer{} + enc := xml2.ChildEncoder(b, parentEnc) + if err := enc.EncodeToken(se); err != nil { + return nil, fmt.Errorf("encoding start element failed: %v", err) + } + if err := copyTag(enc, dec); err != nil { + return nil, fmt.Errorf("copyTag failed: %s", err) + } + enc.Flush() + return b, nil +} + +func copyTag(enc *xml2.Encoder, dec *xml.Decoder) error { + for { + t, err := dec.Token() + if err != nil { + return fmt.Errorf("extract token failed: %v", err) + } + if err := enc.EncodeToken(t); err != nil { + return fmt.Errorf("encoding token %v failed: %v", t, err) + } + switch t.(type) { + case xml.StartElement: + if err := copyTag(enc, dec); err != nil { + return err + } + case xml.EndElement: + return nil + } + } +} + +func sendVR(ctx context.Context, vrC chan<- *res.ValuesResource, vr *res.ValuesResource) bool { + select { + case vrC <- vr: + case <-ctx.Done(): + return false + } + return true +} + +func hasChildType(dec *xml.Decoder, lookup map[xml.Name]res.Type, want res.Type) (bool, error) { + for { + t, err := dec.Token() + if err != nil { + return false, fmt.Errorf("extract token failed: %v", err) + } + switch t.(type) { + case xml.StartElement: + if rt, ok := lookup[t.(xml.StartElement).Name]; ok { + if rt == want { + return true, nil + } + } + // when tag is not in the lookup or the type is unknown or "wanted", skip it. + dec.Skip() + case xml.EndElement: + return false, nil + } + } +} + +func createPartitions(typedOutputs []string) (map[res.Type][]io.Writer, error) { + partitions := make(map[res.Type][]io.Writer) + for _, tAndOP := range typedOutputs { + tOP := strings.SplitN(tAndOP, ":", 2) + // no shard count override specified + if len(tOP) == 1 { + return nil, fmt.Errorf("got malformed typed output path %q wanted the following format \"<type>:<file path>\"", tAndOP) + } + t, err := res.ParseType(tOP[0]) + if err != nil { + return nil, fmt.Errorf("got err while trying to parse %s to a res type: %v", tOP[0], err) + } + op := tOP[1] + if err := os.MkdirAll(path.Dir(op), 0744); err != nil { + return nil, fmt.Errorf("%s: mkdir failed: %v", op, err) + } + f, err := os.OpenFile(op, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644) + if err != nil { + return nil, fmt.Errorf("open/create failed: %v", err) + } + partitions[t] = append(partitions[t], f) + } + return partitions, nil +} + +// Run is the entry point for bucketize. +func Run() { + if resPaths == nil || typedOutputs == nil { + log.Fatal("Flags -res_paths and -typed_outputs must be specified.") + } + + resFiles, err := walk.Files(resPaths) + if err != nil { + log.Fatalf("Got error getting the resource paths: %v", err) + } + resFileIdxs := make(map[string]int) + for i, resFile := range resFiles { + resFileIdxs[resFile] = i + } + + p, err := createPartitions(typedOutputs) + if err != nil { + log.Fatalf("Got error creating partitions: %v", err) + } + + ps, err := makePartitionSession(p, shard.FNV, resFileIdxs) + if err != nil { + log.Fatalf("Got error making partition session: %v", err) + } + + m, err := makeArchiver(resFiles, ps) + if err != nil { + log.Fatalf("Got error making archiver: %v", err) + } + + if err := m.Archive(context.Background()); err != nil { + log.Fatalf("Got error archiving: %v", err) + } +} diff --git a/src/tools/ak/bucketize/bucketize_bin.go b/src/tools/ak/bucketize/bucketize_bin.go new file mode 100644 index 0000000..f9ea517 --- /dev/null +++ b/src/tools/ak/bucketize/bucketize_bin.go @@ -0,0 +1,29 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The bucketize_bin is a command line tool to bucketize Android resources. +package main + +import ( + "flag" + + _ "src/common/golang/flagfile" + "src/tools/ak/bucketize/bucketize" +) + +func main() { + bucketize.Init() + flag.Parse() + bucketize.Run() +} diff --git a/src/tools/ak/bucketize/bucketize_test.go b/src/tools/ak/bucketize/bucketize_test.go new file mode 100644 index 0000000..3864164 --- /dev/null +++ b/src/tools/ak/bucketize/bucketize_test.go @@ -0,0 +1,483 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bucketize + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "reflect" + "strings" + "testing" + + "src/common/golang/shard" + "src/common/golang/walk" + "src/tools/ak/res/res" +) + +func TestNormalizeResPaths(t *testing.T) { + // Create a temporary directory to house the fake workspace. + tmp, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("Can't make temp directory: %v", err) + } + defer os.RemoveAll(tmp) + + var resPaths []string + fp1 := path.Join(tmp, "foo") + _, err = os.Create(fp1) + if err != nil { + t.Fatalf("Got error while trying to create %s: %v", fp1, err) + } + resPaths = append(resPaths, fp1) + + dp1 := path.Join(tmp, "bar", "baz", "qux") + if err != os.MkdirAll(dp1, 0777) { + t.Fatalf("Got error while trying to create %s: %v", dp1, err) + } + resPaths = append(resPaths, dp1) + + // Create a file nested in the directory that is passed in as a resPath. This file will get + // injected between fp1 and fp3 because the directory is defined in the middle. Hence, + // files added to the directory will appear between fp1 and fp3. This behavior is intended. + fInDP1 := path.Join(dp1, "quux") + _, err = os.Create(fInDP1) + if err != nil { + t.Fatalf("Got error while trying to create %s: %v", fInDP1, err) + } + + fp3 := path.Join(tmp, "bar", "corge") + _, err = os.Create(fp3) + if err != nil { + t.Fatalf("Got error while trying to create %s: %v", fp3, err) + } + resPaths = append(resPaths, fp3) + + gotFiles, err := walk.Files(resPaths) + if err != nil { + t.Fatalf("Got error getting the resource paths: %v", err) + } + gotFileIdxs := make(map[string]int) + for i, gotFile := range gotFiles { + gotFileIdxs[gotFile] = i + } + + wantFiles := []string{fp1, fInDP1, fp3} + if !reflect.DeepEqual(gotFiles, wantFiles) { + t.Errorf("DeepEqual(\n%#v\n,\n%#v\n): returned false", gotFiles, wantFiles) + } + + wantFileIdxs := map[string]int{fp1: 0, fInDP1: 1, fp3: 2} + if !reflect.DeepEqual(gotFileIdxs, wantFileIdxs) { + t.Errorf("DeepEqual(\n%#v\n,\n%#v\n): returned false", gotFileIdxs, wantFileIdxs) + } +} + +func TestArchiverWithPartitionSession(t *testing.T) { + order := make(map[string]int) + ps, err := makePartitionSession(map[res.Type][]io.Writer{}, shard.FNV, order) + if err != nil { + t.Fatalf("MakePartitionSesion got err: %v", err) + } + if _, err := makeArchiver([]string{}, ps); err != nil { + t.Errorf("MakeArchiver got err: %v", err) + } +} + +func TestArchiveNoValues(t *testing.T) { + ctx, cxlFn := context.WithCancel(context.Background()) + defer cxlFn() + a, err := makeArchiver([]string{}, &mockPartitioner{}) + if err != nil { + t.Fatalf("MakeArchiver got error: %v", err) + } + a.Archive(ctx) +} + +func TestInternalArchive(t *testing.T) { + tcs := []struct { + name string + p Partitioner + pis []*res.PathInfo + vrs []*res.ValuesResource + ras []ResourcesAttribute + errs []error + wantErr bool + }{ + { + name: "MultipleResPathInfosAndValuesResources", + p: &mockPartitioner{}, + pis: []*res.PathInfo{{Path: "foo"}}, + vrs: []*res.ValuesResource{ + {Src: &res.PathInfo{Path: "bar"}}, + {Src: &res.PathInfo{Path: "baz"}}, + }, + errs: []error{}, + }, + { + name: "NoValues", + p: &mockPartitioner{}, + pis: []*res.PathInfo{}, + vrs: []*res.ValuesResource{}, + errs: []error{}, + }, + { + name: "ErrorOccurred", + p: &mockPartitioner{}, + pis: []*res.PathInfo{{Path: "foo"}}, + vrs: []*res.ValuesResource{}, + errs: []error{fmt.Errorf("failure")}, + wantErr: true, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + piC := make(chan *res.PathInfo) + go func() { + defer close(piC) + for _, pi := range tc.pis { + piC <- pi + } + }() + vrC := make(chan *res.ValuesResource) + go func() { + defer close(vrC) + for _, vr := range tc.vrs { + vrC <- vr + } + }() + raC := make(chan *ResourcesAttribute) + go func() { + defer close(raC) + for _, ra := range tc.ras { + nra := new(ResourcesAttribute) + *nra = ra + raC <- nra + } + }() + errC := make(chan error) + go func() { + defer close(errC) + for _, err := range tc.errs { + errC <- err + } + }() + a, err := makeArchiver([]string{}, tc.p) + if err != nil { + t.Errorf("MakeArchiver got error: %v", err) + return + } + ctx, cxlFn := context.WithCancel(context.Background()) + defer cxlFn() + if err := a.archive(ctx, piC, vrC, raC, errC); err != nil { + if !tc.wantErr { + t.Errorf("archive got unexpected error: %v", err) + } + return + } + }) + } +} + +func TestSyncParseReader(t *testing.T) { + tcs := []struct { + name string + pi *res.PathInfo + content *bytes.Buffer + want map[string]string + wantErr bool + }{ + { + name: "SingleResourcesBlock", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources> + <string name="introduction">hello world</string> + <string name="foo">bar</string> + <attr name="baz" format="reference|color"></attr> + </resources>`), + want: map[string]string{ + "introduction-string": "<string name=\"introduction\">hello world</string>", + "foo-string": "<string name=\"foo\">bar</string>", + "baz-attr": "<attr name=\"baz\" format=\"reference|color\"></attr>", + }, + }, + { + name: "MultipleResourcesBlocks", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources> + <string name="introduction">hello world</string> + <string name="foo">bar</string> + </resources> + <!-- + Subsequent resources sections are ignored, hence the "qux" item will not + materialize in the parsed values. + --> + <resources> + <item name="qux" type="integer">23</item> + </resources>`), + want: map[string]string{ + "introduction-string": "<string name=\"introduction\">hello world</string>", + "foo-string": "<string name=\"foo\">bar</string>", + }, + }, + { + name: "NamespacedResourcesBlock", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources xmlns:foo="bar"> + <string name="namespaced"><foo:bar>hello</foo:bar> world</string> + </resources>`), + want: map[string]string{ + "resource_attribute-xmlns:foo": "bar", + "namespaced-string": "<string name=\"namespaced\"><foo:bar>hello</foo:bar> world</string>", + }, + }, + { + name: "DeclareStyleable", + pi: &res.PathInfo{}, + content: bytes.NewBufferString("<resources><declare-styleable name=\"foo\"><attr name=\"bar\">baz</attr></declare-styleable></resources>"), + want: map[string]string{ + "foo-styleable": "<declare-styleable name=\"foo\"><attr name=\"bar\">baz</attr></declare-styleable>", + "bar-attr": "<attr name=\"bar\">baz</attr>", + }, + }, + { + name: "NamespacedStyleableBlock", + pi: &res.PathInfo{}, + content: bytes.NewBufferString("<resources xmlns:zoo=\"zoo\"><declare-styleable name=\"foo\"><attr name=\"bar\" zoo:qux=\"rux\">baz</attr></declare-styleable></resources>"), + want: map[string]string{ + "resource_attribute-xmlns:zoo": "zoo", + "foo-styleable": "<declare-styleable name=\"foo\"><attr name=\"bar\" zoo:qux=\"rux\">baz</attr></declare-styleable>", + "bar-attr": "<attr name=\"bar\" zoo:qux=\"rux\">baz</attr>", + }, + }, + { + name: "PluralsStringArrayOutputToStringToo", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources> + <string-array name="foo"><item>bar</item><item>baz</item></string-array> + <plurals name="corge"><item quantity="one">qux</item><item quantity="other">quux</item></plurals> + </resources>`), + want: map[string]string{ + "foo-array": "<string-array name=\"foo\"><item>bar</item><item>baz</item></string-array>", + "corge-plurals": "<plurals name=\"corge\"><item quantity=\"one\">qux</item><item quantity=\"other\">quux</item></plurals>", + }, + }, + { + name: "AttrWithFlagOrEnumChildren", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources> + <attr name="foo"><enum name="bar" value="0" /><enum name="baz" value="10" /></attr> + <attr name="qux"><flag name="quux" value="0x4" /></attr> + </resources>`), + want: map[string]string{ + "foo-attr": "<attr name=\"foo\"><enum name=\"bar\" value=\"0\"></enum><enum name=\"baz\" value=\"10\"></enum></attr>", + "qux-attr": "<attr name=\"qux\"><flag name=\"quux\" value=\"0x4\"></flag></attr>", + }, + }, + { + name: "Style", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources> + <style name="foo"><item>bar</item><item>baz</item></style> + </resources>`), + want: map[string]string{ + "foo-style": "<style name=\"foo\"><item>bar</item><item>baz</item></style>", + }, + }, + { + name: "ArraysGoToStingAndInteger", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources> + <array name="foo"><item>bar</item><item>1</item></array> + </resources>`), + want: map[string]string{ + "foo-array": "<array name=\"foo\"><item>bar</item><item>1</item></array>", + }, + }, + { + name: "NoContent", + pi: &res.PathInfo{}, + content: &bytes.Buffer{}, + want: map[string]string{}, + }, + { + name: "EmptyResources", + pi: &res.PathInfo{}, + content: bytes.NewBufferString("<resources></resources>"), + want: map[string]string{}, + }, + { + name: "IgnoredContent", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(` + <!--ignore my comment--> + <ignore_tag /> + ignore random string. + <resources> + <!--ignore this comment too--> + <attr name="foo">bar<baz>qux</baz></attr> + ignore this random string too. + <!-- following are a list of ignored tags --> + <eat-comment /> + <skip /> + </resources>`), + want: map[string]string{ + "foo-attr": "<attr name=\"foo\">bar<baz>qux</baz></attr>", + }, + }, + { + name: "TagMissingNameAttribute", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources><string>MissingNameAttr</string></resources>`), + wantErr: true, + }, + { + name: "ItemTagMissingTypeAttribute", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources><item name="MissingTypeAttr">bar</item></resources>`), + wantErr: true, + }, + { + name: "ItemTagUnknownTypeAttribute", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources><item name="UnknownType" type="foo" /></resources>`), + wantErr: true, + }, + { + name: "UnhandledTag", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources><foo name="bar"/></resources>`), + wantErr: true, + }, + { + name: "MalFormedXml_OpenResourcesTag", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources>`), + wantErr: true, + }, + { + name: "MalFormedXml_Unabalanced", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources><attr name="unbalanced"><b></attr></resources>`), + wantErr: true, + }, + { + name: "NamespaceUsedWithoutNamespaceDefinition", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(`<resources><string name="ohno"><bad:b>Oh no!</bad:b></string></resources>`), + wantErr: true, + }, + { + // Verify parent Encoder is properly shadowing the xml file. + name: "NamespaceUsedOutsideOfDefinition", + pi: &res.PathInfo{}, + content: bytes.NewBufferString(` + <resources> + <string name="foo" xmlns:bar="baz">qux</string> + <string name="ohno"><foo:b>Oh no!</foo:b></string> + </resources>`), + wantErr: true, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + ctx, cxlFn := context.WithCancel(context.Background()) + defer cxlFn() + vrC := make(chan *res.ValuesResource) + raC := make(chan *ResourcesAttribute) + errC := make(chan error) + go func() { + defer close(vrC) + defer close(raC) + defer close(errC) + syncParseReader(ctx, tc.pi, xml.NewDecoder(tc.content), vrC, raC, errC) + }() + got := make(map[string]string) + errMs := make([]string, 0) + for errC != nil || vrC != nil { + select { + case e, ok := <-errC: + if !ok { + errC = nil + } + if e != nil { + errMs = append(errMs, e.Error()) + } + case ra, ok := <-raC: + if !ok { + raC = nil + } + if ra != nil { + a := ra.Attribute + got[fmt.Sprintf("resource_attribute-%s:%s", a.Name.Space, a.Name.Local)] = a.Value + } + case vr, ok := <-vrC: + if !ok { + vrC = nil + } + if vr != nil { + got[fmt.Sprintf("%s-%s", vr.N.Name, vr.N.Type.String())] = string(vr.Payload) + } + } + } + + // error handling + if tc.wantErr { + if len(errMs) == 0 { + t.Errorf("syncParseReader expected an error.") + } + return + } + if len(errMs) > 0 { + t.Errorf("syncParserReader got unexpected error(s): \n%s", strings.Join(errMs, "\n")) + return + } + + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("DeepEqual(\n%#v\n,\n%#v\n): returned false", got, tc.want) + } + }) + } +} + +// mockPartitioner is a Partitioner mock used for testing. +type mockPartitioner struct { + strPI []res.PathInfo + cvVR []res.ValuesResource + ra []*ResourcesAttribute +} + +func (mp *mockPartitioner) Close() error { + return nil +} + +func (mp *mockPartitioner) CollectPathResource(src res.PathInfo) { + mp.strPI = append(mp.strPI, src) +} + +func (mp *mockPartitioner) CollectValues(vr *res.ValuesResource) error { + mp.cvVR = append(mp.cvVR, res.ValuesResource{vr.Src, vr.N, vr.Payload}) + return nil +} + +func (mp *mockPartitioner) CollectResourcesAttribute(ra *ResourcesAttribute) { + mp.ra = append(mp.ra, ra) +} diff --git a/src/tools/ak/bucketize/partitioner.go b/src/tools/ak/bucketize/partitioner.go new file mode 100644 index 0000000..97a328d --- /dev/null +++ b/src/tools/ak/bucketize/partitioner.go @@ -0,0 +1,319 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bucketize + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "os" + "path" + "path/filepath" + "sort" + "strings" + + "src/common/golang/shard" + "src/common/golang/xml2" + "src/tools/ak/res/res" +) + +// Helper struct to sort paths by index +type indexedPaths struct { + order map[string]int + ps []string +} + +type byPathIndex indexedPaths + +func (b byPathIndex) Len() int { return len(b.ps) } +func (b byPathIndex) Swap(i, j int) { b.ps[i], b.ps[j] = b.ps[j], b.ps[i] } +func (b byPathIndex) Less(i, j int) bool { + iIdx := pathIdx(b.ps[i], b.order) + jIdx := pathIdx(b.ps[j], b.order) + // Files exist in the same directory + if iIdx == jIdx { + return b.ps[i] < b.ps[j] + } + return iIdx < jIdx +} + +// Helper struct to sort valuesKeys by index +type indexedValuesKeys struct { + order map[string]int + ks []valuesKey +} + +type byValueKeyIndex indexedValuesKeys + +func (b byValueKeyIndex) Len() int { return len(b.ks) } +func (b byValueKeyIndex) Swap(i, j int) { b.ks[i], b.ks[j] = b.ks[j], b.ks[i] } +func (b byValueKeyIndex) Less(i, j int) bool { + iIdx := pathIdx(b.ks[i].sourcePath.Path, b.order) + jIdx := pathIdx(b.ks[j].sourcePath.Path, b.order) + // Files exist in the same directory + if iIdx == jIdx { + return b.ks[i].sourcePath.Path < b.ks[j].sourcePath.Path + } + return iIdx < jIdx +} + +type valuesKey struct { + sourcePath res.PathInfo + resType res.Type +} + +// PartitionSession consumes resources and partitions them into archives by the resource type. +// The typewise partitions can be further sharded by the provided shardFn +type PartitionSession struct { + typedOutput map[res.Type][]*zip.Writer + sharder shard.Func + collectedVals map[valuesKey]map[string][]byte + collectedPaths map[string]res.PathInfo + collectedRAs map[string][]xml.Attr + resourceOrder map[string]int +} + +// Partitioner takes the provided resource values and paths and stores the data sharded +type Partitioner interface { + Close() error + CollectValues(vr *res.ValuesResource) error + CollectPathResource(src res.PathInfo) + CollectResourcesAttribute(attr *ResourcesAttribute) +} + +// makePartitionSession creates a PartitionSession that writes to the given outputs. +func makePartitionSession(outputs map[res.Type][]io.Writer, sharder shard.Func, resourceOrder map[string]int) (*PartitionSession, error) { + typeToArchs := make(map[res.Type][]*zip.Writer) + for t, ws := range outputs { + archs := make([]*zip.Writer, 0, len(ws)) + for _, w := range ws { + archs = append(archs, zip.NewWriter(w)) + } + typeToArchs[t] = archs + } + return &PartitionSession{ + typeToArchs, + sharder, + make(map[valuesKey]map[string][]byte), + make(map[string]res.PathInfo), + make(map[string][]xml.Attr), + resourceOrder, + }, nil +} + +// Close finalizes all archives in this partition session. +func (ps *PartitionSession) Close() error { + if err := ps.flushCollectedPaths(); err != nil { + return fmt.Errorf("got error flushing collected paths: %v", err) + } + if err := ps.flushCollectedVals(); err != nil { + return fmt.Errorf("got error flushing collected values: %v", err) + } + // close archives. + for _, as := range ps.typedOutput { + for _, a := range as { + if err := a.Close(); err != nil { + return fmt.Errorf("%s: could not close: %v", a, err) + } + } + } + return nil +} + +// CollectPathResource takes a file system resource and tracks it so that it can be stored in an output partition and shard. +func (ps *PartitionSession) CollectPathResource(src res.PathInfo) { + // store the path only if the type is accepted by the underlying partitions. + if ps.isTypeAccepted(src.Type) { + ps.collectedPaths[src.Path] = src + } +} + +// CollectValues stores the xml representation of a particular resource from a particular file. +func (ps *PartitionSession) CollectValues(vr *res.ValuesResource) error { + // store the value only if the type is accepted by the underlying partitions. + if ps.isTypeAccepted(vr.N.Type) { + // Don't store style attr's from other packages + if !(vr.N.Type == res.Attr && vr.N.Package != "res-auto") { + k := valuesKey{*vr.Src, vr.N.Type} + if tv, ok := ps.collectedVals[k]; !ok { + ps.collectedVals[k] = make(map[string][]byte) + ps.collectedVals[k][vr.N.String()] = vr.Payload + } else { + if p, ok := tv[vr.N.String()]; !ok { + ps.collectedVals[k][vr.N.String()] = vr.Payload + } else if len(p) < len(vr.Payload) { + ps.collectedVals[k][vr.N.String()] = vr.Payload + } else if len(p) == len(vr.Payload) && bytes.Compare(p, vr.Payload) != 0 { + return fmt.Errorf("different values for resource %q", vr.N.String()) + } + } + } + } + return nil +} + +// CollectResourcesAttribute stores the xml attributes of the resources tag from a particular file. +func (ps *PartitionSession) CollectResourcesAttribute(ra *ResourcesAttribute) { + ps.collectedRAs[ra.ResFile.Path] = append(ps.collectedRAs[ra.ResFile.Path], ra.Attribute) +} + +func (ps *PartitionSession) isTypeAccepted(t res.Type) bool { + _, ok := ps.typedOutput[t] + return ok +} + +func (ps *PartitionSession) flushCollectedPaths() error { + // sort keys so that data is written to the archives in a deterministic order + // specifically the same order in which they were declared + ks := make([]string, 0, len(ps.collectedPaths)) + for k := range ps.collectedPaths { + ks = append(ks, k) + } + sort.Sort(byPathIndex(indexedPaths{order: ps.resourceOrder, ps: ks})) + for _, k := range ks { + v := ps.collectedPaths[k] + f, err := os.Open(v.Path) + if err != nil { + return fmt.Errorf("%s: could not be opened for reading: %v", v.Path, err) + } + if err := ps.storePathResource(v, f); err != nil { + return fmt.Errorf("%s: got error storing path resource: %v", v.Path, err) + } + f.Close() + } + return nil +} + +func (ps *PartitionSession) storePathResource(src res.PathInfo, r io.Reader) error { + p := path.Base(src.Path) + if dot := strings.Index(p, "."); dot == 0 { + // skip files where the name starts with a ".", these are already ignored by aapt + return nil + } else if dot > 0 { + p = p[:dot] + } + fqn, err := res.ParseName(p, src.Type) + if err != nil { + return fmt.Errorf("%s: %q could not be parsed into a res name: %v", src.Path, p, err) + } + arch, err := ps.archiveFor(fqn) + if err != nil { + return fmt.Errorf("%s: could not get partitioned archive: %v", src.Path, err) + } + w, err := arch.Create(pathResSuffix(src.Path)) + if err != nil { + return fmt.Errorf("%s: could not create writer: %v", src.Path, err) + } + if _, err = io.Copy(w, r); err != nil { + return fmt.Errorf("%s: could not copy into archive: %v", src.Path, err) + } + return nil +} + +func (ps *PartitionSession) archiveFor(fqn res.FullyQualifiedName) (*zip.Writer, error) { + archs, ok := ps.typedOutput[fqn.Type] + if !ok { + return nil, fmt.Errorf("%s: do not have output stream for this res type", fqn.Type) + } + shard := ps.sharder(fqn.String(), len(archs)) + if shard > len(archs) || 0 > shard { + return nil, fmt.Errorf("%v: bad sharder f(%v, %d) -> %d must be [0,%d)", ps.sharder, fqn, len(archs), shard, len(archs)) + } + return archs[shard], nil +} + +var ( + resXMLHeader = []byte("<?xml version='1.0' encoding='utf-8'?>") + resXMLFooter = []byte("</resources>") +) + +func (ps *PartitionSession) flushCollectedVals() error { + // sort keys so that data is written to the archives in a deterministic order + // specifically the same order in which blaze provides them + ks := make([]valuesKey, 0, len(ps.collectedVals)) + for k := range ps.collectedVals { + ks = append(ks, k) + } + sort.Sort(byValueKeyIndex(indexedValuesKeys{order: ps.resourceOrder, ks: ks})) + for _, k := range ks { + as, ok := ps.typedOutput[k.resType] + if !ok { + return fmt.Errorf("%s: no output for res type", k.resType) + } + ws := make([]io.Writer, 0, len(as)) + // For each given source file, create a corresponding file in each of the shards. A file in a particular shard may be empty, if none of the resources defined in the source file ended up in that shard. + for _, a := range as { + w, err := a.Create(pathResSuffix(k.sourcePath.Path)) + if err != nil { + return fmt.Errorf("%s: could not create entry: %v", k.sourcePath.Path, err) + } + if _, err = w.Write(resXMLHeader); err != nil { + return fmt.Errorf("%s: could not write xml header: %v", k.sourcePath.Path, err) + } + // Write the resources open tag, with the attributes collected. + b := bytes.Buffer{} + xml2.NewEncoder(&b).EncodeToken(xml.StartElement{ + Name: res.ResourcesTagName, + Attr: ps.collectedRAs[k.sourcePath.Path], + }) + if _, err = w.Write(b.Bytes()); err != nil { + return fmt.Errorf("%s: could not write resources tag %q: %v", k.sourcePath.Path, b.String(), err) + } + ws = append(ws, w) + } + v := ps.collectedVals[k] + var keys []string + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + for _, fqn := range keys { + p := v[fqn] + shard := ps.sharder(fqn, len(ws)) + if shard < 0 || shard >= len(ws) { + return fmt.Errorf("%v: bad sharder f(%s, %d) -> %d must be [0,%d)", ps.sharder, fqn, len(ws), shard, len(ws)) + } + if _, err := ws[shard].Write(p); err != nil { + return fmt.Errorf("%s: writing resource %s failed: %v", k.sourcePath.Path, fqn, err) + } + } + for _, w := range ws { + if _, err := w.Write(resXMLFooter); err != nil { + return fmt.Errorf("%s: could not write xml footer: %v", k.sourcePath.Path, err) + } + } + } + return nil +} + +func pathIdx(path string, order map[string]int) int { + if idx, ok := order[path]; ok == true { + return idx + } + // TODO(mauriciogg): maybe replace with prefix search + // list of resources might contain directories so exact match might not exist + dirPos := strings.LastIndex(path, "/res/") + idx, _ := order[path[0:dirPos+4]] + return idx +} + +func pathResSuffix(path string) string { + // returns the relative resource path from the full path + // e.g. /foo/bar/res/values/strings.xml -> res/values/strings.xml + parentDir := filepath.Dir(filepath.Dir(filepath.Dir(path))) + return strings.TrimPrefix(path, parentDir+string(filepath.Separator)) +} diff --git a/src/tools/ak/bucketize/partitioner_test.go b/src/tools/ak/bucketize/partitioner_test.go new file mode 100644 index 0000000..846718e --- /dev/null +++ b/src/tools/ak/bucketize/partitioner_test.go @@ -0,0 +1,349 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bucketize + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "reflect" + "sort" + "strconv" + "strings" + "testing" + + "src/common/golang/shard" + "src/tools/ak/res/res" +) + +func TestInternalStorePathResource(t *testing.T) { + // test internal storePathResource and skip the creation of real files. + tcs := []struct { + name string + inFiles map[string]string + partitions map[res.Type][]io.Writer + shardFn shard.Func + want map[res.Type][][]string + wantErr bool + }{ + { + name: "MultipleResTypeFilesWithShardsOfDifferentSizes", + inFiles: map[string]string{ + "res/drawable/2-foo.xml": "all", + "res/layout/0-bar.xml": "your", + "res/color/0-baz.xml": "base", + "res/layout/1-qux.xml": "are", + "res/drawable/0-quux.xml": "belong", + "res/color/0-corge.xml": "to", + "res/color/0-grault.xml": "us", + "res/layout/0-garply.xml": "!", + }, + shardFn: shard.Func(func(fqn string, shardCount int) int { + // sharding strategy is built into the file name as "<shard num>-foo.bar" (i.e. 8-baz.xml) + name := strings.Split(fqn, "/")[1] + ai := strings.SplitN(name, "-", 2)[0] + shard, err := strconv.Atoi(ai) + if err != nil { + t.Fatalf("Atoi(%s) got err: %v", ai, err) + } + return shard + }), + partitions: map[res.Type][]io.Writer{ + res.Drawable: {&bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}}, + res.Color: {&bytes.Buffer{}}, + res.Layout: {&bytes.Buffer{}, &bytes.Buffer{}}, + }, + want: map[res.Type][][]string{ + res.Drawable: {{"res/drawable/0-quux.xml"}, {}, {"res/drawable/2-foo.xml"}}, + res.Color: {{"res/color/0-baz.xml", "res/color/0-corge.xml", "res/color/0-grault.xml"}}, + res.Layout: {{"res/layout/0-bar.xml", "res/layout/0-garply.xml"}, {"res/layout/1-qux.xml"}}, + }, + }, + { + name: "IgnoredFilePatterns", + inFiles: map[string]string{ + "res/drawable/.ignore": "me", + }, + shardFn: shard.FNV, + partitions: map[res.Type][]io.Writer{res.Drawable: {&bytes.Buffer{}}}, + wantErr: true, + }, + { + name: "NoFiles", + inFiles: map[string]string{}, + shardFn: shard.FNV, + partitions: map[res.Type][]io.Writer{res.Drawable: {&bytes.Buffer{}}}, + want: map[res.Type][][]string{res.Drawable: {{}}}, + }, + } + + order := make(map[string]int) + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + ps, err := makePartitionSession(tc.partitions, tc.shardFn, order) + if err != nil { + t.Errorf("MakePartitionSession(%v, %v, %d) got err: %v", tc.partitions, tc.shardFn, 0, err) + return + } + + for k, v := range tc.inFiles { + pi, err := res.ParsePath(k) + if err != nil { + if !tc.wantErr { + t.Fatalf("ParsePath(%s) got err: %v", k, err) + } + return + } + if err := ps.storePathResource(pi, strings.NewReader(v)); err != nil { + t.Fatalf("storePathResource got unexpected err: %v", err) + } + } + + if err := ps.Close(); err != nil { + t.Errorf("partition Close() got err: %v", err) + return + } + + // validate data outputted to the partitions + got := make(map[res.Type][][]string) + for rt, shards := range tc.partitions { + shardPaths := make([][]string, 0, len(shards)) + for _, shard := range shards { + br := bytes.NewReader(shard.(*bytes.Buffer).Bytes()) + rr, err := zip.NewReader(br, br.Size()) + if err != nil { + t.Errorf("NewReader(%v, %d) got err: %v", br, br.Size(), err) + return + } + paths := make([]string, 0, len(rr.File)) + for _, f := range rr.File { + paths = append(paths, f.Name) + c, err := readAll(f) + if err != nil { + t.Errorf("readAll got err: %v", err) + return + } + if tc.inFiles[f.Name] != c { + t.Errorf("error copying data for %s got %q but wanted %q", f.Name, c, tc.inFiles[f.Name]) + return + } + } + sort.Strings(paths) + shardPaths = append(shardPaths, paths) + } + got[rt] = shardPaths + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("DeepEqual(\n%#v\n,\n%#v\n): returned false", got, tc.want) + } + }) + } +} + +func TestCollectValues(t *testing.T) { + tcs := []struct { + name string + pathVPsMap map[string]map[res.FullyQualifiedName][]byte + pathRAMap map[string][]xml.Attr + partitions map[res.Type][]io.Writer + want map[res.Type][][]string + wantErr bool + }{ + { + name: "MultipleResTypesShardsResources", + partitions: map[res.Type][]io.Writer{ + res.Attr: {&bytes.Buffer{}, &bytes.Buffer{}}, + res.String: {&bytes.Buffer{}, &bytes.Buffer{}}, + res.Color: {&bytes.Buffer{}, &bytes.Buffer{}}, + }, + pathVPsMap: map[string]map[res.FullyQualifiedName][]byte{ + "res/values/strings.xml": { + res.FullyQualifiedName{Package: "res-auto", Type: res.String, Name: "foo"}: []byte("<string name='foo'>bar</string>"), + res.FullyQualifiedName{Package: "android", Type: res.String, Name: "baz"}: []byte("<string name='baz'>qux</string>"), + res.FullyQualifiedName{Package: "res-auto", Type: res.Attr, Name: "quux"}: []byte("<attr name='quux'>corge</attr>"), + }, + "res/values/attr.xml": { + res.FullyQualifiedName{Package: "android", Type: res.Attr, Name: "foo"}: []byte("<attr name='android:foo'>bar</attr>"), + }, + "baz/res/values/attr.xml": { + res.FullyQualifiedName{Package: "android", Type: res.Attr, Name: "bazfoo"}: []byte("<attr name='android:bazfoo'>qix</attr>"), + }, + "baz/res/values/strings.xml": { + res.FullyQualifiedName{Package: "android", Type: res.String, Name: "baz"}: []byte("<string name='baz'>qux</string>"), + }, + "foo/res/values/attr.xml": { + res.FullyQualifiedName{Package: "android", Type: res.Attr, Name: "foofoo"}: []byte("<attr name='android:foofoo'>qex</attr>"), + }, + "foo/res/values/color.xml": { + res.FullyQualifiedName{Package: "android", Type: res.Color, Name: "foobar"}: []byte("<color name='foobar'>#FFFFFFFF</color>"), + }, + "dir/res/values/strings.xml": { + res.FullyQualifiedName{Package: "android", Type: res.String, Name: "dirbaz"}: []byte("<string name='dirbaz'>qux</string>"), + }, + "dir/res/values/color.xml": { + res.FullyQualifiedName{Package: "android", Type: res.Color, Name: "dirfoobar"}: []byte("<color name='dirfoobar'>#FFFFFFFF</color>"), + }, + }, + pathRAMap: map[string][]xml.Attr{ + "res/values/strings.xml": { + xml.Attr{Name: xml.Name{Space: "xmlns", Local: "ns1"}, Value: "path1"}, + xml.Attr{Name: xml.Name{Space: "xmlns", Local: "ns2"}, Value: "path2"}, + }, + }, + want: map[res.Type][][]string{ + res.Attr: { + { + "res/values/strings.xml", "<?xml version='1.0' encoding='utf-8'?><resources xmlns:ns1=\"path1\" xmlns:ns2=\"path2\"><attr name='quux'>corge</attr></resources>", + }, + { + "res/values/strings.xml", "<?xml version='1.0' encoding='utf-8'?><resources xmlns:ns1=\"path1\" xmlns:ns2=\"path2\"></resources>", + }, + }, + res.String: { + { + "res/values/strings.xml", "<?xml version='1.0' encoding='utf-8'?><resources xmlns:ns1=\"path1\" xmlns:ns2=\"path2\"><string name='baz'>qux</string><string name='foo'>bar</string></resources>", + "res/values/strings.xml", "<?xml version='1.0' encoding='utf-8'?><resources><string name='dirbaz'>qux</string></resources>", + "res/values/strings.xml", "<?xml version='1.0' encoding='utf-8'?><resources><string name='baz'>qux</string></resources>", + }, + { + "res/values/strings.xml", "<?xml version='1.0' encoding='utf-8'?><resources xmlns:ns1=\"path1\" xmlns:ns2=\"path2\"></resources>", + "res/values/strings.xml", "<?xml version='1.0' encoding='utf-8'?><resources></resources>", + "res/values/strings.xml", "<?xml version='1.0' encoding='utf-8'?><resources></resources>", + }, + }, + res.Color: { + { + "res/values/color.xml", "<?xml version='1.0' encoding='utf-8'?><resources><color name='foobar'>#FFFFFFFF</color></resources>", + "res/values/color.xml", "<?xml version='1.0' encoding='utf-8'?><resources><color name='dirfoobar'>#FFFFFFFF</color></resources>", + }, + { + "res/values/color.xml", "<?xml version='1.0' encoding='utf-8'?><resources></resources>", + "res/values/color.xml", "<?xml version='1.0' encoding='utf-8'?><resources></resources>", + }, + }, + }, + }, + { + name: "NoValuesPayloads", + pathVPsMap: map[string]map[res.FullyQualifiedName][]byte{ + "res/values/strings.xml": {}, + }, + partitions: map[res.Type][]io.Writer{res.String: {&bytes.Buffer{}}}, + want: map[res.Type][][]string{res.String: {{}}}, + }, + { + name: "ResTypeValuesResTypeMismatch", + pathVPsMap: map[string]map[res.FullyQualifiedName][]byte{ + "res/values/strings.xml": { + res.FullyQualifiedName{ + Package: "res-auto", + Type: res.String, + Name: "foo", + }: []byte("<string name='foo'>bar</string>"), + }, + }, + partitions: map[res.Type][]io.Writer{res.Attr: {&bytes.Buffer{}}}, + want: map[res.Type][][]string{res.Attr: {{}}}, + }, + } + + shardFn := func(name string, shardCount int) int { return 0 } + order := map[string]int{ + "foo/res/values/attr.xml": 0, + "foo/res/values/color.xml": 1, + "res/values/attr.xml": 2, + "res/values/strings.xml": 3, + "dir/res": 4, + "baz/res/values/attr.xml": 5, + "baz/res/values/strings.xml": 6, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + ps, err := makePartitionSession(tc.partitions, shardFn, order) + if err != nil { + t.Errorf("makePartitionSession(%v, %v, %d) got err: %v", tc.partitions, shard.FNV, 0, err) + return + } + for p, vps := range tc.pathVPsMap { + pi, err := res.ParsePath(p) + if err != nil { + t.Errorf("ParsePath(%s) got err: %v", p, err) + return + } + for fqn, p := range vps { + ps.CollectValues(&res.ValuesResource{Src: &pi, N: fqn, Payload: p}) + } + } + for p, as := range tc.pathRAMap { + pi, err := res.ParsePath(p) + if err != nil { + t.Errorf("ParsePath(%s) got err: %v", p, err) + return + } + for _, a := range as { + ps.CollectResourcesAttribute(&ResourcesAttribute{ResFile: &pi, Attribute: a}) + } + } + if err := ps.Close(); err != nil { + t.Errorf("partition Close() got err: %v", err) + return + } + + // validate data outputted to the partitions. + got := make(map[res.Type][][]string) + for rt, shards := range tc.partitions { + shardPaths := make([][]string, 0, len(shards)) + for _, shard := range shards { + br := bytes.NewReader(shard.(*bytes.Buffer).Bytes()) + rr, err := zip.NewReader(br, br.Size()) + if err != nil { + t.Errorf("NewReader(%v, %d) got err: %v", br, br.Size(), err) + return + } + paths := make([]string, 0, len(rr.File)) + for _, f := range rr.File { + c, err := readAll(f) + if err != nil { + t.Errorf("readAll got err: %v", err) + return + } + paths = append(paths, f.Name, c) + } + shardPaths = append(shardPaths, paths) + } + got[rt] = shardPaths + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("DeepEqual(\n%#v\n,\n%#v\n): returned false", got, tc.want) + } + }) + } +} + +func readAll(f *zip.File) (string, error) { + rc, err := f.Open() + if err != nil { + return "", fmt.Errorf("%q: Open got err: %v", f.Name, err) + } + defer rc.Close() + body, err := ioutil.ReadAll(rc) + if err != nil { + return "", fmt.Errorf("%q: ReadAll got err: %v", f.Name, err) + } + return string(body), nil +} diff --git a/src/tools/ak/bucketize/pipe.go b/src/tools/ak/bucketize/pipe.go new file mode 100644 index 0000000..7162232 --- /dev/null +++ b/src/tools/ak/bucketize/pipe.go @@ -0,0 +1,154 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bucketize + +import ( + "context" + "fmt" + "strings" + "sync" + + "src/tools/ak/res/res" +) + +type contextKey int + +const ( + ctxErr contextKey = 0 +) + +// errorf returns a formatted error with any context sensitive information prefixed to the error +func errorf(ctx context.Context, fmts string, a ...interface{}) error { + if s, ok := ctx.Value(ctxErr).(string); ok { + return fmt.Errorf(strings.Join([]string{s, fmts}, ""), a...) + } + return fmt.Errorf(fmts, a...) +} + +// prefixErr returns a context which adds a prefix to error messages. +func prefixErr(ctx context.Context, add string) context.Context { + if s, ok := ctx.Value(ctxErr).(string); ok { + return context.WithValue(ctx, ctxErr, strings.Join([]string{s, add}, "")) + } + return context.WithValue(ctx, ctxErr, add) +} + +func separatePathInfosByValues(ctx context.Context, pis []*res.PathInfo) (<-chan *res.PathInfo, <-chan *res.PathInfo) { + valuesPIC := make(chan *res.PathInfo) + nonValuesPIC := make(chan *res.PathInfo) + go func() { + defer close(valuesPIC) + defer close(nonValuesPIC) + for _, pi := range pis { + if pi.Type.Kind() == res.Value || pi.Type.Kind() == res.Both && strings.HasPrefix(pi.TypeDir, "values") { + select { + case valuesPIC <- pi: + case <-ctx.Done(): + return + } + } else { + select { + case nonValuesPIC <- pi: + case <-ctx.Done(): + return + } + } + } + }() + return valuesPIC, nonValuesPIC +} + +func mergeValuesResourceStreams(ctx context.Context, vrCs []<-chan *res.ValuesResource) <-chan *res.ValuesResource { + vrC := make(chan *res.ValuesResource) + var wg sync.WaitGroup + wg.Add(len(vrCs)) + output := func(c <-chan *res.ValuesResource) { + defer wg.Done() + for vr := range c { + select { + case vrC <- vr: + case <-ctx.Done(): + return + } + } + } + for _, c := range vrCs { + go output(c) + } + go func() { + wg.Wait() + close(vrC) + }() + return vrC +} + +func mergeResourcesAttributeStreams(ctx context.Context, raCs []<-chan *ResourcesAttribute) <-chan *ResourcesAttribute { + raC := make(chan *ResourcesAttribute) + var wg sync.WaitGroup + wg.Add(len(raCs)) + output := func(c <-chan *ResourcesAttribute) { + defer wg.Done() + for ra := range c { + select { + case raC <- ra: + case <-ctx.Done(): + return + } + } + } + for _, c := range raCs { + go output(c) + } + go func() { + wg.Wait() + close(raC) + }() + return raC +} + +// mergeErrStreams fans in multiple error streams into a single stream. +func mergeErrStreams(ctx context.Context, errCs []<-chan error) <-chan error { + errC := make(chan error) + var wg sync.WaitGroup + wg.Add(len(errCs)) + output := func(c <-chan error) { + defer wg.Done() + for e := range c { + select { + case errC <- e: + case <-ctx.Done(): + return + } + } + } + for _, rc := range errCs { + go output(rc) + } + go func() { + wg.Wait() + close(errC) + }() + return errC +} + +// sendErr attempts to send the provided error to the provided chan, however is the context is canceled, it will return false. +func sendErr(ctx context.Context, errC chan<- error, err error) bool { + select { + case <-ctx.Done(): + return false + case errC <- err: + return true + } +} diff --git a/src/tools/ak/bucketize/pipe_test.go b/src/tools/ak/bucketize/pipe_test.go new file mode 100644 index 0000000..81456ce --- /dev/null +++ b/src/tools/ak/bucketize/pipe_test.go @@ -0,0 +1,75 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bucketize + +import ( + "context" + "errors" + "reflect" + "testing" +) + +func TestPrefixErr(t *testing.T) { + tests := []struct { + ctx context.Context + fmts string + args []interface{} + want error + }{ + { + ctx: context.Background(), + fmts: "Hello world", + want: errors.New("Hello world"), + }, + { + ctx: prefixErr(context.Background(), "file: foo: "), + fmts: "Hello world: %d", + args: []interface{}{1}, + want: errors.New("file: foo: Hello world: 1"), + }, + { + ctx: prefixErr(prefixErr(context.Background(), "file: foo: "), "tag: <resources>: "), + fmts: "Hello world: %d", + args: []interface{}{1}, + want: errors.New("file: foo: tag: <resources>: Hello world: 1"), + }, + } + for _, tc := range tests { + got := errorf(tc.ctx, tc.fmts, tc.args...) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("Errorf(%v, %v, %v): %v wanted %v", tc.ctx, tc.fmts, tc.args, got, tc.want) + } + } +} + +func TestMergeErrStreams(t *testing.T) { + ctx := context.Background() + sendClose := func(e error, eC chan<- error) { + defer close(eC) + eC <- e + } + in1 := make(chan error) + in2 := make(chan error) + go sendClose(errors.New("hi"), in1) + go sendClose(errors.New("hello"), in2) + merged := mergeErrStreams(ctx, []<-chan error{in1, in2}) + var rcv []error + for r := range merged { + rcv = append(rcv, r) + } + if len(rcv) != 2 { + t.Errorf("got: %v on merged stream, wanted only 2 elements", rcv) + } +} diff --git a/src/tools/ak/compile/BUILD b/src/tools/ak/compile/BUILD new file mode 100644 index 0000000..6b7f246 --- /dev/null +++ b/src/tools/ak/compile/BUILD @@ -0,0 +1,38 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description: +# Package for compile module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_binary( + name = "compile_bin", + srcs = ["compile_bin.go"], + deps = [ + ":compile", + "//src/common/golang:flagfile", + ], +) + +go_library( + name = "compile", + srcs = [ + "compile.go", + ], + importpath = "src/tools/ak/compile/compile", + deps = [ + "//src/common/golang:ziputils", + "//src/tools/ak:types", + ], +) + +go_test( + name = "compile_test", + size = "small", + srcs = [ + "compile_test.go", + ], + embed = [":compile"], +) diff --git a/src/tools/ak/compile/compile.go b/src/tools/ak/compile/compile.go new file mode 100644 index 0000000..39d55be --- /dev/null +++ b/src/tools/ak/compile/compile.go @@ -0,0 +1,135 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package compile is a thin wrapper around aapt2 to compile android resources. +package compile + +import ( + "flag" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + "src/common/golang/ziputils" + "src/tools/ak/types" +) + +var ( + // Cmd defines the command to run compile + Cmd = types.Command{ + Init: Init, + Run: Run, + Desc: desc, + Flags: []string{ + "aapt2", + "in", + "out", + }, + } + + in string + aapt2 string + out string + + initOnce sync.Once + + dirPerm os.FileMode = 0755 + dirReplacer = strings.NewReplacer("sr-rLatn", "b+sr+Latn", "es-419", "b+es+419") + archiveSuffix = ".zip" +) + +// Init initializes compile. +func Init() { + initOnce.Do(func() { + flag.StringVar(&aapt2, "aapt2", "", "Path to the aapt2 binary.") + flag.StringVar(&in, "in", "", "Input res bucket/dir to compile.") + flag.StringVar(&out, "out", "", "The compiled resource archive.") + }) +} + +func desc() string { + return "Compile android resources directory." +} + +// Run is the entry point for compile. +func Run() { + if in == "" || aapt2 == "" || out == "" { + log.Fatal("Flags -in and -aapt2 and -out must be specified.") + } + + fi, err := os.Stat(in) + if err != nil { + log.Fatal(err) + } + + resDir := in + if !fi.IsDir() { + if strings.HasSuffix(resDir, archiveSuffix) { + // We are dealing with a resource archive. + td, err := ioutil.TempDir("", "-res") + if err != nil { + log.Fatal(err) + } + + resDir = filepath.Join(td, "res/") + if err := os.MkdirAll(resDir, dirPerm); err != nil { + log.Fatal(err) + } + if err := ziputils.Unzip(in, td); err != nil { + log.Fatal(err) + } + } else { + // We are compiling a single file, but we need to provide dir. + resDir = filepath.Dir(filepath.Dir(resDir)) + } + } + + if err := sanitizeDirs(resDir, dirReplacer); err != nil { + log.Fatal(err) + } + + cmd := exec.Command(aapt2, []string{"compile", "--legacy", "-o", out, "--dir", resDir}...) + if out, err := cmd.CombinedOutput(); err != nil { + log.Fatalf("error compiling resources for resource directory %s: %v\n%s", resDir, err, string(out)) + } +} + +// sanitizeDirs renames the directories that aapt is unable to parse +func sanitizeDirs(dir string, r *strings.Replacer) error { + src, err := os.Open(dir) + if err != nil { + return err + } + defer src.Close() + + fs, err := src.Readdir(-1) + if err != nil { + return err + } + + for _, f := range fs { + if f.Mode().IsDir() { + if qd := r.Replace(f.Name()); qd != f.Name() { + if err := os.Rename(filepath.Join(dir, f.Name()), filepath.Join(dir, qd)); err != nil { + return err + } + } + } + } + return nil +} diff --git a/src/tools/ak/compile/compile_bin.go b/src/tools/ak/compile/compile_bin.go new file mode 100644 index 0000000..c9efd76 --- /dev/null +++ b/src/tools/ak/compile/compile_bin.go @@ -0,0 +1,29 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The compile_bin is a command line tool to wrap around aapt2 rough edges. +package main + +import ( + "flag" + + _ "src/common/golang/flagfile" + "src/tools/ak/compile/compile" +) + +func main() { + compile.Init() + flag.Parse() + compile.Run() +} diff --git a/src/tools/ak/compile/compile_test.go b/src/tools/ak/compile/compile_test.go new file mode 100644 index 0000000..e639cb7 --- /dev/null +++ b/src/tools/ak/compile/compile_test.go @@ -0,0 +1,107 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compile + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" +) + +func TestDirReplacer(t *testing.T) { + + qualified := []string{ + "res/values-en-rGB/strings.xml", + "res/values-es-rMX/strings.xml", + "res/values-sr-rLatn/strings.xml", + "res/values-sr-rLatn-xhdpi/strings.xml", + "res/values-es-419/strings.xml", + "res/values-es-419-xhdpi/strings.xml"} + + expected := []string{ + "res/values-en-rGB/strings.xml", + "res/values-es-rMX/strings.xml", + "res/values-b+sr+Latn/strings.xml", + "res/values-b+sr+Latn-xhdpi/strings.xml", + "res/values-b+es+419/strings.xml", + "res/values-b+es+419-xhdpi/strings.xml"} + + var actual []string + for _, d := range qualified { + actual = append(actual, dirReplacer.Replace(d)) + } + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("dirReplacer.Replace(%v) = %v want %v", qualified, actual, expected) + } +} + +func TestSanitizeDirs(t *testing.T) { + base, err := ioutil.TempDir("", "res-") + dirs := []string{ + "values", + "values-bas-foo", + "values-foo-rNOTGOOD", + "values-foo-rNOBUENO-baz", + } + for _, dir := range dirs { + if err := os.Mkdir(filepath.Join(base, dir), 0777); err != nil { + t.Fatal(err) + } + } + + var expected sort.StringSlice + expected = append(expected, []string{ + "values", + "values-bas-foo", + "values-foo-rVERY-GOOD", + "values-foo-rMUCHO-BUENO-baz"}...) + + r := strings.NewReplacer("NOTGOOD", "VERY-GOOD", "NOBUENO", "MUCHO-BUENO") + if err := sanitizeDirs(base, r); err != nil { + t.Fatalf("sanitizeDirs(%s, %v) failed %v", base, r, err) + } + + src, err := os.Open(base) + if err != nil { + t.Fatal(err) + } + defer src.Close() + + fs, err := src.Readdir(-1) + if err != nil { + t.Fatal(err) + } + + actual := make(map[string]bool) + for _, f := range fs { + actual[f.Name()] = true + } + + for _, dir := range dirs { + expected := r.Replace(dir) + if expected != dir && actual[dir] { + t.Errorf("sanitizeDirs(%s) = %v got invalid dir %s. Expected %s ", base, actual, dir, expected) + } + if _, ok := actual[expected]; !ok { + t.Errorf("sanitizeDirs(%s) = %v missing dir %s", base, actual, expected) + } + } + +} diff --git a/src/tools/ak/dex/BUILD b/src/tools/ak/dex/BUILD new file mode 100644 index 0000000..80e4900 --- /dev/null +++ b/src/tools/ak/dex/BUILD @@ -0,0 +1,28 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Package for dex compilation module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_binary( + name = "dex_bin", + srcs = ["dex_bin.go"], + deps = [ + ":dex", + "//src/common/golang:flagfile", + ], +) + +go_library( + name = "dex", + srcs = ["dex.go"], + importpath = "src/tools/ak/dex/dex", + deps = [ + "//src/common/golang:flags", + "//src/common/golang:shard", + "//src/common/golang:ziputils", + "//src/tools/ak:types", + ], +) diff --git a/src/tools/ak/dex/dex.go b/src/tools/ak/dex/dex.go new file mode 100644 index 0000000..c024e44 --- /dev/null +++ b/src/tools/ak/dex/dex.go @@ -0,0 +1,281 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package dex provides a thin wrapper around d8 to handle corner cases +package dex + +import ( + "archive/zip" + "bufio" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + "src/common/golang/flags" + "src/common/golang/shard" + "src/common/golang/ziputils" + "src/tools/ak/types" +) + +var ( + // Cmd defines the command to run + Cmd = types.Command{ + Init: Init, + Run: Run, + Desc: desc, + Flags: []string{ + "desugar", + "android_jar", + "desugar_core_libs", + "classpath", + "d8", + "intermediate", + "in", + "out", + }, + } + + tmp struct { + Dir string + } + + // Flag variables + desugar, androidJar, d8, in string + classpaths, outs, outputDir flags.StringList + desugarCoreLibs, intermediate bool + + initOnce sync.Once +) + +// Init initializes manifest flags +func Init() { + initOnce.Do(func() { + flag.StringVar(&desugar, "desugar", "", "Path to desugar tool") + flag.StringVar(&androidJar, "android_jar", "", "Required for desugar, path to android.jar") + flag.Var(&classpaths, "classpath", "(Optional) Path to library resource(s) for desugar") + flag.BoolVar(&desugarCoreLibs, "desugar_core_libs", false, "Desugar Java 8 core libs, default false") + flag.StringVar(&d8, "d8", "", "Path to d8 dexer") + flag.BoolVar(&intermediate, "intermediate", false, "Compile for later merging, default false") + flag.StringVar(&in, "in", "", "Path to input") + flag.Var(&outs, "out", "Path to output, if more than one specified, output is sharded across files.") + }) +} + +func desc() string { + return "Dex converts Java byte code to Dex code." +} + +// Run is the main entry point +func Run() { + if desugar != "" && androidJar == "" { + log.Fatal("--android_jar is required for desugaring") + } + if d8 == "" || in == "" || outs == nil { + log.Fatal("Missing required flags. Must specify --d8 --in --out") + } + sc := len(outs) + if sc > 256 { + log.Fatalf("%d: is an unreasonable shard count (want [1 to 256])", sc) + } + + var err error + tmp.Dir, err = ioutil.TempDir("", "dex") + if err != nil { + log.Fatalf("Error creating temp dir: %v", err) + } + defer os.RemoveAll(tmp.Dir) + + notEmpty, err := hasCode(in) + if err != nil { + log.Fatal(err) + } + + if notEmpty { + jar := in + if desugar != "" { + jar = filepath.Join(tmp.Dir, "desugared.jar") + if err = desugarJar(in, jar); err != nil { + log.Fatalf("Error desugaring %v: %v", in, err) + } + } + if sc == 1 { + if err = dex(jar, outs[0]); err != nil { + log.Fatalf("Dex error: %v", err) + } + } else { + out := filepath.Join(tmp.Dir, "dexed.zip") + if err = dex(jar, out); err != nil { + log.Fatalf("Dex error: %v", err) + } + if err = zipShard(out, outs); err != nil { + log.Fatalf("ZipShard error: %v", err) + } + } + } else { + for _, out := range outs { + if err := ziputils.EmptyZip(out); err != nil { + log.Fatalf("Error creating empty zip archive: %v", err) + } + } + } +} + +func createFlagFile(args []string) (string, error) { + f, err := ioutil.TempFile(tmp.Dir, "flags") + if err != nil { + return "", err + } + for _, arg := range args { + if _, err := f.WriteString(arg + "\n"); err != nil { + return "", err + } + } + if err := f.Close(); err != nil { + return "", err + } + return f.Name(), nil +} + +func hasCode(f string) (bool, error) { + reader, err := zip.OpenReader(f) + if err != nil { + return false, fmt.Errorf("Opening zip %q failed: %v", f, err) + } + defer reader.Close() + + for _, file := range reader.File { + ext := filepath.Ext(file.Name) + if ext == ".class" || ext == ".dex" { + return true, nil + } + } + return false, nil +} + +func desugarJar(in, out string) error { + args := []string{ + "--input", + in, + "--bootclasspath_entry", + androidJar, + "--output", + out, + } + if desugarCoreLibs { + args = append(args, "--desugar_supported_core_libs") + } + for _, cp := range classpaths { + args = append(args, "--classpath_entry", cp) + } + return runCmd(desugar, args) +} + +func dex(in, out string) error { + args := []string{ + "--min-api", + "21", + "--no-desugaring", + "--output", + out, + } + if intermediate { + args = append(args, "--file-per-class") + args = append(args, "--intermediate") + } + args = append(args, in) + return runCmd(d8, args) +} + +func runCmd(cmd string, args []string) error { + flagFile, err := createFlagFile(args) + if err != nil { + return fmt.Errorf("Error creating flag file: %v", err) + } + output, err := exec.Command(cmd, "@"+flagFile).CombinedOutput() + if err != nil { + return fmt.Errorf("%v:\n%s", err, output) + } + return nil +} + +func zipShard(input string, outs []string) error { + zr, err := zip.OpenReader(input) + if err != nil { + return fmt.Errorf("%s: cannot open for input: %v", input, err) + } + defer zr.Close() + + if len(outs) < 2 { + log.Fatalf("Need at least two output shards)") + } + + zws := make([]*zip.Writer, len(outs)) + for i, out := range outs { + outDir := filepath.Dir(out) + if _, err := os.Stat(outDir); os.IsNotExist(err) { + if err := os.MkdirAll(outDir, 0755); err != nil { + return fmt.Errorf("%s: could not make dir: %v", input, outDir) + } + } + outF, err := os.Create(out) + if err != nil { + return fmt.Errorf("%s: could not create output file: %s %v", out, outDir, err) + } + w := bufio.NewWriterSize(outF, 2<<16) + zw := zip.NewWriter(w) + defer func() error { + if err := zw.Close(); err != nil { + return fmt.Errorf("%s: closing zip failed: %v", out, err) + } + if err := w.Flush(); err != nil { + return fmt.Errorf("%s: flushing output file failed: %v", out, err) + } + if err := outF.Close(); err != nil { + return fmt.Errorf("%s: closing output file failed: %v", out, err) + } + return nil + }() + zws[i] = zw + } + + err = shard.ZipShard(&zr.Reader, zws, shardFn) + if err != nil { + return fmt.Errorf("%s: sharder failed: %v", input, err) + } + return nil +} + +func shardFn(name string, shardCount int) int { + // Sharding function which ensures that a class and all its inner classes are + // placed in the same shard. An important side effect of this is that all D8 + // synthetics are in the same shard as their context, as a synthetic is named + // <context>$$ExternalSyntheticXXXN. + index := len(name) + if strings.HasSuffix(name, ".dex") { + index -= 4 + } else { + log.Fatalf("Name expected to end with '.dex', was: %s", name) + } + trimIndex := strings.IndexAny(name, "$-") + if trimIndex > -1 { + index = trimIndex + } + return shard.FNV(name[:index], shardCount) +} diff --git a/src/tools/ak/dex/dex_bin.go b/src/tools/ak/dex/dex_bin.go new file mode 100644 index 0000000..3114d70 --- /dev/null +++ b/src/tools/ak/dex/dex_bin.go @@ -0,0 +1,29 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// dex_bin is a command line tool that wraps d8 to handle corner cases. +package main + +import ( + "flag" + + _ "src/common/golang/flagfile" + "src/tools/ak/dex/dex" +) + +func main() { + dex.Init() + flag.Parse() + dex.Run() +} diff --git a/src/tools/ak/extractaar/BUILD b/src/tools/ak/extractaar/BUILD new file mode 100644 index 0000000..a4da9ce --- /dev/null +++ b/src/tools/ak/extractaar/BUILD @@ -0,0 +1,44 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description: +# Package for extractaar module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_library( + name = "extractaar", + srcs = [ + "buildozer.go", + "extractaar.go", + "validator.go", + ], + importpath = "src/tools/ak/extractaar/extractaar", + deps = [ + "//src/tools/ak:types", + ], +) + +go_binary( + name = "extractaar_bin", + srcs = ["extractaar_bin.go"], + deps = [ + ":extractaar", + "//src/common/golang:flagfile", + ], +) + +go_test( + name = "extractaar_test", + size = "small", + srcs = [ + "extractaar_test.go", + "validator_test.go", + ], + embed = [":extractaar"], + deps = [ + "@com_github_google_go_cmp//cmp:go_default_library", + "@com_github_google_go_cmp//cmp/cmpopts:go_default_library", + ], +) diff --git a/src/tools/ak/extractaar/buildozer.go b/src/tools/ak/extractaar/buildozer.go new file mode 100644 index 0000000..f7f52a5 --- /dev/null +++ b/src/tools/ak/extractaar/buildozer.go @@ -0,0 +1,48 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extractaar + +import ( + "fmt" + "strings" +) + +// BuildozerError represent a rule configuration error fixable with a buildozer command. +type BuildozerError struct { + Msg string + RuleAttr string + NewValue string +} + +func mergeBuildozerErrors(label string, errs []*BuildozerError) string { + var msg strings.Builder + msg.WriteString(fmt.Sprintf("error(s) found while processing aar '%s':\n", label)) + var buildozerCommand strings.Builder + buildozerCommand.WriteString("Use the following command to fix the target:\nbuildozer ") + useBuildozer := false + for _, err := range errs { + msg.WriteString(fmt.Sprintf("\t- %s\n", err.Msg)) + if err.NewValue != "" { + useBuildozer = true + buildozerCommand.WriteString(fmt.Sprintf("'set %s %s' ", err.RuleAttr, err.NewValue)) + } + } + buildozerCommand.WriteString(label) + + if useBuildozer { + msg.WriteString(buildozerCommand.String()) + } + return msg.String() +} diff --git a/src/tools/ak/extractaar/extractaar.go b/src/tools/ak/extractaar/extractaar.go new file mode 100644 index 0000000..2a431fa --- /dev/null +++ b/src/tools/ak/extractaar/extractaar.go @@ -0,0 +1,286 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package extractaar extracts files from an aar. +package extractaar + +import ( + "archive/zip" + "errors" + "flag" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + "sync" + + "src/tools/ak/types" +) + +// A tristate may be true, false, or unset +type tristate int + +func (t tristate) isSet() bool { + return t == tsTrue || t == tsFalse +} + +func (t tristate) value() bool { + return t == tsTrue +} + +const ( + tsTrue = 1 + tsFalse = -1 + + manifest = iota + res + assets +) + +var ( + // Cmd defines the command to run the extractor. + Cmd = types.Command{ + Init: Init, + Run: Run, + Desc: desc, + Flags: []string{ + "aar", "label", + "out_manifest", "out_res_dir", "out_assets_dir", + "has_res", "has_assets", + }, + } + + aar string + label string + outputManifest string + outputResDir string + outputAssetsDir string + hasRes int + hasAssets int + + initOnce sync.Once +) + +// Init initializes the extractor. +func Init() { + initOnce.Do(func() { + flag.StringVar(&aar, "aar", "", "Path to the aar") + flag.StringVar(&label, "label", "", "Target's label") + flag.StringVar(&outputManifest, "out_manifest", "", "Output manifest") + flag.StringVar(&outputResDir, "out_res_dir", "", "Output resources directory") + flag.StringVar(&outputAssetsDir, "out_assets_dir", "", "Output assets directory") + flag.IntVar(&hasRes, "has_res", 0, "Whether the aar has resources") + flag.IntVar(&hasAssets, "has_assets", 0, "Whether the aar has assets") + }) +} + +func desc() string { + return "Extracts files from an AAR" +} + +type aarFile struct { + path string + relPath string +} + +func (file *aarFile) String() string { + return fmt.Sprintf("%s:%s", file.path, file.relPath) +} + +type toCopy struct { + src string + dest string +} + +// Run runs the extractor +func Run() { + if err := doWork(aar, label, outputManifest, outputResDir, outputAssetsDir, hasRes, hasAssets); err != nil { + log.Fatal(err) + } +} + +func doWork(aar, label, outputManifest, outputResDir, outputAssetsDir string, hasRes, hasAssets int) error { + tmpDir, err := os.MkdirTemp("", "extractaar_") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + + files, err := extractAAR(aar, tmpDir) + if err != nil { + return err + } + + validators := map[int]validator{ + manifest: manifestValidator{dest: outputManifest}, + res: resourceValidator{dest: outputResDir, hasRes: tristate(hasRes), ruleAttr: "has_res"}, + assets: resourceValidator{dest: outputAssetsDir, hasRes: tristate(hasAssets), ruleAttr: "has_assets"}, + } + + var filesToCopy []*toCopy + var validationErrs []*BuildozerError + for fileType, files := range groupAARFiles(files) { + validatedFiles, err := validators[fileType].validate(files) + if err != nil { + validationErrs = append(validationErrs, err) + continue + } + filesToCopy = append(filesToCopy, validatedFiles...) + } + + if len(validationErrs) != 0 { + return errors.New(mergeBuildozerErrors(label, validationErrs)) + } + + for _, file := range filesToCopy { + if err := copyFile(file.src, file.dest); err != nil { + return err + } + } + + // TODO(ostonge): Add has_res/has_assets attr to avoid having to do this + // We need to create at least one file so that Bazel does not complain + // that the output tree artifact was not created. + if err := createIfEmpty(outputResDir, "res/values/empty.xml", "<resources/>"); err != nil { + return err + } + // aapt will ignore this file and not print an error message, because it + // thinks that it is a swap file + if err := createIfEmpty(outputAssetsDir, "assets/empty_asset_generated_by_bazel~", ""); err != nil { + return err + } + return nil +} + +func groupAARFiles(aarFiles []*aarFile) map[int][]*aarFile { + // Map of file type to channel of aarFile + filesMap := make(map[int][]*aarFile) + for _, fileType := range []int{manifest, res, assets} { + filesMap[fileType] = make([]*aarFile, 0) + } + + for _, file := range aarFiles { + if file.relPath == "AndroidManifest.xml" { + filesMap[manifest] = append(filesMap[manifest], file) + } else if strings.HasPrefix(file.relPath, "res"+string(os.PathSeparator)) { + filesMap[res] = append(filesMap[res], file) + } else if strings.HasPrefix(file.relPath, "assets"+string(os.PathSeparator)) { + filesMap[assets] = append(filesMap[assets], file) + } + // TODO(ostonge): support jar and aidl files + } + return filesMap +} + +func extractAAR(aar string, dest string) ([]*aarFile, error) { + reader, err := zip.OpenReader(aar) + if err != nil { + return nil, err + } + defer reader.Close() + + var files []*aarFile + for _, f := range reader.File { + if f.FileInfo().IsDir() { + continue + } + extractedPath := filepath.Join(dest, f.Name) + if err := extractFile(f, extractedPath); err != nil { + return nil, err + } + files = append(files, &aarFile{path: extractedPath, relPath: f.Name}) + } + return files, nil +} + +func extractFile(file *zip.File, dest string) error { + if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil { + return err + } + outFile, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE, file.Mode()) + if err != nil { + return err + } + defer outFile.Close() + + rc, err := file.Open() + if err != nil { + return err + } + defer rc.Close() + + _, err = io.Copy(outFile, rc) + if err != nil { + return err + } + return nil +} + +func copyFile(name, dest string) error { + in, err := os.Open(name) + if err != nil { + return err + } + defer in.Close() + + if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil { + return err + } + out, err := os.Create(dest) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + return nil +} + +func dirIsEmpty(dir string) (bool, error) { + f, err := os.Open(dir) + if os.IsNotExist(err) { + return true, nil + } + if err != nil { + return false, err + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, nil + } + return false, err +} + +// Create the file with the content if the directory is empty or does not exists +func createIfEmpty(dir, filename, content string) error { + isEmpty, err := dirIsEmpty(dir) + if err != nil { + return err + } + if isEmpty { + dest := filepath.Join(dir, filename) + if err := os.MkdirAll(filepath.Dir(dest), os.ModePerm); err != nil { + return err + } + return os.WriteFile(dest, []byte(content), 0644) + } + return nil +} diff --git a/src/tools/ak/extractaar/extractaar_bin.go b/src/tools/ak/extractaar/extractaar_bin.go new file mode 100644 index 0000000..bc7488b --- /dev/null +++ b/src/tools/ak/extractaar/extractaar_bin.go @@ -0,0 +1,29 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// extract_aar_bin is a command line tool that extracts files from an aar. +package main + +import ( + "flag" + + _ "src/common/golang/flagfile" + "src/tools/ak/extractaar/extractaar" +) + +func main() { + extractaar.Init() + flag.Parse() + extractaar.Run() +} diff --git a/src/tools/ak/extractaar/extractaar_test.go b/src/tools/ak/extractaar/extractaar_test.go new file mode 100644 index 0000000..e0595c2 --- /dev/null +++ b/src/tools/ak/extractaar/extractaar_test.go @@ -0,0 +1,73 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extractaar + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestGroupAARFiles(t *testing.T) { + tests := []struct { + name string + files []*aarFile + expectedMap map[int][]*aarFile + }{ + { + name: "empty aar", + files: []*aarFile{}, + expectedMap: map[int][]*aarFile{ + manifest: []*aarFile{}, + res: []*aarFile{}, + assets: []*aarFile{}, + }, + }, + { + name: "simple aar", + files: []*aarFile{ + &aarFile{relPath: "AndroidManifest.xml"}, + &aarFile{relPath: "res/values/strings.xml"}, + &aarFile{relPath: "lint.jar"}, + &aarFile{relPath: "proguard.txt"}, + &aarFile{relPath: "classes.jar"}, + &aarFile{relPath: "assetsdir/values.txt"}, + &aarFile{relPath: "libs/foo.jar"}, + &aarFile{relPath: "resource/some/file.txt"}, + &aarFile{relPath: "assets/some/asset.png"}, + }, + expectedMap: map[int][]*aarFile{ + manifest: []*aarFile{ + &aarFile{relPath: "AndroidManifest.xml"}, + }, + res: []*aarFile{ + &aarFile{relPath: "res/values/strings.xml"}, + }, + assets: []*aarFile{ + &aarFile{relPath: "assets/some/asset.png"}, + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + filesMap := groupAARFiles(tc.files) + if diff := cmp.Diff(tc.expectedMap, filesMap, cmp.AllowUnexported(aarFile{})); diff != "" { + t.Errorf("groupAARFiles(%v) returned diff (-want, +got):\n%v", tc.files, diff) + } + }) + } +} diff --git a/src/tools/ak/extractaar/validator.go b/src/tools/ak/extractaar/validator.go new file mode 100644 index 0000000..2a0d845 --- /dev/null +++ b/src/tools/ak/extractaar/validator.go @@ -0,0 +1,77 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extractaar + +import ( + "fmt" + "path/filepath" + "strings" +) + +func boolToString(b bool) string { + return strings.Title(fmt.Sprintf("%t", b)) +} + +type validator interface { + validate(files []*aarFile) ([]*toCopy, *BuildozerError) +} + +type manifestValidator struct { + dest string +} + +func (v manifestValidator) validate(files []*aarFile) ([]*toCopy, *BuildozerError) { + var filesToCopy []*toCopy + seen := false + for _, file := range files { + if seen { + return nil, &BuildozerError{Msg: "More than one manifest was found"} + } + seen = true + filesToCopy = append(filesToCopy, &toCopy{src: file.path, dest: v.dest}) + } + if !seen { + return nil, &BuildozerError{Msg: "No manifest was found"} + } + return filesToCopy, nil +} + +type resourceValidator struct { + dest string + ruleAttr string + hasRes tristate +} + +func (v resourceValidator) validate(files []*aarFile) ([]*toCopy, *BuildozerError) { + var filesToCopy []*toCopy + seen := false + for _, file := range files { + seen = true + filesToCopy = append(filesToCopy, + &toCopy{src: file.path, dest: filepath.Join(v.dest, file.relPath)}, + ) + } + if v.hasRes.isSet() { + if seen != v.hasRes.value() { + var not string + if !seen { + not = "not " + } + msg := fmt.Sprintf("%s attribute is %s, but files were %sfound", v.ruleAttr, boolToString(v.hasRes.value()), not) + return nil, &BuildozerError{Msg: msg, RuleAttr: v.ruleAttr, NewValue: boolToString(seen)} + } + } + return filesToCopy, nil +} diff --git a/src/tools/ak/extractaar/validator_test.go b/src/tools/ak/extractaar/validator_test.go new file mode 100644 index 0000000..24b7003 --- /dev/null +++ b/src/tools/ak/extractaar/validator_test.go @@ -0,0 +1,175 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package extractaar + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func TestValidateManifest(t *testing.T) { + tests := []struct { + name string + files []*aarFile + dest string + expectedFiles []*toCopy + }{ + { + name: "one manifest", + files: []*aarFile{ + &aarFile{path: "/tmp/aar/AndroidManifest.xml"}, + }, + dest: "/dest/outputManifest.xml", + expectedFiles: []*toCopy{ + &toCopy{src: "/tmp/aar/AndroidManifest.xml", dest: "/dest/outputManifest.xml"}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + validator := manifestValidator{dest: tc.dest} + files, err := validator.validate(tc.files) + if err != nil { + t.Fatalf("manifestValidator.validate(%s) unexpected error: %v", tc.files, err) + } + if diff := cmp.Diff(tc.expectedFiles, files, cmp.AllowUnexported(toCopy{})); diff != "" { + t.Errorf("manifestValidator.validate(%s) returned diff (-want, +got):\n%v", tc.files, diff) + } + }) + } +} + +func TestValidateManifestError(t *testing.T) { + tests := []struct { + name string + files []*aarFile + }{ + { + name: "no manifest", + files: []*aarFile{}, + }, + { + name: "multiple manifests", + files: []*aarFile{ + &aarFile{path: "/tmp/aar/AndroidManifest.xml"}, + &aarFile{path: "/tmp/aar/SecondAndroidManifest.xml"}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + validator := manifestValidator{} + if _, err := validator.validate(tc.files); err == nil { + t.Errorf("manifestValidator.validate(%s) expected error but test succeeded: %v", tc.files, err) + } + }) + } +} + +func TestValidateResources(t *testing.T) { + tests := []struct { + name string + files []*aarFile + dest string + hasRes tristate + expectedFiles []*toCopy + }{ + { + name: "has resources with valid hasRes attribute", + files: []*aarFile{ + &aarFile{path: "/tmp/aar/res/values/strings.xml", relPath: "res/values/strings.xml"}, + &aarFile{path: "/tmp/aar/res/layout/activity.xml", relPath: "res/layout/activity.xml"}, + }, + hasRes: tristate(1), + dest: "/dest/outputres", + expectedFiles: []*toCopy{ + &toCopy{src: "/tmp/aar/res/values/strings.xml", dest: "/dest/outputres/res/values/strings.xml"}, + &toCopy{src: "/tmp/aar/res/layout/activity.xml", dest: "/dest/outputres/res/layout/activity.xml"}, + }, + }, + { + name: "does not have resources with valid hasRes attribute", + files: []*aarFile{}, + hasRes: tristate(0), + dest: "/dest/outputres", + expectedFiles: nil, + }, + { + name: "no resources and checks disabled", + files: []*aarFile{}, + hasRes: tristate(-1), + dest: "/dest/outputres", + expectedFiles: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + validator := resourceValidator{dest: tc.dest, hasRes: tc.hasRes} + files, err := validator.validate(tc.files) + if err != nil { + t.Fatalf("resourceValidator.validate(%s) unexpected error: %v", tc.files, err) + } + if diff := cmp.Diff(tc.expectedFiles, files, cmp.AllowUnexported(toCopy{})); diff != "" { + t.Errorf("resourceValidator.validate(%s) returned diff (-want, +got):\n%v", tc.files, diff) + } + }) + } +} + +func TestValidateResourcesError(t *testing.T) { + tests := []struct { + name string + files []*aarFile + hasRes tristate + ruleAttr string + expectedError *BuildozerError + }{ + { + name: "has resources with invalid hasRes attribute", + files: []*aarFile{ + &aarFile{path: "/tmp/aar/res/values/strings.xml", relPath: "res/values/strings.xml"}, + &aarFile{path: "/tmp/aar/res/layout/activity.xml", relPath: "res/layout/activity.xml"}, + }, + hasRes: tristate(-1), + ruleAttr: "test", + expectedError: &BuildozerError{RuleAttr: "test", NewValue: "True"}, + }, + { + name: "no resources with invalid hasRes attribute", + files: []*aarFile{}, + hasRes: tristate(1), + ruleAttr: "test", + expectedError: &BuildozerError{RuleAttr: "test", NewValue: "False"}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + validator := resourceValidator{ruleAttr: tc.ruleAttr, hasRes: tc.hasRes} + _, err := validator.validate(tc.files) + if err == nil { + t.Fatalf("resourceValidator.validate(%s) expected error but test succeeded: %v", tc.files, err) + } + if diff := cmp.Diff(tc.expectedError, err, cmpopts.IgnoreFields(BuildozerError{}, "Msg")); diff != "" { + t.Errorf("resourceValidator.validate(%s) returned diff (-want, +got):\n%v", tc.files, diff) + } + }) + } +} diff --git a/src/tools/ak/finalrjar/BUILD b/src/tools/ak/finalrjar/BUILD new file mode 100644 index 0000000..db487da --- /dev/null +++ b/src/tools/ak/finalrjar/BUILD @@ -0,0 +1,35 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description: +# Package for final R.jar module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_library( + name = "finalrjar", + srcs = ["finalrjar.go"], + importpath = "src/tools/ak/finalrjar/finalrjar", + deps = [ + "//src/common/golang:ziputils", + "//src/tools/ak:types", + ], +) + +go_binary( + name = "finalrjar_bin", + srcs = ["finalrjar_bin.go"], + deps = [ + ":finalrjar", + "//src/common/golang:flagfile", + ], +) + +go_test( + name = "finalrjar_test", + size = "small", + srcs = ["finalrjar_test.go"], + embed = [":finalrjar"], + deps = ["@com_github_google_go_cmp//cmp:go_default_library"], +) diff --git a/src/tools/ak/finalrjar/finalrjar.go b/src/tools/ak/finalrjar/finalrjar.go new file mode 100644 index 0000000..89d73a4 --- /dev/null +++ b/src/tools/ak/finalrjar/finalrjar.go @@ -0,0 +1,451 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package finalrjar generates a valid final R.jar. +package finalrjar + +import ( + "archive/zip" + "bufio" + "flag" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + + "src/common/golang/ziputils" + "src/tools/ak/types" +) + +var ( + // Cmd defines the command. + Cmd = types.Command{ + Init: Init, + Run: Run, + Desc: desc, + Flags: []string{"package", "r_txts", "out_r_java", "root_pkg", "jdk", "jartool", "target_label"}, + } + + // Variables to hold flag values. + pkg string + rtxts string + outputRJar string + rootPackage string + jdk string + jartool string + targetLabel string + + initOnce sync.Once + + resTypes = []string{ + "anim", + "animator", + "array", + "attr", + "^attr-private", + "bool", + "color", + "configVarying", + "dimen", + "drawable", + "fraction", + "font", + "id", + "integer", + "interpolator", + "layout", + "menu", + "mipmap", + "navigation", + "plurals", + "raw", + "string", + "style", + "styleable", + "transition", + "xml", + } + + javaReserved = map[string]bool{ + "abstract": true, + "assert": true, + "boolean": true, + "break": true, + "byte": true, + "case": true, + "catch": true, + "char": true, + "class": true, + "const": true, + "continue": true, + "default": true, + "do": true, + "double": true, + "else": true, + "enum": true, + "extends": true, + "false": true, + "final": true, + "finally": true, + "float": true, + "for": true, + "goto": true, + "if": true, + "implements": true, + "import": true, + "instanceof": true, + "int": true, + "interface": true, + "long": true, + "native": true, + "new": true, + "null": true, + "package": true, + "private": true, + "protected": true, + "public": true, + "return": true, + "short": true, + "static": true, + "strictfp": true, + "super": true, + "switch": true, + "synchronized": true, + "this": true, + "throw": true, + "throws": true, + "transient": true, + "true": true, + "try": true, + "void": true, + "volatile": true, + "while": true} +) + +type rtxtFile interface { + io.Reader + io.Closer +} + +type resource struct { + ID string + resType string + varType string +} + +func (r *resource) String() string { + return fmt.Sprintf("{%s %s %s}", r.varType, r.resType, r.ID) +} + +// Init initializes finalrjar action. +func Init() { + initOnce.Do(func() { + flag.StringVar(&pkg, "package", "", "Package for the R.jar") + flag.StringVar(&rtxts, "r_txts", "", "Comma separated list of R.txt files") + flag.StringVar(&outputRJar, "out_rjar", "", "Output R.jar path") + flag.StringVar(&rootPackage, "root_pkg", "mi.rjava", "Package to use for root R.java") + flag.StringVar(&jdk, "jdk", "", "Jdk path") + flag.StringVar(&jartool, "jartool", "", "Jartool path") + flag.StringVar(&targetLabel, "target_label", "", "The target label") + }) +} + +func desc() string { + return "finalrjar creates a platform conform R.jar from R.txt files" +} + +// Run is the entry point for finalrjar. Will exit on error. +func Run() { + if err := doWork(pkg, rtxts, outputRJar, rootPackage, jdk, jartool, targetLabel); err != nil { + log.Fatalf("error creating final R.jar: %v", err) + } +} + +func doWork(pkg, rtxts, outputRJar, rootPackage, jdk, jartool, targetLabel string) error { + pkgParts := strings.Split(pkg, ".") + // Check if the package is invalid. + if hasJavaReservedWord(pkgParts) { + return ziputils.EmptyZip(outputRJar) + } + + rtxtFiles, err := openRtxts(strings.Split(rtxts, ",")) + if err != nil { + return err + } + + resC := getIds(rtxtFiles) + // Resources need to be grouped by type to write the R.java classes. + resMap := groupResByType(resC) + + srcDir, err := os.MkdirTemp("", "rjar") + if err != nil { + return err + } + defer os.RemoveAll(srcDir) + + rJava, outRJava, err := createTmpRJava(srcDir, pkgParts) + if err != nil { + return err + } + defer outRJava.Close() + + rootPkgParts := strings.Split(rootPackage, ".") + rootRJava, outRootRJava, err := createTmpRJava(srcDir, rootPkgParts) + if err != nil { + return err + } + defer outRootRJava.Close() + + if err := writeRJavas(outRJava, outRootRJava, resMap, pkg, rootPackage); err != nil { + return err + } + + fullRJar := filepath.Join(srcDir, "R.jar") + if err := compileRJar([]string{rJava, rootRJava}, fullRJar, jdk, jartool, targetLabel); err != nil { + return err + } + + return filterZip(fullRJar, outputRJar, filepath.Join(rootPkgParts...)) +} + +func getIds(rtxtFiles []rtxtFile) <-chan *resource { + // Sending all res to the same channel, even duplicates. + resC := make(chan *resource) + var wg sync.WaitGroup + wg.Add(len(rtxtFiles)) + + for _, file := range rtxtFiles { + go func(file rtxtFile) { + defer wg.Done() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + // Each line is in the following format: + // [int|int[]] resType resID value + // Ex: int anim abc_fade_in 0 + parts := strings.Split(line, " ") + if len(parts) < 3 { + continue + } + // Aapt2 will sometime add resources containing the char '$'. + // Those should be ignored - they are derived from an actual resource. + if strings.Contains(parts[2], "$") { + continue + } + resC <- &resource{ID: parts[2], resType: parts[1], varType: parts[0]} + } + file.Close() + }(file) + } + + go func() { + wg.Wait() + close(resC) + }() + + return resC +} + +func groupResByType(resC <-chan *resource) map[string][]*resource { + // Set of resType.ID seen to ignore duplicates from different R.txt files. + // Resources of different types can have the same ID, so we merge the values + // to get a unique string. Ex: integer.btn_background_alpa + seen := make(map[string]bool) + + // Map of resource type to list of resources. + resMap := make(map[string][]*resource) + for res := range resC { + uniqueID := fmt.Sprintf("%s.%s", res.resType, res.ID) + if _, ok := seen[uniqueID]; ok { + continue + } + seen[uniqueID] = true + resMap[res.resType] = append(resMap[res.resType], res) + } + return resMap +} + +func writeRJavas(outRJava, outRootRJava io.Writer, resMap map[string][]*resource, pkg, rootPackage string) error { + // The R.java points to the same resources ID in the root R.java. + // The root R.java uses 0 or null for simplicity and does not use final fields to avoid inlining. + // That way we can strip it from the compiled R.jar later and replace it with the real one. + rJavaWriter := bufio.NewWriter(outRJava) + rJavaWriter.WriteString(fmt.Sprintf("package %s;\n", pkg)) + rJavaWriter.WriteString("public class R {\n") + rootRJavaWriter := bufio.NewWriter(outRootRJava) + rootRJavaWriter.WriteString(fmt.Sprintf("package %s;\n", rootPackage)) + rootRJavaWriter.WriteString("public class R {\n") + + for _, resType := range resTypes { + if resources, ok := resMap[resType]; ok { + rJavaWriter.WriteString(fmt.Sprintf(" public static class %s {\n", resType)) + rootRJavaWriter.WriteString(fmt.Sprintf(" public static class %s {\n", resType)) + rootID := fmt.Sprintf("%s.R.%s.", rootPackage, resType) + + // Sorting resources before writing to class + sort.Slice(resources, func(i, j int) bool { + return resources[i].ID < resources[j].ID + }) + for _, res := range resources { + defaultValue := "0" + if res.varType == "int[]" { + defaultValue = "null" + } + rJavaWriter.WriteString(fmt.Sprintf(" public static final %s %s=%s%s;\n", res.varType, res.ID, rootID, res.ID)) + rootRJavaWriter.WriteString(fmt.Sprintf(" public static %s %s=%s;\n", res.varType, res.ID, defaultValue)) + } + rJavaWriter.WriteString(" }\n") + rootRJavaWriter.WriteString(" }\n") + } + } + rJavaWriter.WriteString("}\n") + rootRJavaWriter.WriteString("}\n") + + if err := rJavaWriter.Flush(); err != nil { + return err + } + return rootRJavaWriter.Flush() +} + +func createTmpRJava(srcDir string, pkgParts []string) (string, *os.File, error) { + pkgDir := filepath.Join(append([]string{srcDir}, pkgParts...)...) + if err := os.MkdirAll(pkgDir, 0777); err != nil { + return "", nil, err + } + file := filepath.Join(pkgDir, "R.java") + out, err := os.Create(file) + return file, out, err +} + +func openRtxts(filePaths []string) ([]rtxtFile, error) { + var rtxtFiles []rtxtFile + for _, filePath := range filePaths { + in, err := os.Open(filePath) + if err != nil { + return nil, err + } + rtxtFiles = append(rtxtFiles, in) + } + return rtxtFiles, nil + +} + +func createOuput(output string) (io.Writer, error) { + if _, err := os.Lstat(output); err == nil { + if err := os.Remove(output); err != nil { + return nil, err + } + } + if err := os.MkdirAll(filepath.Dir(output), 0777); err != nil { + return nil, err + } + + return os.Create(output) +} + +func filterZip(in, output, ignorePrefix string) error { + w, err := createOuput(output) + if err != nil { + return err + } + + zipOut := zip.NewWriter(w) + defer zipOut.Close() + + zipIn, err := zip.OpenReader(in) + if err != nil { + return err + } + defer zipIn.Close() + + for _, f := range zipIn.File { + // Ignoring the dummy root R.java. + if strings.HasPrefix(f.Name, ignorePrefix) { + continue + } + reader, err := f.Open() + if err != nil { + return err + } + if err := writeToZip(zipOut, reader, f.Name, f.Method); err != nil { + return err + } + if err := reader.Close(); err != nil { + return err + } + } + return nil +} + +func writeToZip(out *zip.Writer, in io.Reader, name string, method uint16) error { + writer, err := out.CreateHeader(&zip.FileHeader{ + Name: name, + Method: method, + }) + if err != nil { + return err + } + + if !strings.HasSuffix(name, "/") { + if _, err := io.Copy(writer, in); err != nil { + return err + } + } + return nil +} + +func compileRJar(srcs []string, rjar, jdk, jartool string, targetLabel string) error { + control, err := os.CreateTemp("", "control") + if err != nil { + return err + } + defer os.Remove(control.Name()) + + args := []string{"--javacopts", + "-source", "8", + "-target", "8", + "-nowarn", "--", "--sources"} + args = append(args, srcs...) + args = append(args, + "--strict_java_deps", "ERROR", + "--output", rjar) + if len(targetLabel) > 0 { + args = append(args, "--target_label", targetLabel) + } + if _, err := fmt.Fprint(control, strings.Join(args, "\n")); err != nil { + return err + } + if err := control.Sync(); err != nil { + return err + } + c, err := exec.Command(jdk, "-jar", jartool, fmt.Sprintf("@%s", control.Name())).CombinedOutput() + if err != nil { + return fmt.Errorf("error compiling R.jar (using command: %s): %v", c, err) + } + return nil +} + +func hasJavaReservedWord(parts []string) bool { + for _, p := range parts { + if javaReserved[p] { + return true + } + } + return false +} diff --git a/src/tools/ak/finalrjar/finalrjar_bin.go b/src/tools/ak/finalrjar/finalrjar_bin.go new file mode 100644 index 0000000..eff376c --- /dev/null +++ b/src/tools/ak/finalrjar/finalrjar_bin.go @@ -0,0 +1,29 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// finalrjar_bin is the command line tool to create an R.jar +package main + +import ( + "flag" + + _ "src/common/golang/flagfile" + "src/tools/ak/finalrjar/finalrjar" +) + +func main() { + finalrjar.Init() + flag.Parse() + finalrjar.Run() +} diff --git a/src/tools/ak/finalrjar/finalrjar_test.go b/src/tools/ak/finalrjar/finalrjar_test.go new file mode 100644 index 0000000..fffd1b9 --- /dev/null +++ b/src/tools/ak/finalrjar/finalrjar_test.go @@ -0,0 +1,366 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package finalrjar generates a valid final R.jar. +package finalrjar + +import ( + "bytes" + "sort" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +type fakeFile struct { + reader *strings.Reader +} + +func (f fakeFile) Read(b []byte) (int, error) { + return f.reader.Read(b) +} + +func (f fakeFile) Close() error { + return nil +} + +func TestGetIds(t *testing.T) { + tests := []struct { + name string + rtxtFiles []*strings.Reader + expectedResources []*resource + }{ + { + name: "one R.txt", + rtxtFiles: []*strings.Reader{ + strings.NewReader( + `int anim abc_fade_in 0 +int anim abc_fade_out 0 +int attr actionBarDivider 0 +int bool abc_action_bar_embed_tabs 0 +int color abc_background_cache_hint_selector_material_dark 0 +int[] color abc_background_cache_hint_selector_material_light 0 +int color abc_btn_colored_borderless_text_material 0 +int dimen tooltip_y_offset_non_touch 0 +int dimen $avd_hide_password__0 0 +int[] dimen tooltip_y_offset_touch 0 +int drawable abc_ab_share_pack_mtrl_alpha 0`), + }, + expectedResources: []*resource{ + &resource{ID: "abc_ab_share_pack_mtrl_alpha", resType: "drawable", varType: "int"}, + &resource{ID: "abc_action_bar_embed_tabs", resType: "bool", varType: "int"}, + &resource{ID: "abc_background_cache_hint_selector_material_dark", resType: "color", varType: "int"}, + &resource{ID: "abc_background_cache_hint_selector_material_light", resType: "color", varType: "int[]"}, + &resource{ID: "abc_btn_colored_borderless_text_material", resType: "color", varType: "int"}, + &resource{ID: "abc_fade_in", resType: "anim", varType: "int"}, + &resource{ID: "abc_fade_out", resType: "anim", varType: "int"}, + &resource{ID: "actionBarDivider", resType: "attr", varType: "int"}, + &resource{ID: "tooltip_y_offset_non_touch", resType: "dimen", varType: "int"}, + &resource{ID: "tooltip_y_offset_touch", resType: "dimen", varType: "int[]"}, + }, + }, + { + name: "multiple R.txt files", + rtxtFiles: []*strings.Reader{ + strings.NewReader( + `int styleable toolbar_logo 0 +int[] style widget_appcompat_dark 0`), + strings.NewReader( + `int layout custom_dialog 0 +int interpolator btn_checkbox 0`), + strings.NewReader( + `int id view_tree 0 +int integer cancel_button_image_alpha 0`), + }, + expectedResources: []*resource{ + &resource{ID: "btn_checkbox", resType: "interpolator", varType: "int"}, + &resource{ID: "cancel_button_image_alpha", resType: "integer", varType: "int"}, + &resource{ID: "custom_dialog", resType: "layout", varType: "int"}, + &resource{ID: "toolbar_logo", resType: "styleable", varType: "int"}, + &resource{ID: "view_tree", resType: "id", varType: "int"}, + &resource{ID: "widget_appcompat_dark", resType: "style", varType: "int[]"}, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + rtxts := make([]rtxtFile, 0, len(tc.rtxtFiles)) + for _, f := range tc.rtxtFiles { + file := fakeFile{reader: f} + file.reader.Seek(0, 0) + rtxts = append(rtxts, file) + } + + resC := getIds(rtxts) + receivedResources := make([]*resource, 0) + for res := range resC { + receivedResources = append(receivedResources, res) + } + sort.Slice(receivedResources, func(i, j int) bool { + return receivedResources[i].ID < receivedResources[j].ID + }) + + if diff := cmp.Diff(tc.expectedResources, receivedResources, cmp.AllowUnexported(resource{})); diff != "" { + t.Errorf("getIds(%v) returned diff (-want, +got):\n%v", rtxts, diff) + } + }) + } + +} + +func TestSortResByType(t *testing.T) { + tests := []struct { + name string + resources []*resource + expectedMap map[string][]*resource + }{ + { + name: "simple list of resources", + resources: []*resource{ + &resource{ID: "btn_checkbox", resType: "interpolator", varType: "int"}, + &resource{ID: "cancel_button_image_alpha", resType: "integer", varType: "int"}, + &resource{ID: "custom_dialog", resType: "id", varType: "int"}, + &resource{ID: "toolbar_logo", resType: "interpolator", varType: "int"}, + &resource{ID: "view_tree", resType: "id", varType: "int"}, + &resource{ID: "widget_appcompat_dark", resType: "layout", varType: "int[]"}, + }, + expectedMap: map[string][]*resource{ + "interpolator": []*resource{ + &resource{ID: "btn_checkbox", resType: "interpolator", varType: "int"}, + &resource{ID: "toolbar_logo", resType: "interpolator", varType: "int"}, + }, + "integer": []*resource{ + &resource{ID: "cancel_button_image_alpha", resType: "integer", varType: "int"}, + }, + "id": []*resource{ + &resource{ID: "custom_dialog", resType: "id", varType: "int"}, + &resource{ID: "view_tree", resType: "id", varType: "int"}, + }, + "layout": []*resource{ + &resource{ID: "widget_appcompat_dark", resType: "layout", varType: "int[]"}, + }, + }, + }, + { + name: "list of resources with duplicates", + resources: []*resource{ + &resource{ID: "btn_checkbox", resType: "interpolator", varType: "int"}, + &resource{ID: "btn_checkbox", resType: "interpolator", varType: "int"}, + &resource{ID: "cancel_button_image_alpha", resType: "integer", varType: "int"}, + &resource{ID: "custom_dialog", resType: "id", varType: "int"}, + &resource{ID: "toolbar_logo", resType: "interpolator", varType: "int"}, + &resource{ID: "toolbar_logo", resType: "attr", varType: "int"}, + &resource{ID: "view_tree", resType: "id", varType: "int"}, + &resource{ID: "cancel_button_image_alpha", resType: "integer", varType: "int"}, + &resource{ID: "widget_appcompat_dark", resType: "layout", varType: "int[]"}, + }, + expectedMap: map[string][]*resource{ + "attr": []*resource{ + &resource{ID: "toolbar_logo", resType: "attr", varType: "int"}, + }, + "interpolator": []*resource{ + &resource{ID: "btn_checkbox", resType: "interpolator", varType: "int"}, + &resource{ID: "toolbar_logo", resType: "interpolator", varType: "int"}, + }, + "integer": []*resource{ + &resource{ID: "cancel_button_image_alpha", resType: "integer", varType: "int"}, + }, + "id": []*resource{ + &resource{ID: "custom_dialog", resType: "id", varType: "int"}, + &resource{ID: "view_tree", resType: "id", varType: "int"}, + }, + "layout": []*resource{ + &resource{ID: "widget_appcompat_dark", resType: "layout", varType: "int[]"}, + }, + }, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + resC := make(chan *resource) + go func() { + for _, res := range tc.resources { + resC <- res + } + close(resC) + }() + resMap := groupResByType(resC) + + if diff := cmp.Diff(tc.expectedMap, resMap, cmp.AllowUnexported(resource{})); diff != "" { + t.Errorf("groupResByType(%v) returned diff (-want, +got):\n%v", tc.resources, diff) + } + }) + } + +} + +func TestWriteRJavas(t *testing.T) { + tests := []struct { + name string + resMap map[string][]*resource + pkg string + rootPackage string + expectedRJava string + expectedRootRJava string + }{ + { + name: "simple map of resources", + resMap: map[string][]*resource{ + "interpolator": []*resource{ + &resource{ID: "btn_checkbox", resType: "interpolator", varType: "int"}, + &resource{ID: "toolbar_logo", resType: "interpolator", varType: "int"}, + }, + "integer": []*resource{ + &resource{ID: "cancel_button_image_alpha", resType: "integer", varType: "int"}, + }, + "id": []*resource{ + &resource{ID: "view_tree", resType: "id", varType: "int"}, + &resource{ID: "custom_dialog", resType: "id", varType: "int"}, + }, + "layout": []*resource{ + &resource{ID: "widget_appcompat_dark", resType: "layout", varType: "int[]"}, + }, + }, + pkg: "com.google.android.apps.sample", + rootPackage: "mi.rjava", + expectedRJava: `package com.google.android.apps.sample; +public class R { + public static class id { + public static final int custom_dialog=mi.rjava.R.id.custom_dialog; + public static final int view_tree=mi.rjava.R.id.view_tree; + } + public static class integer { + public static final int cancel_button_image_alpha=mi.rjava.R.integer.cancel_button_image_alpha; + } + public static class interpolator { + public static final int btn_checkbox=mi.rjava.R.interpolator.btn_checkbox; + public static final int toolbar_logo=mi.rjava.R.interpolator.toolbar_logo; + } + public static class layout { + public static final int[] widget_appcompat_dark=mi.rjava.R.layout.widget_appcompat_dark; + } +} +`, + expectedRootRJava: `package mi.rjava; +public class R { + public static class id { + public static int custom_dialog=0; + public static int view_tree=0; + } + public static class integer { + public static int cancel_button_image_alpha=0; + } + public static class interpolator { + public static int btn_checkbox=0; + public static int toolbar_logo=0; + } + public static class layout { + public static int[] widget_appcompat_dark=null; + } +} +`, + }, + { + name: "with empty class", + resMap: map[string][]*resource{ + "interpolator": []*resource{ + &resource{ID: "toolbar_logo", resType: "interpolator", varType: "int"}, + &resource{ID: "btn_checkbox", resType: "interpolator", varType: "int"}, + }, + "integer": []*resource{ + &resource{ID: "cancel_button_image_alpha", resType: "integer", varType: "int"}, + }, + "layout": []*resource{ + &resource{ID: "widget_appcompat_dark", resType: "layout", varType: "int[]"}, + }, + }, + pkg: "com.google.android.apps.empty", + rootPackage: "mi.rjava", + expectedRJava: `package com.google.android.apps.empty; +public class R { + public static class integer { + public static final int cancel_button_image_alpha=mi.rjava.R.integer.cancel_button_image_alpha; + } + public static class interpolator { + public static final int btn_checkbox=mi.rjava.R.interpolator.btn_checkbox; + public static final int toolbar_logo=mi.rjava.R.interpolator.toolbar_logo; + } + public static class layout { + public static final int[] widget_appcompat_dark=mi.rjava.R.layout.widget_appcompat_dark; + } +} +`, + expectedRootRJava: `package mi.rjava; +public class R { + public static class integer { + public static int cancel_button_image_alpha=0; + } + public static class interpolator { + public static int btn_checkbox=0; + public static int toolbar_logo=0; + } + public static class layout { + public static int[] widget_appcompat_dark=null; + } +} +`, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var rJavaBuffer bytes.Buffer + var rootRJavaBuffer bytes.Buffer + if err := writeRJavas(&rJavaBuffer, &rootRJavaBuffer, tc.resMap, tc.pkg, tc.rootPackage); err != nil { + t.Fatalf("writeRJavas(%v, %s, %s) unexpected error: %v", tc.resMap, tc.pkg, tc.rootPackage, err) + } + if diff := cmp.Diff(tc.expectedRJava, rJavaBuffer.String()); diff != "" { + t.Errorf("writeRJavas(%v, %s, %s) returned diff for R.java (-want, +got):\n%v", tc.resMap, tc.pkg, tc.rootPackage, diff) + } + if diff := cmp.Diff(tc.expectedRootRJava, rootRJavaBuffer.String()); diff != "" { + t.Errorf("writeRJavas(%v, %s, %s) returned diff for root R.java(-want, +got):\n%v", tc.resMap, tc.pkg, tc.rootPackage, diff) + } + }) + } + +} + +func TestHasReservedKeywords(t *testing.T) { + tests := []struct { + name string + pkg string + expected bool + }{ + { + name: "valid package", + pkg: "com.google.android.apps.sampleapp.lib", + expected: false, + }, + { + name: "valid package", + pkg: "com.google.android.static.sampleapp.lib", + expected: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + pkgParts := strings.Split(tc.pkg, ".") + invalid := hasJavaReservedWord(pkgParts) + if invalid != tc.expected { + t.Errorf("hasJavaReservedWord(%v) returned %v, want %v", pkgParts, invalid, tc.expected) + } + }) + } + +} diff --git a/src/tools/ak/generatemanifest/BUILD b/src/tools/ak/generatemanifest/BUILD new file mode 100644 index 0000000..2da8e4a --- /dev/null +++ b/src/tools/ak/generatemanifest/BUILD @@ -0,0 +1,34 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Package for manifest generation module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_binary( + name = "generatemanifest_bin", + srcs = ["generatemanifest_bin.go"], + deps = [ + ":generatemanifest", + "//src/common/golang:flagfile", + ], +) + +go_library( + name = "generatemanifest", + srcs = ["generatemanifest.go"], + importpath = "src/tools/ak/generatemanifest/generatemanifest", + deps = [ + "//src/common/golang:flags", + "//src/tools/ak:types", + ], +) + +go_test( + name = "generatemanifest_test", + size = "small", + srcs = ["generatemanifest_test.go"], + embed = [":generatemanifest"], + deps = ["@com_github_google_go_cmp//cmp:go_default_library"], +) diff --git a/src/tools/ak/generatemanifest/generatemanifest.go b/src/tools/ak/generatemanifest/generatemanifest.go new file mode 100644 index 0000000..a42a227 --- /dev/null +++ b/src/tools/ak/generatemanifest/generatemanifest.go @@ -0,0 +1,188 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package generatemanifest is a command line tool to generate an empty AndroidManifest +package generatemanifest + +import ( + "bufio" + "encoding/xml" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "strconv" + "sync" + + "src/common/golang/flags" + "src/tools/ak/types" +) + +// Structs used for reading the manifest xml file +type manifestTag struct { + XMLName xml.Name `xml:"manifest"` + UsesSdk usesSdkTag `xml:"uses-sdk"` +} + +type usesSdkTag struct { + XMLName xml.Name `xml:"uses-sdk"` + MinSdk string `xml:"minSdkVersion,attr"` +} + +type result struct { + minSdk int + err error +} + +const manifestContent string = `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="%s"> + <uses-sdk android:minSdkVersion="%d" /> + <application/> +</manifest> +` + +var ( + // Cmd defines the command to run + Cmd = types.Command{ + Init: Init, + Run: Run, + Desc: desc, + Flags: []string{ + "out", + "java_package", + "manifests", + "minsdk", + }, + } + + // Flag variables + out, javaPackage string + minSdk int + manifests flags.StringList + + initOnce sync.Once +) + +// Init initializes manifest flags +func Init() { + initOnce.Do(func() { + flag.StringVar(&out, "out", "", "Path to output manifest generated with the max min sdk value found from --manifests.") + flag.StringVar(&javaPackage, "java_package", "com.default", "(optional) Java package to use for the manifest.") + flag.IntVar(&minSdk, "minsdk", 14, "(optional) Default min sdk to support.") + flag.Var(&manifests, "manifests", "(optional) Manifests(s) to get min sdk from.") + }) +} + +func desc() string { + return "Generates an empty AndroidManifest.xml with a minSdk value. The min sdk is selected " + + "by taking the max value found between the manifests and the minsdk flag." +} + +// Run is the main entry point +func Run() { + if out == "" { + log.Fatal("Missing required flag. Must specify --out") + } + + var manifestFiles []io.ReadCloser + for _, manifest := range manifests { + manifestFile, err := os.Open(manifest) + if err != nil { + log.Fatalf("error opening manifest %s: %v", manifest, err) + } + manifestFiles = append(manifestFiles, manifestFile) + } + defer func(manifestFiles []io.ReadCloser) { + for _, manifestFile := range manifestFiles { + manifestFile.Close() + } + }(manifestFiles) + + extractedMinSdk, err := extractMinSdk(manifestFiles, minSdk) + if err != nil { + log.Fatalf("error extracting min sdk from manifests: %v", err) + } + + outFile, err := os.Create(out) + if err != nil { + log.Fatalf("error opening output manifest: %v", err) + } + defer outFile.Close() + if err := writeManifest(outFile, javaPackage, extractedMinSdk); err != nil { + log.Fatalf("error writing output manifest: %v", err) + } +} + +// The min sdk is selected by taking the max value found +// between the manifests and the minsdk flag +func extractMinSdk(manifests []io.ReadCloser, defaultSdk int) (int, error) { + // Extracting minSdk values in goroutines + results := make(chan result, len(manifests)) + var wg sync.WaitGroup + wg.Add(len(manifests)) + for _, manifestFile := range manifests { + go func(manifestFile io.Reader) { + res := extractMinSdkFromManifest(manifestFile) + results <- res + wg.Done() + }(manifestFile) + } + wg.Wait() + close(results) + + // Finding max value from channel + minSdk := defaultSdk + for result := range results { + if result.err != nil { + return 0, result.err + } + minSdk = max(minSdk, result.minSdk) + } + return minSdk, nil +} + +func extractMinSdkFromManifest(reader io.Reader) result { + manifestBytes, err := ioutil.ReadAll(reader) + if err != nil { + return result{minSdk: 0, err: err} + } + usesSdk := usesSdkTag{MinSdk: ""} + manifest := manifestTag{UsesSdk: usesSdk} + if err := xml.Unmarshal(manifestBytes, &manifest); err != nil { + return result{minSdk: 0, err: err} + } + + // MinSdk value could be a placeholder, we ignore it if that's the case + value, err := strconv.Atoi(manifest.UsesSdk.MinSdk) + if err != nil { + return result{minSdk: 0, err: nil} + } + return result{minSdk: value, err: nil} +} + +func writeManifest(outManifest io.Writer, javaPackage string, minSdk int) error { + manifestWriter := bufio.NewWriter(outManifest) + manifestWriter.WriteString(fmt.Sprintf(manifestContent, javaPackage, minSdk)) + return manifestWriter.Flush() +} + +func max(a, b int) int { + if a < b { + return b + } + return a +} diff --git a/src/tools/ak/generatemanifest/generatemanifest_bin.go b/src/tools/ak/generatemanifest/generatemanifest_bin.go new file mode 100644 index 0000000..668a42a --- /dev/null +++ b/src/tools/ak/generatemanifest/generatemanifest_bin.go @@ -0,0 +1,29 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// generatemanifest_bin is a command line tool to generate an empty AndroidManifest from dependencies +package main + +import ( + "flag" + + _ "src/common/golang/flagfile" + "src/tools/ak/generatemanifest/generatemanifest" +) + +func main() { + generatemanifest.Init() + flag.Parse() + generatemanifest.Run() +} diff --git a/src/tools/ak/generatemanifest/generatemanifest_test.go b/src/tools/ak/generatemanifest/generatemanifest_test.go new file mode 100644 index 0000000..ade3d4e --- /dev/null +++ b/src/tools/ak/generatemanifest/generatemanifest_test.go @@ -0,0 +1,220 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package generatemanifest is a command line tool to generate an empty AndroidManifest +package generatemanifest + +import ( + "io" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +type fakeFile struct { + reader *strings.Reader +} + +func (f fakeFile) Read(b []byte) (int, error) { + return f.reader.Read(b) +} + +func (f fakeFile) Close() error { + return nil +} + +func TestExtractMinSdk(t *testing.T) { + tests := []struct { + name string + manifests []*strings.Reader + defaultMinSdk int + expectedMinSdk int + }{ + { + name: "one manifest", + manifests: []*strings.Reader{ + strings.NewReader( + `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.default"> + <uses-sdk android:minSdkVersion="20" /> +</manifest>`)}, + defaultMinSdk: 14, + expectedMinSdk: 20, + }, + { + name: "one manifest, lower then default min sdk", + manifests: []*strings.Reader{ + strings.NewReader( + `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.default"> + <uses-sdk android:minSdkVersion="20" /> +</manifest>`)}, + defaultMinSdk: 30, + expectedMinSdk: 30, + }, + { + name: "multiple manifests", + manifests: []*strings.Reader{ + strings.NewReader( + `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.default"> + <uses-sdk android:minSdkVersion="20" /> +</manifest>`), + strings.NewReader( + `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.default"> + <uses-sdk android:minSdkVersion="5" /> +</manifest>`), + strings.NewReader( + `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.default"> + <uses-sdk android:minSdkVersion="30" /> +</manifest>`), + }, + defaultMinSdk: 14, + expectedMinSdk: 30, + }, + { + name: "multiple manifests, all lower than default min sdk", + manifests: []*strings.Reader{ + strings.NewReader( + `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.default"> + <uses-sdk android:minSdkVersion="1" /> +</manifest>`), + strings.NewReader( + `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.default"> + <uses-sdk android:minSdkVersion="2" /> +</manifest>`), + strings.NewReader( + `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.default"> + <uses-sdk android:minSdkVersion="3" /> +</manifest>`), + }, + defaultMinSdk: 4, + expectedMinSdk: 4, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + files := make([]io.ReadCloser, 0, len(tc.manifests)) + for _, f := range tc.manifests { + file := fakeFile{reader: f} + file.reader.Seek(0, 0) + files = append(files, file) + } + minSdk, err := extractMinSdk(files, tc.defaultMinSdk) + if err != nil { + t.Fatalf("extractMinSdk(%v, %d) failed with err: %v", files, tc.defaultMinSdk, err) + } + if diff := cmp.Diff(tc.expectedMinSdk, minSdk); diff != "" { + t.Errorf("extractMinSdkFromManifest(%v) returned diff (-want, +got):\n%v", files, diff) + } + }) + } + +} + +func TestExtractMinSdkFromManifest(t *testing.T) { + tests := []struct { + name string + manifest *strings.Reader + expectedMinSdk int + }{ + { + name: "minimal manifest", + manifest: strings.NewReader( + `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.default"> + <uses-sdk android:minSdkVersion="1" /> + <application/> +</manifest>`), + expectedMinSdk: 1, + }, + { + name: "manifest with placeholder", + manifest: strings.NewReader( + `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.default"> + <uses-sdk android:minSdkVersion="${minSdkVersion}" /> + <application/> +</manifest>`), + expectedMinSdk: 0, + }, + { + name: "empty manifest", + manifest: strings.NewReader( + `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.default"> +</manifest>`), + expectedMinSdk: 0, + }, + { + name: "manifest with various elements", + manifest: strings.NewReader( + `<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.default"> + <uses-permission android:name="android.permission.INTERNET" /> + <application android:label="@string/app_name" + android:name="com.default.SomeApp" + android:icon="@drawable/some_icon" + android:theme="@style/a_theme" + android:banner="@drawable/banner"> + <service android:name="com.default.MyService" + android:exported="true"> + <intent-filter> + <action android:name="com.default.IntentFilter" /> + </intent-filter> + </service> + </application> + <uses-sdk android:minSdkVersion="25" /> +</manifest>`), + expectedMinSdk: 25, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + file := fakeFile{reader: tc.manifest} + file.reader.Seek(0, 0) + results := make(chan result) + go func(results chan result) { + res := extractMinSdkFromManifest(file) + results <- res + }(results) + result := <-results + if result.err != nil { + t.Fatalf("extractMinSdkFromManifest(%v) failed with err: %v", file, result.err) + } + if diff := cmp.Diff(tc.expectedMinSdk, result.minSdk); diff != "" { + t.Errorf("extractMinSdkFromManifest(%v) returned diff (-want, +got):\n%v", file, diff) + } + }) + } + +} diff --git a/src/tools/ak/link/BUILD b/src/tools/ak/link/BUILD new file mode 100644 index 0000000..e52fb72 --- /dev/null +++ b/src/tools/ak/link/BUILD @@ -0,0 +1,31 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description: +# Package for link module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_binary( + name = "link_bin", + srcs = ["link_bin.go"], + deps = [ + ":link", + "//src/common/golang:flagfile", + ], +) + +go_library( + name = "link", + srcs = [ + "link.go", + ], + importpath = "src/tools/ak/link/link", + deps = [ + "//src/common/golang:flags", + "//src/common/golang:walk", + "//src/common/golang:ziputils", + "//src/tools/ak:types", + ], +) diff --git a/src/tools/ak/link/link.go b/src/tools/ak/link/link.go new file mode 100644 index 0000000..4850ae7 --- /dev/null +++ b/src/tools/ak/link/link.go @@ -0,0 +1,123 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package link is a thin wrapper around aapt2 to link android resources. +package link + +import ( + "flag" + "io/ioutil" + "log" + "os/exec" + "sync" + + "src/common/golang/flags" + "src/common/golang/walk" + "src/common/golang/ziputils" + "src/tools/ak/types" +) + +var ( + // Cmd defines the command to run link. + Cmd = types.Command{ + Init: Init, + Run: Run, + Desc: desc, + Flags: []string{ + "aapt2", + "sdk_jar", + "manifest", + "res_dirs", + "asset_dirs", + "pkg", + "src_jar", + "out", + }, + } + + aapt2 string + sdkJar string + manifest string + resDirs flags.StringList + assetDirs flags.StringList + pkg string + srcJar string + out string + + initOnce sync.Once +) + +// Init initializes link. +func Init() { + initOnce.Do(func() { + flag.StringVar(&aapt2, "aapt2", "", "Path to the aapt2 binary.") + flag.StringVar(&sdkJar, "sdk_jar", "", "Path to the android jar.") + flag.StringVar(&manifest, "manifest", "", "Path to the application AndroidManifest.xml.") + flag.Var(&resDirs, "res_dirs", "List of resource archives to link.") + flag.Var(&assetDirs, "asset_dirs", "Paths to asset directories..") + flag.StringVar(&pkg, "pkg", "", "Package for R.java.") + flag.StringVar(&srcJar, "src_jar", "", "R java source jar path.") + flag.StringVar(&out, "out", "", "Output path for linked archive.") + }) +} + +func desc() string { + return "Link compiled Android resources." +} + +// Run is the entry point for link. +func Run() { + if aapt2 == "" || + sdkJar == "" || + manifest == "" || + resDirs == nil || + pkg == "" || + srcJar == "" || + out == "" { + log.Fatal("Flags -aapt2 -sdk_jar -manifest -res_dirs -pkg -src_jar and -out must be specified.") + } + + // Note that relative order between directories needs to be respected by traversal function. + // I.e. all files in dir n most come before all files in directory n+1. + resArchives, err := walk.Files(resDirs) + if err != nil { + log.Fatalf("error getting resource archives: %v", err) + } + + rjavaDir, err := ioutil.TempDir("", "rjava") + if err != nil { + log.Fatalf("error creating temp dir: %v", err) + } + + args := []string{ + "link", "--manifest", manifest, "--auto-add-overlay", "--no-static-lib-packages", + "--java", rjavaDir, "--custom-package", pkg, "-I", sdkJar} + + for _, r := range resArchives { + args = append(args, "-R", r) + } + + for _, a := range assetDirs { + args = append(args, "-A", a) + } + + args = append(args, "-o", out) + + if out, err := exec.Command(aapt2, args...).CombinedOutput(); err != nil { + log.Fatalf("error linking Android resources: %v\n %s", err, string(out)) + } + if err := ziputils.Zip(rjavaDir, srcJar); err != nil { + log.Fatalf("error unable to create resources src jar: %v", err) + } +} diff --git a/src/tools/ak/link/link_bin.go b/src/tools/ak/link/link_bin.go new file mode 100644 index 0000000..26efddb --- /dev/null +++ b/src/tools/ak/link/link_bin.go @@ -0,0 +1,29 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The link_bin is a command line tool to wrap around aapt2 rough edges. +package main + +import ( + "flag" + + _ "src/common/golang/flagfile" + "src/tools/ak/link/link" +) + +func main() { + link.Init() + flag.Parse() + link.Run() +} diff --git a/src/tools/ak/liteparse/BUILD b/src/tools/ak/liteparse/BUILD new file mode 100644 index 0000000..10f1482 --- /dev/null +++ b/src/tools/ak/liteparse/BUILD @@ -0,0 +1,58 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description: +# Package for parse module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_library( + name = "liteparse", + srcs = [ + "liteparse.go", + "non_values_parse.go", + "values_parse.go", + ], + importpath = "src/tools/ak/liteparse/liteparse", + deps = [ + "//src/common/golang:flags", + "//src/common/golang:walk", + "//src/tools/ak:types", + "//src/tools/ak/res", + "//src/tools/ak/res/proto:res_data_go_proto", + "//src/tools/ak/res/proto:res_meta_go_proto", + "//src/tools/ak/res/respipe", + "//src/tools/ak/res/resxml", + "@org_golang_google_protobuf//proto", + ], +) + +go_test( + name = "liteparse_test", + size = "small", + srcs = [ + "liteparse_test.go", + "non_values_parse_test.go", + "values_parse_test.go", + ], + data = glob(["testdata/**"]), + embed = [":liteparse"], + deps = [ + "//src/common/golang:runfilelocation", + "//src/tools/ak/res", + "//src/tools/ak/res/proto:res_data_go_proto", + "//src/tools/ak/res/respipe", + "//src/tools/ak/res/resxml", + "@com_github_google_go_cmp//cmp:go_default_library", + ], +) + +go_binary( + name = "liteparse_bin", + srcs = ["liteparse_bin.go"], + deps = [ + ":liteparse", + "//src/common/golang:flagfile", + ], +) diff --git a/src/tools/ak/liteparse/liteparse.go b/src/tools/ak/liteparse/liteparse.go new file mode 100644 index 0000000..9ab50d8 --- /dev/null +++ b/src/tools/ak/liteparse/liteparse.go @@ -0,0 +1,436 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package liteparse does a light parsing of android resources files that can be used at a later +// stage to generate R.java files. +package liteparse + +import ( + "bytes" + "context" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "strings" + "sync" + + "src/common/golang/flags" + "src/common/golang/walk" + rdpb "src/tools/ak/res/proto/res_data_go_proto" + "src/tools/ak/res/res" + "src/tools/ak/res/respipe/respipe" + "src/tools/ak/res/resxml/resxml" + "src/tools/ak/types" + "google.golang.org/protobuf/proto" +) + +var ( + // Cmd defines the command to run the res parser. + Cmd = types.Command{ + Init: Init, + Run: Run, + Desc: desc, + Flags: []string{"resourceFiles", "rPbOutput"}, + } + + resourceFiles flags.StringList + rPbOutput string + pkg string + + initOnce sync.Once +) + +const ( + numParsers = 25 +) + +// Init initializes parse. Flags here need to match flags in AndroidResourceParsingAction. +func Init() { + initOnce.Do(func() { + flag.Var(&resourceFiles, "res_files", "Resource files and asset directories to parse.") + flag.StringVar(&rPbOutput, "out", "", "Path to the output proto file.") + flag.StringVar(&pkg, "pkg", "", "Java package name.") + }) +} + +func desc() string { + return "Lite parses the resource files to generate an R.pb." +} + +// Run runs the parser. +func Run() { + rscs := ParseAll(context.Background(), resourceFiles, pkg) + b, err := proto.Marshal(rscs) + if err != nil { + log.Fatal(err) + } + if err = ioutil.WriteFile(rPbOutput, b, 0644); err != nil { + log.Fatal(err) + } +} + +type resourceFile struct { + pathInfo *res.PathInfo + contents []byte +} + +// ParseAll parses all the files in resPaths, which can contain both files and directories, +// and returns pb. +func ParseAll(ctx context.Context, resPaths []string, packageName string) *rdpb.Resources { + resFiles, err := walk.Files(resPaths) + if err != nil { + log.Fatal(err) + } + pifs, rscs, err := initializeFileParse(resFiles, packageName) + if err != nil { + log.Fatal(err) + } + if len(pifs) == 0 { + return rscs + } + + piC := make(chan *res.PathInfo, len(pifs)) + for _, pi := range pifs { + piC <- pi + } + close(piC) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + resC, errC := ResParse(ctx, piC) + rscs.Resource, err = processResAndErr(resC, errC) + if err != nil { + cancel() + log.Fatal(err) + } + return rscs +} + +// ResParse consumes a stream of resource paths and converts them into resource protos. These +// protos will only have the minimal name/type info set. +func ResParse(ctx context.Context, piC <-chan *res.PathInfo) (<-chan *rdpb.Resource, <-chan error) { + parserC := make(chan *res.PathInfo) + var parsedResCs []<-chan *rdpb.Resource + var parsedErrCs []<-chan error + + for i := 0; i < numParsers; i++ { + parsedResC, parsedErrC := xmlParser(ctx, parserC) + parsedResCs = append(parsedResCs, parsedResC) + parsedErrCs = append(parsedErrCs, parsedErrC) + } + pathResC := make(chan *rdpb.Resource) + pathErrC := make(chan error) + go func() { + defer close(pathResC) + defer close(pathErrC) + defer close(parserC) + + for pi := range piC { + np, err := needsParse(pi) + if err != nil { + pathErrC <- err + return + } else if np { + parserC <- pi + } + if !parsePathInfo(ctx, pi, pathResC, pathErrC) { + return + } + } + }() + parsedResCs = append(parsedResCs, pathResC) + parsedErrCs = append(parsedErrCs, pathErrC) + resC := respipe.MergeResStreams(ctx, parsedResCs) + errC := respipe.MergeErrStreams(ctx, parsedErrCs) + + return resC, errC +} + +// ParseAllContents parses all resource files with paths and contents and returns pb representing +// the R class that is generated from the files with the package packageName. +// paths and contents must have the same length, and a file with paths[i] file path +// has file contents contents[i]. +func ParseAllContents(ctx context.Context, paths []string, contents [][]byte, packageName string) (*rdpb.Resources, error) { + if len(paths) != len(contents) { + return nil, fmt.Errorf("length of paths (%v) and contents (%v) are not equal", len(paths), len(contents)) + } + pifs, rscs, err := initializeFileParse(paths, packageName) + if err != nil { + return nil, err + } + if len(pifs) == 0 { + return rscs, nil + } + + var rfC []*resourceFile + for i, pi := range pifs { + rfC = append(rfC, &resourceFile{ + pathInfo: pi, + contents: contents[i], + }) + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + resC, errC := resParseContents(ctx, rfC) + rscs.Resource, err = processResAndErr(resC, errC) + if err != nil { + return nil, err + } + return rscs, nil +} + +// resParseContents consumes resource files and converts them into resource protos. +// These protos will only have the minimal name/type info set. +// The returned channels will be consumed by processRessAndErr. +func resParseContents(ctx context.Context, rfC []*resourceFile) (<-chan *rdpb.Resource, <-chan error) { + parserC := make(chan *resourceFile) + var parsedResCs []<-chan *rdpb.Resource + var parsedErrCs []<-chan error + + for i := 0; i < numParsers; i++ { + parsedResC, parsedErrC := xmlParserContents(ctx, parserC) + parsedResCs = append(parsedResCs, parsedResC) + parsedErrCs = append(parsedErrCs, parsedErrC) + } + pathResC := make(chan *rdpb.Resource) + pathErrC := make(chan error) + go func() { + defer close(pathResC) + defer close(pathErrC) + defer close(parserC) + + for _, rf := range rfC { + if needsParseContents(rf.pathInfo, bytes.NewReader(rf.contents)) { + parserC <- rf + } + if !parsePathInfo(ctx, rf.pathInfo, pathResC, pathErrC) { + return + } + } + }() + parsedResCs = append(parsedResCs, pathResC) + parsedErrCs = append(parsedErrCs, pathErrC) + resC := respipe.MergeResStreams(ctx, parsedResCs) + errC := respipe.MergeErrStreams(ctx, parsedErrCs) + + return resC, errC +} + +// initializeFileParse returns a slice of all PathInfos of files contained in each file path, +// which must be a file (not a directory). It also returns Resources with packageName. +func initializeFileParse(filePaths []string, packageName string) ([]*res.PathInfo, *rdpb.Resources, error) { + rscs := &rdpb.Resources{ + Pkg: packageName, + } + + pifs, err := res.MakePathInfos(filePaths) + if err != nil { + return nil, nil, err + } + + return pifs, rscs, nil +} + +// parsePathInfo attempts to parse the PathInfo and send the provided Resource and error to the +// provided chan. If the context is canceled, returns false, and otherwise, returns true. +func parsePathInfo(ctx context.Context, pi *res.PathInfo, pathResC chan<- *rdpb.Resource, pathErrC chan<- error) bool { + if rawName, ok := pathAsRes(pi); ok { + fqn, err := res.ParseName(rawName, pi.Type) + if err != nil { + return respipe.SendErr(ctx, pathErrC, respipe.Errorf(ctx, "%s: name parse failed: %v", pi.Path, err)) + } + r := new(rdpb.Resource) + if err := fqn.SetResource(r); err != nil { + return respipe.SendErr(ctx, pathErrC, respipe.Errorf(ctx, "%s: name->proto failed: %v", fqn, err)) + } + return respipe.SendRes(ctx, pathResC, r) + } + return true +} + +// processResAndErr processes the res and err channels and returns the resources if successful +// or the first encountered error. +func processResAndErr(resC <-chan *rdpb.Resource, errC <-chan error) ([]*rdpb.Resource, error) { + parseErrChan := make(chan error, 1) + go func() { + for err := range errC { + if err != nil { + parseErrChan <- err + return + } + } + }() + + doneChan := make(chan struct{}, 1) + var res []*rdpb.Resource + go func() { + for r := range resC { + res = append(res, r) + } + doneChan <- struct{}{} + }() + + select { + case err := <-parseErrChan: + return nil, err + case <-doneChan: + } + + return res, nil +} + +// xmlParser consumes a stream of paths that need to have their xml contents parsed into resource +// protos. We only need to get names and types - so the parsing is very quick. +func xmlParser(ctx context.Context, piC <-chan *res.PathInfo) (<-chan *rdpb.Resource, <-chan error) { + resC := make(chan *rdpb.Resource) + errC := make(chan error) + go func() { + defer close(resC) + defer close(errC) + for p := range piC { + if !syncParse(respipe.PrefixErr(ctx, fmt.Sprintf("%s xml-parse: ", p.Path)), p, resC, errC) { + // ctx must have been canceled - exit. + return + } + } + }() + return resC, errC +} + +// xmlParserContents consumes a stream of resource files that need to have their xml contents +// parsed into resource protos. We only need to get names and types - so the parsing is very quick. +func xmlParserContents(ctx context.Context, rfC <-chan *resourceFile) (<-chan *rdpb.Resource, <-chan error) { + resC := make(chan *rdpb.Resource) + errC := make(chan error) + go func() { + defer close(resC) + defer close(errC) + for rf := range rfC { + if !syncParseContents(respipe.PrefixErr(ctx, fmt.Sprintf("%s xml-parse: ", rf.pathInfo.Path)), rf.pathInfo, bytes.NewReader(rf.contents), resC, errC) { + // ctx must have been canceled - exit. + return + } + } + }() + return resC, errC +} + +func syncParse(ctx context.Context, p *res.PathInfo, resC chan<- *rdpb.Resource, errC chan<- error) bool { + f, err := os.Open(p.Path) + if err != nil { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "open failed: %v", err)) + } + defer f.Close() + return syncParseContents(ctx, p, f, resC, errC) +} + +func syncParseContents(ctx context.Context, p *res.PathInfo, fileReader io.Reader, resC chan<- *rdpb.Resource, errC chan<- error) bool { + parsedResC, mergedErrC := parseContents(ctx, p, fileReader) + for parsedResC != nil || mergedErrC != nil { + select { + case r, ok := <-parsedResC: + if !ok { + parsedResC = nil + continue + } + if !respipe.SendRes(ctx, resC, r) { + return false + } + case e, ok := <-mergedErrC: + if !ok { + mergedErrC = nil + continue + } + if !respipe.SendErr(ctx, errC, e) { + return false + } + } + + } + return true +} + +func parseContents(ctx context.Context, filePathInfo *res.PathInfo, fileReader io.Reader) (resC <-chan *rdpb.Resource, errC <-chan error) { + xmlC, xmlErrC := resxml.StreamDoc(ctx, fileReader) + var parsedErrC <-chan error + if filePathInfo.Type == res.ValueType { + ctx := respipe.PrefixErr(ctx, "mini-values-parse: ") + resC, parsedErrC = valuesParse(ctx, xmlC) + } else { + ctx := respipe.PrefixErr(ctx, "mini-non-values-parse: ") + resC, parsedErrC = nonValuesParse(ctx, xmlC) + } + errC = respipe.MergeErrStreams(ctx, []<-chan error{parsedErrC, xmlErrC}) + return resC, errC +} + +// needsParse determines if a path needs to have a values / nonvalues xml parser run to extract +// resource information. +func needsParse(pi *res.PathInfo) (bool, error) { + r, err := os.Open(pi.Path) + if err != nil { + return false, fmt.Errorf("Unable to open file %s: %s", pi.Path, err) + } + defer r.Close() + + return needsParseContents(pi, r), nil +} + +// needsParseContents determines if a path with the corresponding reader for contents needs to have a +// values / nonvalues xml parser run to extract resource information. +func needsParseContents(pi *res.PathInfo, r io.Reader) bool { + if pi.Type == res.Raw { + return false + } + if filepath.Ext(pi.Path) == ".xml" { + return true + } + if filepath.Ext(pi.Path) == "" { + var header [5]byte + _, err := io.ReadFull(r, header[:]) + if err != nil && err != io.EOF { + log.Fatal("Unable to read file %s: %s", pi.Path, err) + } + if string(header[:]) == "<?xml" { + return true + } + } + return false +} + +// pathAsRes determines if a particular res.PathInfo is also a standalone resource. +func pathAsRes(pi *res.PathInfo) (string, bool) { + if pi.Type.Kind() == res.Value || (pi.Type.Kind() == res.Both && strings.HasPrefix(pi.TypeDir, "values")) { + return "", false + } + p := path.Base(pi.Path) + // Only split on last index of dot when the resource is of RAW type. + // Some drawable resources (Nine-Patch files) ends with .9.png which should not + // be included in the resource name. + if dot := strings.LastIndex(p, "."); dot >= 0 && pi.Type == res.Raw { + return p[:dot], true + } + if dot := strings.Index(p, "."); dot >= 0 { + return p[:dot], true + } + return p, true +} diff --git a/src/tools/ak/liteparse/liteparse_bin.go b/src/tools/ak/liteparse/liteparse_bin.go new file mode 100644 index 0000000..cb76afe --- /dev/null +++ b/src/tools/ak/liteparse/liteparse_bin.go @@ -0,0 +1,30 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// liteparse_bin is a command line tool that does a light parsing of android resources files that +// can be used at a later stage to generate R.java files. +package main + +import ( + "flag" + + _ "src/common/golang/flagfile" + "src/tools/ak/liteparse/liteparse" +) + +func main() { + liteparse.Init() + flag.Parse() + liteparse.Run() +} diff --git a/src/tools/ak/liteparse/liteparse_test.go b/src/tools/ak/liteparse/liteparse_test.go new file mode 100644 index 0000000..94d4181 --- /dev/null +++ b/src/tools/ak/liteparse/liteparse_test.go @@ -0,0 +1,381 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package liteparse + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "reflect" + "sort" + "testing" + + "src/common/golang/runfilelocation" + rdpb "src/tools/ak/res/proto/res_data_go_proto" + "src/tools/ak/res/res" + "src/tools/ak/res/respipe/respipe" + "github.com/google/go-cmp/cmp" +) + +const ( + testdata = "src/tools/ak/liteparse/testdata/" +) + +func TestPathAsRes(t *testing.T) { + tests := []struct { + arg string + name string + ok bool + }{ + { + "foo/bar/res/values/strings.xml", + "", + false, + }, + { + "foo/bar/res/values-ldpi-v19/strings.xml", + "", + false, + }, + { + "foo/bar/res/layout-en-US-v19/hello_america.xml", + "hello_america", + true, + }, + { + "foo/bar/res/xml-land/perfs.xml", + "perfs", + true, + }, + { + "foo/bar/res/drawable-land/eagle.png", + "eagle", + true, + }, + { + "foo/bar/res/raw/vid.1080p.png", + "vid.1080p", + true, + }, + { + "foo/bar/res/drawable-land/circle.9.png", + "circle", + true, + }, + } + + for _, tc := range tests { + pi, err := res.ParsePath(tc.arg) + if err != nil { + t.Errorf("res.ParsePath(%q) returns %v unexpectedly", tc.arg, err) + continue + } + rawName, ok := pathAsRes(&pi) + if tc.name != rawName || ok != tc.ok { + t.Errorf("pathAsRes(%v) got %q, %t want %q, %t", pi, rawName, ok, tc.name, tc.ok) + } + } +} + +func TestNeedsParse(t *testing.T) { + tests := []struct { + arg string + content string + want bool + }{ + { + "foo/bar/res/values/strings.xml", + "", + true, + }, + { + "foo/bar/res/values-ldpi-v19/strings.xml", + "", + true, + }, + { + "foo/bar/res/layout-en-US-v19/hello_america.xml", + "", + true, + }, + { + "foo/bar/res/xml-land/perfs.xml", + "", + true, + }, + { + "foo/bar/res/drawable-land/eagle.png", + "", + false, + }, + { + "foo/bar/res/drawable-land/eagle", + "", + false, + }, + { + "foo/bar/res/drawable-land/eagle_xml", + "<?xml version=\"1.0\" encoding=\"utf-8\"?></xml>", + true, + }, + { + "foo/bar/res/drawable-land/eagle_txt", + "some non-xml file", + false, + }, + } + + for _, tc := range tests { + f := createTestFile(tc.arg, tc.content) + defer os.Remove(f) + pi, err := res.ParsePath(f) + if err != nil { + t.Errorf("res.ParsePath(%s) returns %v unexpectedly", f, err) + continue + } + got, err := needsParse(&pi) + if err != nil { + t.Errorf("needsParse(%v) returns %v unexpectedly", pi, err) + } + if got != tc.want { + t.Errorf("needsParse(%v) got %t want %t", pi, got, tc.want) + } + } +} + +func createTestFile(path, content string) string { + dir := filepath.Dir(path) + tmpDir, err := ioutil.TempDir("", "test") + if err != nil { + log.Fatal(err) + } + err = os.MkdirAll(tmpDir+"/"+dir, os.ModePerm) + if err != nil { + log.Fatal(err) + } + f, err := os.Create(tmpDir + "/" + path) + if err != nil { + log.Fatal(err) + } + if _, err := f.Write([]byte(content)); err != nil { + log.Fatal(err) + } + if err := f.Close(); err != nil { + log.Fatal(err) + } + return f.Name() +} + +func TestParse(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + testRes := createResFile("res") + piC, pathErrC := respipe.EmitPathInfosDir(ctx, testRes) + resC, parseErrC := ResParse(ctx, piC) + errC := respipe.MergeErrStreams(ctx, []<-chan error{pathErrC, parseErrC}) + var parsedNames []string + for resC != nil || errC != nil { + select { + case r, ok := <-resC: + if !ok { + resC = nil + continue + } + pn, err := res.ParseName(r.GetName(), res.Type(r.ResourceType)) + if err != nil { + t.Errorf("res.ParseName(%q, %v) unexpected err: %v", r.GetName(), r.ResourceType, err) + fmt.Printf("parsename err: %v\n", err) + continue + } + parsedNames = append(parsedNames, pn.String()) + case e, ok := <-errC: + if !ok { + errC = nil + continue + } + t.Errorf("Unexpected err: %v", e) + } + } + sort.Strings(parsedNames) + expectedNames := []string{ + "res-auto:attr/bg", + "res-auto:attr/size", + "res-auto:drawable/foo", + "res-auto:id/item1", + "res-auto:id/item2", + "res-auto:id/large", + "res-auto:id/response", + "res-auto:id/small", + "res-auto:menu/simple", + "res-auto:raw/garbage", + "res-auto:string/exlusive", + "res-auto:string/greeting", + "res-auto:string/lonely", + "res-auto:string/title", + "res-auto:string/title2", + "res-auto:string/version", + "res-auto:string/version", // yes duplicated (appears in 2 different files, dupes get handled later in the pipeline) + "res-auto:styleable/absPieChart", + } + if !reflect.DeepEqual(parsedNames, expectedNames) { + t.Errorf("%s: has these resources: %s expected: %s", testRes, parsedNames, expectedNames) + } +} + +func TestParseAll(t *testing.T) { + tests := []struct { + resfiles []string + pkg string + want *rdpb.Resources + }{ + { + resfiles: createResfiles([]string{}), + pkg: "", + want: createResources("", []rdpb.Resource_Type{}, []string{}), + }, + { + resfiles: createResfiles([]string{"mini-1"}), + pkg: "example", + want: createResources("example", []rdpb.Resource_Type{rdpb.Resource_STRING}, []string{"greeting"}), + }, + { + resfiles: createResfiles([]string{"mini-2"}), + pkg: "com.example", + want: createResources("com.example", []rdpb.Resource_Type{rdpb.Resource_XML, rdpb.Resource_ID}, []string{"foo", "foobar"}), + }, + { + resfiles: createResfiles([]string{"res/drawable-ldpi/foo.9.png", "res/menu/simple.xml"}), + pkg: "com.example", + want: createResources("com.example", + []rdpb.Resource_Type{rdpb.Resource_DRAWABLE, rdpb.Resource_MENU, rdpb.Resource_ID, rdpb.Resource_ID}, + []string{"foo", "simple", "item1", "item2"}), + }, + } + + for _, tc := range tests { + if got := ParseAll(context.Background(), tc.resfiles, tc.pkg); !resourcesEqual(got, tc.want) { + t.Errorf("ParseAll(%v, %v) = {%v}, want {%v}", tc.resfiles, tc.pkg, got, tc.want) + } + } +} + +func TestParseAllContents(t *testing.T) { + tests := []struct { + resfiles []string + pkg string + want *rdpb.Resources + }{ + { + resfiles: createResfiles([]string{}), + pkg: "", + want: createResources("", []rdpb.Resource_Type{}, []string{}), + }, + { + resfiles: createResfiles([]string{"mini-1/res/values/strings.xml"}), + pkg: "example", + want: createResources("example", []rdpb.Resource_Type{rdpb.Resource_STRING}, []string{"greeting"}), + }, + { + resfiles: createResfiles([]string{"mini-2/res/xml/foo.xml"}), + pkg: "com.example", + want: createResources("com.example", []rdpb.Resource_Type{rdpb.Resource_XML, rdpb.Resource_ID}, []string{"foo", "foobar"}), + }, + { + resfiles: createResfiles([]string{"res/drawable-ldpi/foo.9.png", "res/menu/simple.xml"}), + pkg: "com.example", + want: createResources("com.example", + []rdpb.Resource_Type{rdpb.Resource_DRAWABLE, rdpb.Resource_MENU, rdpb.Resource_ID, rdpb.Resource_ID}, + []string{"foo", "simple", "item1", "item2"}), + }, + } + + for _, tc := range tests { + allContents := getAllContents(t, tc.resfiles) + got, err := ParseAllContents(context.Background(), tc.resfiles, allContents, tc.pkg) + if err != nil { + t.Errorf("ParseAllContents(%v, %v) failed with error %v", tc.resfiles, tc.pkg, err) + } + if !resourcesEqual(got, tc.want) { + t.Errorf("ParseAllContents(%v, %v) = {%v}, want {%v}", tc.resfiles, tc.pkg, got, tc.want) + } + } +} + +// createResFile creates filename with the testdata as the base +func createResFile(filename string) string { + fullPath := testdata + filename + resFilePath, err := runfilelocation.Find(fullPath) + if err != nil { + log.Fatalf("Could not find the runfile at %v", resFilePath) + } + return resFilePath +} + +// createResfiles creates filenames with the testdata as the base +func createResfiles(filenames []string) []string { + var resfiles []string + for _, filename := range filenames { + resfiles = append(resfiles, createResFile(filename)) + } + return resfiles +} + +func getAllContents(t *testing.T, paths []string) [][]byte { + var allContents [][]byte + for _, path := range paths { + contents, err := os.ReadFile(path) + if err != nil { + t.Errorf("cannot read file %v: %v", path, err) + } + allContents = append(allContents, contents) + } + return allContents +} + +// createResources creates rdpb.Resources with package name pkg and resources {names[i], resource[i]} +func createResources(pkg string, resources []rdpb.Resource_Type, names []string) *rdpb.Resources { + rscs := &rdpb.Resources{ + Pkg: pkg, + } + for i := 0; i < len(names); i++ { + r := &rdpb.Resource{Name: names[i], ResourceType: resources[i]} + rscs.Resource = append(rscs.Resource, r) + } + return rscs +} + +// resourcesEqual checks if the two resources have the same package names and resources +func resourcesEqual(rscs1 *rdpb.Resources, rscs2 *rdpb.Resources) bool { + return rscs1.Pkg == rscs2.Pkg && cmp.Equal(createResourcesMap(rscs1), createResourcesMap(rscs2)) +} + +// createResourcesMap creates a map of resources contained in rscs that maps the rdpb.Resource_Type to the names and the number of times the name appears. +func createResourcesMap(rscs *rdpb.Resources) map[rdpb.Resource_Type]map[string]int { + m := make(map[rdpb.Resource_Type]map[string]int) + for _, r := range rscs.Resource { + if _, ok := m[r.GetResourceType()]; !ok { + m[r.GetResourceType()] = make(map[string]int) + m[r.GetResourceType()][r.GetName()] = 1 + } else if _, ok := m[r.GetResourceType()][r.GetName()]; !ok { + m[r.GetResourceType()][r.GetName()] = 1 + } else { + m[r.GetResourceType()][r.GetName()]++ + } + } + return m +} diff --git a/src/tools/ak/liteparse/non_values_parse.go b/src/tools/ak/liteparse/non_values_parse.go new file mode 100644 index 0000000..248264c --- /dev/null +++ b/src/tools/ak/liteparse/non_values_parse.go @@ -0,0 +1,61 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package liteparse + +import ( + "context" + "strings" + + rdpb "src/tools/ak/res/proto/res_data_go_proto" + "src/tools/ak/res/res" + "src/tools/ak/res/respipe/respipe" + "src/tools/ak/res/resxml/resxml" +) + +// nonValuesParse searches a non-values xml document for ID declarations. It creates ID +// resources for any declarations it finds. +func nonValuesParse(ctx context.Context, xmlC <-chan resxml.XMLEvent) (<-chan *rdpb.Resource, <-chan error) { + resC := make(chan *rdpb.Resource) + errC := make(chan error) + go func() { + defer close(resC) + defer close(errC) + for xe := range xmlC { + for _, a := range resxml.Attrs(xe) { + if strings.HasPrefix(a.Value, res.GeneratedIDPrefix) { + unparsed := strings.Replace(a.Value, res.GeneratedIDPrefix, "@id", 1) + fqn, err := res.ParseName(unparsed, res.ID) + if err != nil { + if !respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%s: unparsable id attribute: %+v: %v", a.Value, xe, err)) { + return + } + continue + } + r := new(rdpb.Resource) + if err := fqn.SetResource(r); err != nil { + if !respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%s: name->proto failed: %+v", fqn, err)) { + return + } + continue + } + if !respipe.SendRes(ctx, resC, r) { + return + } + } + } + } + }() + return resC, errC +} diff --git a/src/tools/ak/liteparse/non_values_parse_test.go b/src/tools/ak/liteparse/non_values_parse_test.go new file mode 100644 index 0000000..9f6d3d6 --- /dev/null +++ b/src/tools/ak/liteparse/non_values_parse_test.go @@ -0,0 +1,88 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package liteparse + +import ( + "bytes" + "context" + "reflect" + "testing" + + "src/tools/ak/res/res" + "src/tools/ak/res/respipe/respipe" + "src/tools/ak/res/resxml/resxml" +) + +func TestResNonValuesParse(t *testing.T) { + tests := []struct { + doc string + wanted []string + }{ + { + `<?xml version="1.0" encoding="utf-8"?> + <LinearLayout xmlns:android="http://schemas.android.com/apk/res/android" + android:layout_width="match_parent"> + <!-- a comment --> + <TextView android:id="@+id/new_tv" android:layout_height="match_parent"/> + <LinearLayout android:id="@+id/bad_layouts" android:layout_height="match_parent"> + <Something myAttr="@+id/id_here_too"/> + <LinearLayout android:id="@+id/really_bad_layouts"/> + </LinearLayout> + + </LinearLayout> + `, + []string{ + "res-auto:id/new_tv", + "res-auto:id/bad_layouts", + "res-auto:id/id_here_too", + "res-auto:id/really_bad_layouts", + }, + }, + } + + for _, tc := range tests { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + xmlC, xmlErrC := resxml.StreamDoc(ctx, bytes.NewBufferString(tc.doc)) + resC, parseErrC := nonValuesParse(ctx, xmlC) + errC := respipe.MergeErrStreams(ctx, []<-chan error{xmlErrC, parseErrC}) + var parsedNames []string + for resC != nil || errC != nil { + select { + case r, ok := <-resC: + if !ok { + resC = nil + continue + } + pn, err := res.ParseName(r.GetName(), res.Type(r.ResourceType)) + if err != nil { + t.Errorf("res.ParseName(%s, %v) unexpected err: %v", r.GetName(), r.ResourceType, err) + } + parsedNames = append(parsedNames, pn.String()) + case e, ok := <-errC: + if !ok { + errC = nil + continue + } + t.Errorf("unexpected error: %v", e) + } + } + + if !reflect.DeepEqual(parsedNames, tc.wanted) { + t.Errorf("nonValuesParse of: %s got: %s wanted: %s", tc.doc, parsedNames, tc.wanted) + } + } + +} diff --git a/src/tools/ak/liteparse/testdata/mini-1/res/values/strings.xml b/src/tools/ak/liteparse/testdata/mini-1/res/values/strings.xml new file mode 100644 index 0000000..26d1276 --- /dev/null +++ b/src/tools/ak/liteparse/testdata/mini-1/res/values/strings.xml @@ -0,0 +1,3 @@ +<resources> + <string name="greeting">hello world</string> +</resources> diff --git a/src/tools/ak/liteparse/testdata/mini-2/res/xml/foo.xml b/src/tools/ak/liteparse/testdata/mini-2/res/xml/foo.xml new file mode 100644 index 0000000..26c82a3 --- /dev/null +++ b/src/tools/ak/liteparse/testdata/mini-2/res/xml/foo.xml @@ -0,0 +1,3 @@ +<doc> + <myelement id="@+id/foobar"/> +</doc> diff --git a/src/tools/ak/liteparse/testdata/res/drawable-ldpi/foo.9.png b/src/tools/ak/liteparse/testdata/res/drawable-ldpi/foo.9.png new file mode 100644 index 0000000..f6d1995 --- /dev/null +++ b/src/tools/ak/liteparse/testdata/res/drawable-ldpi/foo.9.png @@ -0,0 +1,2 @@ + +dont parse me ! diff --git a/src/tools/ak/liteparse/testdata/res/menu/simple.xml b/src/tools/ak/liteparse/testdata/res/menu/simple.xml new file mode 100644 index 0000000..e9e33cb --- /dev/null +++ b/src/tools/ak/liteparse/testdata/res/menu/simple.xml @@ -0,0 +1,6 @@ +<menu xmlns:android="http://schemas.android.com/apk/res/android"> + <item android:id="@+id/item1" + android:title="@string/title"/> + <item android:id="@+id/item2" + android:title="@string/title2"/> +</menu> diff --git a/src/tools/ak/liteparse/testdata/res/raw/garbage.xml b/src/tools/ak/liteparse/testdata/res/raw/garbage.xml new file mode 100644 index 0000000..4babd14 --- /dev/null +++ b/src/tools/ak/liteparse/testdata/res/raw/garbage.xml @@ -0,0 +1,6 @@ + +<Just> +</try> +<to> +</parse> +<me> diff --git a/src/tools/ak/liteparse/testdata/res/values-v19/strings.xml b/src/tools/ak/liteparse/testdata/res/values-v19/strings.xml new file mode 100644 index 0000000..5839831 --- /dev/null +++ b/src/tools/ak/liteparse/testdata/res/values-v19/strings.xml @@ -0,0 +1,5 @@ +<resources> +<string name="exlusive">to_v19</string> +<string name="version">beta</string> +</resources> + diff --git a/src/tools/ak/liteparse/testdata/res/values/other.xml b/src/tools/ak/liteparse/testdata/res/values/other.xml new file mode 100644 index 0000000..796472a --- /dev/null +++ b/src/tools/ak/liteparse/testdata/res/values/other.xml @@ -0,0 +1,3 @@ +<resources> +<string name="lonely">string</string> +</resources> diff --git a/src/tools/ak/liteparse/testdata/res/values/vals.xml b/src/tools/ak/liteparse/testdata/res/values/vals.xml new file mode 100644 index 0000000..47d8e20 --- /dev/null +++ b/src/tools/ak/liteparse/testdata/res/values/vals.xml @@ -0,0 +1,15 @@ +<resources> +<string name="greeting">Hello <b> dear user </b></string> +<item type="id" name="response"/> +<declare-styleable name="absPieChart"> + <attr name="bg" format="color"/> + <attr name="android:gravity"/> +</declare-styleable> +<attr name="size"> + <enum name="small"/> + <enum name="large"/> +</attr> +<string name="version">alpha</string> +<string name="title">title</string> +<string name="title2">title2</string> +</resources> diff --git a/src/tools/ak/liteparse/values_parse.go b/src/tools/ak/liteparse/values_parse.go new file mode 100644 index 0000000..048133b --- /dev/null +++ b/src/tools/ak/liteparse/values_parse.go @@ -0,0 +1,226 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package liteparse + +import ( + "context" + "encoding/xml" + "fmt" + + rdpb "src/tools/ak/res/proto/res_data_go_proto" + rmpb "src/tools/ak/res/proto/res_meta_go_proto" + "src/tools/ak/res/res" + "src/tools/ak/res/respipe/respipe" + "src/tools/ak/res/resxml/resxml" +) + +// valuesParse handles all tags beneath <resources> and extracts the associated +// ResourceType/names. Any encountered resources or errors are passed back on the returned channels. +func valuesParse(ctx context.Context, xmlC <-chan resxml.XMLEvent) (<-chan *rdpb.Resource, <-chan error) { + resC := make(chan *rdpb.Resource) + errC := make(chan error) + go func() { + defer close(resC) + defer close(errC) + for { + xe, ok := resxml.ConsumeUntil(res.ResourcesTagName, xmlC) + if !ok { + return + } + resChildrenC := resxml.ForwardChildren(ctx, xe, xmlC) + for xe := range resChildrenC { + se, ok := xe.Token.(xml.StartElement) + if !ok { + // we ignore all non-start elements during a mini-parse. + continue + } + + tagChildrenC := resxml.ForwardChildren(ctx, xe, resChildrenC) + ctx := respipe.PrefixErr(ctx, fmt.Sprintf("tag-name: %s at: %d: ", se.Name, xe.Offset)) + if t, ok := res.ResourcesTagToType[se.Name.Local]; ok { + if !minResChildParse(ctx, xe, t, tagChildrenC, resC, errC) { + return + } + } else if resxml.SloppyMatches(se.Name, res.ItemTagName) { + if !itemParse(ctx, xe, tagChildrenC, resC, errC) { + return + } + } + for range tagChildrenC { + // exhaust any children beneath this tag, we did not need them in the mini-parse. + } + } + } + }() + return resC, errC +} + +// itemParse handles <item name="xxxx" type="yyy"></item> tags that are children of <resources/> +func itemParse(ctx context.Context, xe resxml.XMLEvent, childC <-chan resxml.XMLEvent, resC chan<- *rdpb.Resource, errC chan<- error) bool { + name, err := extractName(xe) + if err != nil { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%v: expected to encounter name attribute: %v", xe, err)) + } + var tv string + for _, a := range resxml.Attrs(xe) { + if resxml.SloppyMatches(res.TypeAttrName, a.Name) { + tv = a.Value + } + } + if tv == "" { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%v: needs type atttribute", xe)) + } + t, err := res.ParseType(tv) + if err != nil { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%q: cannot convert to type: %v", tv, err)) + } + fqn, err := res.ParseName(name, t) + if err != nil { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%q / type: %s: convert to fqn: %v", name, t, err)) + } + r := new(rdpb.Resource) + if err := fqn.SetResource(r); err != nil { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%v: name->proto failed: %v", fqn, err)) + } + return respipe.SendRes(ctx, resC, r) +} + +// Returns the value of the name attribute or an error. +func extractName(xe resxml.XMLEvent) (string, error) { + for _, a := range resxml.Attrs(xe) { + if resxml.SloppyMatches(res.NameAttrName, a.Name) { + return a.Value, nil + } + } + return "", fmt.Errorf("Expected to encounter name attribute within: %v", resxml.Attrs(xe)) +} + +// minResChildParse handles a single top-level tag beneath <resources> and extracts all ResourceTypes/Names beneath it. It returns false if it detects that the context is done. +func minResChildParse(ctx context.Context, xe resxml.XMLEvent, t res.Type, childC <-chan resxml.XMLEvent, resC chan<- *rdpb.Resource, errC chan<- error) bool { + name, err := extractName(xe) + if err != nil { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%#v: needs name attribute: %v", xe, err)) + } + + fqn, err := res.ParseName(name, t) + if err != nil { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%s: invalid name: %v", name, err)) + } + + r := new(rdpb.Resource) + if err := fqn.SetResource(r); err != nil { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%v: name->proto failed: %v", fqn, err)) + } + if fqn.Type == res.Styleable { + md, ok := parseStyleableChildren(ctx, childC, resC, errC) + if !ok { + return false + } + if err := fqn.SetMetaData(md); err != nil { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%v: could not set stylablemeta: %v", fqn, err)) + } + r.StyleableValue = md + } + if fqn.Type == res.Attr && !parseAttrChildren(ctx, childC, resC, errC) { + return false + } + + return respipe.SendRes(ctx, resC, r) +} + +// parseAttrChildren looks at the children of an <attr> tag and determines if any of them creates resources. +// If it realizes that the provided ctx is canceled, it returns true, otherwise false. +func parseAttrChildren(ctx context.Context, xmlC <-chan resxml.XMLEvent, resC chan<- *rdpb.Resource, errC chan<- error) bool { + for c := range xmlC { + ce, ok := c.Token.(xml.StartElement) + if !ok { + // do not care about non-start element events. + continue + } + if !resxml.SloppyMatches(res.EnumTagName, ce.Name) && !resxml.SloppyMatches(res.FlagTagName, ce.Name) { + // only want <enum> or <flag> elements + continue + } + + enumFlagName, err := extractName(c) + if err != nil { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%v: flag / enum should have had a name attribute: %v", ce, err)) + } + cFqn, err := res.ParseName(enumFlagName, res.ID) + if err != nil { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%v: could not parse child of <attr>: %v", ce, err)) + } + cr := new(rdpb.Resource) + if err := cFqn.SetResource(cr); err != nil { + return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%v: name->proto failed: %v", ce, err)) + } + if !respipe.SendRes(ctx, resC, cr) { + return false + } + } + return true +} + +// parseStyleableChildren looks at the children of a <declare-stylable> tag and determines what resources they create. +func parseStyleableChildren(ctx context.Context, xmlC <-chan resxml.XMLEvent, resC chan<- *rdpb.Resource, errC chan<- error) (*rmpb.StyleableMetaData, bool) { + var attrNames []string + for c := range xmlC { + if _, ok := c.Token.(xml.StartElement); !ok { + // skip events besides start element. + continue + } + name, err := extractName(c) + if err != nil { + // being liberal with what we can encounter under a <declare-styleable> tag. + continue + } + attrFqn, err := res.ParseName(name, res.Attr) + if err != nil { + return nil, respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%q: could not parse name to fqn: %v", name, err)) + } + if attrFqn.Type != res.Attr { + return nil, respipe.SendErr( + ctx, errC, respipe.Errorf(ctx, "%v: name->nameid proto failed: %v", attrFqn, res.ErrWrongType)) + } + + attrNames = append(attrNames, attrFqn.String()) + if attrFqn.Package == "android" { + // since we're not generating android attributes (they already exist already) + // omit the resource proto for these attrs. + continue + } + + if attrFqn.Type == res.Attr { + ctx := respipe.PrefixErr(ctx, fmt.Sprintf("%q: <attr> child: ", name)) + childC := resxml.ForwardChildren(ctx, c, xmlC) + if !parseAttrChildren(ctx, childC, resC, errC) { + return nil, false + } + } + + attrR := new(rdpb.Resource) + if err := attrFqn.SetResource(attrR); err != nil { + return nil, respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "%v: name->proto failed: %v", attrFqn, err)) + } + + if !respipe.SendRes(ctx, resC, attrR) { + return nil, false + } + + } + return &rmpb.StyleableMetaData{ + FqnAttributes: attrNames, + }, true +} diff --git a/src/tools/ak/liteparse/values_parse_test.go b/src/tools/ak/liteparse/values_parse_test.go new file mode 100644 index 0000000..a46f45b --- /dev/null +++ b/src/tools/ak/liteparse/values_parse_test.go @@ -0,0 +1,171 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package liteparse + +import ( + "bytes" + "context" + "reflect" + "strings" + "testing" + + "src/tools/ak/res/res" + "src/tools/ak/res/respipe/respipe" + "src/tools/ak/res/resxml/resxml" +) + +func TestResValuesParse(t *testing.T) { + tests := []struct { + doc string + wanted []string + wantedErr []string + }{ + { + doc: `<resources> + <integer name='two'>2</integer> + <string name='embedded_stuff'>hi <b>there</b></string> + </resources>`, + wanted: []string{ + "res-auto:integer/two", + "res-auto:string/embedded_stuff", + }, + }, + { + doc: `<resources> + <fraction name='frac'>12dp</fraction> + <item type='id' name='foo'/> + <id name='two'/> + <bool name='on'>true</bool> + </resources>`, + wanted: []string{ + "res-auto:fraction/frac", + "res-auto:id/foo", + "res-auto:id/two", + "res-auto:bool/on", + }, + }, + { + doc: `<resources> + <color name='red'>#fff</color> + <item name='hundred' type='dimen'>100%</item> + <attr name="custom"> + <enum name="cars" value="21"/> + <enum name="planes" value="42"/> + </attr> + <eat-comment/> + <!-- a comment --> + <attr name='textSize'/> + </resources>`, + wanted: []string{ + "res-auto:color/red", + "res-auto:dimen/hundred", + "res-auto:id/cars", + "res-auto:id/planes", + "res-auto:attr/custom", + "res-auto:attr/textSize", + }, + }, + { + doc: `<resources> + <attr name='touch'> + <flag name="tap" value="0"/> + <flag name="double_tap" value="2"/> + </attr> + <integer-array name='empty'> + </integer-array> + <integer-array name='five'> + <item>1</item> + <item>@integer/two</item> + </integer-array> + </resources>`, + + wanted: []string{ + "res-auto:id/tap", + "res-auto:id/double_tap", + "res-auto:attr/touch", + "res-auto:array/empty", + "res-auto:array/five", + }, + }, + { + + doc: `<resources> + <declare-styleable name='absPieChart'> + <attr name='android:gravity'/> + <attr name='local' format='string'/> + <attr name='overlay'> + <flag name="transparent" value="0"/> + <flag name="awesome" value="2"/> + </attr> + </declare-styleable> + </resources>`, + wanted: []string{ + "res-auto:attr/local", + "res-auto:id/transparent", + "res-auto:id/awesome", + "res-auto:attr/overlay", + "res-auto:styleable/absPieChart", + }, + }, + { + doc: `<resources><string>2</string></resources>`, + wantedErr: []string{"Expected to encounter name attribute"}, + }, + } + + for _, tc := range tests { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + xmlC, xmlErrC := resxml.StreamDoc(ctx, bytes.NewBufferString(tc.doc)) + resC, parseErrC := valuesParse(ctx, xmlC) + errC := respipe.MergeErrStreams(ctx, []<-chan error{xmlErrC, parseErrC}) + var parsedNames []string + var errStrs []string + for resC != nil || errC != nil { + select { + case r, ok := <-resC: + if !ok { + resC = nil + continue + } + pn, err := res.ParseName(r.GetName(), res.Type(r.ResourceType)) + if err != nil { + t.Errorf("res.ParseName(%s, %v) unexpected err: %v", r.GetName(), r.ResourceType, err) + } + parsedNames = append(parsedNames, pn.String()) + case e, ok := <-errC: + if !ok { + errC = nil + continue + } + errStrs = append(errStrs, e.Error()) + } + + } + + if !reflect.DeepEqual(parsedNames, tc.wanted) { + t.Errorf("valuesParse of: %s got: %s wanted: %s", tc.doc, parsedNames, tc.wanted) + } + if len(errStrs) != len(tc.wantedErr) { + t.Errorf("%s: unexpected amount of errs: %v wanted: %v", tc.doc, errStrs, tc.wantedErr) + continue + } + for i, e := range errStrs { + if !strings.Contains(e, tc.wantedErr[i]) { + t.Errorf("doc: %q got err: %s should contain: %s", tc.doc, e, tc.wantedErr[i]) + } + } + } +} diff --git a/src/tools/ak/manifest/BUILD b/src/tools/ak/manifest/BUILD new file mode 100644 index 0000000..b8031a8 --- /dev/null +++ b/src/tools/ak/manifest/BUILD @@ -0,0 +1,29 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Package for manifest compilation module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_binary( + name = "manifest_bin", + srcs = ["manifest_bin.go"], + deps = [ + ":manifest", + "//src/common/golang:flagfile", + ], +) + +go_library( + name = "manifest", + srcs = [ + "manifest.go", + ], + importpath = "src/tools/ak/manifest/manifest", + deps = [ + "//src/common/golang:flags", + "//src/tools/ak:manifestutils", + "//src/tools/ak:types", + ], +) diff --git a/src/tools/ak/manifest/manifest.go b/src/tools/ak/manifest/manifest.go new file mode 100644 index 0000000..c519545 --- /dev/null +++ b/src/tools/ak/manifest/manifest.go @@ -0,0 +1,158 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package manifest provides a thin wrapper around aapt2 to compile an AndroidManifest.xml +package manifest + +import ( + "archive/zip" + "bytes" + "flag" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "sync" + + "src/common/golang/flags" + "src/tools/ak/manifestutils" + "src/tools/ak/types" +) + +const errMsg string = ` ++----------------------------------------------------------- +| Error while compiling AndroidManifest.xml +| If your build succeeds with Blaze/Bazel build, this is most +| likely due to the stricter aapt2 used by mobile-install +` + + ` ++----------------------------------------------------------- +ERROR: %s +` + +var ( + // Cmd defines the command to run + Cmd = types.Command{ + Init: Init, + Run: Run, + Desc: desc, + Flags: []string{ + "aapt2", + "manifest", + "out", + "sdk_jar", + "res", + "attr", + }, + } + + // Flag variables + aapt2, manifest, out, sdkJar, res string + attr flags.StringList + + initOnce sync.Once +) + +// Init initializes manifest flags +func Init() { + initOnce.Do(func() { + flag.StringVar(&aapt2, "aapt2", "", "Path to aapt2") + flag.StringVar(&manifest, "manifest", "", "Path to manifest") + flag.StringVar(&out, "out", "", "Path to output") + flag.StringVar(&sdkJar, "sdk_jar", "", "Path to sdk jar") + flag.StringVar(&res, "res", "", "Path to res") + flag.Var(&attr, "attr", "(optional) attr(s) to set. {element}:{attr}:{value}.") + }) +} + +func desc() string { + return "Compile an AndroidManifest.xml" +} + +// Run is the main entry point +func Run() { + if aapt2 == "" || manifest == "" || out == "" || sdkJar == "" || res == "" { + log.Fatal("Missing required flags. Must specify --aapt2 --manifest --out --sdk_jar --res") + } + + aaptOut, err := ioutil.TempFile("", "manifest_apk") + if err != nil { + log.Fatalf("Creating temp file failed: %v", err) + } + defer os.Remove(aaptOut.Name()) + + manifestPath := manifest + if len(attr) > 0 { + patchedManifest, err := ioutil.TempFile("", "AndroidManifest_patched.xml") + if err != nil { + log.Fatalf("Creating temp file failed: %v", err) + } + defer os.Remove(patchedManifest.Name()) + manifestPath = patchManifest(manifest, patchedManifest, attr) + } + + stdoutStderr, err := exec.Command(aapt2, "link", "-o", aaptOut.Name(), "--manifest", manifestPath, "-I", sdkJar, "-I", res).CombinedOutput() + if err != nil { + log.Fatalf(errMsg, stdoutStderr) + } + + reader, err := zip.OpenReader(aaptOut.Name()) + if err != nil { + log.Fatalf("Opening zip %q failed: %v", aaptOut.Name(), err) + } + defer reader.Close() + + for _, file := range reader.File { + if file.Name == "AndroidManifest.xml" { + err = os.MkdirAll(filepath.Dir(out), os.ModePerm) + if err != nil { + log.Fatalf("Creating output directory for %q failed: %v", out, err) + } + + fileReader, err := file.Open() + if err != nil { + log.Fatalf("Opening file %q inside zip %q failed: %v", file.Name, aaptOut.Name(), err) + } + defer fileReader.Close() + + outFile, err := os.Create(out) + if err != nil { + log.Fatalf("Creating output %q failed: %v", out, err) + } + + if _, err := io.Copy(outFile, fileReader); err != nil { + log.Fatalf("Writing to output %q failed: %v", out, err) + } + + if err = outFile.Close(); err != nil { + log.Fatal(err) + } + break + } + } +} + +func patchManifest(manifest string, patchedManifest *os.File, attrs []string) string { + b, err := ioutil.ReadFile(manifest) + if err != nil { + log.Fatalf("Failed to read manifest: %v", err) + } + err = manifestutils.WriteManifest(patchedManifest, bytes.NewReader(b), manifestutils.CreatePatchElements(attrs)) + if err != nil { + log.Fatalf("Failed to update manifest: %v", err) + } + return patchedManifest.Name() +} diff --git a/src/tools/ak/manifest/manifest_bin.go b/src/tools/ak/manifest/manifest_bin.go new file mode 100644 index 0000000..6248bc8 --- /dev/null +++ b/src/tools/ak/manifest/manifest_bin.go @@ -0,0 +1,29 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// manifest_bin is a command line tool to manifest an AndroidManifest.xml +package main + +import ( + "flag" + + _ "src/common/golang/flagfile" + "src/tools/ak/manifest/manifest" +) + +func main() { + manifest.Init() + flag.Parse() + manifest.Run() +} diff --git a/src/tools/ak/manifestutils.go b/src/tools/ak/manifestutils.go new file mode 100644 index 0000000..bea212b --- /dev/null +++ b/src/tools/ak/manifestutils.go @@ -0,0 +1,148 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package manifestutils provides common methods to interact with and modify AndroidManifest.xml files. +package manifestutils + +import ( + "encoding/xml" + "io" + "log" + "strings" + + "src/common/golang/xml2" +) + +// Constant attribute names used in an AndroidManifest. +const ( + NameSpace = "http://schemas.android.com/apk/res/android" + ElemManifest = "manifest" + AttrPackage = "package" + AttrSplit = "split" + AttrFeatureName = "featureName" + AttrSharedUserID = "sharedUserId" + AttrSharedUserLabel = "sharedUserLabel" + AttrVersionCode = "versionCode" + AttrVersionName = "versionName" +) + +var ( + // NoNSAttrs contains attributes that are not namespaced. + NoNSAttrs = map[string]bool{ + AttrPackage: true, + AttrSplit: true, + AttrFeatureName: true} +) + +// Manifest is the XML root that we want to parse. +type Manifest struct { + XMLName xml.Name `xml:"manifest"` + Package string `xml:"package,attr"` + SharedUserID string `xml:"sharedUserId,attr"` + SharedUserLabel string `xml:"sharedUserLabel,attr"` + VersionCode string `xml:"versionCode,attr"` + VersionName string `xml:"versionName,attr"` + Application Application `xml:"application"` +} + +// Application is the XML tag that we want to parse. +type Application struct { + XMLName xml.Name `xml:"application"` + Name string `xml:"http://schemas.android.com/apk/res/android name,attr"` +} + +// Encoder takes the xml.Token and encodes it, interface allows us to use xml2.Encoder. +type Encoder interface { + EncodeToken(xml.Token) error +} + +// Patch updates an AndroidManifest by patching the attributes of existing elements. +// +// Attributes that are already defined on the element are updated, while missing +// attributes are added to the element's attributes. Elements in patchElems that are +// missing from the manifest are ignored. +func Patch(dec *xml.Decoder, enc Encoder, patchElems map[string]map[string]xml.Attr) error { + for { + t, err := dec.Token() + if err != nil { + if err == io.EOF { + break + } + return err + } + switch tt := t.(type) { + case xml.StartElement: + elem := tt.Name.Local + if attrs, ok := patchElems[elem]; ok { + found := make(map[string]bool) + for i, a := range tt.Attr { + if attr, ok := attrs[a.Name.Local]; a.Name.Space == attr.Name.Space && ok { + found[a.Name.Local] = true + tt.Attr[i] = attr + } + } + for _, attr := range attrs { + if found[attr.Name.Local] { + continue + } + + tt.Attr = append(tt.Attr, attr) + } + } + enc.EncodeToken(tt) + default: + enc.EncodeToken(tt) + } + } + return nil +} + +// WriteManifest writes an AndroidManifest with updates to patched elements. +func WriteManifest(dst io.Writer, src io.Reader, patchElems map[string]map[string]xml.Attr) error { + e := xml2.NewEncoder(dst) + if err := Patch(xml.NewDecoder(src), e, patchElems); err != nil { + return err + } + return e.Flush() +} + +// CreatePatchElements creates an element map from a string array of "element:attr:attr_value" entries. +func CreatePatchElements(attr []string) map[string]map[string]xml.Attr { + patchElems := make(map[string]map[string]xml.Attr) + for _, a := range attr { + pts := strings.Split(a, ":") + if len(pts) < 3 { + log.Fatalf("Failed to parse attr to replace %s", a) + } + + elem := pts[0] + attr := pts[1] + ns := NameSpace + + // https://developer.android.com/guide/topics/manifest/manifest-element + if elem == ElemManifest && NoNSAttrs[attr] { + ns = "" + } + + if ais, ok := patchElems[elem]; ok { + ais[attr] = xml.Attr{ + Name: xml.Name{Space: ns, Local: attr}, Value: pts[2]} + } else { + patchElems[elem] = map[string]xml.Attr{ + attr: xml.Attr{ + Name: xml.Name{Space: ns, Local: attr}, Value: pts[2]}} + } + } + return patchElems +} diff --git a/src/tools/ak/mindex/BUILD b/src/tools/ak/mindex/BUILD new file mode 100644 index 0000000..aeb40ec --- /dev/null +++ b/src/tools/ak/mindex/BUILD @@ -0,0 +1,24 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description: +# Package for mindex module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_binary( + name = "mindex_bin", + srcs = ["mindex_bin.go"], + deps = [ + ":mindex", + "//src/common/golang:flagfile", + ], +) + +go_library( + name = "mindex", + srcs = ["mindex.go"], + importpath = "src/tools/ak/mindex/mindex", + deps = ["//src/tools/ak:types"], +) diff --git a/src/tools/ak/mindex/mindex.go b/src/tools/ak/mindex/mindex.go new file mode 100644 index 0000000..9bdbefa --- /dev/null +++ b/src/tools/ak/mindex/mindex.go @@ -0,0 +1,108 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package mindex writes a minimal dex into a file. +// It encodes the minimal necessary fields to pass muster with the android runtime. +package mindex + +import ( + "flag" + "io/ioutil" + "log" + "sync" + + "src/tools/ak/types" +) + +var ( + // Cmd defines the command to run mindex + Cmd = types.Command{ + Init: Init, + Run: Run, + Desc: desc, + Flags: []string{ + "out", + }, + } + + // Variables that hold flag values + out string + initOnce sync.Once + + // see https://android.googlesource.com/platform/art/+/android-5.0.1_r1/runtime/dex_file.h + + minDex = []byte{ + 0x64, 0x65, 0x78, 0x0A, // 0x00: Dex magic + 0x30, 0x33, 0x35, 0x00, // 0x04: Dex version + 0x44, 0x0E, 0xC1, 0x0F, // 0x08: Adler32 checksum + 0x8B, 0x9B, 0x61, 0xEA, // 0x0C: Sha1 digest + 0xC1, 0x7F, 0x94, 0x5A, // 0x10: Sha1 digest cont. + 0xE9, 0xC0, 0x8A, 0x70, // 0x14: Sha1 digest cont. + 0xFD, 0xED, 0x4F, 0x53, // 0x18: Sha1 digest cont. + 0x0F, 0x10, 0x51, 0x75, // 0x1C: Sha1 digest cont. + + // Header + 0x8C, 0x00, 0x00, 0x00, // 0x20: Files size + 0x70, 0x00, 0x00, 0x00, // 0x24: Header size + 0x78, 0x56, 0x34, 0x12, // 0x28: Endian tag + 0x00, 0x00, 0x00, 0x00, // 0x2C: Link size + 0x00, 0x00, 0x00, 0x00, // 0x30: Link offset + 0x70, 0x00, 0x00, 0x00, // 0x34: Map offset + 0x00, 0x00, 0x00, 0x00, // 0x38: String ids size + 0x00, 0x00, 0x00, 0x00, // 0x3C: String ids offset + 0x00, 0x00, 0x00, 0x00, // 0x40: Type ids size + 0x00, 0x00, 0x00, 0x00, // 0x44: Type ids offset + 0x00, 0x00, 0x00, 0x00, // 0x48: Proto ids size + 0x00, 0x00, 0x00, 0x00, // 0x4C: Proto ids offset + 0x00, 0x00, 0x00, 0x00, // 0x50: Field ids size + 0x00, 0x00, 0x00, 0x00, // 0x54: Field ids offset + 0x00, 0x00, 0x00, 0x00, // 0x58: Method ids size + 0x00, 0x00, 0x00, 0x00, // 0x5C: Method ids offset + 0x00, 0x00, 0x00, 0x00, // 0x60: Class defs size + 0x00, 0x00, 0x00, 0x00, // 0x64: Class defs offset + 0x1C, 0x00, 0x00, 0x00, // 0x68: Data size + 0x70, 0x00, 0x00, 0x00, // 0x6C: Data offset + + // Data + 0x02, 0x00, 0x00, 0x00, // 0x70: Map list size (header and map list) + 0x00, 0x00, 0x00, 0x00, // 0x74: kDexTypeHeaderItem + 0x01, 0x00, 0x00, 0x00, // 0x78: HeaderItem count + 0x00, 0x00, 0x00, 0x00, // 0x7C: Offset 0x00 + 0x00, 0x10, 0x00, 0x00, // 0x80: kDexTypeMapList + 0x01, 0x00, 0x00, 0x00, // 0x84: MapList count + 0x70, 0x00, 0x00, 0x00, // 0x88: Offset 0x70 + } +) + +// Init initializes mindex. +func Init() { + initOnce.Do(func() { + flag.StringVar(&out, "out", "", "Path to output.") + }) +} + +func desc() string { + return "Mindex writes the smallest possible dex to a file." +} + +// Run is the entry point for mindex. +func Run() { + if out == "" { + log.Fatal("Flags -out must be specified.") + } + + if err := ioutil.WriteFile(out, minDex, 0655); err != nil { + log.Fatalf("Error writing minimal dex %v", err) + } +} diff --git a/src/tools/ak/mindex/mindex_bin.go b/src/tools/ak/mindex/mindex_bin.go new file mode 100644 index 0000000..c41dc69 --- /dev/null +++ b/src/tools/ak/mindex/mindex_bin.go @@ -0,0 +1,29 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The mindex_bin is a command line tool to mindex a zip archive. +package main + +import ( + "flag" + + _ "src/common/golang/flagfile" + "src/tools/ak/mindex/mindex" +) + +func main() { + mindex.Init() + flag.Parse() + mindex.Run() +} diff --git a/src/tools/ak/nativelib/BUILD b/src/tools/ak/nativelib/BUILD new file mode 100644 index 0000000..dc9fe12 --- /dev/null +++ b/src/tools/ak/nativelib/BUILD @@ -0,0 +1,40 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description: +# Package for nativelib module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_library( + name = "nativelib", + srcs = ["nativelib.go"], + importpath = "src/tools/ak/nativelib/nativelib", + deps = [ + "//src/common/golang:fileutils", + "//src/common/golang:flags", + "//src/common/golang:ziputils", + "//src/tools/ak:types", + ], +) + +go_binary( + name = "nativelib_bin", + srcs = ["nativelib_bin.go"], + deps = [ + ":nativelib", + "//src/common/golang:flagfile", + ], +) + +go_test( + name = "nativelib_test", + size = "small", + srcs = ["nativelib_test.go"], + data = [ + "//src/tools/ak/nativelib/testdata:dummy_so", + ], + embed = [":nativelib"], + deps = ["//src/common/golang:runfilelocation"], +) diff --git a/src/tools/ak/nativelib/nativelib.go b/src/tools/ak/nativelib/nativelib.go new file mode 100644 index 0000000..d8ca988 --- /dev/null +++ b/src/tools/ak/nativelib/nativelib.go @@ -0,0 +1,162 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package nativelib creates the native library zip. +package nativelib + +import ( + "archive/zip" + "bufio" + "errors" + "flag" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "src/common/golang/fileutils" + "src/common/golang/flags" + "src/common/golang/ziputils" + "src/tools/ak/types" +) + +var ( + // Cmd defines the command to run nativelib. + Cmd = types.Command{ + Init: Init, + Run: Run, + Desc: desc, + Flags: []string{"lib", "native_libs_zip", "out"}, + } + + // Variables to hold flag values + nativeLibs flags.StringList + nativeLibsZip flags.StringList + out string + + initOnce sync.Once +) + +// Init initializes nativelib. +func Init() { + initOnce.Do(func() { + flag.Var(&nativeLibs, "lib", "Path to native lib.") + flag.Var(&nativeLibsZip, "native_libs_zip", "Zip(s) containing native libs.") + flag.StringVar(&out, "out", "", "Native libraries files.") + }) +} + +func desc() string { + return "Nativelib creates the native lib zip." +} + +// Run is the entry point for nativelib. +func Run() { + if nativeLibsZip != nil { + dstDir, err := ioutil.TempDir("", "ziplibs") + if err != nil { + log.Fatalf("Error creating native lib zip: %v", err) + } + + for _, native := range nativeLibsZip { + libs, err := extractLibs(native, dstDir) + if err != nil { + log.Fatalf("Error creating native lib zip: %v", err) + } + nativeLibs = append(nativeLibs, libs...) + } + } + + if err := doWork(nativeLibs, out); err != nil { + log.Fatalf("Error creating native lib zip: %v", err) + } +} + +func extractLibs(libZip, dstDir string) ([]string, error) { + zr, err := zip.OpenReader(libZip) + if err != nil { + return nil, err + } + defer zr.Close() + + libs := []string{} + for _, f := range zr.File { + if f.Mode().IsDir() { + continue + } + arch := filepath.Base(filepath.Dir(f.Name)) + libs = append(libs, fmt.Sprintf("%s:%s", arch, filepath.Join(dstDir, f.Name))) + } + if err := ziputils.Unzip(libZip, dstDir); err != nil { + return nil, err + } + return libs, nil +} + +func doWork(nativeLibs []string, out string) error { + nativeDir, err := ioutil.TempDir("", "nativelib") + if err != nil { + return err + } + defer os.RemoveAll(nativeDir) + nativePaths, err := copyNativeLibs(nativeLibs, nativeDir) + if err != nil { + return err + } + zipFile, err := os.Create(out) + if err != nil { + return err + } + writer := bufio.NewWriter(zipFile) + zipWriter := zip.NewWriter(writer) + sort.Strings(nativePaths) + for _, f := range nativePaths { + p, err := filepath.Rel(nativeDir, f) + if err != nil { + return err + } + ziputils.WriteFile(zipWriter, f, p) + } + zipWriter.Close() + return nil +} + +func copyNativeLibs(nativeLibs []string, dir string) ([]string, error) { + var paths []string + for _, cpuNativeLib := range nativeLibs { + r := strings.SplitN(cpuNativeLib, ":", 2) + if len(r) != 2 { + return nil, errors.New("error parsing native lib") + } + arch := r[0] + nativeLib := r[1] + if arch == "armv7a" { + arch = "armeabi-v7a" + } + libOutDir := filepath.Join(dir, "lib", arch) + if err := os.MkdirAll(libOutDir, 0777); err != nil && !os.IsExist(err) { + return nil, err + } + outNativeLibPath := filepath.Join(libOutDir, filepath.Base(nativeLib)) + if err := fileutils.Copy(nativeLib, outNativeLibPath); err != nil { + return nil, err + } + paths = append(paths, outNativeLibPath) + } + return paths, nil +} diff --git a/src/tools/ak/nativelib/nativelib_bin.go b/src/tools/ak/nativelib/nativelib_bin.go new file mode 100644 index 0000000..58bb6b9 --- /dev/null +++ b/src/tools/ak/nativelib/nativelib_bin.go @@ -0,0 +1,29 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Nativelib_bin is a command line tool to extract native libraries for the shell apk. +package main + +import ( + "flag" + + _ "src/common/golang/flagfile" + "src/tools/ak/nativelib/nativelib" +) + +func main() { + nativelib.Init() + flag.Parse() + nativelib.Run() +} diff --git a/src/tools/ak/nativelib/nativelib_test.go b/src/tools/ak/nativelib/nativelib_test.go new file mode 100644 index 0000000..4f61914 --- /dev/null +++ b/src/tools/ak/nativelib/nativelib_test.go @@ -0,0 +1,123 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package nativelib + +import ( + "archive/zip" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "src/common/golang/runfilelocation" +) + +const ( + expectedName = "lib/x86/dummy.so" + dummyLib = "src/tools/ak/nativelib/testdata/dummy.so" +) + +func makeLibZip(t *testing.T, entry io.Reader, entryName, zipPath string) error { + f, err := os.Create(zipPath) + if err != nil { + return err + } + defer func() { + if err := f.Close(); err != nil { + t.Error(err) + } + }() + + archive := zip.NewWriter(f) + wr, err := archive.CreateHeader(&zip.FileHeader{Name: entryName, Method: zip.Store}) + if err != nil { + return err + } + if _, err := io.Copy(wr, entry); err != nil { + return err + } + return archive.Close() +} + +func TestCreateNativeLibZip(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "shelltest") + if err != nil { + t.Errorf("Error creating temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + out := filepath.Join(tmpDir, "lib.zip") + dummyLibPath, err := runfilelocation.Find(dummyLib) + if err != nil { + t.Errorf("Error finding dummy lib runfile: %v", err) + } + in := []string{"x86:" + dummyLibPath} + if err := doWork(in, out); err != nil { + t.Errorf("Error creating native lib zip: %v", err) + } + + z, err := zip.OpenReader(out) + if err != nil { + t.Fatalf("Error opening output zip: %v", err) + } + defer z.Close() + + if len(z.File) != 1 { + t.Fatalf("Got %d files in zip, expected 1", len(z.File)) + } + + if z.File[0].Name != expectedName { + t.Fatalf("Got .so file %s, expected %s", z.File[0].Name, expectedName) + } +} + +func TestExtractLibs(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "shelltest") + if err != nil { + t.Fatalf("Error creating temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + lib, err := os.Create(filepath.Join(tmpDir, "dmmylib.so")) + if err != nil { + t.Fatalf("Error creating dummy lib: %v", err) + } + + libZip := filepath.Join(tmpDir, "libs.zip") + if err := makeLibZip(t, lib, expectedName, libZip); err != nil { + t.Fatalf("error creating aar lib zip: %v", err) + } + + dstDir, err := ioutil.TempDir("", "ziplibs") + if err != nil { + t.Fatalf("Error extracting creating zip dir: %v", err) + } + defer os.RemoveAll(dstDir) + + libs, err := extractLibs(libZip, dstDir) + if err != nil { + t.Fatalf("Error extracting libs from zip: %v", err) + } + + if len(libs) != 1 { + t.Fatalf("Got %d files in zip, expected 1", len(libs)) + } + expected := fmt.Sprintf("x86:%s", filepath.Join(dstDir, "lib/x86/dummy.so")) + if libs[0] != expected { + t.Fatalf("Got %s lib, expected %s", libs[0], expected) + } + +} diff --git a/src/tools/ak/nativelib/testdata/BUILD b/src/tools/ak/nativelib/testdata/BUILD new file mode 100644 index 0000000..a0615ed0 --- /dev/null +++ b/src/tools/ak/nativelib/testdata/BUILD @@ -0,0 +1,14 @@ +# Creates test data for testing the nativelib action. + +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +package(default_visibility = ["//src/tools/ak/nativelib:__subpackages__"]) + +licenses(["notice"]) + +genrule( + name = "dummy_so", + outs = ["dummy.so"], + cmd = "touch $@", +) diff --git a/src/tools/ak/res/BUILD b/src/tools/ak/res/BUILD new file mode 100644 index 0000000..6cdb6e5 --- /dev/null +++ b/src/tools/ak/res/BUILD @@ -0,0 +1,40 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description: +# Package for res module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_library( + name = "res", + srcs = [ + "naming.go", + "path.go", + "struct.go", + "xml.go", + ], + importpath = "src/tools/ak/res/res", + visibility = [ + "//src/tools/ak:__subpackages__", + "//src/tools/resource_extractor:__subpackages__", + "//tools/android/incremental:__subpackages__", + ], + deps = [ + "//src/tools/ak/res/proto:res_data_go_proto", + "//src/tools/ak/res/proto:res_meta_go_proto", + "@org_golang_google_protobuf//proto", + ], +) + +go_test( + name = "res_test", + size = "small", + srcs = [ + "naming_test.go", + "path_test.go", + "struct_test.go", + ], + embed = [":res"], +) diff --git a/src/tools/ak/res/naming.go b/src/tools/ak/res/naming.go new file mode 100644 index 0000000..178f740 --- /dev/null +++ b/src/tools/ak/res/naming.go @@ -0,0 +1,175 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package res + +import ( + "fmt" + "strings" + + rdpb "src/tools/ak/res/proto/res_data_go_proto" + rmpb "src/tools/ak/res/proto/res_meta_go_proto" + "google.golang.org/protobuf/proto" +) + +// FullyQualifiedName represents the components of a name. +type FullyQualifiedName struct { + Package string + Type Type + Name string +} + +// ValuesResource represents a resource element. +type ValuesResource struct { + Src *PathInfo + N FullyQualifiedName + Payload []byte +} + +// SetResource sets all the name related fields on the top level resource proto. +func (f FullyQualifiedName) SetResource(r *rdpb.Resource) error { + rt, err := f.Type.Enum() + if err != nil { + return err + } + r.ResourceType = rt + r.Name = protoNameSanitizer.Replace(f.Name) + return nil +} + +// SetMetaData sets all name related fields for this style on a StyleableMetaData proto +func (f FullyQualifiedName) SetMetaData(md *rmpb.StyleableMetaData) error { + if f.Type != Styleable { + return ErrWrongType + } + md.Name = proto.String(protoNameSanitizer.Replace(f.Name)) + return nil +} + +var ( + protoNameSanitizer = strings.NewReplacer(".", "_") + javaNameSanitizer = strings.NewReplacer(":", "_", ".", "_") +) + +// JavaName returns a version of the FullyQualifiedName that should be used for resource identifier fields. +func (f FullyQualifiedName) JavaName() (string, error) { + if !f.Type.IsReal() { + return "", ErrWrongType + } + return javaNameSanitizer.Replace(f.Name), nil +} + +// StyleableAttrName creates the java identifier for referencing this attribute in the given +// style. +func StyleableAttrName(styleable, attr FullyQualifiedName) (string, error) { + if styleable.Type != Styleable || attr.Type != Attr { + return "", ErrWrongType + } + js, err := styleable.JavaName() + if err != nil { + return "", err + } + ja, err := attr.JavaName() + if err != nil { + return "", err + } + + if attr.Package == "android" { + return fmt.Sprintf("%s_android_%s", js, ja), nil + } + return fmt.Sprintf("%s_%s", js, ja), nil +} + +// ParseName is given a name string and optional context about the name (what type the name may be) +// and attempts to extract the local name, Type, and package from the unparsed input. The format of +// unparsed names is flexible and not well specified. +// A FullyQualifiedName's String method will emit pkg:type/name which every tool understands, but +// ParseName will encounter input like ?type:pkg/name - an undocumented, but legal way to specify a +// reference to a style. If unparsed is so mangled that a legal name cannot possibly be determined, +// it will return an error. +func ParseName(unparsed string, resType Type) (FullyQualifiedName, error) { + fqn := removeRef(unparsed) + fqn.Type = resType + pkgIdx := strings.Index(fqn.Name, ":") + typeIdx := strings.Index(fqn.Name, "/") + if pkgIdx == 0 || typeIdx == 0 { + return FullyQualifiedName{}, fmt.Errorf("malformed name %q - can not start with ':' or '/'", unparsed) + } + + if typeIdx != -1 { + if pkgIdx != -1 { + if pkgIdx < typeIdx { + // Package, type and name (pkg:type/name) + t, err := ParseType(fqn.Name[pkgIdx+1 : typeIdx]) + if err != nil { + // the name has illegal type in it that we'll never be able to scrub out. + return FullyQualifiedName{}, err + } + fqn.Type = t + fqn.Package = fqn.Name[:pkgIdx] + fqn.Name = fqn.Name[typeIdx+1:] + + } else { + // Package, type and name, type and package swapped (type:pkg/name) + t, err := ParseType(fqn.Name[:typeIdx]) + if err != nil { + // the name has illegal type in it that we'll never be able to scrub out. + return FullyQualifiedName{}, err + } + fqn.Type = t + fqn.Package = fqn.Name[typeIdx+1 : pkgIdx] + fqn.Name = fqn.Name[pkgIdx+1:] + } + } else { + // Only type and name (type/name) + t, err := ParseType(fqn.Name[:typeIdx]) + if err != nil { + // the name has illegal type in it that we'll never be able to scrub out. + return FullyQualifiedName{}, err + } + fqn.Type = t + fqn.Name = fqn.Name[typeIdx+1:] + } + } else { + // Only package and name (pkg:name) + if pkgIdx != -1 { + fqn.Package = fqn.Name[:pkgIdx] + fqn.Name = fqn.Name[pkgIdx+1:] + } + } + + if fqn.Package == "" { + fqn.Package = "res-auto" + } + + if fqn.Type == UnknownType { + return FullyQualifiedName{}, fmt.Errorf("cannot determine type from %q and %v - not a valid name", unparsed, resType) + } + if fqn.Name == "" { + return FullyQualifiedName{}, fmt.Errorf("cannot determine name from %q and %v - not a valid name", unparsed, resType) + } + return fqn, nil +} + +func removeRef(unparsed string) (fqn FullyQualifiedName) { + fqn.Name = unparsed + if len(fqn.Name) > 2 && (strings.HasPrefix(fqn.Name, "@") || strings.HasPrefix(fqn.Name, "?")) { + fqn.Name = fqn.Name[1:] + } + return +} + +func (f FullyQualifiedName) String() string { + return fmt.Sprintf("%s:%s/%s", f.Package, f.Type, f.Name) +} diff --git a/src/tools/ak/res/naming_test.go b/src/tools/ak/res/naming_test.go new file mode 100644 index 0000000..550251c --- /dev/null +++ b/src/tools/ak/res/naming_test.go @@ -0,0 +1,341 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package res + +import ( + "reflect" + "strings" + "testing" +) + +func TestNaming(t *testing.T) { + tests := []struct { + unparsed string + resType Type + want FullyQualifiedName + wantErrPrefix string + }{ + { + "style/InlineProjectStyle", + ValueType, + FullyQualifiedName{ + Name: "InlineProjectStyle", + Type: Style, + Package: "res-auto", + }, + "", + }, + { + "android:style/InlineProjectStyle", + ValueType, + FullyQualifiedName{ + Name: "InlineProjectStyle", + Type: Style, + Package: "android", + }, + "", + }, + { + "@style/InlineProjectStyle", + ValueType, + FullyQualifiedName{ + Name: "InlineProjectStyle", + Type: Style, + Package: "res-auto", + }, + "", + }, + { + "@style/android:InlineProjectStyle", + ValueType, + FullyQualifiedName{ + Name: "InlineProjectStyle", + Type: Style, + Package: "android", + }, + "", + }, + { + "?style/InlineProjectStyle", + ValueType, + FullyQualifiedName{ + Name: "InlineProjectStyle", + Type: Style, + Package: "res-auto", + }, + "", + }, + { + "?style/android:InlineProjectStyle", + ValueType, + FullyQualifiedName{ + Name: "InlineProjectStyle", + Type: Style, + Package: "android", + }, + "", + }, + { + "android:style/Widget.TextView", + ValueType, + FullyQualifiedName{ + Name: "Widget.TextView", + Type: Style, + Package: "android", + }, + "", + }, + { + "@android:style/Widget.TextView", + ValueType, + FullyQualifiedName{ + Name: "Widget.TextView", + Type: Style, + Package: "android", + }, + "", + }, + { + "?android:style/Widget.TextView", + ValueType, + FullyQualifiedName{ + Name: "Widget.TextView", + Type: Style, + Package: "android", + }, + "", + }, + { + "?attr/styleReference", + ValueType, + FullyQualifiedName{ + Name: "styleReference", + Type: Attr, + Package: "res-auto", + }, + "", + }, + { + "?android:attr/textAppearance", + ValueType, + FullyQualifiedName{ + Name: "textAppearance", + Type: Attr, + Package: "android", + }, + "", + }, + { + "?attr/android:textAppearance", + ValueType, + FullyQualifiedName{ + Name: "textAppearance", + Type: Attr, + Package: "android", + }, + "", + }, + { + "@dimen/viewer:progress_bar_height", + ValueType, + FullyQualifiedName{ + Name: "progress_bar_height", + Type: Dimen, + Package: "viewer", + }, + "", + }, + { + "drawable/simple", + Drawable, + FullyQualifiedName{ + Name: "simple", + Type: Drawable, + Package: "res-auto", + }, + "", + }, + { + "android:fraction/name", + ValueType, + FullyQualifiedName{ + Name: "name", + Type: Fraction, + Package: "android", + }, + "", + }, + { + "android:style/foo:with_colon", + ValueType, + FullyQualifiedName{ + Name: "foo:with_colon", + Type: Style, + Package: "android", + }, + "", + }, + { + "color/red", + ValueType, + FullyQualifiedName{ + Name: "red", + Type: Color, + Package: "res-auto", + }, + "", + }, + { + "style/bright:with_colon", + ValueType, + FullyQualifiedName{ + Name: "with_colon", + Type: Style, + Package: "bright", + }, + "", + }, + { + "com.google.android.apps.gmoney:array/available_locales", + ValueType, + FullyQualifiedName{ + Name: "available_locales", + Type: Array, + Package: "com.google.android.apps.gmoney", + }, + "", + }, + { + "@android:string/ok", + ValueType, + FullyQualifiedName{ + Name: "ok", + Type: String, + Package: "android", + }, + "", + }, + { + "@string/android:ok", + ValueType, + FullyQualifiedName{ + Name: "ok", + Type: String, + Package: "android", + }, + "", + }, + { + "name", + String, + FullyQualifiedName{ + Package: "res-auto", + Type: String, + Name: "name", + }, + "", + }, + { + "string/name", + String, + FullyQualifiedName{ + Package: "res-auto", + Type: String, + Name: "name", + }, + "", + }, + { + "android:Theme.Material.Light", + Style, + FullyQualifiedName{ + Package: "android", + Type: Style, + Name: "Theme.Material.Light", + }, + "", + }, + { + "@android:attr/borderlessButtonStyle", + Style, + FullyQualifiedName{ + Package: "android", + Type: Attr, + Name: "borderlessButtonStyle", + }, + "", + }, + { + "@id/:packagelessId", + Style, + FullyQualifiedName{ + Package: "res-auto", + Type: ID, + Name: "packagelessId", + }, + "", + }, + { + "InlineProjectStyle", + UnknownType, + FullyQualifiedName{}, + "cannot determine type", + }, + { + "android:InlineProjectStyle", + UnknownType, + FullyQualifiedName{}, + "cannot determine type", + }, + { + "res-auto:InlineProjectStyle", + UnknownType, + FullyQualifiedName{}, + "cannot determine type", + }, + { + "style/", + ValueType, + FullyQualifiedName{}, + "cannot determine name", + }, + { + ":style/InlineProjectStyle", + ValueType, + FullyQualifiedName{}, + "malformed name", + }, + { + "/InlineProjectStyle", + ValueType, + FullyQualifiedName{}, + "malformed name", + }, + } + + for _, tc := range tests { + got, gotErr := ParseName(tc.unparsed, tc.resType) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("ParseName(%s, %+v): got: %#v want: %#v", tc.unparsed, tc.resType, got, tc.want) + } + + if gotErr != nil && ("" == tc.wantErrPrefix || !strings.HasPrefix(gotErr.Error(), tc.wantErrPrefix)) { + t.Errorf("ParseName(%s, %+v): %v want prefix: %s", tc.unparsed, tc.resType, gotErr, tc.wantErrPrefix) + } + if gotErr == nil && "" != tc.wantErrPrefix { + t.Errorf("ParseName(%s, %+v): got no err want err prefix: %s", tc.unparsed, tc.resType, tc.wantErrPrefix) + } + } +} diff --git a/src/tools/ak/res/path.go b/src/tools/ak/res/path.go new file mode 100644 index 0000000..502dedd --- /dev/null +++ b/src/tools/ak/res/path.go @@ -0,0 +1,108 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package res + +import ( + "errors" + "fmt" + "path" + "strings" +) + +// ErrNotResPath the provided path does not seem to point to a resource file +var ErrNotResPath = errors.New("Not a resource path") + +// ErrSkipResPath the provided path does needs to be skipped. +var ErrSkipResPath = errors.New("resource path that does needs to be skipped") + +// PathInfo contains all information about a resource that can be derived from its location on the filesystem. +type PathInfo struct { + Path string + ResDir string + TypeDir string + Type Type + Qualifier string + Density Density +} + +// ParsePath converts a path string into a PathInfo object if the string points to a resource file. +func ParsePath(p string) (PathInfo, error) { + parent := path.Dir(p) + resDir := path.Dir(parent) + typeDir := path.Base(parent) + + if strings.HasPrefix(path.Base(p), ".") { + return PathInfo{}, ErrSkipResPath + } + + resType, err := ParseValueOrType(strings.Split(typeDir, "-")[0]) + qualifier := extractQualifier(typeDir) + if err != nil { + return PathInfo{}, ErrNotResPath + } + var density Density + for _, q := range strings.Split(qualifier, "-") { + var err error + density, err = ParseDensity(q) + if err != nil { + return PathInfo{}, err + } + if density != UnspecifiedDensity { + break + } + } + return PathInfo{ + Path: p, + ResDir: resDir, + TypeDir: typeDir, + Type: resType, + Qualifier: qualifier, + Density: density, + }, nil +} + +// MakePathInfo converts a path string into a PathInfo object. +func MakePathInfo(p string) (*PathInfo, error) { + pi, err := ParsePath(p) + if err != nil { + return nil, fmt.Errorf("ParsePath failed to parse %q: %v", p, err) + } + return &pi, nil +} + +// MakePathInfos converts a list of path strings into a list of PathInfo objects. +func MakePathInfos(paths []string) ([]*PathInfo, error) { + pis := make([]*PathInfo, 0, len(paths)) + for _, p := range paths { + if strings.HasPrefix(path.Base(p), ".") { + continue + } + pi, err := MakePathInfo(p) + if err != nil { + return nil, err + } + pis = append(pis, pi) + } + return pis, nil +} + +func extractQualifier(s string) string { + base := path.Base(s) + parts := strings.SplitN(base, "-", 2) + if len(parts) > 1 { + return parts[1] + } + return "" +} diff --git a/src/tools/ak/res/path_test.go b/src/tools/ak/res/path_test.go new file mode 100644 index 0000000..24871d7 --- /dev/null +++ b/src/tools/ak/res/path_test.go @@ -0,0 +1,249 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package res + +import ( + "reflect" + "testing" +) + +func TestParsePath(t *testing.T) { + tests := []struct { + arg string + want PathInfo + }{ + { + "/tmp/foobar/values/strings.xml", + PathInfo{ + Path: "/tmp/foobar/values/strings.xml", + Type: ValueType, + TypeDir: "values", + ResDir: "/tmp/foobar", + }, + }, + { + "/tmp/foobar/values-v19/strings.xml", + PathInfo{ + Path: "/tmp/foobar/values-v19/strings.xml", + Type: ValueType, + TypeDir: "values-v19", + ResDir: "/tmp/foobar", + Qualifier: "v19", + }, + }, + { + "/tmp/baz/foobar/layout-es-419/main_activity.xml", + PathInfo{ + Path: "/tmp/baz/foobar/layout-es-419/main_activity.xml", + Type: Layout, + TypeDir: "layout-es-419", + ResDir: "/tmp/baz/foobar", + Qualifier: "es-419", + }, + }, + { + "/tmp/baz/foobar/menu/menu_data.xml", + PathInfo{ + Path: "/tmp/baz/foobar/menu/menu_data.xml", + Type: Menu, + TypeDir: "menu", + ResDir: "/tmp/baz/foobar", + }, + }, + { + "tmp/baz/foobar/drawable-en-ldpi/mercury.png", + PathInfo{ + Path: "tmp/baz/foobar/drawable-en-ldpi/mercury.png", + Type: Drawable, + TypeDir: "drawable-en-ldpi", + ResDir: "tmp/baz/foobar", + Qualifier: "en-ldpi", + Density: 120, + }, + }, + { + "tmp/baz/foobar/drawable-fr-mdpi-nokeys/mars.xml", + PathInfo{ + Path: "tmp/baz/foobar/drawable-fr-mdpi-nokeys/mars.xml", + Type: Drawable, + TypeDir: "drawable-fr-mdpi-nokeys", + ResDir: "tmp/baz/foobar", + Qualifier: "fr-mdpi-nokeys", + Density: 160, + }, + }, + + { + "tmp/baz/foobar/drawable-mcc310-en-rUS-tvdpi/venus.jpg", + PathInfo{ + Path: "tmp/baz/foobar/drawable-mcc310-en-rUS-tvdpi/venus.jpg", + Type: Drawable, + TypeDir: "drawable-mcc310-en-rUS-tvdpi", + ResDir: "tmp/baz/foobar", + Qualifier: "mcc310-en-rUS-tvdpi", + Density: 213, + }, + }, + { + "tmp/baz/foobar/drawable-mcc208-mnc00-fr-rCA-hdpi-12key-dpad/earth.gif", + PathInfo{ + Path: "tmp/baz/foobar/drawable-mcc208-mnc00-fr-rCA-hdpi-12key-dpad/earth.gif", + Type: Drawable, + TypeDir: "drawable-mcc208-mnc00-fr-rCA-hdpi-12key-dpad", + ResDir: "tmp/baz/foobar", + Qualifier: "mcc208-mnc00-fr-rCA-hdpi-12key-dpad", + Density: 240, + }, + }, + { + "tmp/baz/foobar/drawable-xhdpi/neptune.jpg", + PathInfo{ + Path: "tmp/baz/foobar/drawable-xhdpi/neptune.jpg", + Type: Drawable, + TypeDir: "drawable-xhdpi", + ResDir: "tmp/baz/foobar", + Qualifier: "xhdpi", + Density: 320, + }, + }, + { + "tmp/baz/foobar/drawable-xxhdpi/uranus.png", + PathInfo{ + Path: "tmp/baz/foobar/drawable-xxhdpi/uranus.png", + Type: Drawable, + TypeDir: "drawable-xxhdpi", + ResDir: "tmp/baz/foobar", + Qualifier: "xxhdpi", + Density: 480, + }, + }, + { + "tmp/baz/foobar/drawable-xxxhdpi/saturn.xml", + PathInfo{ + Path: "tmp/baz/foobar/drawable-xxxhdpi/saturn.xml", + Type: Drawable, + TypeDir: "drawable-xxxhdpi", + ResDir: "tmp/baz/foobar", + Qualifier: "xxxhdpi", + Density: 640, + }, + }, + { + "tmp/baz/foobar/drawable-anydpi/jupiter.png", + PathInfo{ + Path: "tmp/baz/foobar/drawable-anydpi/jupiter.png", + Type: Drawable, + TypeDir: "drawable-anydpi", + ResDir: "tmp/baz/foobar", + Qualifier: "anydpi", + Density: AnyDPI, + }, + }, + { + "tmp/baz/foobar/drawable-nodpi/sun.gif", + PathInfo{ + Path: "tmp/baz/foobar/drawable-nodpi/sun.gif", + Type: Drawable, + TypeDir: "drawable-nodpi", + ResDir: "tmp/baz/foobar", + Qualifier: "nodpi", + Density: NoDPI, + }, + }, + { + "tmp/baz/foobar/drawable-120dpi/moon.xml", + PathInfo{ + Path: "tmp/baz/foobar/drawable-120dpi/moon.xml", + Type: Drawable, + TypeDir: "drawable-120dpi", + ResDir: "tmp/baz/foobar", + Qualifier: "120dpi", + Density: 120, + }, + }, + } + for _, tc := range tests { + got, err := ParsePath(tc.arg) + if err != nil { + t.Errorf("ParsePath(%s): got err: %s", tc.arg, err) + continue + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("ParsePath(%s): got %+v want: %+v", tc.arg, got, tc.want) + } + } +} + +func TestParsePath_NegativeCases(t *testing.T) { + tests := []struct { + arg string + err error + }{ + {"/foo/bar/baz/strings.xml", ErrNotResPath}, + {"strings.xml", ErrNotResPath}, + } + for _, tc := range tests { + got, err := ParsePath(tc.arg) + if err == nil { + t.Errorf("ParsePath(%s): got: %+v and nil err, want err: %v", tc.arg, got, tc.err) + } + if err != tc.err { + t.Errorf("ParsePath(%s): got err: %v want err: %v", tc.arg, err, tc.err) + } + } +} + +func TestMakePathInfo(t *testing.T) { + paths := []string{ + "/tmp/foobar/values/strings.xml", + "/tmp/foobar/values-v19/strings.xml", + "/tmp/foobar/values-v19/.skip_me.xml", + "/tmp/baz/foobar/menu/menu_data.xml", + "tmp/baz/foobar/drawable-en-ldpi/mercury.png", + "/tmp/foobar/values-v19/.skip_me_as_well.xml", + } + want := []*PathInfo{ + &PathInfo{ + Path: "/tmp/foobar/values/strings.xml", + Type: ValueType, + TypeDir: "values", + ResDir: "/tmp/foobar"}, + &PathInfo{ + Path: "/tmp/foobar/values-v19/strings.xml", + Type: ValueType, + TypeDir: "values-v19", + ResDir: "/tmp/foobar", + Qualifier: "v19"}, + &PathInfo{ + Path: "/tmp/baz/foobar/menu/menu_data.xml", + Type: Menu, + TypeDir: "menu", + ResDir: "/tmp/baz/foobar"}, + &PathInfo{ + Path: "tmp/baz/foobar/drawable-en-ldpi/mercury.png", + Type: Drawable, + TypeDir: "drawable-en-ldpi", + ResDir: "tmp/baz/foobar", + Qualifier: "en-ldpi", + Density: 120}, + } + pInfos, err := MakePathInfos(paths) + if err != nil { + t.Fatalf("MakePathInfos unexpected error: %v", err) + } + if !reflect.DeepEqual(pInfos, want) { + t.Errorf("MakePathInfos: got %+v want: %+v", pInfos, want) + } +} diff --git a/src/tools/ak/res/proto/BUILD b/src/tools/ak/res/proto/BUILD new file mode 100644 index 0000000..adbdf33 --- /dev/null +++ b/src/tools/ak/res/proto/BUILD @@ -0,0 +1,32 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description +# Android resources proto +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +proto_library( + name = "res_meta_proto", + srcs = ["res_meta.proto"], +) + +go_proto_library( + name = "res_meta_go_proto", + importpath = "src/tools/ak/res/proto/res_meta_go_proto", + protos = [":res_meta_proto"], +) + +proto_library( + name = "res_data_proto", + srcs = ["res_data.proto"], + deps = [":res_meta_proto"], +) + +go_proto_library( + name = "res_data_go_proto", + importpath = "src/tools/ak/res/proto/res_data_go_proto", + protos = [":res_data_proto"], + deps = [":res_meta_go_proto"], +) diff --git a/src/tools/ak/res/proto/res_data.proto b/src/tools/ak/res/proto/res_data.proto new file mode 100644 index 0000000..406983a --- /dev/null +++ b/src/tools/ak/res/proto/res_data.proto @@ -0,0 +1,53 @@ +syntax = "proto3"; + +package tools.android.ak.res.proto; + +import "src/tools/ak/res/proto/res_meta.proto"; + +// A Resource file including its values. +// Next ID: 4 +// From frameworks/base/tools/aapt2/Resource.h, +message Resource { + // Next ID: 26 + enum Type { + ANIM = 0; + ANIMATOR = 1; + ARRAY = 2; + ATTR = 3; + ATTR_PRIVATE = 4; + BOOL = 5; + COLOR = 6; + CONFIG_VARYING = 7; + DIMEN = 8; + DRAWABLE = 9; + FONT = 10; + FRACTION = 11; + ID = 12; + INTEGER = 13; + INTERPOLATOR = 14; + LAYOUT = 15; + MENU = 16; + MIPMAP = 17; + NAVIGATION = 18; + PLURALS = 19; + RAW = 20; + STRING = 21; + STYLE = 22; + STYLEABLE = 23; + TRANSITION = 24; + XML = 25; + } + + // name of the resource, e.g.: + // for strings its the "name" attribute of the <string> entry + // for layouts its the layout file name. + string name = 1; + Type resource_type = 2; + StyleableMetaData styleable_value = 3; // set if resource_type = STYLEABLE +} + +// Ideally we could just use a recordio file for this. But not opensource. +message Resources { + string pkg = 1; + repeated Resource resource = 2; +} diff --git a/src/tools/ak/res/proto/res_meta.proto b/src/tools/ak/res/proto/res_meta.proto new file mode 100644 index 0000000..38d2f57 --- /dev/null +++ b/src/tools/ak/res/proto/res_meta.proto @@ -0,0 +1,14 @@ +syntax = "proto2"; + +package tools.android.ak.res.proto; + +// Describes a stylable view. +// Corresponds to R.styleable +// Next ID: 5 +message StyleableMetaData { + // The name of the style - eg AbsListView or PieChart + optional string name = 1; + + // pkg:attr_name form. + repeated string fqn_attributes = 2; +} diff --git a/src/tools/ak/res/respipe/BUILD b/src/tools/ak/res/respipe/BUILD new file mode 100644 index 0000000..ddc3053 --- /dev/null +++ b/src/tools/ak/res/respipe/BUILD @@ -0,0 +1,42 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +licenses(["notice"]) + +go_library( + name = "respipe", + srcs = [ + "errors.go", + "path_emitter.go", + "res_io.go", + "streams.go", + ], + importpath = "src/tools/ak/res/respipe/respipe", + visibility = [ + "//src/tools/ak:__subpackages__", + "//src/tools/resource_extractor:__subpackages__", + "//tools/android/incremental:__subpackages__", + ], + deps = [ + "//src/tools/ak/res", + "//src/tools/ak/res/proto:res_data_go_proto", + "@org_golang_google_protobuf//proto", + ], +) + +go_test( + name = "respipe_test", + size = "small", + srcs = [ + "errors_test.go", + "path_emitter_test.go", + "res_io_test.go", + "streams_test.go", + ], + embed = [":respipe"], + deps = [ + "//src/tools/ak/res", + "//src/tools/ak/res/proto:res_data_go_proto", + "@org_golang_google_protobuf//proto", + ], +) diff --git a/src/tools/ak/res/respipe/errors.go b/src/tools/ak/res/respipe/errors.go new file mode 100644 index 0000000..60cbb02 --- /dev/null +++ b/src/tools/ak/res/respipe/errors.go @@ -0,0 +1,43 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package respipe + +import ( + "fmt" + "strings" + + "context" +) + +const ( + ctxErrPrefix = "err-prefix" +) + +// Errorf returns a formatted error with any context sensitive information prefixed to the error +func Errorf(ctx context.Context, fmts string, a ...interface{}) error { + if s, ok := ctx.Value(ctxErrPrefix).(string); ok { + return fmt.Errorf(strings.Join([]string{s, fmts}, ""), a...) + } + return fmt.Errorf(fmts, a...) +} + +// PrefixErr returns a context which adds a prefix to error messages. +func PrefixErr(ctx context.Context, add string) context.Context { + if s, ok := ctx.Value(ctxErrPrefix).(string); ok { + return context.WithValue(ctx, ctxErrPrefix, strings.Join([]string{s, add}, "")) + } + return context.WithValue(ctx, ctxErrPrefix, add) + +} diff --git a/src/tools/ak/res/respipe/errors_test.go b/src/tools/ak/res/respipe/errors_test.go new file mode 100644 index 0000000..6b7b5b1 --- /dev/null +++ b/src/tools/ak/res/respipe/errors_test.go @@ -0,0 +1,56 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package respipe + +import ( + "errors" + "reflect" + "testing" + + "context" +) + +func TestPrefixErr(t *testing.T) { + tests := []struct { + ctx context.Context + fmts string + args []interface{} + want error + }{ + { + ctx: context.Background(), + fmts: "Hello world", + want: errors.New("Hello world"), + }, + { + ctx: PrefixErr(context.Background(), "file: foo: "), + fmts: "Hello world: %d", + args: []interface{}{1}, + want: errors.New("file: foo: Hello world: 1"), + }, + { + ctx: PrefixErr(PrefixErr(context.Background(), "file: foo: "), "tag: <resources>: "), + fmts: "Hello world: %d", + args: []interface{}{1}, + want: errors.New("file: foo: tag: <resources>: Hello world: 1"), + }, + } + for _, tc := range tests { + got := Errorf(tc.ctx, tc.fmts, tc.args...) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("Errorf(%v, %v, %v): %v wanted %v", tc.ctx, tc.fmts, tc.args, got, tc.want) + } + } +} diff --git a/src/tools/ak/res/respipe/path_emitter.go b/src/tools/ak/res/respipe/path_emitter.go new file mode 100644 index 0000000..f903718 --- /dev/null +++ b/src/tools/ak/res/respipe/path_emitter.go @@ -0,0 +1,94 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package respipe + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "src/tools/ak/res/res" +) + +// EmitPathInfos takes the list of provided PathInfos and emits them via its returned channel. +func EmitPathInfos(ctx context.Context, pis []*res.PathInfo) <-chan *res.PathInfo { + // produce PathInfos from res files + piC := make(chan *res.PathInfo) + go func() { + defer close(piC) + for _, pi := range pis { + select { + case piC <- pi: + case <-ctx.Done(): + return + } + } + }() + return piC +} + +// EmitPathInfosDir descends a provided directory and emits PathInfo objects via its returned +// channel. It also emits any errors encountered during the walk to its error channel. +func EmitPathInfosDir(ctx context.Context, base string) (<-chan *res.PathInfo, <-chan error) { + piC := make(chan *res.PathInfo) + errC := make(chan error) + go func() { + defer close(piC) + defer close(errC) + emit := func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("%s: walk failed: %v", path, err) + } + if info.IsDir() { + // we do not care about dirs. + return nil + } + pi, err := res.ParsePath(path) + if err == res.ErrNotResPath || err == res.ErrSkipResPath { + return nil + } + if err != nil { + if !SendErr(ctx, errC, Errorf(ctx, "%s: unexpected PathInfo failure: %v", path, err)) { + return filepath.SkipDir + } + return nil + } + select { + case <-ctx.Done(): + return filepath.SkipDir + case piC <- &pi: + } + return nil + } + if err := filepath.Walk(base, emit); err != nil { + SendErr(ctx, errC, Errorf(ctx, "%s: walk encountered err: %v", base, err)) + } + }() + return piC, errC +} + +// EmitPathInfosDirs descends a provided directories and emits PathsInfo objects via its returned +// channel. It also emits any errors encountered during the walk to its error channel. +func EmitPathInfosDirs(ctx context.Context, dirs []string) (<-chan *res.PathInfo, <-chan error) { + piCs := make([]<-chan *res.PathInfo, 0, len(dirs)) + errCs := make([]<-chan error, 0, len(dirs)) + for _, rd := range dirs { + piC, piErr := EmitPathInfosDir(ctx, rd) + piCs = append(piCs, piC) + errCs = append(errCs, piErr) + } + return MergePathInfoStreams(ctx, piCs), MergeErrStreams(ctx, errCs) +} diff --git a/src/tools/ak/res/respipe/path_emitter_test.go b/src/tools/ak/res/respipe/path_emitter_test.go new file mode 100644 index 0000000..84f28cc --- /dev/null +++ b/src/tools/ak/res/respipe/path_emitter_test.go @@ -0,0 +1,92 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package respipe + +import ( + "io/ioutil" + "os" + "path" + "reflect" + "sort" + "testing" + + "context" +) + +func TestEmitPathInfosDir(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatalf("%s: make failed: %v", tmpDir, err) + } + defer func() { + if err := os.RemoveAll(tmpDir); err != nil { + t.Errorf("%s: could not remove: %v", tmpDir, err) + } + }() + + touch := func(p string) string { + if err := os.MkdirAll(path.Dir(path.Join(tmpDir, p)), 0744); err != nil { + t.Fatalf("%s: mkdir failed: %v", p, err) + } + f, err := os.OpenFile(path.Join(tmpDir, p), os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + t.Fatalf("%s: touch failed: %v", p, err) + } + defer f.Close() + return f.Name() + } + wantPaths := []string{ + "values/strings.xml", + "values/styles.xml", + "layout-land/hello.xml", + "layout/hello.xml", + "values-v19/styles.xml", + "drawable-ldpi/foo.png", + "raw/data.xml", + "xml/perf.xml", + } + for i, p := range wantPaths { + wantPaths[i] = touch(p) + } + touch("values/.placeholder") + touch("something_random/data.txt") + + ctx, cxlFn := context.WithCancel(context.Background()) + defer cxlFn() + piC, errC := EmitPathInfosDir(ctx, tmpDir) + var gotPaths []string +Loop: + for { + select { + case p, ok := <-piC: + if !ok { + break Loop + } + gotPaths = append(gotPaths, p.Path) + case e, ok := <-errC: + if !ok { + break Loop + } + t.Fatalf("Unexpected failure: %v", e) + + } + } + sort.Strings(gotPaths) + sort.Strings(wantPaths) + if !reflect.DeepEqual(gotPaths, wantPaths) { + t.Errorf("EmitPathInfosDir(): %v wanted: %v", gotPaths, wantPaths) + } + +} diff --git a/src/tools/ak/res/respipe/res_io.go b/src/tools/ak/res/respipe/res_io.go new file mode 100644 index 0000000..d1e608f --- /dev/null +++ b/src/tools/ak/res/respipe/res_io.go @@ -0,0 +1,109 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +package respipe + +import ( + "bufio" + "encoding/binary" + "io" + + "context" + rdpb "src/tools/ak/res/proto/res_data_go_proto" + "google.golang.org/protobuf/proto" +) + +// ResInput sends all protos in the provided reader into the pipeline. +type ResInput struct { + In io.Reader +} + +// Produce returns a channel of resource protos encountered in the input along with a chan of errors encountered while decoding them. +func (ri ResInput) Produce(ctx context.Context) (<-chan *rdpb.Resource, <-chan error) { + resC := make(chan *rdpb.Resource) + errC := make(chan error) + go func() { + defer close(resC) + defer close(errC) + r := bufio.NewReaderSize(ri.In, 2<<16) + var b [4]byte + for { + if _, err := io.ReadFull(r, b[:]); err != nil { + if err != io.EOF { + SendErr(ctx, errC, Errorf(ctx, "read len failed: %v", err)) + } + return + + } + dlen := binary.LittleEndian.Uint32(b[:]) + d := make([]byte, dlen) + if _, err := io.ReadFull(r, d); err != nil { + SendErr(ctx, errC, Errorf(ctx, "read proto failed: %v", err)) + return + } + r := &rdpb.Resource{} + if err := proto.Unmarshal(d, r); err != nil { + SendErr(ctx, errC, Errorf(ctx, "unmarshal proto failed: %v", err)) + return + } + if !SendRes(ctx, resC, r) { + return + } + + } + + }() + return resC, errC +} + +// ResOutput is a sink to a resource pipeline that writes all resource protos it encounters to the given writer. +type ResOutput struct { + Out io.Writer +} + +// Consume takes all resource protos from the provided channel and writes them to ResOutput's writer. +func (ro ResOutput) Consume(ctx context.Context, resChan <-chan *rdpb.Resource) <-chan error { + + errC := make(chan error) + go func() { + defer close(errC) + + w := bufio.NewWriterSize(ro.Out, 2<<16) + defer func() { + if err := w.Flush(); err != nil { + SendErr(ctx, errC, Errorf(ctx, "flush end of data failed: %v", err)) + } + }() + var b [4]byte + for r := range resChan { + d, err := proto.Marshal(r) + if err != nil { + SendErr(ctx, errC, Errorf(ctx, "%#v encoding failed: %v", r, err)) + return + } + binary.LittleEndian.PutUint32(b[:], uint32(len(d))) + if _, err := w.Write(b[:]); err != nil { + SendErr(ctx, errC, Errorf(ctx, "write failed: %v", err)) + return + } + if _, err := w.Write(d); err != nil { + SendErr(ctx, errC, Errorf(ctx, "write failed: %v", err)) + return + } + } + }() + + return errC +} diff --git a/src/tools/ak/res/respipe/res_io_test.go b/src/tools/ak/res/respipe/res_io_test.go new file mode 100644 index 0000000..70ac90a --- /dev/null +++ b/src/tools/ak/res/respipe/res_io_test.go @@ -0,0 +1,85 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package respipe + +import ( + "bytes" + "context" + "testing" + + rdpb "src/tools/ak/res/proto/res_data_go_proto" + "google.golang.org/protobuf/proto" +) + +func TestProduceConsume(t *testing.T) { + var b bytes.Buffer + + ro := ResOutput{Out: &b} + resC := make(chan *rdpb.Resource) + ctx, cxlFn := context.WithCancel(context.Background()) + defer cxlFn() + + errC := ro.Consume(ctx, resC) + ress := []*rdpb.Resource{ + { + Name: "hi", + }, + { + Name: "bye", + }, + { + Name: "foo", + }, + } + for _, r := range ress { + select { + case err := <-errC: + t.Fatalf("Unexpected err: %v", err) + case resC <- r: + } + } + close(resC) + if err := <-errC; err != nil { + t.Fatalf("unexpected err: %v", err) + } + ri := ResInput{In: &b} + + resInC, errC := ri.Produce(ctx) + var got []*rdpb.Resource + for resInC != nil || errC != nil { + select { + case r, ok := <-resInC: + if !ok { + resInC = nil + continue + } + got = append(got, r) + case err, ok := <-errC: + if !ok { + errC = nil + continue + } + t.Fatalf("Unexpected err: %v", err) + } + } + if len(got) != len(ress) { + t.Fatalf("Got %d elements, expected %d", len(got), len(ress)) + } + for i := range ress { + if !proto.Equal(got[i], ress[i]) { + t.Errorf("Got: %+v wanted: %+v", got[i], ress[i]) + } + } +} diff --git a/src/tools/ak/res/respipe/streams.go b/src/tools/ak/res/respipe/streams.go new file mode 100644 index 0000000..33c41ea --- /dev/null +++ b/src/tools/ak/res/respipe/streams.go @@ -0,0 +1,119 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package respipe contains utilities for running pipelines on android resources. +package respipe + +import ( + "context" + "sync" + + rdpb "src/tools/ak/res/proto/res_data_go_proto" + "src/tools/ak/res/res" +) + +// MergePathInfoStreams fans in multiple PathInfo streams into a single stream. +func MergePathInfoStreams(ctx context.Context, piCs []<-chan *res.PathInfo) <-chan *res.PathInfo { + piC := make(chan *res.PathInfo) + var wg sync.WaitGroup + wg.Add(len(piCs)) + output := func(c <-chan *res.PathInfo) { + defer wg.Done() + for r := range c { + select { + case piC <- r: + case <-ctx.Done(): + return + } + } + } + for _, rc := range piCs { + go output(rc) + } + go func() { + wg.Wait() + close(piC) + }() + return piC +} + +// MergeResStreams fans in multiple Resource streams into a single stream. +func MergeResStreams(ctx context.Context, resCs []<-chan *rdpb.Resource) <-chan *rdpb.Resource { + resC := make(chan *rdpb.Resource) + var wg sync.WaitGroup + wg.Add(len(resCs)) + output := func(c <-chan *rdpb.Resource) { + defer wg.Done() + for r := range c { + select { + case resC <- r: + case <-ctx.Done(): + return + } + } + } + for _, rc := range resCs { + go output(rc) + } + go func() { + wg.Wait() + close(resC) + }() + return resC +} + +// MergeErrStreams fans in multiple error streams into a single stream. +func MergeErrStreams(ctx context.Context, errCs []<-chan error) <-chan error { + errC := make(chan error) + var wg sync.WaitGroup + wg.Add(len(errCs)) + output := func(c <-chan error) { + defer wg.Done() + for e := range c { + select { + case errC <- e: + case <-ctx.Done(): + return + } + } + } + for _, rc := range errCs { + go output(rc) + } + go func() { + wg.Wait() + close(errC) + }() + return errC +} + +// SendErr attempts to send the provided error to the provided chan, however is the context is canceled, it will return false. +func SendErr(ctx context.Context, errC chan<- error, err error) bool { + select { + case <-ctx.Done(): + return false + case errC <- err: + return true + } +} + +// SendRes attempts to send the provided resource to the provided chan, however is the context is canceled, it will return false. +func SendRes(ctx context.Context, resC chan<- *rdpb.Resource, r *rdpb.Resource) bool { + select { + case <-ctx.Done(): + return false + case resC <- r: + return true + } +} diff --git a/src/tools/ak/res/respipe/streams_test.go b/src/tools/ak/res/respipe/streams_test.go new file mode 100644 index 0000000..81dd7fc --- /dev/null +++ b/src/tools/ak/res/respipe/streams_test.go @@ -0,0 +1,85 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package respipe + +import ( + "context" + "errors" + "testing" + + rdpb "src/tools/ak/res/proto/res_data_go_proto" + "src/tools/ak/res/res" +) + +func TestMergePathInfoStreams(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sendClose := func(p *res.PathInfo, c chan<- *res.PathInfo) { + defer close(c) + c <- p + } + in1 := make(chan *res.PathInfo) + in2 := make(chan *res.PathInfo) + go sendClose(&res.PathInfo{}, in1) + go sendClose(&res.PathInfo{}, in2) + mergedC := MergePathInfoStreams(ctx, []<-chan *res.PathInfo{in1, in2}) + var rcv []*res.PathInfo + for p := range mergedC { + rcv = append(rcv, p) + } + if len(rcv) != 2 { + t.Errorf("got: %v on merged stream, wanted only 2 elements", rcv) + } +} + +func TestMergeResStreams(t *testing.T) { + ctx := context.Background() + sendClose := func(r *rdpb.Resource, c chan<- *rdpb.Resource) { + defer close(c) + c <- r + } + in1 := make(chan *rdpb.Resource) + in2 := make(chan *rdpb.Resource) + go sendClose(&rdpb.Resource{}, in1) + go sendClose(&rdpb.Resource{}, in2) + merged := MergeResStreams(ctx, []<-chan *rdpb.Resource{in1, in2}) + var rcv []*rdpb.Resource + for r := range merged { + rcv = append(rcv, r) + } + if len(rcv) != 2 { + t.Errorf("got: %v on merged stream, wanted only 2 elements", rcv) + } +} + +func TestMergeErrStreams(t *testing.T) { + ctx := context.Background() + sendClose := func(e error, eC chan<- error) { + defer close(eC) + eC <- e + } + in1 := make(chan error) + in2 := make(chan error) + go sendClose(errors.New("hi"), in1) + go sendClose(errors.New("hello"), in2) + merged := MergeErrStreams(ctx, []<-chan error{in1, in2}) + var rcv []error + for r := range merged { + rcv = append(rcv, r) + } + if len(rcv) != 2 { + t.Errorf("got: %v on merged stream, wanted only 2 elements", rcv) + } +} diff --git a/src/tools/ak/res/resxml/BUILD b/src/tools/ak/res/resxml/BUILD new file mode 100644 index 0000000..c74aa68 --- /dev/null +++ b/src/tools/ak/res/resxml/BUILD @@ -0,0 +1,24 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +licenses(["notice"]) + +go_library( + name = "resxml", + srcs = ["xml_parser.go"], + importpath = "src/tools/ak/res/resxml/resxml", + visibility = ["//src/tools/ak/liteparse:__subpackages__"], + deps = [ + "//src/tools/ak/res/respipe", + ], +) + +go_test( + name = "resxml_test", + size = "small", + srcs = ["xml_parser_test.go"], + embed = [":resxml"], + deps = [ + "//src/tools/ak/res/respipe", + ], +) diff --git a/src/tools/ak/res/resxml/xml_parser.go b/src/tools/ak/res/resxml/xml_parser.go new file mode 100644 index 0000000..ed765fd --- /dev/null +++ b/src/tools/ak/res/resxml/xml_parser.go @@ -0,0 +1,133 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package resxml contains common functions to extract information from xml files and feed that information into the resource processing pipeline. +package resxml + +import ( + "context" + "encoding/xml" + "io" + + "src/tools/ak/res/respipe/respipe" +) + +// XMLEvent wraps an XMLToken and the Offset at which it was encountered. +type XMLEvent struct { + Token xml.Token + Offset int64 +} + +// ConsumeUntil takes xmlEvents from the provided chan and discards them until it finds a StartEvent which matches the provided name. If the channel is exhausted, false is returned. +func ConsumeUntil(name xml.Name, xmlC <-chan XMLEvent) (XMLEvent, bool) { + for xe := range xmlC { + if se, ok := xe.Token.(xml.StartElement); ok { + if SloppyMatches(name, se.Name) { + return xe, true + } + } + } + return XMLEvent{}, false +} + +// ForwardChildren takes the provided StartElement and a channel of XMLEvents and forwards that all events onto the returned XMLEvent channel until the matching EndElement to start is encountered. +func ForwardChildren(ctx context.Context, start XMLEvent, xmlC <-chan XMLEvent) <-chan XMLEvent { + eventC := make(chan XMLEvent, 1) + se := start.Token.(xml.StartElement) + go func() { + defer close(eventC) + count := 1 + for xe := range xmlC { + if e, ok := xe.Token.(xml.StartElement); ok { + if StrictMatches(e.Name, se.Name) { + count++ + } + } + if e, ok := xe.Token.(xml.EndElement); ok { + if StrictMatches(e.Name, se.Name) { + count-- + } + if count == 0 { + return + } + } + if !SendXML(ctx, eventC, xe) { + return + } + } + }() + return eventC + +} + +// StrictMatches considers xml.Names equal if both their space and name matches. +func StrictMatches(n1, n2 xml.Name) bool { + return n1.Local == n2.Local && n1.Space == n2.Space +} + +// SloppyMatches ignores xml.Name Space attributes unless both names specify Space. Otherwise +// only the Local attribute is used for matching. +func SloppyMatches(n1, n2 xml.Name) bool { + if n1.Space != "" && n2.Space != "" { + return StrictMatches(n1, n2) + } + return n1.Local == n2.Local +} + +// StreamDoc parses the provided doc and forwards all xml tokens to the returned XMLEvent chan. +func StreamDoc(ctx context.Context, doc io.Reader) (<-chan XMLEvent, <-chan error) { + eventC := make(chan XMLEvent) + errC := make(chan error) + go func() { + defer close(eventC) + defer close(errC) + decoder := xml.NewDecoder(doc) + // Turns off unknown entities check. Would otherwise fail on resources + // using non-standard XML entities. + decoder.Strict = false + for { + tok, err := decoder.Token() + if err == io.EOF { + return + } + if err != nil { + respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "offset: %d xml error: %v", decoder.InputOffset(), err)) + return + } + tok = xml.CopyToken(tok) + if !SendXML(ctx, eventC, XMLEvent{tok, decoder.InputOffset()}) { + return + } + } + }() + return eventC, errC +} + +// SendXML sends an XMLEvent to the provided channel and returns true, otherwise if the context is done, it returns false. +func SendXML(ctx context.Context, xmlC chan<- XMLEvent, xml XMLEvent) bool { + select { + case <-ctx.Done(): + return false + case xmlC <- xml: + return true + } +} + +// Attrs returns all []xml.Attrs encounted on an XMLEvent. +func Attrs(xe XMLEvent) []xml.Attr { + if se, ok := xe.Token.(xml.StartElement); ok { + return se.Attr + } + return nil +} diff --git a/src/tools/ak/res/resxml/xml_parser_test.go b/src/tools/ak/res/resxml/xml_parser_test.go new file mode 100644 index 0000000..8c39e29 --- /dev/null +++ b/src/tools/ak/res/resxml/xml_parser_test.go @@ -0,0 +1,226 @@ +// Copyright 2022 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resxml + +import ( + "bytes" + "context" + "encoding/xml" + "io" + "reflect" + "testing" + + "src/tools/ak/res/respipe/respipe" +) + +const ( + doc = ` + <Person> + <FullName>Grace R. Emlin</FullName> + <Company>Example Inc.</Company> + <Email where="home"> + <Addr>gre@example.com</Addr> + </Email> + <City>Hanga Rao<Street>1234 Main St.</Street>RandomText</City> + <Email where='work'> + <Addr>gre@work.com</Addr> + </Email> + <Group> + <Value>Friends</Value> + <Value>Squash</Value> + </Group> + <State>Easter Island</State> + </Person> + ` +) + +func TestForwardChildren(t *testing.T) { + ctx, cancel := context.WithCancel(respipe.PrefixErr(context.Background(), "test doc: ")) + defer cancel() + xmlC, errC := StreamDoc(ctx, bytes.NewBufferString(doc)) + xe, ok := ConsumeUntil(xml.Name{Local: "City"}, xmlC) + if !ok { + t.Fatalf("Expected to find: %s in %s", xml.Name{Local: "City"}, doc) + } + childC := ForwardChildren(ctx, xe, xmlC) + wantEvents := []XMLEvent{ + { + Token: xml.CharData("Hanga Rao"), + }, + { + Token: xml.StartElement{Name: xml.Name{Local: "Street"}, Attr: []xml.Attr{}}, + }, + { + Token: xml.CharData("1234 Main St."), + }, + { + Token: xml.EndElement{Name: xml.Name{Local: "Street"}}, + }, + { + Token: xml.CharData("RandomText"), + }, + } + var gotEvents []XMLEvent + for childC != nil || errC != nil { + select { + case xe, ok := <-childC: + if !ok { + childC = nil + cancel() + continue + } + xe.Offset = 0 + gotEvents = append(gotEvents, xe) + case e, ok := <-errC: + if !ok { + errC = nil + continue + } + t.Errorf("unexpected error: %v", e) + } + } + + if !reflect.DeepEqual(wantEvents, gotEvents) { + t.Errorf("Got children: %#v wanted: %#v", gotEvents, wantEvents) + } + +} + +func TestAttrs(t *testing.T) { + tests := []struct { + arg XMLEvent + want []xml.Attr + }{ + { + XMLEvent{ + Token: xml.StartElement{ + Attr: []xml.Attr{ + { + Name: xml.Name{Local: "dog"}, + Value: "shepard", + }, + { + Name: xml.Name{Local: "cat"}, + Value: "cheshire", + }, + }, + }, + }, + []xml.Attr{ + { + Name: xml.Name{Local: "dog"}, + Value: "shepard", + }, + { + Name: xml.Name{Local: "cat"}, + Value: "cheshire", + }, + }, + }, + { + XMLEvent{Token: xml.StartElement{}}, + []xml.Attr(nil), + }, + { + XMLEvent{Token: xml.CharData("foo")}, + []xml.Attr(nil), + }, + } + + for _, tc := range tests { + got := Attrs(tc.arg) + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("Attrs(%#v): %#v wanted %#v", tc.arg, got, tc.want) + } + } +} + +func TestConsumeUntil(t *testing.T) { + ctx, cancel := context.WithCancel(respipe.PrefixErr(context.Background(), "test doc: ")) + defer cancel() + xmlC, errC := StreamDoc(ctx, bytes.NewBufferString(doc)) + + xe, ok := ConsumeUntil(xml.Name{Local: "Email"}, xmlC) + if !ok { + t.Fatalf("Expected to find: %s in %s", xml.Name{Local: "Email"}, doc) + } + if se, ok := xe.Token.(xml.StartElement); ok { + want := []xml.Attr{{xml.Name{Local: "where"}, "home"}} + if !reflect.DeepEqual(want, se.Attr) { + t.Errorf("Got attr: %v wanted: %v", se.Attr, want) + } + } else { + t.Fatalf("Got: %v Expected to stop on a start element", xe) + } + xe, ok = ConsumeUntil(xml.Name{Local: "Email"}, xmlC) + if !ok { + t.Fatalf("Expected to find: %s in %s", xml.Name{Local: "Email"}, doc) + } + if se, ok := xe.Token.(xml.StartElement); ok { + want := []xml.Attr{{xml.Name{Local: "where"}, "work"}} + if !reflect.DeepEqual(want, se.Attr) { + t.Errorf("Got attr: %v wanted: %v", se.Attr, want) + } + } else { + t.Fatalf("Got: %v Expected to stop on a start element", xe) + } + xe, ok = ConsumeUntil(xml.Name{Local: "Email"}, xmlC) + if ok { + t.Fatalf("Expected no more nodes with: %v got: %v in doc: %s", xml.Name{Local: "Email"}, xe, doc) + } + e, ok := <-errC + if ok { + t.Fatalf("Expected no errors during parse: %v", e) + } +} + +func TestStreamDoc(t *testing.T) { + dec := xml.NewDecoder(bytes.NewBufferString(doc)) + var events []XMLEvent + for { + tok, err := dec.Token() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("Unexpected xml parse failure: %v", err) + } + events = append(events, XMLEvent{xml.CopyToken(tok), dec.InputOffset()}) + } + ctx, cancel := context.WithCancel(respipe.PrefixErr(context.Background(), "test doc: ")) + defer cancel() + xmlC, errC := StreamDoc(ctx, bytes.NewBufferString(doc)) + var got []XMLEvent + for xmlC != nil || errC != nil { + select { + case e, ok := <-errC: + if !ok { + errC = nil + continue + } + t.Errorf("Unexpected error: %v", e) + case xe, ok := <-xmlC: + if !ok { + xmlC = nil + continue + } + got = append(got, xe) + } + } + if !reflect.DeepEqual(events, got) { + t.Errorf("StreamDoc() got: %v wanted: %v", got, events) + } + +} diff --git a/src/tools/ak/res/struct.go b/src/tools/ak/res/struct.go new file mode 100644 index 0000000..b58e854 --- /dev/null +++ b/src/tools/ak/res/struct.go @@ -0,0 +1,328 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package res handles understanding and representing information about Android resources. +package res + +import ( + "errors" + "fmt" + "strconv" + "strings" + + rdpb "src/tools/ak/res/proto/res_data_go_proto" +) + +var ( + // ErrWrongType occurs when a type is used in an operation that it does not support. + ErrWrongType = errors.New("this type cannot be used in this operation") +) + +// Type of resource (eg: string, layout, drawable) +type Type rdpb.Resource_Type + +// Enum converts a Type into a enum proto value +func (t Type) Enum() (rdpb.Resource_Type, error) { + if !t.IsSerializable() { + return rdpb.Resource_Type(ValueType), ErrWrongType + } + return rdpb.Resource_Type(t), nil +} + +// IsSerializable indicates that the Type can be converted to a proto (some types are only for in memory operations). +func (t Type) IsSerializable() bool { + for _, a := range nonProtoTypes { + if t == a { + return false + } + } + return true +} + +// NestedClassName is the R.java nested class name for this type (if the type is understood by android). +func (t Type) NestedClassName() (string, error) { + if !t.IsReal() { + return "", ErrWrongType + } + return typeToString[t], nil +} + +// IsReal indicates that the type is known to the android framework. +func (t Type) IsReal() bool { + for _, a := range nonProtoTypes { + if a == t { + return false + } + } + return true +} + +// From frameworks/base/tools/aapt2/Resource.h, except UnknownType and ValueType +// TODO(mauriciogg): use proto definitions and remove ValueType and UnknownType. +const ( + // UnknownType needs to be zero value + UnknownType Type = -2 + ValueType = -1 + + // Anim represents Android Anim resource types. + Anim = Type(rdpb.Resource_ANIM) + // Animator represents Android Animator resource types. + Animator = Type(rdpb.Resource_ANIMATOR) + // Array represents Android Array resource types. + Array = Type(rdpb.Resource_ARRAY) + // Attr represents Android Attr resource types. + Attr = Type(rdpb.Resource_ATTR) + // AttrPrivate represents Android AttrPrivate resource types. + AttrPrivate = Type(rdpb.Resource_ATTR_PRIVATE) + // Bool represents Android Bool resource types. + Bool = Type(rdpb.Resource_BOOL) + // Color represents Android Color resource types. + Color = Type(rdpb.Resource_COLOR) + // ConfigVarying represents Android ConfigVarying resource types, not really a type, but it shows up in some CTS tests + ConfigVarying = Type(rdpb.Resource_CONFIG_VARYING) + // Dimen represents Android Dimen resource types. + Dimen = Type(rdpb.Resource_DIMEN) + // Drawable represents Android Drawable resource types. + Drawable = Type(rdpb.Resource_DRAWABLE) + // Font represents Android Font resource types. + Font = Type(rdpb.Resource_FONT) + // Fraction represents Android Fraction resource types. + Fraction = Type(rdpb.Resource_FRACTION) + // ID represents Android Id resource types. + ID = Type(rdpb.Resource_ID) + // Integer represents Android Integer resource types. + Integer = Type(rdpb.Resource_INTEGER) + // Interpolator represents Android Interpolator resource types. + Interpolator = Type(rdpb.Resource_INTERPOLATOR) + // Layout represents Android Layout resource types. + Layout = Type(rdpb.Resource_LAYOUT) + // Menu represents Android Menu resource types. + Menu = Type(rdpb.Resource_MENU) + // Mipmap represents Android Mipmap resource types. + Mipmap = Type(rdpb.Resource_MIPMAP) + // Navigation represents Android Navigation resource types. + Navigation = Type(rdpb.Resource_NAVIGATION) + // Plurals represents Android Plurals resource types. + Plurals = Type(rdpb.Resource_PLURALS) + // Raw represents Android Raw resource types. + Raw = Type(rdpb.Resource_RAW) + // String represents Android String resource types. + String = Type(rdpb.Resource_STRING) + // Style represents Android Style resource types. + Style = Type(rdpb.Resource_STYLE) + // Styleable represents Android Styleable resource types. + Styleable = Type(rdpb.Resource_STYLEABLE) + // Transition represents Android Transition resource types. + Transition = Type(rdpb.Resource_TRANSITION) + // XML represents Android Xml resource types. + XML = Type(rdpb.Resource_XML) +) + +var ( + // A fixed mapping between the string representation of a type and its Type. + typeToString = map[Type]string{ + Anim: "anim", + Animator: "animator", + Array: "array", + Attr: "attr", + AttrPrivate: "^attr-private", + Bool: "bool", + Color: "color", + ConfigVarying: "configVarying", + Dimen: "dimen", + Drawable: "drawable", + Fraction: "fraction", + Font: "font", + ID: "id", + Integer: "integer", + Interpolator: "interpolator", + Layout: "layout", + Menu: "menu", + Mipmap: "mipmap", + Navigation: "navigation", + Plurals: "plurals", + Raw: "raw", + String: "string", + Style: "style", + Styleable: "styleable", + Transition: "transition", + XML: "xml", + } + stringToType = make(map[string]Type) + // AllTypes is a list of all known resource types. + AllTypes = make([]Type, 0, len(typeToString)) + + // These types are not allowed to be serialized into proto format. + nonProtoTypes = []Type{ValueType, UnknownType} +) + +// Kind indicates what type of resource file emits this resource. A resource can be found in +// res/values folder (and therefore is a Value - which can be represented as a ResourceValue in +// Android) or in folders outside of res/values (such as res/layout) and thus are not ResourceValues +// but rather some external resource (such as an image or parsed xml file). +type Kind uint8 + +const ( + // Unknown should not be encountered. + Unknown Kind = iota + // Value can only be encountered in res/values folders. + Value + // NonValue can not be encountered in res/values folders. + NonValue + // Both is a Kind of Type which may be inside a res/values folder or in another res/ folder. + Both +) + +var ( + kindToString = map[Kind]string{ + Unknown: "Unknown", + Value: "Value", + NonValue: "NonValue", + Both: "Both", + } + + // A fixed mapping between Type and Kind. + TypesToKind = map[Type]Kind{ + Anim: NonValue, + Animator: NonValue, + Array: Value, + Attr: Value, + AttrPrivate: Value, + Bool: Value, + Color: Both, + ConfigVarying: Value, + Dimen: Value, + Drawable: NonValue, + Font: NonValue, + Fraction: Value, + ID: Value, + Integer: Value, + Interpolator: NonValue, + Layout: NonValue, + Menu: NonValue, + Mipmap: NonValue, + Navigation: NonValue, + Plurals: Value, + Raw: NonValue, + String: Value, + Style: Value, + Styleable: Value, + Transition: NonValue, + XML: NonValue, + } +) + +// Density represents the dpi value of a resource. +type Density uint16 + +// From frameworks/base/core/java/Android/content/res/Configuration.java +const ( + // UnspecifiedDensity is a default value indicating no dpi has been specified + UnspecifiedDensity Density = 0 + + // LDPI has a dpi of 120 + LDPI Density = 120 + // MDPI has a dpi of 160 + MDPI Density = 160 + // TVDPI has a dpi of 213 + TVDPI Density = 213 + // HDPI has a dpi of 240 + HDPI Density = 240 + // XhDPI has a dpi of 320 + XhDPI Density = 320 + // XxhDPI has a dpi of 480 + XxhDPI Density = 480 + // XxxhDPI has a dpi of 640 + XxxhDPI Density = 640 + // AnyDPI indicates a resource which can be any dpi. + AnyDPI Density = 0xfffe + // NoDPI indicates the resources have no dpi constraints + NoDPI Density = 0xffff + dpiSuffix = "dpi" +) + +var ( + densityToStr = map[Density]string{ + LDPI: "ldpi", + MDPI: "mdpi", + TVDPI: "tvdpi", + HDPI: "hdpi", + XhDPI: "xhdpi", + XxhDPI: "xxhdpi", + XxxhDPI: "xxxhdpi", + AnyDPI: "anydpi", + NoDPI: "nodpi", + } + strToDensity = make(map[string]Density) +) + +// ParseValueOrType converts a string into a value type or well known type +func ParseValueOrType(s string) (Type, error) { + if s == "values" { + return ValueType, nil + } + return ParseType(s) +} + +// ParseType converts a string into a well known type +func ParseType(s string) (Type, error) { + if t, ok := stringToType[s]; ok { + return t, nil + } + return UnknownType, fmt.Errorf("%s: unknown type", s) +} + +// String for Type structs corresponds to the string format known to Android. +func (t Type) String() string { + if s, ok := typeToString[t]; ok { + return s + } + return fmt.Sprintf("Type(%d)", t) +} + +// Kind indicates the resource kind of this type. +func (t Type) Kind() Kind { + if t == ValueType { + return Value + } + if t, ok := TypesToKind[t]; ok { + return t + } + return Unknown +} + +// ParseDensity converts a string representation of a density into a Density. +func ParseDensity(s string) (Density, error) { + if d, ok := strToDensity[s]; ok { + return d, nil + } + if strings.HasSuffix(s, dpiSuffix) { + parsed, err := strconv.ParseUint(s[0:len(s)-len(dpiSuffix)], 10, 16) + if err != nil { + return 0, fmt.Errorf("%s: unparsable: %v", s, err) + } + return Density(parsed), nil + } + return UnspecifiedDensity, nil +} + +func init() { + for k, v := range typeToString { + AllTypes = append(AllTypes, k) + stringToType[v] = k + } + for k, v := range densityToStr { + strToDensity[v] = k + } +} diff --git a/src/tools/ak/res/struct_test.go b/src/tools/ak/res/struct_test.go new file mode 100644 index 0000000..0a3d4d5 --- /dev/null +++ b/src/tools/ak/res/struct_test.go @@ -0,0 +1,99 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package res + +import ( + "fmt" + "strings" + "testing" +) + +func TestKinds(t *testing.T) { + tests := []struct { + t Type + k Kind + }{ + {t: String, k: Value}, + {t: XML, k: NonValue}, + {t: Drawable, k: NonValue}, + {t: Color, k: Both}, + {t: Menu, k: NonValue}, + {t: Dimen, k: Value}, + {t: UnknownType, k: Unknown}, + } + for _, tc := range tests { + if tc.t.Kind() != tc.k { + t.Errorf("%v.Kind() = %v want: %v", tc.t, tc.t.Kind(), tc.k) + } + } + for _, at := range AllTypes { + if at == UnknownType { + continue + } + if at.Kind() == Unknown { + t.Errorf("%v.Kind() = %v - wanting anything else but that", at, Unknown) + } + } +} + +func TestTypes(t *testing.T) { + tests := []struct { + t Type + s string + }{ + {t: String, s: "string"}, + {t: XML, s: "xml"}, + {t: Drawable, s: "drawable"}, + {t: Color, s: "color"}, + {t: Menu, s: "menu"}, + {t: Dimen, s: "dimen"}, + } + + for _, tc := range tests { + pt, err := ParseType(tc.s) + if tc.t != pt || err != nil { + t.Errorf("ParseType(%s): %v, %v want: %v", tc.s, pt, err, tc.t) + } + } +} + +func TestDensities(t *testing.T) { + tests := []struct { + arg string + want Density + err error + }{ + {arg: "tvdpi", want: TVDPI}, + {arg: "hdpi", want: HDPI}, + {arg: "320dpi", want: 320}, + {arg: "nodpi", want: NoDPI}, + {arg: "en-US", want: UnspecifiedDensity}, + {arg: "12000000dpi", err: fmt.Errorf("%ddpi: unparsable", 12000000)}, + } + + for _, tc := range tests { + got, err := ParseDensity(tc.arg) + if tc.err == nil && err != nil { + t.Errorf("ParseDensity(%s): got err: %s", tc.arg, err) + } + if tc.err != nil && err != nil && !strings.HasPrefix(err.Error(), tc.err.Error()) { + t.Errorf("ParseDensity(%s): got err: %v want err: %v", tc.arg, err, tc.err) + } + + if got != tc.want { + t.Errorf("ParseDensity(%s): Got: %v want: %v", tc.arg, got, tc.want) + } + } +} diff --git a/src/tools/ak/res/xml.go b/src/tools/ak/res/xml.go new file mode 100644 index 0000000..bdf3f4a --- /dev/null +++ b/src/tools/ak/res/xml.go @@ -0,0 +1,87 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package res + +import ( + "encoding/xml" +) + +var ( + // IDAttrName is the android:id attribute xml name. + // It appears anywhere an xml document wishes to associate a tag to a given android id. + IDAttrName = xml.Name{Space: "http://schemas.android.com/apk/res/android", Local: "id"} + + // ResourcesTagName <resources> tag wraps all xml documents in res/values directory. These + // documents are reasonably well structured, and its children _normally_ end up becoming + // ResourceValues in Android. The exception being <declare-styleable> and <attr> which + // define how to interpret and store attributes in xml files outside of res/values. + ResourcesTagName = xml.Name{Local: "resources"} + + // ItemTagName is used in various ways in a <resources> tag. If it is a direct child, it can + // only denote an id resource. Otherwise, it can be a child of array/*-array and denotes the + // type and wraps the value of the item of the array. + ItemTagName = xml.Name{Local: "item"} + + // NameAttrName is an attribute that is expected to be encountered on every tag that is a + // direct child of <resources>. The value of this tag is the name of the resource that is + // being generated. + NameAttrName = xml.Name{Local: "name"} + + // TypeAttrName is the type attribute xml name. + // It appears in the <item> tag when the item wants to specify its type. + TypeAttrName = xml.Name{Local: "type"} + + // EnumTagName <enum> appears beneath <attr/> tags to define valid enum values for an attribute. + EnumTagName = xml.Name{Local: "enum"} + + // FlagTagName <flag> appears beneath <attr/> tags to define valid flag values for an attribute. + FlagTagName = xml.Name{Local: "flag"} + + // ResourcesTagToType maps the child tag name of resources to the resource type it will generate. + ResourcesTagToType = map[string]Type{ + "array": Array, + "integer-array": Array, + "string-array": Array, + "attr": Attr, + "^attr-private": AttrPrivate, + "bool": Bool, + "color": Color, + "configVarying": ConfigVarying, + "dimen": Dimen, + "drawable": Drawable, + "fraction": Fraction, + "id": ID, + "integer": Integer, + "layout": Layout, + "plurals": Plurals, + "string": String, + "style": Style, + "declare-styleable": Styleable, + } + + // ResourcesChildToSkip a map containing child tags that can be skipped while parsing resources. + ResourcesChildToSkip = map[xml.Name]bool{ + {Local: "skip"}: true, + {Local: "eat-comment"}: true, + {Local: "public"}: true, + } +) + +const ( + // GeneratedIDPrefix prefixes an attribute value whose name is IDAttrName, it indicates that + // this id likely does not exist outside of the current document and a new Id Resource for + // this value. + GeneratedIDPrefix = "@+id" +) diff --git a/src/tools/ak/rjar/BUILD b/src/tools/ak/rjar/BUILD new file mode 100644 index 0000000..3928909 --- /dev/null +++ b/src/tools/ak/rjar/BUILD @@ -0,0 +1,44 @@ +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +# Description: +# Package for R.jar module +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +go_library( + name = "rjar", + srcs = ["rjar.go"], + importpath = "src/tools/ak/rjar/rjar", + deps = [ + "//src/common/golang:ziputils", + "//src/tools/ak:types", + ], +) + +go_binary( + name = "rjar_bin", + srcs = ["rjar_bin.go"], + deps = [ + ":rjar", + "//src/common/golang:flagfile", + ], +) + +go_test( + name = "rjar_test", + size = "small", + srcs = ["rjar_test.go"], + data = [ + "//src/tools/ak/rjar/testdata:R", + "//src/tools/ak/rjar/testdata:pkgs", + "@local_jdk//:bin/java", + "@remote_java_tools_for_rules_android//:java_tools/JavaBuilder_deploy.jar", + ], + embed = [":rjar"], + tags = [ + "manual", + "nozapfhahn", + ], +) diff --git a/src/tools/ak/rjar/rjar.go b/src/tools/ak/rjar/rjar.go new file mode 100644 index 0000000..c95dea8 --- /dev/null +++ b/src/tools/ak/rjar/rjar.go @@ -0,0 +1,295 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package rjar generated R.jar. +package rjar + +import ( + "bufio" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + "src/common/golang/ziputils" + "src/tools/ak/types" +) + +var ( + // Cmd defines the command. + Cmd = types.Command{ + Init: Init, + Run: Run, + Desc: desc, + Flags: []string{"rjava", "pkgs", "rjar", "jdk", "jartool", "target_label"}, + } + + // Variables to hold flag values. + rjava string + pkgs string + rjar string + jdk string + jartool string + targetLabel string + + initOnce sync.Once + + javaReserved = map[string]bool{ + "abstract": true, + "assert": true, + "boolean": true, + "break": true, + "byte": true, + "case": true, + "catch": true, + "char": true, + "class": true, + "const": true, + "continue": true, + "default": true, + "do": true, + "double": true, + "else": true, + "enum": true, + "extends": true, + "false": true, + "final": true, + "finally": true, + "float": true, + "for": true, + "goto": true, + "if": true, + "implements": true, + "import": true, + "instanceof": true, + "int": true, + "interface": true, + "long": true, + "native": true, + "new": true, + "null": true, + "package": true, + "private": true, + "protected": true, + "public": true, + "return": true, + "short": true, + "static": true, + "strictfp": true, + "super": true, + "switch": true, + "synchronized": true, + "this": true, + "throw": true, + "throws": true, + "transient": true, + "true": true, + "try": true, + "void": true, + "volatile": true, + "while": true} +) + +// Init initiailizes rjar action. Must be called before google.Init. +func Init() { + initOnce.Do(func() { + flag.StringVar(&rjava, "rjava", "", "Input R.java path") + flag.StringVar(&pkgs, "pkgs", "", "Packages file path") + flag.StringVar(&rjar, "rjar", "", "Output R.jar path") + flag.StringVar(&jdk, "jdk", "", "Jdk path") + flag.StringVar(&jartool, "jartool", "", "Jartool path") + flag.StringVar(&targetLabel, "target_label", "", "The target label") + }) +} + +func desc() string { + return "rjar creates the R.jar" +} + +// Run is the entry point for rjar. Will exit on error. +func Run() { + if err := doWork(rjava, pkgs, rjar, jdk, jartool, targetLabel); err != nil { + log.Fatalf("Error creating R.jar: %v", err) + } +} + +func doWork(rjava, pkgs, rjar, jdk, jartool string, targetLabel string) error { + f, err := os.Stat(rjava) + if os.IsNotExist(err) || (err == nil && f.Size() == 0) { + // If we don't have an input r_java or have an empty r_java just write + // an empty jar apps might not define resources and in some cases (aar + // files) its not possible to know during analysis phase, so this action + // gets executed regardless. + return ziputils.EmptyZip(rjar) + } + if err != nil { + return fmt.Errorf("os.Stat(%s) failed: %v", rjava, err) + } + + srcDir, err := ioutil.TempDir("", "rjar") + if err != nil { + return err + } + defer os.RemoveAll(srcDir) + + var parentPkg, subclassTmpl string + var srcs []string + + filteredPkgs, err := getPkgs(pkgs) + if err != nil { + return err + } + for _, pkg := range filteredPkgs { + pkgParts := strings.Split(pkg, ".") + if hasInvalid(pkgParts) { + continue + } + pkgDir := filepath.Join(append([]string{srcDir}, pkgParts...)...) + err = os.MkdirAll(pkgDir, 0777) + if err != nil { + return err + } + outRJava := filepath.Join(pkgDir, "R.java") + srcs = append(srcs, outRJava) + if parentPkg == "" { + parentPkg = pkg + var classes []string + out, err := os.Create(outRJava) + if err != nil { + return err + } + defer out.Close() + in, err := os.Open(rjava) + if err != nil { + return err + } + defer in.Close() + if _, err := fmt.Fprintf(out, "package %s;", pkg); err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + return err + } + if _, err := in.Seek(0, 0); err != nil { + return err + } + scanner := bufio.NewScanner(in) + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, "public static class ") { + classes = append(classes, strings.Split(strings.Split(line, "public static class ")[1], " ")[0]) + } + } + subclassPts := []string{"package %s;", fmt.Sprintf("public class R extends %s.R {", pkg)} + for _, t := range classes { + subclassPts = append(subclassPts, fmt.Sprintf(" public static class %s extends %s.R.%s {}", t, pkg, t)) + } + subclassPts = append(subclassPts, "}") + subclassTmpl = strings.Join(subclassPts, "\n") + } else { + out, err := os.Create(outRJava) + if err != nil { + return err + } + defer out.Close() + fmt.Fprintf(out, subclassTmpl, pkg) + } + } + if _, err := os.Lstat(rjar); err == nil { + if err := os.Remove(rjar); err != nil { + return err + } + } + if err = os.MkdirAll(filepath.Dir(rjar), 0777); err != nil { + return err + } + return compileRJar(srcs, rjar, jdk, jartool, targetLabel) +} + +func compileRJar(srcs []string, rjar, jdk, jartool string, targetLabel string) error { + control, err := ioutil.TempFile("", "control") + if err != nil { + return err + } + defer os.Remove(control.Name()) + + args := []string{"--javacopts", + "-source", "8", + "-target", "8", + "-nowarn", "--", "--sources"} + args = append(args, srcs...) + args = append(args, []string{ + "--strict_java_deps", "ERROR", + "--output", rjar, + }...) + if len(targetLabel) > 0 { + args = append(args, []string{ + "--target_label", targetLabel, + }...) + } + if _, err := fmt.Fprint(control, strings.Join(args, "\n")); err != nil { + return err + } + if err := control.Sync(); err != nil { + return err + } + c, err := exec.Command(jdk, "-jar", jartool, fmt.Sprintf("@%s", control.Name())).CombinedOutput() + if err != nil { + return fmt.Errorf("%v:\n%s", err, c) + } + return nil +} + +func getPkgs(pkgs string) ([]string, error) { + var filteredPkgs []string + seenPkgs := map[string]bool{} + + f, err := os.Open(pkgs) + if err != nil { + return nil, err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + pkg := strings.TrimSpace(scanner.Text()) + if strings.ContainsAny(pkg, "-$/") || pkg == "" { + continue + } + if seenPkgs[pkg] { + continue + } + filteredPkgs = append(filteredPkgs, pkg) + seenPkgs[pkg] = true + } + if err := scanner.Err(); err != nil { + return nil, err + } + return filteredPkgs, nil +} + +func hasInvalid(parts []string) bool { + for _, p := range parts { + if javaReserved[p] { + return true + } + } + return false +} diff --git a/src/tools/ak/rjar/rjar_bin.go b/src/tools/ak/rjar/rjar_bin.go new file mode 100644 index 0000000..11557f4 --- /dev/null +++ b/src/tools/ak/rjar/rjar_bin.go @@ -0,0 +1,29 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// rjar_bin is the command line tool to create an R.jar +package main + +import ( + "flag" + + _ "src/common/golang/flagfile" + "src/tools/ak/rjar/rjar" +) + +func main() { + rjar.Init() + flag.Parse() + rjar.Run() +} diff --git a/src/tools/ak/rjar/rjar_test.go b/src/tools/ak/rjar/rjar_test.go new file mode 100644 index 0000000..8c29650 --- /dev/null +++ b/src/tools/ak/rjar/rjar_test.go @@ -0,0 +1,82 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rjar + +import ( + "archive/zip" + "io/ioutil" + "os" + "path" + "path/filepath" + "testing" +) + +var ( + expectedClasses = []string{"R.class", "R$attr.class", "R$id.class", "R$layout.class", "R$string.class"} +) + +const ( + java = "local_jdk/bin/java" + testDataBase = "build_bazel_rules_android/src/tools/ak/rjar/testdata" +) + +func TestCreateRJar(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "rjartest") + if err != nil { + t.Fatalf("Error creating temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + out := filepath.Join(tmpDir, "R.jar") + jarDexer := path.Join(os.Getenv("TEST_SRCDIR"), "remote_java_tools_for_rules_android/java_tools/JavaBuilder_deploy.jar") + inJava := dataPath("R.java") + pkgs := dataPath("pkgs.txt") + targetLabel := "//test:test" + + if err := doWork(inJava, pkgs, out, path.Join(os.Getenv("TEST_SRCDIR"), java), jarDexer, targetLabel); err != nil { + t.Fatalf("Error creating R.jar: %v", err) + } + + z, err := zip.OpenReader(out) + if err != nil { + t.Fatalf("Error opening output jar: %v", err) + } + defer z.Close() + + for _, class := range expectedClasses { + if !zipContains(z, filepath.Join("android/support/v7", class)) { + t.Errorf("R.jar does not contain %s", filepath.Join("android/support/v7", class)) + } + if !zipContains(z, filepath.Join("com/google/android/samples/skeletonapp", class)) { + t.Errorf("R.jar does not contain %s", filepath.Join("com/google/android/samples/skeletonapp", class)) + } + if zipContains(z, filepath.Join("com/google/android/package/test", class)) { + t.Errorf("R.jar contains %s", filepath.Join("com/google/android/package/test", class)) + } + } +} + +func dataPath(fn string) string { + return filepath.Join(os.Getenv("TEST_SRCDIR"), testDataBase, fn) +} + +func zipContains(z *zip.ReadCloser, fn string) bool { + for _, f := range z.File { + if fn == f.Name { + return true + } + } + return false +} diff --git a/src/tools/ak/rjar/testdata/BUILD b/src/tools/ak/rjar/testdata/BUILD new file mode 100644 index 0000000..f4831ba --- /dev/null +++ b/src/tools/ak/rjar/testdata/BUILD @@ -0,0 +1,26 @@ +# Creates test data for testing the rjar action. + +load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +package(default_visibility = ["//src/tools/ak/rjar:__subpackages__"]) + +licenses(["notice"]) + +genrule( + name = "pkgs", + outs = ["pkgs.txt"], + cmd = "printf 'android.support.v7\ncom.google.android.samples.skeletonapp\ncom.google.android.package.test' > '$@'", +) + +genrule( + name = "R", + srcs = ["//src/java/com/example/sampleapp:_migrated/lib.srcjar"], + outs = ["R.java"], + cmd = """ + unzip -p $(location //src/java/com/example/sampleapp:_migrated/lib.srcjar) com/example/sampleapp/R.java > '$@' + sed -i -- '/public final class R/,$$!d' '$@' + sed -i -- 's/@Deprecated//g' '$@' + sed -i -- 's/ final / /g' '$@' +""", +) diff --git a/src/tools/ak/types.go b/src/tools/ak/types.go new file mode 100644 index 0000000..6416461 --- /dev/null +++ b/src/tools/ak/types.go @@ -0,0 +1,39 @@ +// Copyright 2018 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package types provides globally used types. +package types + +type initFunc func() +type runFunc func() +type descFunc func() string + +/* +Command is used to specify a command. + + Init: + Entry point to initialize the command. + Run: + Entry point to run the command. + Flags: + (Optional) Flags that are used by the command. + Desc: + A short description of the command. +*/ +type Command struct { + Init initFunc + Run runFunc + Flags []string + Desc descFunc +} diff --git a/src/tools/enforce_min_sdk_floor/BUILD b/src/tools/enforce_min_sdk_floor/BUILD new file mode 100644 index 0000000..221fe8b --- /dev/null +++ b/src/tools/enforce_min_sdk_floor/BUILD @@ -0,0 +1,21 @@ +# Description: +# Package for tool to enforce min SDK floor on AndroidManifests +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +py_binary( + name = "enforce_min_sdk_floor", + srcs = ["enforce_min_sdk_floor.py"], + deps = [ + "@bazel_tools//third_party/py/abseil", + ], +) + +py_test( + name = "enforce_min_sdk_floor_test", + srcs = ["enforce_min_sdk_floor_test.py"], + deps = [ + ":enforce_min_sdk_floor", + ], +) diff --git a/src/tools/enforce_min_sdk_floor/enforce_min_sdk_floor.py b/src/tools/enforce_min_sdk_floor/enforce_min_sdk_floor.py new file mode 100644 index 0000000..d45921a --- /dev/null +++ b/src/tools/enforce_min_sdk_floor/enforce_min_sdk_floor.py @@ -0,0 +1,268 @@ +# pylint: disable=g-direct-third-party-import +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""AndroidManifest tool to enforce a floor on the minSdkVersion attribute. + +Ensures that the minSdkVersion attribute is >= than the specified floor, +and if the attribute is either not specified or less than the floor, +sets it to the floor. +""" + +import os +import sys + +import xml.etree.ElementTree as ET +from absl import app +from absl import flags + +BUMP = "bump" +VALIDATE = "validate" +SET_DEFAULT = "set_default" + +USES_SDK = "uses-sdk" +MIN_SDK_ATTRIB = "{http://schemas.android.com/apk/res/android}minSdkVersion" + +FLAGS = flags.FLAGS + +flags.DEFINE_enum( + "action", + None, + [BUMP, VALIDATE, SET_DEFAULT], + f"Action to perform, either {BUMP}, {VALIDATE}, or {SET_DEFAULT}") +flags.DEFINE_string( + "manifest", + None, + "AndroidManifest.xml of the instrumentation APK") +flags.DEFINE_integer( + "min_sdk_floor", + 0, + "Min SDK floor", + lower_bound=0) +# Needed for SET_DEFAULT +flags.DEFINE_string( + "default_min_sdk", + None, + "Default min SDK") +# Needed for BUMP and SET_DEFAULT +flags.DEFINE_string( + "output", + None, + f"Output AndroidManifest.xml to generate, only needed for {BUMP}") +flags.DEFINE_string("log", None, "Path to write the log to") + + +class MinSdkError(Exception): + """Raised when there is a problem with the min SDK attribute in AndroidManifest.xml.""" + + +def ParseNamespaces(xml_content): + """Parse namespaces first to keep the prefix. + + Args: + xml_content: str, the contents of the AndroidManifest.xml file + """ + ns_parser = ET.XMLPullParser(events=["start-ns"]) + ns_parser.feed(xml_content) + ns_parser.close() + for _, ns_tuple in ns_parser.read_events(): + try: + ET.register_namespace(ns_tuple[0], ns_tuple[1]) + except ValueError: + pass + + +def _BumpMinSdk(xml_content, min_sdk_floor): + """Checks the min SDK in xml_content and replaces with min_sdk_floor if needed. + + Args: + xml_content: str, the contents of the AndroidManifest.xml file + min_sdk_floor: int, the min SDK floor + + Returns: + A tuple with the following elements: + - str: The xml contents of the manifest with the min SDK floor enforced. + This string will be equal to the input if the min SDK is already not less + than the floor. + - str: log message of action taken + """ + if min_sdk_floor == 0: + return xml_content, "No min SDK floor specified. Manifest unchanged." + + ParseNamespaces(xml_content) + + root = ET.fromstring(xml_content) + uses_sdk = root.find(USES_SDK) + if uses_sdk is None: + ET.SubElement(root, USES_SDK, {MIN_SDK_ATTRIB: str(min_sdk_floor)}) + return ( + ET.tostring(root, encoding="utf-8", xml_declaration=True), + "No uses-sdk element found while floor is specified " + + f"({min_sdk_floor}). Min SDK added.") + + min_sdk = uses_sdk.get(MIN_SDK_ATTRIB) + if min_sdk is None: + uses_sdk.set(MIN_SDK_ATTRIB, str(min_sdk_floor)) + return ( + ET.tostring(root, encoding="utf-8", xml_declaration=True), + "No minSdkVersion attribute found while floor is specified" + + f"({min_sdk_floor}). Min SDK added.") + + try: + min_sdk_int = int(min_sdk) + except ValueError: + return ( + xml_content, + f"Placeholder used for the minSdkVersion attribute ({min_sdk}). " + + "Manifest unchanged.") + + if min_sdk_int < min_sdk_floor: + uses_sdk.set(MIN_SDK_ATTRIB, str(min_sdk_floor)) + return ( + ET.tostring(root, encoding="utf-8", xml_declaration=True), + f"minSdkVersion attribute specified in the manifest ({min_sdk}) " + + f"is less than the floor ({min_sdk_floor}). Min SDK replaced.") + return ( + xml_content, + f"minSdkVersion attribute specified in the manifest ({min_sdk}) " + + f"is not less than the floor ({min_sdk_floor}). Manifest unchanged.") + + +def _ValidateMinSdk(xml_content, min_sdk_floor): + """Checks the min SDK in xml_content and raises MinSdkError if it is either not specified or less than the floor. + + Args: + xml_content: str, the contents of the AndroidManifest.xml file + min_sdk_floor: int, the min SDK floor + Returns: + str: log message + Raises: + MinSdkError: The min SDK is less than the specified floor. + """ + if min_sdk_floor == 0: + return "No min SDK floor specified." + + root = ET.fromstring(xml_content) + + uses_sdk = root.find(USES_SDK) + if uses_sdk is None: + raise MinSdkError( + "No uses-sdk element found in manifest " + + f"while floor is specified ({min_sdk_floor}).") + + min_sdk = uses_sdk.get(MIN_SDK_ATTRIB) + if min_sdk is None: + raise MinSdkError( + "No minSdkVersion attribute found in manifest " + + f"while floor is specified ({min_sdk_floor}).") + + try: + min_sdk_int = int(min_sdk) + except ValueError: + return f"Placeholder minSdkVersion = {min_sdk}\n min SDK floor = {min_sdk_floor}" + + if min_sdk_int < min_sdk_floor: + raise MinSdkError( + f"minSdkVersion attribute specified in the manifest ({min_sdk}) " + + f"is less than the floor ({min_sdk_floor}).") + return f"minSdkVersion = {min_sdk}\n min SDK floor = {min_sdk_floor}" + + +def _SetDefaultMinSdk(xml_content, default_min_sdk): + """Checks the min SDK in xml_content and replaces with default_min_sdk if it is not already set. + + Args: + xml_content: str, the contents of the AndroidManifest.xml file + default_min_sdk: str, can be set to either a number or an unreleased version + full name + + Returns: + A tuple with the following elements: + - str: The xml contents of the manifest with the min SDK floor enforced. + This string will be equal to the input if the min SDK is already set. + - str: log message of action taken + """ + if default_min_sdk is None: + return xml_content, ("No default min SDK floor specified. Manifest " + "unchanged.") + + ParseNamespaces(xml_content) + + root = ET.fromstring(xml_content) + uses_sdk = root.find(USES_SDK) + if uses_sdk is None: + ET.SubElement(root, USES_SDK, {MIN_SDK_ATTRIB: default_min_sdk}) + return ( + ET.tostring(root, encoding="utf-8", xml_declaration=True), + "No uses-sdk element found while default is specified. " + + f"Min SDK ({default_min_sdk}) added.") + + min_sdk = uses_sdk.get(MIN_SDK_ATTRIB) + if min_sdk is None: + uses_sdk.set(MIN_SDK_ATTRIB, str(default_min_sdk)) + return ( + ET.tostring(root, encoding="utf-8", xml_declaration=True), + "No minSdkVersion attribute found while default is specified" + + f"({default_min_sdk}). Min SDK set to default.") + + return ( + xml_content, + f"minSdkVersion attribute specified in the manifest ({min_sdk}) " + + ". Manifest unchanged.") + + +def main(unused_argv): + manifest_path = FLAGS.manifest + with open(manifest_path, "rb") as f: + manifest = f.read() + + if FLAGS.action == BUMP: + output_path = FLAGS.output + dirname = os.path.dirname(output_path) + if not os.path.exists(dirname): + os.makedirs(dirname) + + out_contents, log_message = _BumpMinSdk(manifest, FLAGS.min_sdk_floor) + with open(output_path, "wb") as f: + f.write(out_contents) + + elif FLAGS.action == SET_DEFAULT: + output_path = FLAGS.output + dirname = os.path.dirname(output_path) + if not os.path.exists(dirname): + os.makedirs(dirname) + + out_contents, log_message = _SetDefaultMinSdk( + manifest, FLAGS.default_min_sdk + ) + with open(output_path, "wb") as f: + f.write(out_contents) + + elif FLAGS.action == VALIDATE: + try: + log_message = _ValidateMinSdk(manifest, FLAGS.min_sdk_floor) + except MinSdkError as e: + sys.exit(str(e)) + else: + sys.exit(f"Action must be either {BUMP} or {VALIDATE}") + + if FLAGS.log is not None: + log_path = FLAGS.log + dirname = os.path.dirname(log_path) + if not os.path.exists(dirname): + os.makedirs(dirname) + with open(log_path, "w") as f: + f.write(log_message) + +if __name__ == "__main__": + app.run(main) diff --git a/src/tools/enforce_min_sdk_floor/enforce_min_sdk_floor_test.py b/src/tools/enforce_min_sdk_floor/enforce_min_sdk_floor_test.py new file mode 100644 index 0000000..87f9a8d --- /dev/null +++ b/src/tools/enforce_min_sdk_floor/enforce_min_sdk_floor_test.py @@ -0,0 +1,126 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit tests for enforce_min_sdk_floor.py.""" + +import unittest +import xml.etree.ElementTree as ET + +from google3.third_party.bazel_rules.rules_android.src.tools.enforce_min_sdk_floor.enforce_min_sdk_floor import _BumpMinSdk +from google3.third_party.bazel_rules.rules_android.src.tools.enforce_min_sdk_floor.enforce_min_sdk_floor import _SetDefaultMinSdk +from google3.third_party.bazel_rules.rules_android.src.tools.enforce_min_sdk_floor.enforce_min_sdk_floor import _ValidateMinSdk + +from google3.third_party.bazel_rules.rules_android.src.tools.enforce_min_sdk_floor.enforce_min_sdk_floor import MIN_SDK_ATTRIB +from google3.third_party.bazel_rules.rules_android.src.tools.enforce_min_sdk_floor.enforce_min_sdk_floor import MinSdkError +from google3.third_party.bazel_rules.rules_android.src.tools.enforce_min_sdk_floor.enforce_min_sdk_floor import USES_SDK + +MANIFEST_NO_USES_SDK = """<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.example" > +</manifest> +""".encode("utf-8") + +MANIFEST_NO_MIN_SDK = """<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.example" > +<uses-sdk/> +</manifest> +""".encode("utf-8") + +MANIFEST_MIN_SDK_PLACEHOLDER = """<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.example" > +<uses-sdk android:minSdkVersion="${minSdkVersion}" /> +</manifest> +""".encode("utf-8") + +MANIFEST_MIN_SDK = """<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.example" > +<uses-sdk android:minSdkVersion="12" /> +</manifest> +""".encode("utf-8") + + +class EnforceMinSdkFloorTest(unittest.TestCase): + + def test_bump_no_min_sdk_floor(self): + out, _ = _BumpMinSdk(MANIFEST_NO_USES_SDK, 0) + self.assertEqual(out, MANIFEST_NO_USES_SDK) + + def test_bump_no_uses_sdk(self): + out, _ = _BumpMinSdk(MANIFEST_NO_USES_SDK, 11) + min_sdk = ET.fromstring(out).find(USES_SDK).get(MIN_SDK_ATTRIB) + self.assertEqual(min_sdk, "11") + + def test_bump_no_min_sdk_attrib(self): + out, _ = _BumpMinSdk(MANIFEST_NO_MIN_SDK, 7) + min_sdk = ET.fromstring(out).find(USES_SDK).get(MIN_SDK_ATTRIB) + self.assertEqual(min_sdk, "7") + + def test_bump_min_sdk_attrib_placeholder(self): + out, _ = _BumpMinSdk(MANIFEST_MIN_SDK_PLACEHOLDER, 13) + self.assertEqual(out, MANIFEST_MIN_SDK_PLACEHOLDER) + + def test_bump_higher_min_sdk(self): + out, _ = _BumpMinSdk(MANIFEST_MIN_SDK, 10) + self.assertEqual(out, MANIFEST_MIN_SDK) + + def test_bump_lower_min_sdk(self): + out, _ = _BumpMinSdk(MANIFEST_MIN_SDK, 14) + min_sdk = ET.fromstring(out).find(USES_SDK).get(MIN_SDK_ATTRIB) + self.assertEqual(min_sdk, "14") + + def test_set_default_no_uses(self): + out, _ = _SetDefaultMinSdk(MANIFEST_NO_USES_SDK, "11") + min_sdk = ET.fromstring(out).find(USES_SDK).get(MIN_SDK_ATTRIB) + self.assertEqual(min_sdk, "11") + + def test_set_default_no_min_sdk(self): + out, _ = _SetDefaultMinSdk(MANIFEST_NO_USES_SDK, "current") + min_sdk = ET.fromstring(out).find(USES_SDK).get(MIN_SDK_ATTRIB) + self.assertEqual(min_sdk, "current") + + def test_set_default_min_sdk_already_specified(self): + out, _ = _SetDefaultMinSdk(MANIFEST_MIN_SDK, "14") + self.assertEqual(out, MANIFEST_MIN_SDK) + + def test_validate_no_min_sdk_floor(self): + _ = _ValidateMinSdk(MANIFEST_NO_USES_SDK, 0) + + def test_validate_no_uses_sdk(self): + self.assertRaises(MinSdkError, + _ValidateMinSdk, + xml_content=MANIFEST_NO_USES_SDK, + min_sdk_floor=5) + + def test_validate_no_min_sdk_attrib(self): + self.assertRaises(MinSdkError, + _ValidateMinSdk, + xml_content=MANIFEST_NO_MIN_SDK, + min_sdk_floor=19) + + def test_validate_min_sdk_attrib_placeholder(self): + _ = _ValidateMinSdk(MANIFEST_MIN_SDK_PLACEHOLDER, 21) + + def test_validate_higher_min_sdk(self): + _ = _ValidateMinSdk(MANIFEST_MIN_SDK, 8) + + def test_validate_lower_min_sdk(self): + self.assertRaises(MinSdkError, + _ValidateMinSdk, + xml_content=MANIFEST_MIN_SDK, + min_sdk_floor=18) + +if __name__ == "__main__": + unittest.main() diff --git a/toolchains/android/toolchain.bzl b/toolchains/android/toolchain.bzl index 42cdf9d..905238d 100644 --- a/toolchains/android/toolchain.bzl +++ b/toolchains/android/toolchain.bzl @@ -66,12 +66,6 @@ _ATTRS = dict( cfg = "exec", executable = True, ), - android_archive_manifest_package_validator = attr.label( - allow_files = True, - default = "@androidsdk//:fail", - cfg = "exec", - executable = True, - ), android_archive_packages_validator = attr.label( allow_files = True, default = "@androidsdk//:fail", @@ -102,9 +96,15 @@ _ATTRS = dict( default = "//tools/android:bundletool_deploy.jar", executable = True, ), + centralize_r_class_tool = attr.label( + allow_files = True, + cfg = "exec", + default = "@androidsdk//:fail", + executable = True, + ), data_binding_annotation_processor = attr.label( cfg = "exec", - default = "@//tools/android:compiler_annotation_processor", # TODO: processor rules should be moved into rules_android + default = "//tools/android:compiler_annotation_processor", ), data_binding_annotation_template = attr.label( default = "//rules:data_binding_annotation_template.txt", @@ -121,6 +121,12 @@ _ATTRS = dict( default = "@bazel_tools//tools/android:desugar_java8_extra_bootclasspath", executable = True, ), + enforce_min_sdk_floor_tool = attr.label( + allow_files = True, + cfg = "exec", + default = "//src/tools/enforce_min_sdk_floor", + executable = True, + ), idlclass = attr.label( allow_files = True, cfg = "exec", @@ -148,6 +154,12 @@ _ATTRS = dict( default = "@androidsdk//:fail", # TODO: "//src/tools/jdeps", needs Go executable = True, ), + object_method_rewriter = attr.label( + allow_files = True, + cfg = "exec", + default = "@androidsdk//:fail", + executable = True, + ), proguard_allowlister = attr.label( cfg = "exec", default = "@bazel_tools//tools/jdk:proguard_whitelister", diff --git a/toolchains/android_sdk/BUILD b/toolchains/android_sdk/BUILD index d675cbe..66900b8 100644 --- a/toolchains/android_sdk/BUILD +++ b/toolchains/android_sdk/BUILD @@ -16,10 +16,13 @@ toolchain_type( toolchain( name = "android_sdk_tools", - exec_compatible_with = [ - "@bazel_tools//platforms:x86_64", - "@bazel_tools//platforms:linux", - ], + # This causes the toolchain to not be selected under arm, so + # disable for now. + # To be refined as part of https://github.com/bazelbuild/rules_android/issues/72 + #exec_compatible_with = [ + # "@platforms//cpu:x86_64", + # "@platforms//os:linux", + #], # TODO(b/175833893): This causes the toolchain to not be selected, so # disable for now. #target_compatible_with = [ diff --git a/toolchains/emulator/toolchain.bzl b/toolchains/emulator/toolchain.bzl new file mode 100644 index 0000000..6bf2e0f --- /dev/null +++ b/toolchains/emulator/toolchain.bzl @@ -0,0 +1,59 @@ +# Copyright 2019 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Defines the emulator_toolchain rule to allow configuring emulator binaries to use.""" + +EmulatorInfo = provider( + doc = "Information used to launch a specific version of the emulator.", + fields = { + "emulator": "A label for the emulator launcher executable at stable version.", + "emulator_deps": "Additional files required to launch the stable version of emulator.", + "emulator_suffix": "An optional path suffix used to find emulator binary under the emulator label path", + }, +) + +def _emulator_toolchain_impl(ctx): + toolchain_info = platform_common.ToolchainInfo( + info = EmulatorInfo( + emulator = ctx.attr.emulator, + emulator_deps = ctx.attr.emulator_deps, + emulator_suffix = ctx.attr.emulator_suffix, + ), + ) + return [toolchain_info] + +emulator_toolchain = rule( + implementation = _emulator_toolchain_impl, + attrs = { + "emulator": attr.label( + allow_files = True, + cfg = "exec", + mandatory = True, + ), + "emulator_deps": attr.label_list( + allow_files = True, + cfg = "exec", + ), + "emulator_head": attr.label( + allow_files = True, + cfg = "exec", + ), + "emulator_head_deps": attr.label_list( + allow_files = True, + cfg = "exec", + ), + "emulator_suffix": attr.string(default = ""), + "emulator_head_suffix": attr.string(default = ""), + }, +) diff --git a/tools/android/BUILD b/tools/android/BUILD index cb8148a..7a85b59 100644 --- a/tools/android/BUILD +++ b/tools/android/BUILD @@ -30,3 +30,13 @@ java_binary( visibility = ["//visibility:public"], runtime_deps = ["@rules_android_maven//:com_android_tools_build_bundletool"], ) + +java_plugin( + name = "compiler_annotation_processor", + generates_api = True, + processor_class = "android.databinding.annotationprocessor.ProcessDataBinding", + visibility = ["//visibility:public"], + deps = [ + "@bazel_tools//src/tools/android/java/com/google/devtools/build/android:all_android_tools", + ], +) diff --git a/tools/jdk/BUILD b/tools/jdk/BUILD index d83484b..76b1921 100644 --- a/tools/jdk/BUILD +++ b/tools/jdk/BUILD @@ -1,12 +1,9 @@ load("@bazel_tools//tools/jdk:default_java_toolchain.bzl", "default_java_toolchain") -default_java_toolchain( +# AOSP-only change +alias( name = "toolchain_android_only", - bootclasspath = [ - "//tools/android:android_jar", - # TODO(b/175805830): Add this only when desugaring is enabled. - "@bazel_tools//tools/android:desugar_java8_extra_bootclasspath", - ], + actual = "@bazel_tools//tools/jdk:current_java_toolchain", visibility = ["//visibility:public"], ) |
