From 5c6572e53fa2810e45215850842b81b8fa9a22ec Mon Sep 17 00:00:00 2001 From: Jaewoong Jung Date: Mon, 17 Jun 2019 17:40:56 -0700 Subject: Optionally embed NOTICE files in apks. If embed_notices or ALWAYS_EMBED_NOTICES is set, collect NOTICE files from all dependencies of the android_app, merge them with the app's own one (if exists), transform it to HTML, gzip it, and put it as an asset in the final APK output. Bug: 135460391 Test: app_test.go + Built Mainline modules Change-Id: I52d92e2fd19b3f5f396100424665c5cc344190d8 Merged-In: I52d92e2fd19b3f5f396100424665c5cc344190d8 (cherry picked from commit 5b425e2e20c55216c1ed13fcf047b0df33886736) --- Android.bp | 1 + android/module.go | 16 ++++----- android/notices.go | 87 +++++++++++++++++++++++++++++++++++++++++++++++++ apex/apex.go | 20 +----------- apex/apex_test.go | 8 ++--- java/aar.go | 10 +++++- java/app.go | 79 +++++++++++++++++++++++++++++++++++++++------ java/app_test.go | 95 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ java/java_test.go | 4 +++ 9 files changed, 278 insertions(+), 42 deletions(-) create mode 100644 android/notices.go diff --git a/Android.bp b/Android.bp index 0121b70e2..1d65dff04 100644 --- a/Android.bp +++ b/Android.bp @@ -54,6 +54,7 @@ bootstrap_go_package { "android/mutator.go", "android/namespace.go", "android/neverallow.go", + "android/notices.go", "android/onceper.go", "android/override_module.go", "android/package_ctx.go", diff --git a/android/module.go b/android/module.go index 11e9e917a..6a796227d 100644 --- a/android/module.go +++ b/android/module.go @@ -849,14 +849,6 @@ func (a *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) } if a.Enabled() { - a.module.GenerateAndroidBuildActions(ctx) - if ctx.Failed() { - return - } - - a.installFiles = append(a.installFiles, ctx.installFiles...) - a.checkbuildFiles = append(a.checkbuildFiles, ctx.checkbuildFiles...) - notice := proptools.StringDefault(a.commonProperties.Notice, "NOTICE") if m := SrcIsModule(notice); m != "" { a.noticeFile = ctx.ExpandOptionalSource(¬ice, "notice") @@ -864,6 +856,14 @@ func (a *ModuleBase) GenerateBuildActions(blueprintCtx blueprint.ModuleContext) noticePath := filepath.Join(ctx.ModuleDir(), notice) a.noticeFile = ExistentPathForSource(ctx, noticePath) } + + a.module.GenerateAndroidBuildActions(ctx) + if ctx.Failed() { + return + } + + a.installFiles = append(a.installFiles, ctx.installFiles...) + a.checkbuildFiles = append(a.checkbuildFiles, ctx.checkbuildFiles...) } if a == ctx.FinalModule().(Module).base() { diff --git a/android/notices.go b/android/notices.go new file mode 100644 index 000000000..dbb88fc1a --- /dev/null +++ b/android/notices.go @@ -0,0 +1,87 @@ +// Copyright 2019 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package android + +import ( + "path/filepath" + + "github.com/google/blueprint" +) + +func init() { + pctx.SourcePathVariable("merge_notices", "build/soong/scripts/mergenotice.py") + pctx.SourcePathVariable("generate_notice", "build/make/tools/generate-notice-files.py") + + pctx.HostBinToolVariable("minigzip", "minigzip") +} + +var ( + mergeNoticesRule = pctx.AndroidStaticRule("mergeNoticesRule", blueprint.RuleParams{ + Command: `${merge_notices} --output $out $in`, + CommandDeps: []string{"${merge_notices}"}, + Description: "merge notice files into $out", + }) + + generateNoticeRule = pctx.AndroidStaticRule("generateNoticeRule", blueprint.RuleParams{ + Command: `rm -rf $tmpDir $$(dirname $out) && ` + + `mkdir -p $tmpDir $$(dirname $out) && ` + + `${generate_notice} --text-output $tmpDir/NOTICE.txt --html-output $tmpDir/NOTICE.html -t "$title" -s $inputDir && ` + + `${minigzip} -c $tmpDir/NOTICE.html > $out`, + CommandDeps: []string{"${generate_notice}", "${minigzip}"}, + Description: "produce notice file $out", + }, "tmpDir", "title", "inputDir") +) + +func MergeNotices(ctx ModuleContext, mergedNotice WritablePath, noticePaths []Path) { + ctx.Build(pctx, BuildParams{ + Rule: mergeNoticesRule, + Description: "merge notices", + Inputs: noticePaths, + Output: mergedNotice, + }) +} + +func BuildNoticeOutput(ctx ModuleContext, installPath OutputPath, installFilename string, + noticePaths []Path) ModuleOutPath { + // Merge all NOTICE files into one. + // TODO(jungjw): We should just produce a well-formatted NOTICE.html file in a single pass. + // + // generate-notice-files.py, which processes the merged NOTICE file, has somewhat strict rules + // about input NOTICE file paths. + // 1. Their relative paths to the src root become their NOTICE index titles. We want to use + // on-device paths as titles, and so output the merged NOTICE file the corresponding location. + // 2. They must end with .txt extension. Otherwise, they're ignored. + noticeRelPath := InstallPathToOnDevicePath(ctx, installPath.Join(ctx, installFilename+".txt")) + mergedNotice := PathForModuleOut(ctx, filepath.Join("NOTICE_FILES/src", noticeRelPath)) + MergeNotices(ctx, mergedNotice, noticePaths) + + // Transform the merged NOTICE file into a gzipped HTML file. + noticeOutput := PathForModuleOut(ctx, "NOTICE", "NOTICE.html.gz") + tmpDir := PathForModuleOut(ctx, "NOTICE_tmp") + title := "Notices for " + ctx.ModuleName() + ctx.Build(pctx, BuildParams{ + Rule: generateNoticeRule, + Description: "generate notice output", + Input: mergedNotice, + Output: noticeOutput, + Args: map[string]string{ + "tmpDir": tmpDir.String(), + "title": title, + "inputDir": PathForModuleOut(ctx, "NOTICE_FILES/src").String(), + }, + }) + + return noticeOutput +} diff --git a/apex/apex.go b/apex/apex.go index fa4cb48ce..18df4393e 100644 --- a/apex/apex.go +++ b/apex/apex.go @@ -90,12 +90,6 @@ var ( CommandDeps: []string{"${zip2zip}"}, Description: "app bundle", }, "abi") - - apexMergeNoticeRule = pctx.StaticRule("apexMergeNoticeRule", blueprint.RuleParams{ - Command: `${mergenotice} --output $out $inputs`, - CommandDeps: []string{"${mergenotice}"}, - Description: "merge notice files into $out", - }, "inputs") ) var imageApexSuffix = ".apex" @@ -144,8 +138,6 @@ func init() { pctx.HostBinToolVariable("zip2zip", "zip2zip") pctx.HostBinToolVariable("zipalign", "zipalign") - pctx.SourcePathVariable("mergenotice", "build/soong/scripts/mergenotice.py") - android.RegisterModuleType("apex", apexBundleFactory) android.RegisterModuleType("apex_test", testApexBundleFactory) android.RegisterModuleType("apex_defaults", defaultsFactory) @@ -837,32 +829,22 @@ func (a *apexBundle) GenerateAndroidBuildActions(ctx android.ModuleContext) { func (a *apexBundle) buildNoticeFile(ctx android.ModuleContext) { noticeFiles := []android.Path{} - noticeFilesString := []string{} for _, f := range a.filesInfo { if f.module != nil { notice := f.module.NoticeFile() if notice.Valid() { noticeFiles = append(noticeFiles, notice.Path()) - noticeFilesString = append(noticeFilesString, notice.Path().String()) } } } // append the notice file specified in the apex module itself if a.NoticeFile().Valid() { noticeFiles = append(noticeFiles, a.NoticeFile().Path()) - noticeFilesString = append(noticeFilesString, a.NoticeFile().Path().String()) } if len(noticeFiles) > 0 { a.mergedNoticeFile = android.PathForModuleOut(ctx, "NOTICE") - ctx.Build(pctx, android.BuildParams{ - Rule: apexMergeNoticeRule, - Inputs: noticeFiles, - Output: a.mergedNoticeFile, - Args: map[string]string{ - "inputs": strings.Join(noticeFilesString, " "), - }, - }) + android.MergeNotices(ctx, a.mergedNoticeFile, noticeFiles) } } diff --git a/apex/apex_test.go b/apex/apex_test.go index a07a89b8c..200a1c29d 100644 --- a/apex/apex_test.go +++ b/apex/apex_test.go @@ -347,10 +347,10 @@ func TestBasicApex(t *testing.T) { t.Errorf("Could not find all expected symlinks! foo: %t, foo_link_64: %t. Command was %s", found_foo, found_foo_link_64, copyCmds) } - apexMergeNoticeRule := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("apexMergeNoticeRule") - noticeInputs := strings.Split(apexMergeNoticeRule.Args["inputs"], " ") - if len(noticeInputs) != 3 { - t.Errorf("number of input notice files: expected = 3, actual = %d", len(noticeInputs)) + mergeNoticesRule := ctx.ModuleForTests("myapex", "android_common_myapex").Rule("mergeNoticesRule") + noticeInputs := mergeNoticesRule.Inputs.Strings() + if len(noticeInputs) != 4 { + t.Errorf("number of input notice files: expected = 4, actual = %q", len(noticeInputs)) } ensureListContains(t, noticeInputs, "NOTICE") ensureListContains(t, noticeInputs, "custom_notice") diff --git a/java/aar.go b/java/aar.go index c62127664..6273a9b50 100644 --- a/java/aar.go +++ b/java/aar.go @@ -17,6 +17,7 @@ package java import ( "android/soong/android" "fmt" + "path/filepath" "strings" "github.com/google/blueprint" @@ -78,6 +79,7 @@ type aapt struct { rroDirs []rroDir rTxt android.Path extraAaptPackagesFile android.Path + noticeFile android.OptionalPath isLibrary bool uncompressedJNI bool useEmbeddedDex bool @@ -150,10 +152,16 @@ func (a *aapt) aapt2Flags(ctx android.ModuleContext, sdkContext sdkContext, mani assetFiles = append(assetFiles, androidResourceGlob(ctx, dir)...) } + assetDirStrings := assetDirs.Strings() + if a.noticeFile.Valid() { + assetDirStrings = append(assetDirStrings, filepath.Dir(a.noticeFile.Path().String())) + assetFiles = append(assetFiles, a.noticeFile.Path()) + } + linkFlags = append(linkFlags, "--manifest "+manifestPath.String()) linkDeps = append(linkDeps, manifestPath) - linkFlags = append(linkFlags, android.JoinWithPrefix(assetDirs.Strings(), "-A ")) + linkFlags = append(linkFlags, android.JoinWithPrefix(assetDirStrings, "-A ")) linkDeps = append(linkDeps, assetFiles...) // SDK version flags diff --git a/java/app.go b/java/app.go index da8024fd2..8d41b570f 100644 --- a/java/app.go +++ b/java/app.go @@ -18,6 +18,7 @@ package java import ( "path/filepath" + "sort" "strings" "github.com/google/blueprint" @@ -79,6 +80,10 @@ type appProperties struct { // Use_embedded_native_libs still selects whether they are stored uncompressed and aligned or compressed. // True for android_test* modules. AlwaysPackageNativeLibs bool `blueprint:"mutated"` + + // If set, find and merge all NOTICE files that this module and its dependencies have and store + // it in the APK as an asset. + Embed_notices *bool } // android_app properties that can be overridden by override_android_app @@ -335,10 +340,74 @@ func (a *AndroidApp) certificateBuildActions(certificateDeps []Certificate, ctx return append([]Certificate{a.certificate}, certificateDeps...) } +func (a *AndroidApp) noticeBuildActions(ctx android.ModuleContext, installDir android.OutputPath) android.OptionalPath { + if !Bool(a.appProperties.Embed_notices) && !ctx.Config().IsEnvTrue("ALWAYS_EMBED_NOTICES") { + return android.OptionalPath{} + } + + // Collect NOTICE files from all dependencies. + seenModules := make(map[android.Module]bool) + noticePathSet := make(map[android.Path]bool) + + ctx.WalkDepsBlueprint(func(child blueprint.Module, parent blueprint.Module) bool { + if _, ok := child.(android.Module); !ok { + return false + } + module := child.(android.Module) + // Have we already seen this? + if _, ok := seenModules[module]; ok { + return false + } + seenModules[module] = true + + // Skip host modules. + if module.Target().Os.Class == android.Host || module.Target().Os.Class == android.HostCross { + return false + } + + path := module.NoticeFile() + if path.Valid() { + noticePathSet[path.Path()] = true + } + return true + }) + + // If the app has one, add it too. + if a.NoticeFile().Valid() { + noticePathSet[a.NoticeFile().Path()] = true + } + + if len(noticePathSet) == 0 { + return android.OptionalPath{} + } + var noticePaths []android.Path + for path := range noticePathSet { + noticePaths = append(noticePaths, path) + } + sort.Slice(noticePaths, func(i, j int) bool { + return noticePaths[i].String() < noticePaths[j].String() + }) + noticeFile := android.BuildNoticeOutput(ctx, installDir, a.installApkName+".apk", noticePaths) + + return android.OptionalPathForPath(noticeFile) +} + func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { // Check if the install APK name needs to be overridden. a.installApkName = ctx.DeviceConfig().OverridePackageNameFor(a.Name()) + var installDir android.OutputPath + if ctx.ModuleName() == "framework-res" { + // framework-res.apk is installed as system/framework/framework-res.apk + installDir = android.PathForModuleInstall(ctx, "framework") + } else if Bool(a.appProperties.Privileged) { + installDir = android.PathForModuleInstall(ctx, "priv-app", a.installApkName) + } else { + installDir = android.PathForModuleInstall(ctx, "app", a.installApkName) + } + + a.aapt.noticeFile = a.noticeBuildActions(ctx, installDir) + // Process all building blocks, from AAPT to certificates. a.aaptBuildActions(ctx) @@ -374,16 +443,6 @@ func (a *AndroidApp) generateAndroidBuildActions(ctx android.ModuleContext) { a.bundleFile = bundleFile // Install the app package. - var installDir android.OutputPath - if ctx.ModuleName() == "framework-res" { - // framework-res.apk is installed as system/framework/framework-res.apk - installDir = android.PathForModuleInstall(ctx, "framework") - } else if Bool(a.appProperties.Privileged) { - installDir = android.PathForModuleInstall(ctx, "priv-app", a.installApkName) - } else { - installDir = android.PathForModuleInstall(ctx, "app", a.installApkName) - } - ctx.InstallFile(installDir, a.installApkName+".apk", a.outputFile) for _, split := range a.aapt.splits { ctx.InstallFile(installDir, a.installApkName+"_"+split.suffix+".apk", split.path) diff --git a/java/app_test.go b/java/app_test.go index 1f6297c25..d2685107e 100644 --- a/java/app_test.go +++ b/java/app_test.go @@ -968,3 +968,98 @@ func TestOverrideAndroidApp(t *testing.T) { } } } + +func TestEmbedNotice(t *testing.T) { + ctx := testJava(t, cc.GatherRequiredDepsForTest(android.Android)+` + android_app { + name: "foo", + srcs: ["a.java"], + static_libs: ["javalib"], + jni_libs: ["libjni"], + notice: "APP_NOTICE", + embed_notices: true, + } + + // No embed_notice flag + android_app { + name: "bar", + srcs: ["a.java"], + jni_libs: ["libjni"], + notice: "APP_NOTICE", + } + + // No NOTICE files + android_app { + name: "baz", + srcs: ["a.java"], + embed_notices: true, + } + + cc_library { + name: "libjni", + system_shared_libs: [], + stl: "none", + notice: "LIB_NOTICE", + } + + java_library { + name: "javalib", + srcs: [ + ":gen", + ], + } + + genrule { + name: "gen", + tools: ["gentool"], + out: ["gen.java"], + notice: "GENRULE_NOTICE", + } + + java_binary_host { + name: "gentool", + srcs: ["b.java"], + notice: "TOOL_NOTICE", + } + `) + + // foo has NOTICE files to process, and embed_notices is true. + foo := ctx.ModuleForTests("foo", "android_common") + // verify merge notices rule. + mergeNotices := foo.Rule("mergeNoticesRule") + noticeInputs := mergeNotices.Inputs.Strings() + // TOOL_NOTICE should be excluded as it's a host module. + if len(mergeNotices.Inputs) != 3 { + t.Errorf("number of input notice files: expected = 3, actual = %q", noticeInputs) + } + if !inList("APP_NOTICE", noticeInputs) { + t.Errorf("APP_NOTICE is missing from notice files, %q", noticeInputs) + } + if !inList("LIB_NOTICE", noticeInputs) { + t.Errorf("LIB_NOTICE is missing from notice files, %q", noticeInputs) + } + if !inList("GENRULE_NOTICE", noticeInputs) { + t.Errorf("GENRULE_NOTICE is missing from notice files, %q", noticeInputs) + } + // aapt2 flags should include -A so that its contents are put in the APK's /assets. + res := foo.Output("package-res.apk") + aapt2Flags := res.Args["flags"] + e := "-A " + buildDir + "/.intermediates/foo/android_common/NOTICE" + if !strings.Contains(aapt2Flags, e) { + t.Errorf("asset dir flag for NOTICE, %q is missing in aapt2 link flags, %q", e, aapt2Flags) + } + + // bar has NOTICE files to process, but embed_notices is not set. + bar := ctx.ModuleForTests("bar", "android_common") + mergeNotices = bar.MaybeRule("mergeNoticesRule") + if mergeNotices.Rule != nil { + t.Errorf("mergeNotices shouldn't have run for bar") + } + + // baz's embed_notice is true, but it doesn't have any NOTICE files. + baz := ctx.ModuleForTests("baz", "android_common") + mergeNotices = baz.MaybeRule("mergeNoticesRule") + if mergeNotices.Rule != nil { + t.Errorf("mergeNotices shouldn't have run for baz") + } +} diff --git a/java/java_test.go b/java/java_test.go index a5c0aa93e..31b23e764 100644 --- a/java/java_test.go +++ b/java/java_test.go @@ -119,6 +119,10 @@ func testContext(config android.Config, bp string, "b.kt": nil, "a.jar": nil, "b.jar": nil, + "APP_NOTICE": nil, + "GENRULE_NOTICE": nil, + "LIB_NOTICE": nil, + "TOOL_NOTICE": nil, "java-res/a/a": nil, "java-res/b/b": nil, "java-res2/a": nil, -- cgit v1.2.3