diff options
| -rwxr-xr-x | build/mainline_modules_sdks.py | 248 | ||||
| -rw-r--r-- | build/mainline_modules_sdks_test.py | 54 |
2 files changed, 258 insertions, 44 deletions
diff --git a/build/mainline_modules_sdks.py b/build/mainline_modules_sdks.py index 3fe0ddd5..cddd3c93 100755 --- a/build/mainline_modules_sdks.py +++ b/build/mainline_modules_sdks.py @@ -27,6 +27,8 @@ import shutil import subprocess import sys import tempfile +import typing +from typing import Callable, List import zipfile @@ -214,7 +216,7 @@ class SnapshotBuilder: return os.path.join(self.get_mainline_sdks_path(), f"{sdk_name}-{sdk_version}.zip") - def build_snapshots(self, sdk_versions, modules): + def build_snapshots(self, build_release, sdk_versions, modules): # Build the SDKs once for each version. for sdk_version in sdk_versions: # Compute the paths to all the Soong generated sdk snapshot files @@ -225,16 +227,20 @@ class SnapshotBuilder: for sdk in module.sdks ] - # TODO(ngeoffray): remove SOONG_ALLOW_MISSING_DEPENDENCIES, but we - # currently break without it. - # - # Set SOONG_SDK_SNAPSHOT_USE_SRCJAR to generate .srcjars inside sdk - # zip files as expected by prebuilt drop. + # Extra environment variables to pass to the build process. extraEnv = { + # TODO(ngeoffray): remove SOONG_ALLOW_MISSING_DEPENDENCIES, but + # we currently break without it. "SOONG_ALLOW_MISSING_DEPENDENCIES": "true", + # Set SOONG_SDK_SNAPSHOT_USE_SRCJAR to generate .srcjars inside + # sdk zip files as expected by prebuilt drop. "SOONG_SDK_SNAPSHOT_USE_SRCJAR": "true", + # Set SOONG_SDK_SNAPSHOT_VERSION to generate the appropriately + # tagged version of the sdk. "SOONG_SDK_SNAPSHOT_VERSION": sdk_version, } + extraEnv.update(build_release.soong_env) + # Unless explicitly specified in the calling environment set # TARGET_BUILD_VARIANT=user. # This MUST be identical to the TARGET_BUILD_VARIANT used to build @@ -256,6 +262,149 @@ class SnapshotBuilder: self.subprocess_runner.run(cmd, env=env) +# A list of the sdk versions to build. Usually just current but can include a +# numeric version too. +SDK_VERSIONS = [ + # Suitable for overriding the source modules with prefer:true. + # Unlike "unversioned" this mode also adds "@current" suffixed modules + # with the same prebuilts (which are never preferred). + "current", + # Insert additional sdk versions needed for the latest build release. +] + +# The initially empty list of build releases. Every BuildRelease that is created +# automatically appends itself to this list. +ALL_BUILD_RELEASES = [] + + +@dataclasses.dataclass(frozen=True) +class BuildRelease: + """Represents a build release""" + + # The name of the build release, e.g. Q, R, S, T, etc. + name: str + + # The function to call to create the snapshot in the dist, that covers + # building and copying the snapshot into the dist. + creator: Callable[ + ["BuildRelease", "SdkDistProducer", List["MainlineModule"]], None] + + # The sub-directory of dist/mainline-sdks into which the build release + # specific snapshots will be copied. + # + # Defaults to for-<name>-build. + sub_dir: str = None + + # Additional environment variables to pass to Soong when building the + # snapshots for this build release. + # + # Defaults to { + # "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": <name>, + # } + soong_env: typing.Dict[str, str] = None + + # The sdk versions that need to be generated for this build release. + sdk_versions: List[str] = \ + dataclasses.field(default_factory=lambda: SDK_VERSIONS) + + # The position of this instance within the BUILD_RELEASES list. + ordinal: int = dataclasses.field(default=-1, init=False) + + def __post_init__(self): + # The following use object.__setattr__ as this object is frozen and + # attempting to set the fields directly would cause an exception to be + # thrown. + object.__setattr__(self, "ordinal", len(ALL_BUILD_RELEASES)) + # Add this to the end of the list of all build releases. + ALL_BUILD_RELEASES.append(self) + # If no sub_dir was specified then set the default. + if self.sub_dir is None: + object.__setattr__(self, "sub_dir", f"for-{self.name}-build") + # If no soong_env was specified then set the default. + if self.soong_env is None: + object.__setattr__( + self, + "soong_env", + { + # Set SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE to generate a + # snapshot suitable for a specific target build release. + "SOONG_SDK_SNAPSHOT_TARGET_BUILD_RELEASE": self.name, + }) + + def __le__(self, other): + return self.ordinal <= other.ordinal + + +def create_no_dist_snapshot(build_release: BuildRelease, + producer: "SdkDistProducer", + modules: List["MainlineModule"]): + """A place holder dist snapshot creation function that does nothing.""" + print(f"create_no_dist_snapshot for modules {[m.apex for m in modules]}") + return + + +def create_sdk_snapshots_in_Soong(build_release: BuildRelease, + producer: "SdkDistProducer", + modules: List["MainlineModule"]): + """Builds sdks and populates the dist.""" + producer.produce_dist_for_build_release(build_release, modules) + return + + +def reuse_latest_sdk_snapshots(build_release: BuildRelease, + producer: "SdkDistProducer", + modules: List["MainlineModule"]): + """Copies the snapshots from the latest build.""" + producer.populate_dist(build_release, build_release.sdk_versions, modules) + return + + +Q = BuildRelease( + name="Q", + # At the moment we do not generate a snapshot for Q. + creator=create_no_dist_snapshot, +) +R = BuildRelease( + name="R", + # At the moment we do not generate a snapshot for R. + creator=create_no_dist_snapshot, +) +S = BuildRelease( + name="S", + # Generate a snapshot for S using Soong. + creator=create_sdk_snapshots_in_Soong, +) + +# Insert additional BuildRelease definitions for following releases here, +# before LATEST. + +# The build release for the latest build supported by this build, i.e. the +# current build. This must be the last BuildRelease defined in this script, +# before LEGACY_BUILD_RELEASE. +LATEST = BuildRelease( + name="latest", + creator=create_sdk_snapshots_in_Soong, + # There are no build release specific environment variables to pass to + # Soong. + soong_env={}, +) + +# The build release to populate the legacy dist structure that does not specify +# a particular build release. This MUST come after LATEST so that it includes +# all the modules for which sdk snapshot source is available. +LEGACY_BUILD_RELEASE = BuildRelease( + name="legacy", + # There is no build release specific sub directory. + sub_dir="", + # There are no build release specific environment variables to pass to + # Soong. + soong_env={}, + # Do not create new snapshots, simply use the snapshots generated for + # latest. + creator=reuse_latest_sdk_snapshots, +) + + @dataclasses.dataclass(frozen=True) class MainlineModule: """Represents a mainline module""" @@ -265,6 +414,19 @@ class MainlineModule: # The names of the sdk and module_exports. sdks: list[str] + # The first build release in which the SDK snapshot for this module is + # needed. + # + # Note: This is not necessarily the same build release in which the SDK + # source was first included. So, a module that was added in build T + # could potentially be used in an S release and so its SDK will need + # to be made available for S builds. + # + # Defaults to the latest build, i.e. the build on which this script is run + # as the snapshot is assumed to be needed in the build containing the sdk + # source. + first_release: BuildRelease = LATEST + # The configuration variable, defaults to ANDROID:module_build_from_source configVar: ConfigVar = ConfigVar( namespace="ANDROID", @@ -288,6 +450,10 @@ class MainlineModule: configBpDefFile=self.configBpDefFile), ] + def is_required_for(self, target_build_release): + """True if this module is required for the target build release.""" + return self.first_release <= target_build_release + # List of mainline modules. MAINLINE_MODULES = [ @@ -298,6 +464,7 @@ MAINLINE_MODULES = [ "art-module-test-exports", "art-module-host-exports", ], + first_release=S, # Override the config... fields. configVar=ConfigVar( namespace="art_module", @@ -313,50 +480,50 @@ MAINLINE_MODULES = [ "conscrypt-module-test-exports", "conscrypt-module-host-exports", ], + first_release=Q, ), MainlineModule( apex="com.android.ipsec", sdks=["ipsec-module-sdk"], + first_release=S, ), MainlineModule( apex="com.android.media", sdks=["media-module-sdk"], + first_release=R, ), MainlineModule( apex="com.android.mediaprovider", sdks=["mediaprovider-module-sdk"], + first_release=R, ), MainlineModule( apex="com.android.permission", sdks=["permission-module-sdk"], + first_release=R, ), MainlineModule( apex="com.android.sdkext", sdks=["sdkextensions-sdk"], + first_release=R, ), MainlineModule( apex="com.android.os.statsd", sdks=["statsd-module-sdk"], + first_release=R, ), MainlineModule( apex="com.android.tethering", sdks=["tethering-module-sdk"], + first_release=R, ), MainlineModule( apex="com.android.wifi", sdks=["wifi-module-sdk"], + first_release=R, ), ] -# A list of the sdk versions to build. Usually just current but can include a -# numeric version too. -SDK_VERSIONS = [ - # Suitable for overriding the source modules with prefer:true. - # Unlike "unversioned" this mode also adds "@current" suffixed modules - # with the same prebuilts (which are never preferred). - "current", -] - @dataclasses.dataclass class SdkDistProducer: @@ -381,14 +548,39 @@ class SdkDistProducer: # transformed to document where the changes came from. script: str = sys.argv[0] - def produce_dist(self, modules): - sdk_versions = SDK_VERSIONS - self.build_sdks(sdk_versions, modules) - self.populate_dist(sdk_versions, modules) + # The path to the mainline-sdks dist directory. + # + # Initialized in __post_init__(). + mainline_sdks_dir: str = dataclasses.field(init=False) + + def __post_init__(self): + self.mainline_sdks_dir = os.path.join(self.dist_dir, "mainline-sdks") + + def prepare(self): + # Clear the mainline-sdks dist directory. + shutil.rmtree(self.mainline_sdks_dir, ignore_errors=True) + + def produce_dist(self, modules, build_releases): + # Prepare the dist directory for the sdks. + self.prepare() + + for build_release in build_releases: + # Only build modules that are required for this build release. + filtered_modules = [ + m for m in modules if m.is_required_for(build_release) + ] + if filtered_modules: + print(f"Building SDK snapshots for {build_release.name}" + f" build release") + build_release.creator(build_release, self, filtered_modules) + self.populate_stubs(modules) - def build_sdks(self, sdk_versions, modules): - self.snapshot_builder.build_snapshots(sdk_versions, modules) + def produce_dist_for_build_release(self, build_release, modules): + sdk_versions = build_release.sdk_versions + self.snapshot_builder.build_snapshots(build_release, sdk_versions, + modules) + self.populate_dist(build_release, sdk_versions, modules) def unzip_current_stubs(self, sdk_name, apex_name): """Unzips stubs for "current" into {producer.dist_dir}/stubs/{apex}.""" @@ -414,10 +606,9 @@ class SdkDistProducer: if sdk.endswith("-sdk"): self.unzip_current_stubs(sdk, apex) - def populate_dist(self, sdk_versions, modules): - # Clear and populate the mainline-sdks dist directory. - sdks_dist_dir = os.path.join(self.dist_dir, "mainline-sdks") - shutil.rmtree(sdks_dist_dir, ignore_errors=True) + def populate_dist(self, build_release, sdk_versions, modules): + build_release_dist_dir = os.path.join(self.mainline_sdks_dir, + build_release.sub_dir) for module in modules: apex = module.apex @@ -431,8 +622,8 @@ class SdkDistProducer: f" ^[^-]+-(module-)?(sdk|host-exports|test-exports)" ) - sdk_dist_dir = os.path.join(sdks_dist_dir, sdk_version, - apex, subdir) + sdk_dist_dir = os.path.join(build_release_dist_dir, + sdk_version, apex, subdir) sdk_path = self.snapshot_builder.get_sdk_path( sdk, sdk_version) self.dist_sdk_snapshot_zip(sdk_path, sdk_dist_dir, @@ -559,7 +750,8 @@ def main(): producer = create_producer() modules = filter_modules(MAINLINE_MODULES) - producer.produce_dist(modules) + + producer.produce_dist(modules, ALL_BUILD_RELEASES) if __name__ == "__main__": diff --git a/build/mainline_modules_sdks_test.py b/build/mainline_modules_sdks_test.py index cf6b28cd..d4d6faaa 100644 --- a/build/mainline_modules_sdks_test.py +++ b/build/mainline_modules_sdks_test.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for mainline_modules_sdks.py.""" +import dataclasses from pathlib import Path import os import tempfile @@ -42,7 +43,7 @@ class FakeSnapshotBuilder(mm.SnapshotBuilder): z.writestr("sdk_library/public/lib.jar", "") z.writestr("sdk_library/public/api.txt", "") - def build_snapshots(self, sdk_versions, modules): + def build_snapshots(self, build_release, sdk_versions, modules): # Create input file structure. sdks_out_dir = Path(self.get_mainline_sdks_path()) sdks_out_dir.mkdir(parents=True, exist_ok=True) @@ -75,13 +76,21 @@ class TestProduceDist(unittest.TestCase): out_dir=tmp_out_dir, ) + build_releases = [ + mm.Q, + mm.R, + mm.S, + mm.LATEST, + mm.LEGACY_BUILD_RELEASE, + ] + producer = mm.SdkDistProducer( subprocess_runner=subprocess_runner, snapshot_builder=snapshot_builder, dist_dir=tmp_dist_dir, ) - producer.produce_dist(modules) + producer.produce_dist(modules, build_releases) files = [] for abs_dir, _, filenames in os.walk(tmp_dist_dir): @@ -89,20 +98,33 @@ class TestProduceDist(unittest.TestCase): for f in filenames: files.append(os.path.join(rel_dir, f)) # pylint: disable=line-too-long - self.assertEqual([ - "mainline-sdks/current/com.android.art/host-exports/art-module-host-exports-current.zip", - "mainline-sdks/current/com.android.art/sdk/art-module-sdk-current.zip", - "mainline-sdks/current/com.android.art/test-exports/art-module-test-exports-current.zip", - "mainline-sdks/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip", - "stubs/com.android.art/sdk_library/public/api.txt", - "stubs/com.android.art/sdk_library/public/lib.jar", - "stubs/com.android.art/sdk_library/public/removed.txt", - "stubs/com.android.art/sdk_library/public/source.srcjar", - "stubs/com.android.ipsec/sdk_library/public/api.txt", - "stubs/com.android.ipsec/sdk_library/public/lib.jar", - "stubs/com.android.ipsec/sdk_library/public/removed.txt", - "stubs/com.android.ipsec/sdk_library/public/source.srcjar", - ], sorted(files)) + self.assertEqual( + [ + # Legacy copy of the snapshots, for use by tools that don't support build specific snapshots. + "mainline-sdks/current/com.android.art/host-exports/art-module-host-exports-current.zip", + "mainline-sdks/current/com.android.art/sdk/art-module-sdk-current.zip", + "mainline-sdks/current/com.android.art/test-exports/art-module-test-exports-current.zip", + "mainline-sdks/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip", + # Build specific snapshots. + "mainline-sdks/for-S-build/current/com.android.art/host-exports/art-module-host-exports-current.zip", + "mainline-sdks/for-S-build/current/com.android.art/sdk/art-module-sdk-current.zip", + "mainline-sdks/for-S-build/current/com.android.art/test-exports/art-module-test-exports-current.zip", + "mainline-sdks/for-S-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip", + "mainline-sdks/for-latest-build/current/com.android.art/host-exports/art-module-host-exports-current.zip", + "mainline-sdks/for-latest-build/current/com.android.art/sdk/art-module-sdk-current.zip", + "mainline-sdks/for-latest-build/current/com.android.art/test-exports/art-module-test-exports-current.zip", + "mainline-sdks/for-latest-build/current/com.android.ipsec/sdk/ipsec-module-sdk-current.zip", + # Legacy stubs directory containing unpacked java_sdk_library artifacts. + "stubs/com.android.art/sdk_library/public/api.txt", + "stubs/com.android.art/sdk_library/public/lib.jar", + "stubs/com.android.art/sdk_library/public/removed.txt", + "stubs/com.android.art/sdk_library/public/source.srcjar", + "stubs/com.android.ipsec/sdk_library/public/api.txt", + "stubs/com.android.ipsec/sdk_library/public/lib.jar", + "stubs/com.android.ipsec/sdk_library/public/removed.txt", + "stubs/com.android.ipsec/sdk_library/public/source.srcjar", + ], + sorted(files)) def pathToTestData(relative_path): |