From 2178bb2de916409bf1f1d0f57ed4202a83429ec0 Mon Sep 17 00:00:00 2001 From: Karl Shaffer Date: Thu, 15 Apr 2021 14:45:11 -0400 Subject: Implement the prepare_bazel_test_env script The prepare_bazel_test_env script allows for the import of Soong-built test targets into a pseudo-Bazel test environment for the development of test infrastructure. Test: As of this CL, the script can be executed successfully and tested manually with the following steps: bazel run //build/pesto/experiments/prepare_bazel_test_env -- -v generate bazel run //build/pesto/experiments/prepare_bazel_test_env -- -v sync bazel test --test_output=all //platform_testing/tests/example/native:hello_world_test Change-Id: Ia66b67546e2a2f11c5c6c3870a700d3356318461 --- experiments/prepare_bazel_test_env/bazelenv.py | 643 +++++++++++++++++++++ .../prepare_bazel_test_env.py | 91 +++ 2 files changed, 734 insertions(+) create mode 100644 experiments/prepare_bazel_test_env/bazelenv.py create mode 100644 experiments/prepare_bazel_test_env/prepare_bazel_test_env.py (limited to 'experiments') diff --git a/experiments/prepare_bazel_test_env/bazelenv.py b/experiments/prepare_bazel_test_env/bazelenv.py new file mode 100644 index 0000000..e42e767 --- /dev/null +++ b/experiments/prepare_bazel_test_env/bazelenv.py @@ -0,0 +1,643 @@ +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Core logic for generating, syncing, and cleaning up a Bazel environment.""" +import abc +import datetime +import logging +import os +import pathlib +import re +import shutil +import subprocess +from typing import Set, Dict, List + +# Regex for BUILD files used to identify them since they can be named +# BUILD or BUILD.bazel. +BUILD_FILENAME_REGEX = re.compile("(BUILD|BUILD.bazel)") + + +class Error(Exception): + """Base Error that all other errors the system throws are descendants of.""" + pass + + +class SoongExecutionError(Error): + """Raised when Soong fails to build provided targets.""" + pass + + +class Soong: + """Interface for the Soong build system. + + Attributes: + soong_workspace: the top of the Android workspace that Soong + will be operating in. + soong_executable: the path to the executable for the soong_ui.bash + launcher. + """ + soong_workspace: pathlib.Path + soong_executable: pathlib.Path + + def __init__(self, soong_workspace: pathlib.Path): + self.soong_workspace = soong_workspace + + self.soong_executable = self.soong_workspace.joinpath( + "build/soong/soong_ui.bash") + if not self.soong_executable.exists(): + raise SoongExecutionError( + "Unable to find Soong executable, expected location: %s" % + self.soong_executable) + + def build(self, build_targets: Set[str]) -> None: + """Builds the provided set of targets with Soong. + + Of note, there is no verification for the targets that get passed in, + rather that responsibility passes to Soong which will fail to build if a + target is invalid. + + Args: + build_targets: a set of targets to build with Soong. + """ + cmd_args = [ + str(self.soong_executable), "--build-mode", "--all-modules", + f"--dir={self.soong_workspace}" + ] + cmd_args.extend(build_targets) + + logging.info("Building targets with Soong: %s", build_targets) + logging.info("Please be patient, this may take a while...") + logging.debug("Soong Command is: %s", " ".join(cmd_args)) + + try: + subprocess.run(cmd_args, + cwd=self.soong_workspace, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as cpe: + raise SoongExecutionError( + "There was an error during the Soong build process. Please " + "correct the error with Soong, and try running this script " + "again. Soong output follows:\n\n" + f"{cpe.stdout.decode('utf-8')}") from cpe + + logging.info("Soong command completed successfully.") + + +class Resource(abc.ABC): + """ + Represents a resource file that is used for scaffolding the Bazel env. + + Resource is an abstract class. Inheriting classes must provide the following + attributes which are used by the default method implementations below. + + Attributes: + stage_path: the path where this resource should be written when staged. + workspace_path: the path where this resource should be synced to in the + workspace. + """ + stage_path: pathlib.Path + workspace_path: pathlib.Path + + def build_targets(self) -> set: + return set() + + @abc.abstractmethod + def stage(self, _): + """Writes a resource to its stage location.""" + pass + + @abc.abstractmethod + def sync(self): + """Syncs a resource to its workspace location.""" + + # Overwrite any existing file in the workspace when synced. + self.workspace_path.unlink(missing_ok=True) + self.workspace_path.symlink_to(self.stage_path) + + @abc.abstractmethod + def clean(self): + """Cleans a resource from its workspace location.""" + self.workspace_path.unlink(missing_ok=True) + + +class StaticResource(Resource): + """Resource representing a static file to be copied. + + Attributes: + resource_path: path to the resource on disk. + """ + def __init__(self, stage_path: pathlib.Path, workspace_path: pathlib.Path, + resource_path: pathlib.Path): + self.stage_path = stage_path + self.workspace_path = workspace_path + self._resource_path = resource_path + + def stage(self, _): + _verify_directory(self.stage_path.parent) + shutil.copy(self._resource_path, self.stage_path) + _make_executable_if_script(self.stage_path) + + def sync(self): + super().sync() + + def clean(self): + super().sync() + + def __repr__(self): + return (f"StaticResource(stage_path={self.stage_path}, " + f"workspace_path={self.workspace_path}, " + f"resource_path={self._resource_path})") + + +class TemplateError(Error): + """Raised when there is an issue while templating a template file.""" + pass + + +class TemplateResource(Resource): + """Resource that represents a file to be templated. + + When staged, the resource is templated using the "mapping" provided + to the stage function. + + Attributes: + resource_path: path to the resource on disk. + """ + resource_path: pathlib.Path + + # Key within templates that identifies a Soong target to build for a + # provided template. + SOONG_TARGET_KEY = "SOONG_TARGET" + _KEY_VALUE_SEP = ":" + + # For a provided template, lines matching this regex are ignored when + # loading the template. + # + # This enables to contain metadata that, while visible to the script, is + # not visible in the generated templates. This is currently used with the + # SOONG_TARGET key/value pairs in templates. + _IGNORE_LINE_REGEX = re.compile(f"({SOONG_TARGET_KEY})") + + def __init__(self, stage_path: pathlib.Path, workspace_path: pathlib.Path, + resource_path: pathlib.Path): + self.stage_path = stage_path + self.workspace_path = workspace_path + self.resource_path = resource_path + + def __repr__(self): + return (f"TemplateResource(stage_path={self.stage_path}, " + f"workspace_path={self.workspace_path}, " + f"resource_path={self.resource_path})") + + def stage(self, mapping: Dict[str, str]): + _verify_directory(self.stage_path.parent) + lines = self.resource_path.open().readlines() + lines = [ + line for line in lines if not self._IGNORE_LINE_REGEX.search(line) + ] + + try: + output = "".join(lines).format_map(mapping) + except KeyError as ke: + raise TemplateError( + f"Malformed template file: {self.resource_path}") from ke + + with self.stage_path.open("w") as output_file: + output_file.write(output) + _make_executable_if_script(self.stage_path) + + def sync(self): + super().sync() + + def clean(self): + super().clean() + + @classmethod + def read_value_from_template_var(cls, line: str) -> str: + value = line.split(cls._KEY_VALUE_SEP)[-1] + return value.strip() + + +class BuildTemplateResource(Resource): + """Resource that represents a BUILD file. + + This operates in the same way as a TemplateResource, however also + sets up the prebuilts directory needed by BUILD files. + + Attributes: + _template_resource: underlying TemplateResource for this resource. + global_prebuilts_dir: the directory in the filesystem where prebuilts + live, this directory is symlinked to by a directory adjacent + to the BUILD file represented by this Resource. + prebuilts_stage_path: the path of the staged prebuilts directory + which is a symlink to the global_prebuilts_dir. + prebuilts_workspace_path: the path of the workspace prebuilts directory, + which is a symlink to the global_prebuilts_dir. + """ + _template_resource: TemplateResource + global_prebuilts_dir: pathlib.Path + prebuilts_stage_path: pathlib.Path + prebuilts_workspace_path: pathlib.Path + + def __init__(self, stage_path: pathlib.Path, workspace_path: pathlib.Path, + resource_path: pathlib.Path, + global_prebuilts_dir: pathlib.Path, prebuilts_dir_name: str): + self.stage_path = stage_path + self.workspace_path = workspace_path + + self._template_resource = TemplateResource(stage_path, workspace_path, + resource_path) + self.global_prebuilts_dir = global_prebuilts_dir + self.prebuilts_stage_path = self.stage_path.parent.joinpath( + prebuilts_dir_name) + self.prebuilts_workspace_path = self.workspace_path.parent.joinpath( + prebuilts_dir_name) + + def __repr__(self): + return ("BuildTemplateResource(" + f"template_resource={self._template_resource}, " + f"global_prebuilts_dir={self.global_prebuilts_dir}, " + f"prebuilts_stage_path={self.prebuilts_stage_path}, " + f"prebuilts_workspace_path={self.prebuilts_workspace_path})") + + def build_targets(self) -> set: + """Overrides build_targets() to read targets from the BUILD template.""" + targets = set() + with self._template_resource.resource_path.open() as build_file: + while line := build_file.readline(): + if self._template_resource.SOONG_TARGET_KEY in line: + targets.add( + self._template_resource.read_value_from_template_var( + line)) + + return targets + + def stage(self, mapping: Dict[str, str]): + """Overrides stage() to stage a BUILD resource. + + Delegates most actions to the _template_resource, while ensuring that + a generated prebuilts directory is also staged. + """ + self._template_resource.stage(mapping) + self.prebuilts_stage_path.symlink_to(self.global_prebuilts_dir, + target_is_directory=True) + + def sync(self): + """Overrides sync() to sync a BUILD resource with prebuilts. + + Delegates to the _template_resource while also ensuring that the + generated prebuilts directory is written to the workspace. + """ + self._template_resource.sync() + + # Overwrite the existing prebuilts directory, if it exists. + self.prebuilts_workspace_path.unlink(missing_ok=True) + self.prebuilts_workspace_path.symlink_to(self.global_prebuilts_dir, + target_is_directory=True) + + def clean(self): + """Overrides clean() to clean a BUILD resource. + + Delegates most actions to the _template_resource, while also ensuring + that the generated prebuilts directory is removed from the workspace. + """ + self._template_resource.clean() + self.prebuilts_workspace_path.unlink(missing_ok=True) + + +def _make_executable_if_script(path: pathlib.Path) -> None: + """Makes the provided path executable if it is a script. + Args: + path: the path to check and conditionally make executable. + """ + if path.name.endswith(".sh"): + # Grant full permissions (read/write/execute) for current user and + # read/write permissions for group. + path.chmod(mode=0o750) + + +class ResourcesNotFoundError(Error): + """Raised when the required resources are not found.""" + pass + + +class Resources: + """Manages and loads resources from disk. + + Attributes: + workspace_base_path: the path to the root of the workspace where + staged files will be synced to. + gendir_base_path: the path to the root of the staging directory. + global_prebuilts_dir_path: the path to the root of the global + prebuilts directory to which all generated prebuilts directories will + symlink to. + prebuilts_dir_name: the name to use for generated prebuilts + directories. + static_path: the path to the static resources directory. + template_path: the path to the template resources directory. + """ + workspace_base_path: pathlib.Path + gendir_base_path: pathlib.Path + global_prebuilts_dir_path: pathlib.Path + prebuilts_dir_name: str + static_path: pathlib.Path + template_path: pathlib.Path + + # Name of the directory where script runfiles are located. + _DATA_DIRNAME = "data" + + # Name of the directory where the templates should be located. + _TEMPLATES_DIRNAME = "templates" + + # Name of the directory where static files (to be copied to the environment) + # should be located. + _STATIC_DIRNAME = "static" + + # If the script is executed from the root of the runfiles directory, this is + # where the data should be. + _RESOURCES_RUNFILES_BASEPATH = pathlib.Path( + "build/pesto/experiments/prepare_bazel_test_env", _DATA_DIRNAME) + + # File extension for templates, determining whether or not a given file is a + # template. + _TEMPLATE_FILE_EXT = ".template" + + def __init__(self, + workspace_base_path: pathlib.Path, + gendir_base_path: pathlib.Path, + global_prebuilts_dir_path: pathlib.Path, + prebuilts_dir_name: str, + path: pathlib.Path = _RESOURCES_RUNFILES_BASEPATH): + logging.debug("Resources(path=%s)", path) + + self.workspace_base_path = workspace_base_path + self.gendir_base_path = gendir_base_path + self.global_prebuilts_dir_path = global_prebuilts_dir_path + self.prebuilts_dir_name = prebuilts_dir_name + + path = path.resolve() + + self.template_path = path.joinpath(Resources._TEMPLATES_DIRNAME) + self.static_path = path.joinpath(Resources._STATIC_DIRNAME) + + if not self.template_path.exists() or not self.static_path.exists(): + raise ResourcesNotFoundError("Unable to find resources at path " + f"{path}, expected the following " + "directories: " + f"{Resources._TEMPLATES_DIRNAME}, " + f"{Resources._STATIC_DIRNAME}") + + def __repr__(self): + return ("Resources(" + f"template_path={self.template_path}" + f"static_path={self.static_path})") + + def stage(self, mapping: Dict[str, str]) -> None: + for resource in self.load(): + logging.debug("Staging resource: %s", resource) + resource.stage(mapping) + + def sync(self) -> None: + for resource in self.load(): + logging.debug("Syncing resource: %s", resource) + resource.sync() + + def clean(self) -> None: + for resource in self.load(): + logging.debug("Cleaning resource: %s", resource) + resource.clean() + + def build_targets(self) -> Set[str]: + return { + t + for resource in self.load() for t in resource.build_targets() + } + + def load(self) -> List[Resource]: + """Loads the Resources used to scaffold the Bazel env. + + Returns: + a list of Resource objects representing the files + that should be used to template the environment. + """ + resources = [] + + # Add all templates. + for p in self._template_resource_paths(): + template_relpath = Resources._strip_template_identifier_from_path( + p.relative_to(self.template_path)) + stage_path = self.gendir_base_path.joinpath(template_relpath) + workspace_path = self.workspace_base_path.joinpath( + template_relpath) + + if BUILD_FILENAME_REGEX.match(template_relpath.name): + resources.append( + BuildTemplateResource( + stage_path=stage_path, + workspace_path=workspace_path, + resource_path=p, + global_prebuilts_dir=self.global_prebuilts_dir_path, + prebuilts_dir_name=self.prebuilts_dir_name)) + else: + resources.append( + TemplateResource(stage_path=stage_path, + workspace_path=workspace_path, + resource_path=p)) + + # Add all static files. + for p in self._static_resource_paths(): + static_relpath = p.relative_to(self.static_path) + stage_path = self.gendir_base_path.joinpath(static_relpath) + workspace_path = self.workspace_base_path.joinpath(static_relpath) + resources.append( + StaticResource(stage_path=stage_path, + workspace_path=workspace_path, + resource_path=p)) + + return resources + + def _static_resource_paths(self) -> List[pathlib.Path]: + return [p for p in self.static_path.glob("**/*") if p.is_file()] + + def _template_resource_paths(self) -> List[pathlib.Path]: + return [p for p in self.template_path.glob("**/*") if p.is_file()] + + @staticmethod + def _strip_template_identifier_from_path(p: pathlib.Path): + """Strips the template file extension from a provided path.""" + if p.name.endswith(Resources._TEMPLATE_FILE_EXT): + p = p.with_name(p.name[:len(p.name) - + len(Resources._TEMPLATE_FILE_EXT)]) + return p + + +class AndroidBuildEnvironmentError(Error): + """Raised when the Android Build Environment is not properly set.""" + pass + + +class BazelTestEnvGenerator: + """Context for the Bazel environment generation. + + This class provides access to locations within the filesystem pertinent to the + current execution as members that can be accessed by users of the class. + + Attributes: + workspace_top: the top of the codebase, which also serves as the top of + the Bazel WORKSPACE. + host_out: Host artifact staging directory location. + host_testcases: Host testcase staging directory location. + product_out: Product/Target artifact staging directory location. + target_testcases: Product/Target testcase staging directory location. + staging_dir: Staging directory for generated Bazel artifacts. + prebuilts_dir_name: Name of the directory that should be used for the + directories that prebuilts are placed into when synced into the source + tree. + global_prebuilts_dir_path: The absolute path to the global prebuilts + directory. + year: the current calendar year. + gen_dir: Subdirectory of the staging dir where the Bazel environment + should be generated to. + """ + workspace_top: pathlib.Path + host_out: pathlib.Path + host_testcases: pathlib.Path + product_out: pathlib.Path + target_testcases: pathlib.Path + staging_dir: pathlib.Path + prebuilts_dir_name: str = ".soong_prebuilts" + prebuilts_dir_path: pathlib.Path + gen_dir: pathlib.Path + year: str + _resources: Resources + _soong: Soong + + # Name of the subdirectory, within the output directory, where the + # prebuilts directory is scaffolded. + # + # This directory then serves as the target of symlinks from across the + # source tree that gives any Bazel target access to the Soong staging + # directories. + GLOBAL_PREBUILTS_DIR_PATH = "prebuilts" + + def __init__(self, env_dict: Dict[str, str] = os.environ): + try: + self.workspace_top = pathlib.Path(env_dict["ANDROID_BUILD_TOP"]) + self.host_out = pathlib.Path(env_dict["ANDROID_HOST_OUT"]) + self.host_testcases = pathlib.Path( + env_dict["ANDROID_HOST_OUT_TESTCASES"]) + self.product_out = pathlib.Path(env_dict["ANDROID_PRODUCT_OUT"]) + self.target_testcases = pathlib.Path( + env_dict["ANDROID_TARGET_OUT_TESTCASES"]) + except KeyError as e: + raise AndroidBuildEnvironmentError( + "Missing expected environment variable.") from e + + self.staging_dir = pathlib.Path(self.workspace_top, + "out/pesto-environment") + self.prebuilts_dir_name = BazelTestEnvGenerator.prebuilts_dir_name + self.global_prebuilts_dir_path = self.staging_dir.joinpath( + self.GLOBAL_PREBUILTS_DIR_PATH) + self.year = str(datetime.date.today().year) + self.gen_dir = pathlib.Path(self.staging_dir, "gen") + + self._resources = Resources(self.workspace_top, self.gen_dir, + self.global_prebuilts_dir_path, + self.prebuilts_dir_name) + self._soong = Soong(self.workspace_top) + + def __repr__(self): + return "GenerationContext(%s)" % vars(self) + + def generate(self): + logging.info("Starting generation of Bazel environment.") + + self._soong.build(self._resources.build_targets()) + + logging.debug("Creating fresh staging dir at: %s", self.staging_dir) + _verify_directory(self.staging_dir, clean=True) + + logging.debug("Creating fresh gen dir at: %s", self.gen_dir) + _verify_directory(self.gen_dir, clean=True) + + logging.debug("Creating global prebuilts directory at: %s", + self.global_prebuilts_dir_path) + _verify_directory(self.global_prebuilts_dir_path, clean=True) + + # Symlink the build system provided staging directories to the + # global prebuilts directory. + self.global_prebuilts_dir_path.joinpath("host").symlink_to( + self.host_out) + self.global_prebuilts_dir_path.joinpath("host_testcases").symlink_to( + self.host_testcases) + self.global_prebuilts_dir_path.joinpath("product").symlink_to( + self.product_out) + self.global_prebuilts_dir_path.joinpath("target_testcases").symlink_to( + self.target_testcases) + + # Load and process each resource into the gen directory. + self._resources.stage(mapping=vars(self)) + + logging.info( + "Generation of Bazel environment to staging directory " + "(%s) completed successfully.", self.staging_dir) + + def sync(self): + logging.info( + "Starting synchronization of generated environment to source tree." + ) + + if not self.staging_dir.exists(): + raise FileNotFoundError("Staging directory does not exist, " + "the generate function should be called " + " to create this directory.") + + self._resources.sync() + + logging.info( + "Successfully synchronized generated Bazel environment from " + "%s to %s", self.gen_dir, self.workspace_top) + + def clean(self): + logging.info("Starting clean of generated environment.") + + logging.info("Cleaning up synchronized files from the source tree.") + # For all of our configured templates, attempt to find the corresponding + # location in the source tree and remove them. + self._resources.clean() + + logging.info("Cleaning up staging directory: %s", self.staging_dir) + try: + shutil.rmtree(self.staging_dir) + except FileNotFoundError: + logging.debug("Staging directory not found during cleanup " + "and may have already been removed.") + logging.info("Successfully cleaned up generated environment.") + + +def _verify_directory(directory: pathlib.Path, clean: bool = False) -> None: + """Verifies that the provided directory exists, creating it if it does not. + + Args: + directory: path to the directory to create. + clean: whether or not the existing directory should be removed if found or + reused. + """ + if directory.exists() and clean: + logging.debug("Cleaning existing directory %s", directory) + shutil.rmtree(directory) + + logging.debug("Verifying directory exists at %s", directory) + directory.mkdir(parents=True, exist_ok=True) diff --git a/experiments/prepare_bazel_test_env/prepare_bazel_test_env.py b/experiments/prepare_bazel_test_env/prepare_bazel_test_env.py new file mode 100644 index 0000000..b086b00 --- /dev/null +++ b/experiments/prepare_bazel_test_env/prepare_bazel_test_env.py @@ -0,0 +1,91 @@ +# Copyright (C) 2021 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Binary that generates a simulated Bazel environment in the Android source. + +The script utilizes an internal repository of templates to determine the targets +to build, builds them via Soong, then imports them into a Bazel environment +all relying on templated BUILD files. These files can then be placed +directly within the Android source tree to simulate what a real Bazel +environment would look like. +""" +import argparse +import logging + +import bazelenv + +_LOG_PRINT_FORMAT = ("%(asctime)s %(filename)s:%(lineno)s:%(levelname)s: " + "%(message)s") +_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" + + +def _configure_logging(verbose: bool) -> None: + """Configures logging for the application. + + Args: + verbose: if True, all messages are logged, otherwise only INFO and above + are logged. + """ + logging.basicConfig(format=_LOG_PRINT_FORMAT, datefmt=_LOG_DATE_FORMAT) + level = logging.DEBUG if verbose else logging.INFO + logging.root.setLevel(level) + + +def _create_arg_parser(): + parser = argparse.ArgumentParser(description=( + "Prepares a simulated Bazel environment that can be used to " + "execute tests in a Bazel environment based on Soong " + "produced artifacts.")) + + parser.add_argument("-v", + "--verbose", + help="Enables verbose logging.", + action="store_true") + + subparsers = parser.add_subparsers(dest="action", required=True) + + # For each subparser, provide a default 'func' argument that calls the + # corresponding method on the generator instance. + subparsers.add_parser( + "generate", + help="Generates the Bazel environment to the staging directory." + ).set_defaults(func=lambda g: g.generate()) + + subparsers.add_parser( + "sync", + help="Synchronizes the staged Bazel environment to the source tree." + ).set_defaults(func=lambda g: g.sync()) + + subparsers.add_parser( + "clean", + help= + ("Cleans up the Bazel environment by clearing anything that has been " + "synced to the source tree as well as the staging directory itself." + )).set_defaults(func=lambda g: g.clean()) + + return parser + + +if __name__ == "__main__": + args = _create_arg_parser().parse_args() + logging.debug("prepare_bazel_test_env(%s)", args) + + _configure_logging(args.verbose) + + try: + generator = bazelenv.BazelTestEnvGenerator() + args.func(generator) + except bazelenv.Error: + logging.exception( + "A known error occurred, check the error description " + "or logs for more details.") -- cgit v1.2.3