diff options
author | Cole Faust <colefaust@google.com> | 2022-04-15 20:39:20 +0000 |
---|---|---|
committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2022-04-15 20:39:20 +0000 |
commit | e0acede611e2afbec75bfbe60ad6c1493a4230d6 (patch) | |
tree | 8c34b0e59240b604846117c931317ca6f69843b8 | |
parent | 4dd7394e95debb0ade345874c07fcff817dcde29 (diff) | |
parent | d12b9299da313448ed5efd4c3f37904e9634ecc8 (diff) | |
download | bazel-e0acede611e2afbec75bfbe60ad6c1493a4230d6.tar.gz |
Add rbc dashboard script am: c59332e477 am: 20d696c5f5 am: d12b9299da
Original change: https://android-review.googlesource.com/c/platform/build/bazel/+/2065467
Change-Id: I3d7c66a2a211f8b547f9bf24f7b2b0b6b6982f8c
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>
-rwxr-xr-x | ci/rbc_dashboard.py | 467 |
1 files changed, 467 insertions, 0 deletions
diff --git a/ci/rbc_dashboard.py b/ci/rbc_dashboard.py new file mode 100755 index 00000000..2e3ef1b9 --- /dev/null +++ b/ci/rbc_dashboard.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +"""Generates a dashboard for the current RBC product/board config conversion status.""" +# pylint: disable=line-too-long + +import argparse +import asyncio +import dataclasses +import datetime +import os +import re +import shutil +import socket +import subprocess +import sys +import time +from typing import List, Tuple +import xml.etree.ElementTree as ET + +_PRODUCT_REGEX = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)(?:-(user|userdebug|eng))?') + + +@dataclasses.dataclass(frozen=True) +class Product: + """Represents a TARGET_PRODUCT and TARGET_BUILD_VARIANT.""" + product: str + variant: str + + def __post_init__(self): + if not _PRODUCT_REGEX.match(str(self)): + raise ValueError(f'Invalid product name: {self}') + + def __str__(self): + return self.product + '-' + self.variant + + +@dataclasses.dataclass(frozen=True) +class ProductResult: + baseline_success: bool + product_success: bool + board_success: bool + product_has_diffs: bool + board_has_diffs: bool + + def success(self) -> bool: + return not self.baseline_success or ( + self.product_success and self.board_success + and not self.product_has_diffs and not self.board_has_diffs) + + +@dataclasses.dataclass(frozen=True) +class Directories: + out: str + out_baseline: str + out_product: str + out_board: str + results: str + + +def get_top() -> str: + path = '.' + while not os.path.isfile(os.path.join(path, 'build/soong/soong_ui.bash')): + if os.path.abspath(path) == '/': + sys.exit('Could not find android source tree root.') + path = os.path.join(path, '..') + return os.path.abspath(path) + + +def get_build_var(variable, product: Product) -> str: + """Returns the result of the shell command get_build_var.""" + env = { + **os.environ, + 'TARGET_PRODUCT': product.product, + 'TARGET_BUILD_VARIANT': product.variant, + } + return subprocess.run([ + 'build/soong/soong_ui.bash', + '--dumpvar-mode', + variable + ], check=True, capture_output=True, env=env, text=True).stdout.strip() + + +async def run_jailed_command(args: List[str], out_dir: str, env=None) -> bool: + """Runs a command, saves its output to out_dir/build.log, and returns if it succeeded.""" + with open(os.path.join(out_dir, 'build.log'), 'wb') as f: + result = await asyncio.create_subprocess_exec( + 'prebuilts/build-tools/linux-x86/bin/nsjail', + '-q', + '--cwd', + os.getcwd(), + '-e', + '-B', + '/', + '-B', + f'{os.path.abspath(out_dir)}:{os.path.abspath("out")}', + '--time_limit', + '0', + '--skip_setsid', + '--keep_caps', + '--disable_clone_newcgroup', + '--disable_clone_newnet', + '--rlimit_as', + 'soft', + '--rlimit_core', + 'soft', + '--rlimit_cpu', + 'soft', + '--rlimit_fsize', + 'soft', + '--rlimit_nofile', + 'soft', + '--proc_rw', + '--hostname', + socket.gethostname(), + '--', + *args, stdout=f, stderr=subprocess.STDOUT, env=env) + return await result.wait() == 0 + + +async def run_build(flags: List[str], out_dir: str) -> bool: + return await run_jailed_command([ + 'build/soong/soong_ui.bash', + '--make-mode', + *flags, + '--skip-ninja', + 'nothing' + ], out_dir) + + +async def run_config(product: Product, rbc_product: bool, rbc_board: bool, out_dir: str) -> bool: + """Runs config.mk and saves results to out/rbc_variable_dump.txt.""" + env = { + 'OUT_DIR': 'out', + 'TMPDIR': 'tmp', + 'BUILD_DATETIME_FILE': 'out/build_date.txt', + 'CALLED_FROM_SETUP': 'true', + 'TARGET_PRODUCT': product.product, + 'TARGET_BUILD_VARIANT': product.variant, + 'RBC_PRODUCT_CONFIG': 'true' if rbc_product else '', + 'RBC_BOARD_CONFIG': 'true' if rbc_board else '', + 'RBC_DUMP_CONFIG_FILE': 'out/rbc_variable_dump.txt', + } + return await run_jailed_command([ + 'prebuilts/build-tools/linux-x86/bin/ckati', + '-f', + 'build/make/core/config.mk' + ], out_dir, env=env) + + +async def has_diffs(success: bool, file_pairs: List[Tuple[str]], results_folder: str) -> bool: + """Returns true if the two out folders provided have differing ninja files.""" + if not success: + return False + results = [] + for pair in file_pairs: + name = 'soong_build.ninja' if pair[0].endswith('soong/build.ninja') else os.path.basename(pair[0]) + with open(os.path.join(results_folder, name)+'.diff', 'wb') as f: + results.append((await asyncio.create_subprocess_exec( + 'diff', + pair[0], + pair[1], + stdout=f, stderr=subprocess.STDOUT)).wait()) + + for return_code in await asyncio.gather(*results): + if return_code != 0: + return True + return False + + +def generate_html_row(num: int, product: Product, results: ProductResult): + def generate_status_cell(success: bool, diffs: bool) -> str: + message = 'Success' + if diffs: + message = 'Results differed' + if not success: + message = 'Build failed' + return f'<td style="background-color: {"lightgreen" if success and not diffs else "salmon"}">{message}</td>' + + return f''' + <tr> + <td>{num}</td> + <td>{product if results.success() and results.baseline_success else f'<a href="{product}/">{product}</a>'}</td> + {generate_status_cell(results.baseline_success, False)} + {generate_status_cell(results.product_success, results.product_has_diffs)} + {generate_status_cell(results.board_success, results.board_has_diffs)} + </tr> + ''' + + +def get_branch() -> str: + try: + tree = ET.parse('.repo/manifests/default.xml') + default_tag = tree.getroot().find('default') + return default_tag.get('remote') + '/' + default_tag.get('revision') + except Exception as e: # pylint: disable=broad-except + print(str(e), file=sys.stderr) + return 'Unknown' + + +def cleanup_empty_files(path): + if os.path.isfile(path): + if os.path.getsize(path) == 0: + os.remove(path) + elif os.path.isdir(path): + for subfile in os.listdir(path): + cleanup_empty_files(os.path.join(path, subfile)) + if not os.listdir(path): + os.rmdir(path) + + +async def test_one_product(product: Product, dirs: Directories) -> ProductResult: + """Runs the builds and tests for differences for a single product.""" + baseline_success, product_success, board_success = await asyncio.gather( + run_build([ + f'TARGET_PRODUCT={product.product}', + f'TARGET_BUILD_VARIANT={product.variant}', + ], dirs.out_baseline), + run_build([ + f'TARGET_PRODUCT={product.product}', + f'TARGET_BUILD_VARIANT={product.variant}', + 'RBC_PRODUCT_CONFIG=1', + ], dirs.out_product), + run_build([ + f'TARGET_PRODUCT={product.product}', + f'TARGET_BUILD_VARIANT={product.variant}', + 'RBC_BOARD_CONFIG=1', + ], dirs.out_board), + ) + + product_dashboard_folder = os.path.join(dirs.results, str(product)) + os.mkdir(product_dashboard_folder) + os.mkdir(product_dashboard_folder+'/baseline') + os.mkdir(product_dashboard_folder+'/product') + os.mkdir(product_dashboard_folder+'/board') + + if not baseline_success: + shutil.copy2(os.path.join(dirs.out_baseline, 'build.log'), + f'{product_dashboard_folder}/baseline/build.log') + if not product_success: + shutil.copy2(os.path.join(dirs.out_product, 'build.log'), + f'{product_dashboard_folder}/product/build.log') + if not board_success: + shutil.copy2(os.path.join(dirs.out_board, 'build.log'), + f'{product_dashboard_folder}/board/build.log') + + files = [f'build-{product.product}.ninja', f'build-{product.product}-package.ninja', 'soong/build.ninja'] + product_files = [(os.path.join(dirs.out_baseline, x), os.path.join(dirs.out_product, x)) for x in files] + board_files = [(os.path.join(dirs.out_baseline, x), os.path.join(dirs.out_board, x)) for x in files] + product_has_diffs, board_has_diffs = await asyncio.gather( + has_diffs(baseline_success and product_success, product_files, product_dashboard_folder+'/product'), + has_diffs(baseline_success and board_success, board_files, product_dashboard_folder+'/board')) + + # delete files that contain the product name in them to save space, + # otherwise the ninja files end up filling up the whole harddrive + for out_folder in [dirs.out_baseline, dirs.out_product, dirs.out_board]: + for subfolder in ['', 'soong']: + folder = os.path.join(out_folder, subfolder) + for file in os.listdir(folder): + if os.path.isfile(os.path.join(folder, file)) and product.product in file: + os.remove(os.path.join(folder, file)) + + cleanup_empty_files(product_dashboard_folder) + + return ProductResult(baseline_success, product_success, board_success, product_has_diffs, board_has_diffs) + + +async def test_one_product_quick(product: Product, dirs: Directories) -> ProductResult: + """Runs the builds and tests for differences for a single product.""" + baseline_success, product_success, board_success = await asyncio.gather( + run_config( + product, + False, + False, + dirs.out_baseline), + run_config( + product, + True, + False, + dirs.out_product), + run_config( + product, + False, + True, + dirs.out_board), + ) + + product_dashboard_folder = os.path.join(dirs.results, str(product)) + os.mkdir(product_dashboard_folder) + os.mkdir(product_dashboard_folder+'/baseline') + os.mkdir(product_dashboard_folder+'/product') + os.mkdir(product_dashboard_folder+'/board') + + if not baseline_success: + shutil.copy2(os.path.join(dirs.out_baseline, 'build.log'), + f'{product_dashboard_folder}/baseline/build.log') + if not product_success: + shutil.copy2(os.path.join(dirs.out_product, 'build.log'), + f'{product_dashboard_folder}/product/build.log') + if not board_success: + shutil.copy2(os.path.join(dirs.out_board, 'build.log'), + f'{product_dashboard_folder}/board/build.log') + + files = ['rbc_variable_dump.txt'] + product_files = [(os.path.join(dirs.out_baseline, x), os.path.join(dirs.out_product, x)) for x in files] + board_files = [(os.path.join(dirs.out_baseline, x), os.path.join(dirs.out_board, x)) for x in files] + product_has_diffs, board_has_diffs = await asyncio.gather( + has_diffs(baseline_success and product_success, product_files, product_dashboard_folder+'/product'), + has_diffs(baseline_success and board_success, board_files, product_dashboard_folder+'/board')) + + cleanup_empty_files(product_dashboard_folder) + + return ProductResult(baseline_success, product_success, board_success, product_has_diffs, board_has_diffs) + + +async def main(): + parser = argparse.ArgumentParser( + description='Generates a dashboard of the starlark product configuration conversion.') + parser.add_argument('products', nargs='*', + help='list of products to test. If not given, all ' + + 'products will be tested. ' + + 'Example: aosp_arm64-userdebug') + parser.add_argument('--quick', action='store_true', + help='Run a quick test. This will only run config.mk and ' + + 'diff the make variables at the end of it, instead of ' + + 'diffing the full ninja files.') + parser.add_argument('--exclude', nargs='+', default=[], + help='Exclude these producs from the build. Useful if not ' + + 'supplying a list of products manually.') + parser.add_argument('--results-directory', + help='Directory to store results in. Defaults to $(OUT_DIR)/rbc_dashboard. ' + + 'Warning: will be cleared!') + args = parser.parse_args() + + if args.results_directory: + args.results_directory = os.path.abspath(args.results_directory) + + os.chdir(get_top()) + + def str_to_product(p: str) -> Product: + match = _PRODUCT_REGEX.fullmatch(p) + if not match: + sys.exit(f'Invalid product name: {p}. Example: aosp_arm64-userdebug') + return Product(match.group(1), match.group(2) if match.group(2) else 'userdebug') + + products = [str_to_product(p) for p in args.products] + + if not products: + products = list(map(lambda x: Product(x, 'userdebug'), get_build_var( + 'all_named_products', Product('aosp_arm64', 'userdebug')).split())) + + excluded = [str_to_product(p) for p in args.exclude] + products = [p for p in products if p not in excluded] + + for i, product in enumerate(products): + for j, product2 in enumerate(products): + if i != j and product.product == product2.product: + sys.exit(f'Product {product.product} cannot be repeated.') + + out_dir = get_build_var('OUT_DIR', Product('aosp_arm64', 'userdebug')) + + dirs = Directories( + out=out_dir, + out_baseline=os.path.join(out_dir, 'rbc_out_baseline'), + out_product=os.path.join(out_dir, 'rbc_out_product'), + out_board=os.path.join(out_dir, 'rbc_out_board'), + results=args.results_directory if args.results_directory else os.path.join(out_dir, 'rbc_dashboard')) + + for folder in [dirs.out_baseline, dirs.out_product, dirs.out_board, dirs.results]: + # delete and recreate the out directories. You can't reuse them for + # a particular product, because after we delete some product-specific + # files inside the out dir to save space, the build will fail if you + # try to build the same product again. + shutil.rmtree(folder, ignore_errors=True) + os.makedirs(folder) + + # When running in quick mode, we still need to build + # mk2rbc/rbcrun/AndroidProducts.mk.list, so run a get_build_var command to do + # that in each folder. + if args.quick: + commands = [] + for folder in [dirs.out_baseline, dirs.out_product, dirs.out_board]: + commands.append(run_jailed_command([ + 'build/soong/soong_ui.bash', + '--dumpvar-mode', + 'TARGET_PRODUCT' + ], folder)) + for success in await asyncio.gather(*commands): + if not success: + sys.exit('Failed to setup output directories') + + with open(os.path.join(dirs.results, 'index.html'), 'w') as f: + f.write(f''' + <body> + <h2>RBC Product/Board conversion status</h2> + Generated on {datetime.date.today()} for branch {get_branch()} + <table> + <tr> + <th>#</th> + <th>product</th> + <th>baseline</th> + <th>RBC product config</th> + <th>RBC board config</th> + </tr>\n''') + f.flush() + + all_results = [] + start_time = time.time() + print(f'{"Current product":31.31} | {"Time Elapsed":>16} | {"Per each":>8} | {"ETA":>16} | Status') + print('-' * 91) + for i, product in enumerate(products): + if i > 0: + elapsed_time = time.time() - start_time + time_per_product = elapsed_time / i + eta = time_per_product * (len(products) - i) + elapsed_time_str = str(datetime.timedelta(seconds=int(elapsed_time))) + time_per_product_str = str(datetime.timedelta(seconds=int(time_per_product))) + eta_str = str(datetime.timedelta(seconds=int(eta))) + print(f'{f"{i+1}/{len(products)} {product}":31.31} | {elapsed_time_str:>16} | {time_per_product_str:>8} | {eta_str:>16} | ', end='', flush=True) + else: + print(f'{f"{i+1}/{len(products)} {product}":31.31} | {"":>16} | {"":>8} | {"":>16} | ', end='', flush=True) + + if not args.quick: + result = await test_one_product(product, dirs) + else: + result = await test_one_product_quick(product, dirs) + + all_results.append(result) + + if result.success(): + print('Success') + else: + print('Failure') + + f.write(generate_html_row(i+1, product, result)) + f.flush() + + baseline_successes = len([x for x in all_results if x.baseline_success]) + product_successes = len([x for x in all_results if x.product_success and not x.product_has_diffs]) + board_successes = len([x for x in all_results if x.board_success and not x.board_has_diffs]) + f.write(f''' + <tr> + <td></td> + <td># Successful</td> + <td>{baseline_successes}</td> + <td>{product_successes}</td> + <td>{board_successes}</td> + </tr> + <tr> + <td></td> + <td># Failed</td> + <td>N/A</td> + <td>{baseline_successes - product_successes}</td> + <td>{baseline_successes - board_successes}</td> + </tr> + </table> + Finished running successfully. + </body>\n''') + + print('Success!') + print('file://'+os.path.abspath(os.path.join(dirs.results, 'index.html'))) + + for result in all_results: + if result.baseline_success and not result.success(): + print('There were one or more failing products. See the html report for details.') + sys.exit(1) + +if __name__ == '__main__': + asyncio.run(main()) |