Add rbc dashboard script am: c59332e477 am: 20d696c5f5 am: d12b9299da am: e0acede611

Original change: https://android-review.googlesource.com/c/platform/build/bazel/+/2065467

Change-Id: Ia65ace10e17b4ca1e0a59181f74ee9ffeb956e04
Signed-off-by: Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>

1 files changed, 467 insertions, 0 deletions

diff --git a/ci/rbc_dashboard.py b/ci/rbc_dashboard.py
new file mode 100755
index 00000000..2e3ef1b9
--- /dev/null
+++ b/ci/rbc_dashboard.py
@@ -0,0 +1,467 @@
+#!/usr/bin/env python3
+"""Generates a dashboard for the current RBC product/board config conversion status."""
+# pylint: disable=line-too-long
+
+import argparse
+import asyncio
+import dataclasses
+import datetime
+import os
+import re
+import shutil
+import socket
+import subprocess
+import sys
+import time
+from typing import List, Tuple
+import xml.etree.ElementTree as ET
+
+_PRODUCT_REGEX = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)(?:-(user|userdebug|eng))?')
+
+
+@dataclasses.dataclass(frozen=True)
+class Product:
+  """Represents a TARGET_PRODUCT and TARGET_BUILD_VARIANT."""
+  product: str
+  variant: str
+
+  def __post_init__(self):
+    if not _PRODUCT_REGEX.match(str(self)):
+      raise ValueError(f'Invalid product name: {self}')
+
+  def __str__(self):
+    return self.product + '-' + self.variant
+
+
+@dataclasses.dataclass(frozen=True)
+class ProductResult:
+  baseline_success: bool
+  product_success: bool
+  board_success: bool
+  product_has_diffs: bool
+  board_has_diffs: bool
+
+  def success(self) -> bool:
+    return not self.baseline_success or (
+        self.product_success and self.board_success
+        and not self.product_has_diffs and not self.board_has_diffs)
+
+
+@dataclasses.dataclass(frozen=True)
+class Directories:
+  out: str
+  out_baseline: str
+  out_product: str
+  out_board: str
+  results: str
+
+
+def get_top() -> str:
+  path = '.'
+  while not os.path.isfile(os.path.join(path, 'build/soong/soong_ui.bash')):
+    if os.path.abspath(path) == '/':
+      sys.exit('Could not find android source tree root.')
+    path = os.path.join(path, '..')
+  return os.path.abspath(path)
+
+
+def get_build_var(variable, product: Product) -> str:
+  """Returns the result of the shell command get_build_var."""
+  env = {
+      **os.environ,
+      'TARGET_PRODUCT': product.product,
+      'TARGET_BUILD_VARIANT': product.variant,
+  }
+  return subprocess.run([
+      'build/soong/soong_ui.bash',
+      '--dumpvar-mode',
+      variable
+  ], check=True, capture_output=True, env=env, text=True).stdout.strip()
+
+
+async def run_jailed_command(args: List[str], out_dir: str, env=None) -> bool:
+  """Runs a command, saves its output to out_dir/build.log, and returns if it succeeded."""
+  with open(os.path.join(out_dir, 'build.log'), 'wb') as f:
+    result = await asyncio.create_subprocess_exec(
+        'prebuilts/build-tools/linux-x86/bin/nsjail',
+        '-q',
+        '--cwd',
+        os.getcwd(),
+        '-e',
+        '-B',
+        '/',
+        '-B',
+        f'{os.path.abspath(out_dir)}:{os.path.abspath("out")}',
+        '--time_limit',
+        '0',
+        '--skip_setsid',
+        '--keep_caps',
+        '--disable_clone_newcgroup',
+        '--disable_clone_newnet',
+        '--rlimit_as',
+        'soft',
+        '--rlimit_core',
+        'soft',
+        '--rlimit_cpu',
+        'soft',
+        '--rlimit_fsize',
+        'soft',
+        '--rlimit_nofile',
+        'soft',
+        '--proc_rw',
+        '--hostname',
+        socket.gethostname(),
+        '--',
+        *args, stdout=f, stderr=subprocess.STDOUT, env=env)
+    return await result.wait() == 0
+
+
+async def run_build(flags: List[str], out_dir: str) -> bool:
+  return await run_jailed_command([
+      'build/soong/soong_ui.bash',
+      '--make-mode',
+      *flags,
+      '--skip-ninja',
+      'nothing'
+  ], out_dir)
+
+
+async def run_config(product: Product, rbc_product: bool, rbc_board: bool, out_dir: str) -> bool:
+  """Runs config.mk and saves results to out/rbc_variable_dump.txt."""
+  env = {
+      'OUT_DIR': 'out',
+      'TMPDIR': 'tmp',
+      'BUILD_DATETIME_FILE': 'out/build_date.txt',
+      'CALLED_FROM_SETUP': 'true',
+      'TARGET_PRODUCT': product.product,
+      'TARGET_BUILD_VARIANT': product.variant,
+      'RBC_PRODUCT_CONFIG': 'true' if rbc_product else '',
+      'RBC_BOARD_CONFIG': 'true' if rbc_board else '',
+      'RBC_DUMP_CONFIG_FILE': 'out/rbc_variable_dump.txt',
+  }
+  return await run_jailed_command([
+      'prebuilts/build-tools/linux-x86/bin/ckati',
+      '-f',
+      'build/make/core/config.mk'
+  ], out_dir, env=env)
+
+
+async def has_diffs(success: bool, file_pairs: List[Tuple[str]], results_folder: str) -> bool:
+  """Returns true if the two out folders provided have differing ninja files."""
+  if not success:
+    return False
+  results = []
+  for pair in file_pairs:
+    name = 'soong_build.ninja' if pair[0].endswith('soong/build.ninja') else os.path.basename(pair[0])
+    with open(os.path.join(results_folder, name)+'.diff', 'wb') as f:
+      results.append((await asyncio.create_subprocess_exec(
+          'diff',
+          pair[0],
+          pair[1],
+          stdout=f, stderr=subprocess.STDOUT)).wait())
+
+  for return_code in await asyncio.gather(*results):
+    if return_code != 0:
+      return True
+  return False
+
+
+def generate_html_row(num: int, product: Product, results: ProductResult):
+  def generate_status_cell(success: bool, diffs: bool) -> str:
+    message = 'Success'
+    if diffs:
+      message = 'Results differed'
+    if not success:
+      message = 'Build failed'
+    return f'<td style="background-color: {"lightgreen" if success and not diffs else "salmon"}">{message}</td>'
+
+  return f'''
+  <tr>
+    <td>{num}</td>
+    <td>{product if results.success() and results.baseline_success else f'<a href="{product}/">{product}</a>'}</td>
+    {generate_status_cell(results.baseline_success, False)}
+    {generate_status_cell(results.product_success, results.product_has_diffs)}
+    {generate_status_cell(results.board_success, results.board_has_diffs)}
+  </tr>
+  '''
+
+
+def get_branch() -> str:
+  try:
+    tree = ET.parse('.repo/manifests/default.xml')
+    default_tag = tree.getroot().find('default')
+    return default_tag.get('remote') + '/' + default_tag.get('revision')
+  except Exception as e:  # pylint: disable=broad-except
+    print(str(e), file=sys.stderr)
+    return 'Unknown'
+
+
+def cleanup_empty_files(path):
+  if os.path.isfile(path):
+    if os.path.getsize(path) == 0:
+      os.remove(path)
+  elif os.path.isdir(path):
+    for subfile in os.listdir(path):
+      cleanup_empty_files(os.path.join(path, subfile))
+    if not os.listdir(path):
+      os.rmdir(path)
+
+
+async def test_one_product(product: Product, dirs: Directories) -> ProductResult:
+  """Runs the builds and tests for differences for a single product."""
+  baseline_success, product_success, board_success = await asyncio.gather(
+      run_build([
+          f'TARGET_PRODUCT={product.product}',
+          f'TARGET_BUILD_VARIANT={product.variant}',
+      ], dirs.out_baseline),
+      run_build([
+          f'TARGET_PRODUCT={product.product}',
+          f'TARGET_BUILD_VARIANT={product.variant}',
+          'RBC_PRODUCT_CONFIG=1',
+      ], dirs.out_product),
+      run_build([
+          f'TARGET_PRODUCT={product.product}',
+          f'TARGET_BUILD_VARIANT={product.variant}',
+          'RBC_BOARD_CONFIG=1',
+      ], dirs.out_board),
+  )
+
+  product_dashboard_folder = os.path.join(dirs.results, str(product))
+  os.mkdir(product_dashboard_folder)
+  os.mkdir(product_dashboard_folder+'/baseline')
+  os.mkdir(product_dashboard_folder+'/product')
+  os.mkdir(product_dashboard_folder+'/board')
+
+  if not baseline_success:
+    shutil.copy2(os.path.join(dirs.out_baseline, 'build.log'),
+                 f'{product_dashboard_folder}/baseline/build.log')
+  if not product_success:
+    shutil.copy2(os.path.join(dirs.out_product, 'build.log'),
+                 f'{product_dashboard_folder}/product/build.log')
+  if not board_success:
+    shutil.copy2(os.path.join(dirs.out_board, 'build.log'),
+                 f'{product_dashboard_folder}/board/build.log')
+
+  files = [f'build-{product.product}.ninja', f'build-{product.product}-package.ninja', 'soong/build.ninja']
+  product_files = [(os.path.join(dirs.out_baseline, x), os.path.join(dirs.out_product, x)) for x in files]
+  board_files = [(os.path.join(dirs.out_baseline, x), os.path.join(dirs.out_board, x)) for x in files]
+  product_has_diffs, board_has_diffs = await asyncio.gather(
+      has_diffs(baseline_success and product_success, product_files, product_dashboard_folder+'/product'),
+      has_diffs(baseline_success and board_success, board_files, product_dashboard_folder+'/board'))
+
+  # delete files that contain the product name in them to save space,
+  # otherwise the ninja files end up filling up the whole harddrive
+  for out_folder in [dirs.out_baseline, dirs.out_product, dirs.out_board]:
+    for subfolder in ['', 'soong']:
+      folder = os.path.join(out_folder, subfolder)
+      for file in os.listdir(folder):
+        if os.path.isfile(os.path.join(folder, file)) and product.product in file:
+          os.remove(os.path.join(folder, file))
+
+  cleanup_empty_files(product_dashboard_folder)
+
+  return ProductResult(baseline_success, product_success, board_success, product_has_diffs, board_has_diffs)
+
+
+async def test_one_product_quick(product: Product, dirs: Directories) -> ProductResult:
+  """Runs the builds and tests for differences for a single product."""
+  baseline_success, product_success, board_success = await asyncio.gather(
+      run_config(
+          product,
+          False,
+          False,
+          dirs.out_baseline),
+      run_config(
+          product,
+          True,
+          False,
+          dirs.out_product),
+      run_config(
+          product,
+          False,
+          True,
+          dirs.out_board),
+  )
+
+  product_dashboard_folder = os.path.join(dirs.results, str(product))
+  os.mkdir(product_dashboard_folder)
+  os.mkdir(product_dashboard_folder+'/baseline')
+  os.mkdir(product_dashboard_folder+'/product')
+  os.mkdir(product_dashboard_folder+'/board')
+
+  if not baseline_success:
+    shutil.copy2(os.path.join(dirs.out_baseline, 'build.log'),
+                 f'{product_dashboard_folder}/baseline/build.log')
+  if not product_success:
+    shutil.copy2(os.path.join(dirs.out_product, 'build.log'),
+                 f'{product_dashboard_folder}/product/build.log')
+  if not board_success:
+    shutil.copy2(os.path.join(dirs.out_board, 'build.log'),
+                 f'{product_dashboard_folder}/board/build.log')
+
+  files = ['rbc_variable_dump.txt']
+  product_files = [(os.path.join(dirs.out_baseline, x), os.path.join(dirs.out_product, x)) for x in files]
+  board_files = [(os.path.join(dirs.out_baseline, x), os.path.join(dirs.out_board, x)) for x in files]
+  product_has_diffs, board_has_diffs = await asyncio.gather(
+      has_diffs(baseline_success and product_success, product_files, product_dashboard_folder+'/product'),
+      has_diffs(baseline_success and board_success, board_files, product_dashboard_folder+'/board'))
+
+  cleanup_empty_files(product_dashboard_folder)
+
+  return ProductResult(baseline_success, product_success, board_success, product_has_diffs, board_has_diffs)
+
+
+async def main():
+  parser = argparse.ArgumentParser(
+      description='Generates a dashboard of the starlark product configuration conversion.')
+  parser.add_argument('products', nargs='*',
+                      help='list of products to test. If not given, all '
+                      + 'products will be tested. '
+                      + 'Example: aosp_arm64-userdebug')
+  parser.add_argument('--quick', action='store_true',
+                      help='Run a quick test. This will only run config.mk and '
+                      + 'diff the make variables at the end of it, instead of '
+                      + 'diffing the full ninja files.')
+  parser.add_argument('--exclude', nargs='+', default=[],
+                      help='Exclude these producs from the build. Useful if not '
+                      + 'supplying a list of products manually.')
+  parser.add_argument('--results-directory',
+                      help='Directory to store results in. Defaults to $(OUT_DIR)/rbc_dashboard. '
+                      + 'Warning: will be cleared!')
+  args = parser.parse_args()
+
+  if args.results_directory:
+    args.results_directory = os.path.abspath(args.results_directory)
+
+  os.chdir(get_top())
+
+  def str_to_product(p: str) -> Product:
+    match = _PRODUCT_REGEX.fullmatch(p)
+    if not match:
+      sys.exit(f'Invalid product name: {p}. Example: aosp_arm64-userdebug')
+    return Product(match.group(1), match.group(2) if match.group(2) else 'userdebug')
+
+  products = [str_to_product(p) for p in args.products]
+
+  if not products:
+    products = list(map(lambda x: Product(x, 'userdebug'), get_build_var(
+        'all_named_products', Product('aosp_arm64', 'userdebug')).split()))
+
+  excluded = [str_to_product(p) for p in args.exclude]
+  products = [p for p in products if p not in excluded]
+
+  for i, product in enumerate(products):
+    for j, product2 in enumerate(products):
+      if i != j and product.product == product2.product:
+        sys.exit(f'Product {product.product} cannot be repeated.')
+
+  out_dir = get_build_var('OUT_DIR', Product('aosp_arm64', 'userdebug'))
+
+  dirs = Directories(
+      out=out_dir,
+      out_baseline=os.path.join(out_dir, 'rbc_out_baseline'),
+      out_product=os.path.join(out_dir, 'rbc_out_product'),
+      out_board=os.path.join(out_dir, 'rbc_out_board'),
+      results=args.results_directory if args.results_directory else os.path.join(out_dir, 'rbc_dashboard'))
+
+  for folder in [dirs.out_baseline, dirs.out_product, dirs.out_board, dirs.results]:
+    # delete and recreate the out directories. You can't reuse them for
+    # a particular product, because after we delete some product-specific
+    # files inside the out dir to save space, the build will fail if you
+    # try to build the same product again.
+    shutil.rmtree(folder, ignore_errors=True)
+    os.makedirs(folder)
+
+  # When running in quick mode, we still need to build
+  # mk2rbc/rbcrun/AndroidProducts.mk.list, so run a get_build_var command to do
+  # that in each folder.
+  if args.quick:
+    commands = []
+    for folder in [dirs.out_baseline, dirs.out_product, dirs.out_board]:
+      commands.append(run_jailed_command([
+          'build/soong/soong_ui.bash',
+          '--dumpvar-mode',
+          'TARGET_PRODUCT'
+      ], folder))
+    for success in await asyncio.gather(*commands):
+      if not success:
+        sys.exit('Failed to setup output directories')
+
+  with open(os.path.join(dirs.results, 'index.html'), 'w') as f:
+    f.write(f'''
+      <body>
+        <h2>RBC Product/Board conversion status</h2>
+        Generated on {datetime.date.today()} for branch {get_branch()}
+        <table>
+          <tr>
+            <th>#</th>
+            <th>product</th>
+            <th>baseline</th>
+            <th>RBC product config</th>
+            <th>RBC board config</th>
+          </tr>\n''')
+    f.flush()
+
+    all_results = []
+    start_time = time.time()
+    print(f'{"Current product":31.31} | {"Time Elapsed":>16} | {"Per each":>8} | {"ETA":>16} | Status')
+    print('-' * 91)
+    for i, product in enumerate(products):
+      if i > 0:
+        elapsed_time = time.time() - start_time
+        time_per_product = elapsed_time / i
+        eta = time_per_product * (len(products) - i)
+        elapsed_time_str = str(datetime.timedelta(seconds=int(elapsed_time)))
+        time_per_product_str = str(datetime.timedelta(seconds=int(time_per_product)))
+        eta_str = str(datetime.timedelta(seconds=int(eta)))
+        print(f'{f"{i+1}/{len(products)} {product}":31.31} | {elapsed_time_str:>16} | {time_per_product_str:>8} | {eta_str:>16} | ', end='', flush=True)
+      else:
+        print(f'{f"{i+1}/{len(products)} {product}":31.31} | {"":>16} | {"":>8} | {"":>16} | ', end='', flush=True)
+
+      if not args.quick:
+        result = await test_one_product(product, dirs)
+      else:
+        result = await test_one_product_quick(product, dirs)
+
+      all_results.append(result)
+
+      if result.success():
+        print('Success')
+      else:
+        print('Failure')
+
+      f.write(generate_html_row(i+1, product, result))
+      f.flush()
+
+    baseline_successes = len([x for x in all_results if x.baseline_success])
+    product_successes = len([x for x in all_results if x.product_success and not x.product_has_diffs])
+    board_successes = len([x for x in all_results if x.board_success and not x.board_has_diffs])
+    f.write(f'''
+          <tr>
+            <td></td>
+            <td># Successful</td>
+            <td>{baseline_successes}</td>
+            <td>{product_successes}</td>
+            <td>{board_successes}</td>
+          </tr>
+          <tr>
+            <td></td>
+            <td># Failed</td>
+            <td>N/A</td>
+            <td>{baseline_successes - product_successes}</td>
+            <td>{baseline_successes - board_successes}</td>
+          </tr>
+        </table>
+        Finished running successfully.
+      </body>\n''')
+
+  print('Success!')
+  print('file://'+os.path.abspath(os.path.join(dirs.results, 'index.html')))
+
+  for result in all_results:
+    if result.baseline_success and not result.success():
+      print('There were one or more failing products. See the html report for details.')
+      sys.exit(1)
+
+if __name__ == '__main__':
+  asyncio.run(main())