1 files changed, 573 insertions, 0 deletions
diff --git a/llvm_tools/update_chromeos_llvm_hash.py b/llvm_tools/update_chromeos_llvm_hash.py
new file mode 100755
index 00000000..e28fe690
--- /dev/null
+++ b/llvm_tools/update_chromeos_llvm_hash.py
@@ -0,0 +1,573 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Updates the LLVM hash and uprevs the build of the specified
+
+packages.
+
+For each package, a temporary repo is created and the changes are uploaded
+for review.
+"""
+
+from __future__ import print_function
+from datetime import datetime
+from enum import Enum
+
+import argparse
+import os
+import re
+import subprocess
+
+from failure_modes import FailureModes
+import chroot
+import get_llvm_hash
+import git
+import llvm_patch_management
+
+
+# Specify which LLVM hash to update
+class LLVMVariant(Enum):
+  """Represent the LLVM hash in an ebuild file to update."""
+
+  current = 'LLVM_HASH'
+  next = 'LLVM_NEXT_HASH'
+
+
+# If set to `True`, then the contents of `stdout` after executing a command will
+# be displayed to the terminal.
+verbose = False
+
+
+def GetCommandLineArgs():
+  """Parses the command line for the optional command line arguments.
+
+  Returns:
+    The log level to use when retrieving the LLVM hash or google3 LLVM version,
+    the chroot path to use for executing chroot commands,
+    a list of a package or packages to update their LLVM next hash,
+    and the LLVM version to use when retrieving the LLVM hash.
+  """
+
+  # Default path to the chroot if a path is not specified.
+  cros_root = os.path.expanduser('~')
+  cros_root = os.path.join(cros_root, 'chromiumos')
+
+  # Create parser and add optional command-line arguments.
+  parser = argparse.ArgumentParser(
+      description="Updates the build's hash for llvm-next.")
+
+  # Add argument for a specific chroot path.
+  parser.add_argument(
+      '--chroot_path',
+      default=cros_root,
+      help='the path to the chroot (default: %(default)s)')
+
+  # Add argument for specific builds to uprev and update their llvm-next hash.
+  parser.add_argument(
+      '--update_packages',
+      default=['sys-devel/llvm'],
+      required=False,
+      nargs='+',
+      help='the ebuilds to update their hash for llvm-next ' \
+          '(default: %(default)s)')
+
+  # Add argument for whether to display command contents to `stdout`.
+  parser.add_argument(
+      '--verbose',
+      action='store_true',
+      help='display contents of a command to the terminal '
+      '(default: %(default)s)')
+
+  # Add argument for the LLVM hash to update
+  parser.add_argument(
+      '--is_llvm_next',
+      action='store_true',
+      help='which llvm hash to update. If specified, update LLVM_NEXT_HASH. '
+      'Otherwise, update LLVM_HASH')
+
+  # Add argument for the LLVM version to use.
+  parser.add_argument(
+      '--llvm_version',
+      type=get_llvm_hash.is_svn_option,
+      required=True,
+      help='which git hash to use. Either a svn revision, or one '
+      'of %s' % sorted(get_llvm_hash.KNOWN_HASH_SOURCES))
+
+  # Add argument for the mode of the patch management when handling patches.
+  parser.add_argument(
+      '--failure_mode',
+      default=FailureModes.FAIL.value,
+      choices=[FailureModes.FAIL.value, FailureModes.CONTINUE.value,
+               FailureModes.DISABLE_PATCHES.value,
+               FailureModes.REMOVE_PATCHES.value],
+      help='the mode of the patch manager when handling failed patches ' \
+          '(default: %(default)s)')
+
+  # Add argument for the patch metadata file.
+  parser.add_argument(
+      '--patch_metadata_file',
+      default='PATCHES.json',
+      help='the .json file that has all the patches and their '
+      'metadata if applicable (default: PATCHES.json inside $FILESDIR)')
+
+  # Parse the command line.
+  args_output = parser.parse_args()
+
+  # FIXME: We shouldn't be using globals here, but until we fix it, make pylint
+  # stop complaining about it.
+  # pylint: disable=global-statement
+  global verbose
+
+  verbose = args_output.verbose
+
+  return args_output
+
+
+def GetEbuildPathsFromSymLinkPaths(symlinks):
+  """Reads the symlink(s) to get the ebuild path(s) to the package(s).
+
+  Args:
+    symlinks: A list of absolute path symlink/symlinks that point
+    to the package's ebuild.
+
+  Returns:
+    A dictionary where the key is the absolute path of the symlink and the value
+    is the absolute path to the ebuild that was read from the symlink.
+
+  Raises:
+    ValueError: Invalid symlink(s) were provided.
+  """
+
+  # A dictionary that holds:
+  #   key: absolute symlink path
+  #   value: absolute ebuild path
+  resolved_paths = {}
+
+  # Iterate through each symlink.
+  #
+  # For each symlink, check that it is a valid symlink,
+  # and then construct the ebuild path, and
+  # then add the ebuild path to the dict.
+  for cur_symlink in symlinks:
+    if not os.path.islink(cur_symlink):
+      raise ValueError('Invalid symlink provided: %s' % cur_symlink)
+
+    # Construct the absolute path to the ebuild.
+    ebuild_path = os.path.realpath(cur_symlink)
+
+    if cur_symlink not in resolved_paths:
+      resolved_paths[cur_symlink] = ebuild_path
+
+  return resolved_paths
+
+
+def UpdateEbuildLLVMHash(ebuild_path, llvm_variant, git_hash, svn_version):
+  """Updates the LLVM hash in the ebuild.
+
+  The build changes are staged for commit in the temporary repo.
+
+  Args:
+    ebuild_path: The absolute path to the ebuild.
+    llvm_variant: Which LLVM hash to update.
+    git_hash: The new git hash.
+    svn_version: The SVN-style revision number of git_hash.
+
+  Raises:
+    ValueError: Invalid ebuild path provided or failed to stage the commit
+    of the changes or failed to update the LLVM hash.
+  """
+
+  # Iterate through each ebuild.
+  #
+  # For each ebuild, read the file in
+  # advance and then create a temporary file
+  # that gets updated with the new LLVM hash
+  # and revision number and then the ebuild file
+  # gets updated to the temporary file.
+
+  if not os.path.isfile(ebuild_path):
+    raise ValueError('Invalid ebuild path provided: %s' % ebuild_path)
+
+  temp_ebuild_file = '%s.temp' % ebuild_path
+
+  with open(ebuild_path) as ebuild_file:
+    # write updates to a temporary file in case of interrupts
+    with open(temp_ebuild_file, 'w') as temp_file:
+      for cur_line in ReplaceLLVMHash(ebuild_file, llvm_variant, git_hash,
+                                      svn_version):
+        temp_file.write(cur_line)
+
+  os.rename(temp_ebuild_file, ebuild_path)
+
+  # Get the path to the parent directory.
+  parent_dir = os.path.dirname(ebuild_path)
+
+  # Stage the changes.
+  subprocess.check_output(['git', '-C', parent_dir, 'add', ebuild_path])
+
+
+def ReplaceLLVMHash(ebuild_lines, llvm_variant, git_hash, svn_version):
+  """Updates the LLVM git hash.
+
+  Args:
+    ebuild_lines: The contents of the ebuild file.
+    llvm_variant: The LLVM hash to update.
+    git_hash: The new git hash.
+    svn_version: The SVN-style revision number of git_hash.
+  """
+  is_updated = False
+  llvm_regex = re.compile('^' + re.escape(llvm_variant.value) +
+                          '=\"[a-z0-9]+\"')
+  for cur_line in ebuild_lines:
+    if not is_updated and llvm_regex.search(cur_line):
+      # Update the git hash and revision number.
+      cur_line = '%s=\"%s\" # r%d\n' % (llvm_variant.value, git_hash,
+                                        svn_version)
+
+      is_updated = True
+
+    yield cur_line
+
+  if not is_updated:
+    raise ValueError('Failed to update %s' % llvm_variant.value)
+
+
+def UprevEbuildSymlink(symlink):
+  """Uprevs the symlink's revision number.
+
+  Increases the revision number by 1 and stages the change in
+  the temporary repo.
+
+  Args:
+    symlink: The absolute path of an ebuild symlink.
+
+  Raises:
+    ValueError: Failed to uprev the symlink or failed to stage the changes.
+  """
+
+  if not os.path.islink(symlink):
+    raise ValueError('Invalid symlink provided: %s' % symlink)
+
+  new_symlink, is_changed = re.subn(
+      r'r([0-9]+).ebuild',
+      lambda match: 'r%s.ebuild' % str(int(match.group(1)) + 1),
+      symlink,
+      count=1)
+
+  if not is_changed:
+    raise ValueError('Failed to uprev the symlink.')
+
+  # rename the symlink
+  subprocess.check_output(
+      ['git', '-C',
+       os.path.dirname(symlink), 'mv', symlink, new_symlink])
+
+
+def UprevEbuildToVersion(symlink, svn_version):
+  """Uprevs the ebuild's revision number.
+
+  Increases the revision number by 1 and stages the change in
+  the temporary repo.
+
+  Args:
+    symlink: The absolute path of an ebuild symlink.
+    svn_version: The SVN-style revision number of git_hash.
+
+  Raises:
+    ValueError: Failed to uprev the ebuild or failed to stage the changes.
+  """
+
+  if not os.path.islink(symlink):
+    raise ValueError('Invalid symlink provided: %s' % symlink)
+
+  ebuild = os.path.realpath(symlink)
+  # llvm
+  package = os.path.basename(os.path.dirname(symlink))
+  if not package:
+    raise ValueError('Tried to uprev an unknown package')
+  # llvm
+  if package == 'llvm':
+    new_ebuild, is_changed = re.subn(
+        r'pre([0-9]+)_p([0-9]+)',
+        'pre%s_p%s' % (svn_version, \
+            datetime.today().strftime('%Y%m%d')),
+        ebuild,
+        count=1)
+  # any other package
+  else:
+    new_ebuild, is_changed = re.subn(
+        r'pre([0-9]+)', 'pre%s' % svn_version, ebuild, count=1)
+
+  if not is_changed:  # failed to increment the revision number
+    raise ValueError('Failed to uprev the ebuild.')
+
+  symlink_dir = os.path.dirname(symlink)
+
+  # Rename the ebuild
+  subprocess.check_output(['git', '-C', symlink_dir, 'mv', ebuild, new_ebuild])
+
+  # Create a symlink of the renamed ebuild
+  new_symlink = new_ebuild[:-len('.ebuild')] + '-r1.ebuild'
+  subprocess.check_output(['ln', '-s', '-r', new_ebuild, new_symlink])
+
+  if not os.path.islink(new_symlink):
+    raise ValueError('Invalid symlink name: %s' % new_ebuild[:-len('.ebuild')])
+
+  subprocess.check_output(['git', '-C', symlink_dir, 'add', new_symlink])
+
+  # Remove the old symlink
+  subprocess.check_output(['git', '-C', symlink_dir, 'rm', symlink])
+
+
+def CreatePathDictionaryFromPackages(chroot_path, update_packages):
+  """Creates a symlink and ebuild path pair dictionary from the packages.
+
+  Args:
+    chroot_path: The absolute path to the chroot.
+    update_packages: The filtered packages to be updated.
+
+  Returns:
+    A dictionary where the key is the absolute path to the symlink
+    of the package and the value is the absolute path to the ebuild of
+    the package.
+  """
+
+  # Construct a list containing the chroot file paths of the package(s).
+  chroot_file_paths = chroot.GetChrootEbuildPaths(chroot_path, update_packages)
+
+  # Construct a list containing the symlink(s) of the package(s).
+  symlink_file_paths = chroot.ConvertChrootPathsToAbsolutePaths(
+      chroot_path, chroot_file_paths)
+
+  # Create a dictionary where the key is the absolute path of the symlink to
+  # the package and the value is the absolute path to the ebuild of the package.
+  return GetEbuildPathsFromSymLinkPaths(symlink_file_paths)
+
+
+def RemovePatchesFromFilesDir(patches):
+  """Removes the patches from $FILESDIR of a package.
+
+  Args:
+    patches: A list of absolute pathes of patches to remove
+
+  Raises:
+    ValueError: Failed to remove a patch in $FILESDIR.
+  """
+
+  for patch in patches:
+    subprocess.check_output(
+        ['git', '-C', os.path.dirname(patch), 'rm', '-f', patch])
+
+
+def StagePatchMetadataFileForCommit(patch_metadata_file_path):
+  """Stages the updated patch metadata file for commit.
+
+  Args:
+    patch_metadata_file_path: The absolute path to the patch metadata file.
+
+  Raises:
+    ValueError: Failed to stage the patch metadata file for commit or invalid
+    patch metadata file.
+  """
+
+  if not os.path.isfile(patch_metadata_file_path):
+    raise ValueError(
+        'Invalid patch metadata file provided: %s' % patch_metadata_file_path)
+
+  # Cmd to stage the patch metadata file for commit.
+  subprocess.check_output([
+      'git', '-C',
+      os.path.dirname(patch_metadata_file_path), 'add', patch_metadata_file_path
+  ])
+
+
+def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages):
+  """Stages the patch results of the packages to the commit message.
+
+  Args:
+    package_info_dict: A dictionary where the key is the package name and the
+    value is a dictionary that contains information about the patches of the
+    package (key).
+    commit_messages: The commit message that has the updated ebuilds and
+    upreving information.
+  """
+
+  # For each package, check if any patches for that package have
+  # changed, if so, add which patches have changed to the commit
+  # message.
+  for package_name, patch_info_dict in package_info_dict.items():
+    if patch_info_dict['disabled_patches'] or \
+        patch_info_dict['removed_patches'] or \
+        patch_info_dict['modified_metadata']:
+      cur_package_header = '\nFor the package %s:' % package_name
+      commit_messages.append(cur_package_header)
+
+    # Add to the commit message that the patch metadata file was modified.
+    if patch_info_dict['modified_metadata']:
+      patch_metadata_path = patch_info_dict['modified_metadata']
+      commit_messages.append('The patch metadata file %s was modified' %
+                             os.path.basename(patch_metadata_path))
+
+      StagePatchMetadataFileForCommit(patch_metadata_path)
+
+    # Add each disabled patch to the commit message.
+    if patch_info_dict['disabled_patches']:
+      commit_messages.append('The following patches were disabled:')
+
+      for patch_path in patch_info_dict['disabled_patches']:
+        commit_messages.append(os.path.basename(patch_path))
+
+    # Add each removed patch to the commit message.
+    if patch_info_dict['removed_patches']:
+      commit_messages.append('The following patches were removed:')
+
+      for patch_path in patch_info_dict['removed_patches']:
+        commit_messages.append(os.path.basename(patch_path))
+
+      RemovePatchesFromFilesDir(patch_info_dict['removed_patches'])
+
+  return commit_messages
+
+
+def UpdatePackages(packages, llvm_variant, git_hash, svn_version, chroot_path,
+                   patch_metadata_file, mode, git_hash_source,
+                   extra_commit_msg):
+  """Updates an LLVM hash and uprevs the ebuild of the packages.
+
+  A temporary repo is created for the changes. The changes are
+  then uploaded for review.
+
+  Args:
+    packages: A list of all the packages that are going to be updated.
+    llvm_variant: The LLVM hash to update.
+    git_hash: The new git hash.
+    svn_version: The SVN-style revision number of git_hash.
+    chroot_path: The absolute path to the chroot.
+    patch_metadata_file: The name of the .json file in '$FILESDIR/' that has
+    the patches and its metadata.
+    mode: The mode of the patch manager when handling an applicable patch
+    that failed to apply.
+      Ex: 'FailureModes.FAIL'
+    git_hash_source: The source of which git hash to use based off of.
+      Ex: 'google3', 'tot', or <version> such as 365123
+    extra_commit_msg: extra test to append to the commit message.
+
+  Returns:
+    A nametuple that has two (key, value) pairs, where the first pair is the
+    Gerrit commit URL and the second pair is the change list number.
+  """
+
+  # Determines whether to print the result of each executed command.
+  llvm_patch_management.verbose = verbose
+
+  # Construct a dictionary where the key is the absolute path of the symlink to
+  # the package and the value is the absolute path to the ebuild of the package.
+  paths_dict = CreatePathDictionaryFromPackages(chroot_path, packages)
+
+  repo_path = os.path.dirname(next(iter(paths_dict.values())))
+
+  branch = 'update-' + llvm_variant.value + '-' + git_hash
+
+  git.CreateBranch(repo_path, branch)
+
+  try:
+    commit_message_header = 'llvm'
+    if llvm_variant == LLVMVariant.next:
+      commit_message_header = 'llvm-next'
+    if git_hash_source in get_llvm_hash.KNOWN_HASH_SOURCES:
+      commit_message_header += (
+          '/%s: upgrade to %s (r%d)' % (git_hash_source, git_hash, svn_version))
+    else:
+      commit_message_header += (
+          ': upgrade to %s (r%d)' % (git_hash, svn_version))
+
+    commit_messages = [
+        commit_message_header + '\n',
+        'The following packages have been updated:',
+    ]
+
+    # Holds the list of packages that are updating.
+    packages = []
+
+    # Iterate through the dictionary.
+    #
+    # For each iteration:
+    # 1) Update the ebuild's LLVM hash.
+    # 2) Uprev the ebuild (symlink).
+    # 3) Add the modified package to the commit message.
+    for symlink_path, ebuild_path in paths_dict.items():
+      path_to_ebuild_dir = os.path.dirname(ebuild_path)
+
+      UpdateEbuildLLVMHash(ebuild_path, llvm_variant, git_hash, svn_version)
+
+      if llvm_variant == LLVMVariant.current:
+        UprevEbuildToVersion(symlink_path, svn_version)
+      else:
+        UprevEbuildSymlink(symlink_path)
+
+      cur_dir_name = os.path.basename(path_to_ebuild_dir)
+      parent_dir_name = os.path.basename(os.path.dirname(path_to_ebuild_dir))
+
+      packages.append('%s/%s' % (parent_dir_name, cur_dir_name))
+      commit_messages.append('%s/%s' % (parent_dir_name, cur_dir_name))
+
+    # Handle the patches for each package.
+    package_info_dict = llvm_patch_management.UpdatePackagesPatchMetadataFile(
+        chroot_path, svn_version, patch_metadata_file, packages, mode)
+
+    # Update the commit message if changes were made to a package's patches.
+    commit_messages = StagePackagesPatchResultsForCommit(
+        package_info_dict, commit_messages)
+
+    if extra_commit_msg:
+      commit_messages.append(extra_commit_msg)
+
+    change_list = git.UploadChanges(repo_path, branch, commit_messages)
+
+  finally:
+    git.DeleteBranch(repo_path, branch)
+
+  return change_list
+
+
+def main():
+  """Updates the LLVM next hash for each package.
+
+  Raises:
+    AssertionError: The script was run inside the chroot.
+  """
+
+  chroot.VerifyOutsideChroot()
+
+  args_output = GetCommandLineArgs()
+
+  llvm_variant = LLVMVariant.current
+  if args_output.is_llvm_next:
+    llvm_variant = LLVMVariant.next
+
+  git_hash_source = args_output.llvm_version
+
+  git_hash, svn_version = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption(
+      git_hash_source)
+
+  change_list = UpdatePackages(
+      args_output.update_packages,
+      llvm_variant,
+      git_hash,
+      svn_version,
+      args_output.chroot_path,
+      args_output.patch_metadata_file,
+      FailureModes(args_output.failure_mode),
+      git_hash_source,
+      extra_commit_msg=None)
+
+  print('Successfully updated packages to %s (%d)' % (git_hash, svn_version))
+  print('Gerrit URL: %s' % change_list.url)
+  print('Change list number: %d' % change_list.cl_number)
+
+
+if __name__ == '__main__':
+  main()