path: root/chromium/tools/merge_to_android.py

#!/usr/bin/python
#
# Copyright (C) 2012 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Merge master-chromium to master within the Android tree."""

import logging
import optparse
import os
import re
import shutil
import subprocess
import sys

import merge_common


AUTOGEN_MESSAGE = 'This commit was generated by merge_to_master.py.'
WEBVIEW_PROJECT = 'frameworks/webview'


def _GetAbsPath(project):
  """Returns the full path to a given project (either Chromium or Android)."""
  if project in merge_common.ALL_PROJECTS:
    abs_path = os.path.join(merge_common.REPOSITORY_ROOT, project)
  else:
    abs_path = os.path.join(os.environ['ANDROID_BUILD_TOP'], project)
  if not os.path.exists(abs_path):
    raise merge_common.MergeError('Cannot find path ' + abs_path)
  return abs_path


def _CheckoutSingleProject(project, target_branch):
  """Checks out the tip of the target_branch into a local branch (merge-to-XXX).

  Args:
    project: a Chromium project (., third_party/foo) or frameworks/webview.
    target_branch: name of the target branch (in the goog remote).
  """
  dest_dir = _GetAbsPath(project)
  tracking_branch = 'goog/' + target_branch
  logging.debug('Check out %-45s at %-16s', project, tracking_branch)
  merge_common.GetCommandStdout(['git', 'remote', 'update', 'goog'],
                                cwd=dest_dir)
  merge_common.GetCommandStdout(['git', 'checkout',
                                 '-b', 'merge-to-' + target_branch,
                                 '-t', tracking_branch], cwd=dest_dir)


def _FetchSingleProject(project, remote, remote_ref):
  """Fetches a remote ref for the given project and returns the fetched SHA.

  Args:
    project: a Chromium project (., third_party/foo) or frameworks/webview.
    remote: Git remote name (goog for most projects, history for squashed ones).
    remote_ref: the remote ref to fetch (e.g., refs/archive/chromium-XXX).

  Returns:
    The SHA1 of the FETCH_HEAD.
  """
  dest_dir = _GetAbsPath(project)
  logging.debug('Fetch     %-45s %s:%s', project, remote, remote_ref)
  merge_common.GetCommandStdout(['git', 'fetch', remote, remote_ref],
                                cwd=dest_dir)
  return merge_common.GetCommandStdout(['git', 'rev-parse', 'FETCH_HEAD'],
                                       cwd=dest_dir).strip()


def _MergeSingleProject(project, merge_sha, revision, target_branch, flatten):
  """Merges a single project at a given SHA.

  Args:
    project: a Chromium project (., third_party/foo) or frameworks/webview.
    merge_sha: the SHA to merge.
    revision: Abbrev. commitish in the main Chromium repository.
    target_branch: name of the target branch.
    flatten: True: squash history while merging; False: perform a normal merge.
  """
  dest_dir = _GetAbsPath(project)
  if flatten:
    # Make the previous merges into grafts so we can do a correct merge.
    old_sha = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'],
                                            cwd=dest_dir).strip()
    merge_log = os.path.join(dest_dir, '.merged-revisions')
    if os.path.exists(merge_log):
      shutil.copyfile(merge_log,
                      os.path.join(dest_dir, '.git', 'info', 'grafts'))

  # Early out if there is nothing to merge.
  if not merge_common.GetCommandStdout(['git', 'rev-list', '-1',
                                        'HEAD..' + merge_sha], cwd=dest_dir):
    logging.debug('No new commits to merge in project %s', project)
    return

  logging.debug('Merging project %s (flatten: %s)...', project, flatten)
  merge_cmd = ['git', 'merge', '--no-commit']
  merge_cmd += ['--squash'] if flatten else ['--no-ff']
  merge_cmd += [merge_sha]
  # Merge conflicts cause 'git merge' to return 1, so ignore errors
  merge_common.GetCommandStdout(merge_cmd, cwd=dest_dir, ignore_errors=True)

  if flatten:
    dirs_to_prune = merge_common.PRUNE_WHEN_FLATTENING.get(project, [])
    if dirs_to_prune:
      merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', '-rf'] +
                                    dirs_to_prune, cwd=dest_dir)

  if project in merge_common.ALL_PROJECTS:
    commit_msg = 'Merge from Chromium at DEPS revision %s' % revision
  else:
    commit_msg = 'Merge master-chromium into %s at %s' % (target_branch,
                                                          revision)
  commit_msg += '\n\n' + AUTOGEN_MESSAGE
  merge_common.CheckNoConflictsAndCommitMerge(commit_msg, cwd=dest_dir)

  if flatten:
    # Generate the new grafts file and commit it on top of the merge.
    new_sha = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'],
                                            cwd=dest_dir).strip()
    with open(merge_log, 'a+') as f:
      f.write('%s %s %s\n' % (new_sha, old_sha, merge_sha))
    merge_common.GetCommandStdout(['git', 'add', '.merged-revisions'],
                                  cwd=dest_dir)
    merge_common.GetCommandStdout(
        ['git', 'commit', '-m',
         'Record Chromium merge at DEPS revision %s\n\n%s' %
         (revision, AUTOGEN_MESSAGE)], cwd=dest_dir)


def _IsAncestor(ref1, ref2, cwd):
  """Checks whether ref1 is a ancestor of ref2 in the given Git repo."""
  cmd = ['git', 'merge-base', '--is-ancestor', ref1, ref2]
  ret = subprocess.call(cmd, cwd=cwd)
  if ret == 0:
    return True
  elif ret == 1:
    return False
  else:
    raise merge_common.CommandError(ret, ' '.join(cmd), cwd, 'N/A', 'N/A')


def _MergeChromiumProjects(revision, target_branch, repo_shas=None,
                           force=False):
  """Merges the Chromium projects from master-chromium to target_branch.

  The larger projects' histories are flattened in the process.
  When repo_shas != None, it checks that the SHAs of the projects in the
  archive match exactly the SHAs of the projects in repo.prop.

  Args:
    revision: Abbrev. commitish in the main Chromium repository.
    target_branch: target branch name to merge and push to.
    repo_shas: optional dict. of expected revisions (only for --repo-prop).
    force: True: merge anyways using the SHAs from repo.prop; False: bail out if
                 projects mismatch (archive vs repo.prop).
  """
  # Sync and checkout ToT for all projects (creating the merge-to-XXX branch)
  # and fetch the archive snapshot.
  fetched_shas = {}
  remote_ref = 'refs/archive/chromium-%s' % revision
  for project in merge_common.PROJECTS_WITH_FLAT_HISTORY:
    _CheckoutSingleProject(project, target_branch)
    fetched_shas[project] = _FetchSingleProject(project, 'history', remote_ref)
  for project in merge_common.PROJECTS_WITH_FULL_HISTORY:
    _CheckoutSingleProject(project, target_branch)
    fetched_shas[project] = _FetchSingleProject(project, 'goog', remote_ref)

  if repo_shas:
    project_shas_mismatch = False
    for project, merge_sha in fetched_shas.items():  # the dict can be modified.
      expected_sha = repo_shas.get(project)
      if expected_sha != merge_sha:
        logging.warn('The SHA for project %s specified in the repo.prop (%s) '
                     'and the one in the archive (%s) differ.',
                     project, expected_sha, merge_sha)
        dest_dir = _GetAbsPath(project)
        if expected_sha is None:
          reason = 'cannot find a SHA in the repo.pro for %s' % project
        elif _IsAncestor(merge_sha, expected_sha, cwd=dest_dir):
          reason = 'the SHA in repo.prop is ahead of the SHA in the archive. '
          log_cmd = ['git', 'log', '--oneline', '--graph', '--max-count=10',
                     '%s..%s' % (merge_sha, expected_sha)]
          log_cmd_output = merge_common.GetCommandStdout(log_cmd, cwd=dest_dir)
          reason += 'showing partial log (%s): \n %s' % (' '.join(log_cmd),
                                                         log_cmd_output)
        elif _IsAncestor(expected_sha, merge_sha, cwd=dest_dir):
          reason = 'The SHA is already merged in the archive'
        else:
          reason = 'The project history diverged. Consult your Git historian.'

        project_shas_mismatch = True
        if force:
          logging.debug('Merging the SHA in repo.prop anyways (due to --force)')
          fetched_shas[project] = expected_sha
        else:
          logging.debug('Reason: %s', reason)
    if not force and project_shas_mismatch:
      raise merge_common.MergeError(
          'The revision of some projects in the archive is different from the '
          'one provided in build.prop. See the log for more details. Re-run '
          'with --force to continue.')

  for project in merge_common.PROJECTS_WITH_FLAT_HISTORY:
    _MergeSingleProject(project, fetched_shas[project], revision, target_branch,
                        flatten=True)
  for project in merge_common.PROJECTS_WITH_FULL_HISTORY:
    _MergeSingleProject(project, fetched_shas[project], revision, target_branch,
                        flatten=False)


def _GetNearestUpstreamAbbrevSHA(reference='history/master-chromium'):
  """Returns the abbrev. upstream SHA which closest to the given reference."""
  logging.debug('Getting upstream SHA for %s...', reference)
  merge_common.GetCommandStdout(['git', 'remote', 'update', 'history'])
  upstream_commit = merge_common.Abbrev(merge_common.GetCommandStdout([
      'git', 'merge-base', 'history/upstream-master', reference]))

  # Pedantic check: look for the existence of a merge commit which contains the
  # |upstream_commit| in its message and is its children.
  merge_parents = merge_common.GetCommandStdout([
      'git', 'rev-list', reference, '--grep', upstream_commit, '--merges',
      '--parents', '-1'])
  if upstream_commit not in merge_parents:
    raise merge_common.MergeError(
        'Found upstream commit %s, but the merge child (%s) could not be found '
        'or is not a parent of the upstream SHA')
  logging.debug('Found nearest Chromium revision %s', upstream_commit)
  return upstream_commit


def _MergeWithRepoProp(repo_prop_file, target_branch, force):
  """Performs a merge using a repo.prop file (from Android build waterfall).

  This does NOT merge (unless forced with force=True) the pinned
  revisions in repo.prop, as a repo.prop can snapshot an intermediate state
  (between two automerger cycles). Instead, this looks up the archived snapshot
  (generated by the chromium->master-chromium auto-merger) which is closest to
  the given repo.prop (following the main Chromium project) and merges that one.
  If the projects revisions don't match, it fails with detailed error messages.

  Args:
    repo_prop_file: Path to a downloaded repo.prop file.
    target_branch: name of the target branch to merget to.
    force: ignores the aforementioned check and merged anyways.
  """
  chromium_sha = None
  webview_sha = None
  repo_shas = {}  # 'project/path' -> 'sha'
  with open(repo_prop_file) as prop:
    for line in prop:
      repo, sha = line.split()
      # Translate the Android repo paths into the relative project paths used in
      # merge_common (e.g., platform/external/chromium_org/foo -> foo).
      m = (
          re.match(r'^platform/(frameworks/.+)$', repo) or
          re.match(r'^platform/external/chromium_org/?(.*?)(-history)?$', repo))
      if m:
        project = m.group(1) if m.group(1) else '.'  # '.' = Main project.
        repo_shas[project] = sha

  chromium_sha = repo_shas.get('.')
  webview_sha = repo_shas.get(WEBVIEW_PROJECT)
  if not chromium_sha or not webview_sha:
    raise merge_common.MergeError('SHAs for projects not found; '
                                  'invalid build.prop?')

  # Check that the revisions in repo.prop and the on in the archive match.
  archived_chromium_revision = _GetNearestUpstreamAbbrevSHA(chromium_sha)
  logging.info('Merging Chromium at %s and WebView at %s',
               archived_chromium_revision, webview_sha)
  _MergeChromiumProjects(archived_chromium_revision, target_branch, repo_shas,
                         force)

  _CheckoutSingleProject(WEBVIEW_PROJECT, target_branch)
  _MergeSingleProject(WEBVIEW_PROJECT, webview_sha,
                      archived_chromium_revision, target_branch, flatten=False)


def Push(target_branch):
  """Push the finished snapshot to the Android repository.

  Creates first a CL for frameworks/webview (if the merge-to-XXX branch exists)
  then wait for user confirmation and pushes the Chromium merges. This is to
  give an opportunity to get a +2 for  frameworks/webview and then push both
  frameworks/webview and the Chromium projects atomically(ish).

  Args:
    target_branch: name of the target branch (in the goog remote).
  """
  merge_branch = 'merge-to-%s' % target_branch

  # Create a Gerrit CL for the frameworks/webview project (if needed).
  dest_dir = _GetAbsPath(WEBVIEW_PROJECT)
  did_upload_webview_cl = False
  if merge_common.GetCommandStdout(['git', 'branch', '--list', merge_branch],
                                   cwd=dest_dir):
    # Check that there was actually something to merge.
    merge_range = 'goog/%s..%s' % (target_branch, merge_branch)
    if merge_common.GetCommandStdout(['git', 'rev-list', '-1', merge_range],
                                     cwd=dest_dir):
      logging.info('Uploading a merge CL for %s...', WEBVIEW_PROJECT)
      refspec = '%s:refs/for/%s' % (merge_branch, target_branch)
      upload = merge_common.GetCommandStdout(['git', 'push', 'goog', refspec],
                                             cwd=dest_dir)
      logging.info(upload)
      did_upload_webview_cl = True

  prompt_msg = 'About push the Chromium projects merge. '
  if not did_upload_webview_cl:
    logging.info('No merge CL needed for %s.', WEBVIEW_PROJECT)
  else:
    prompt_msg += ('At this point you should have the CL +2-ed and merge it '
                   'together with this push.')
  prompt_msg += '\nPress "y" to continue: '
  if raw_input(prompt_msg) != 'y':
    logging.warn('Push aborted by the user!')
    return

  logging.debug('Pushing Chromium projects to %s ...', target_branch)
  refspec = '%s:%s' % (merge_branch, target_branch)
  for path in merge_common.ALL_PROJECTS:
    logging.debug('Pushing %s', path)
    dest_dir = _GetAbsPath(path)
    # Delete the graft before pushing otherwise git will attempt to push all the
    # grafted-in objects to the server as well as the ones we want.
    graftfile = os.path.join(dest_dir, '.git', 'info', 'grafts')
    if os.path.exists(graftfile):
      os.remove(graftfile)
    merge_common.GetCommandStdout(['git', 'push', 'goog', refspec],
                                  cwd=dest_dir)


def main():
  parser = optparse.OptionParser(usage='%prog [options]')
  parser.epilog = ('Takes the current master-chromium branch of the Chromium '
                   'projects in Android and merges them into master to publish '
                   'them.')
  parser.add_option(
      '', '--revision',
      default=None,
      help=('Merge to the specified archived master-chromium revision (abbrev. '
            'SHA or release version) rather than using HEAD. e.g., '
            '--revision=a1b2c3d4e5f6 or --revision=38.0.2125.24'))
  parser.add_option(
      '', '--repo-prop',
      default=None, metavar='FILE',
      help=('Merge to the revisions specified in this repo.prop file.'))
  parser.add_option(
      '', '--force',
      default=False, action='store_true',
      help=('Skip history checks and merged anyways (only for --repo-prop).'))
  parser.add_option(
      '', '--push',
      default=False, action='store_true',
      help=('Push the result of a previous merge to the server.'))
  parser.add_option(
      '', '--target',
      default='master', metavar='BRANCH',
      help=('Target branch to push to. Defaults to master.'))
  (options, args) = parser.parse_args()
  if args:
    parser.print_help()
    return 1

  logging.basicConfig(format='%(message)s', level=logging.DEBUG,
                      stream=sys.stdout)

  if options.push:
    Push(options.target)
  elif options.repo_prop:
    _MergeWithRepoProp(os.path.expanduser(options.repo_prop),
                       options.target, options.force)
  elif options.revision:
    _MergeChromiumProjects(options.revision, options.target)
  else:
    first_upstream_sha = _GetNearestUpstreamAbbrevSHA()
    _MergeChromiumProjects(first_upstream_sha, options.target)

  return 0

if __name__ == '__main__':
  sys.exit(main())