diff options
Diffstat (limited to 'chromium/tools/merge_from_chromium.py')
| -rwxr-xr-x | chromium/tools/merge_from_chromium.py | 581 |
1 files changed, 0 insertions, 581 deletions
diff --git a/chromium/tools/merge_from_chromium.py b/chromium/tools/merge_from_chromium.py deleted file mode 100755 index 3145387..0000000 --- a/chromium/tools/merge_from_chromium.py +++ /dev/null @@ -1,581 +0,0 @@ -#!/usr/bin/python -# -# Copyright (C) 2012 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Merge Chromium into the Android tree.""" - -import logging -import optparse -import os -import re -import sys - -import merge_common - - -# We need to import this *after* merging from upstream to get the latest -# version. Set it to none here to catch uses before it's imported. -webview_licenses = None - - -AUTOGEN_MESSAGE = 'This commit was generated by merge_from_chromium.py.' -SRC_GIT_BRANCH = 'refs/remotes/history/upstream-master' - - -def _ReadGitFile(sha1, path, git_url=None, git_branch=None): - """Reads a file from a (possibly remote) git project at a specific revision. - - Args: - sha1: The SHA1 at which to read. - path: The relative path of the file to read. - git_url: The URL of the git server, if reading a remote project. - git_branch: The branch to fetch, if reading a remote project. - Returns: - The contents of the specified file. - """ - if git_url: - merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url, git_branch]) - return merge_common.GetCommandStdout(['git', 'show', '%s:%s' % (sha1, path)]) - - -def _ParseDEPS(deps_content): - """Parses the DEPS file from Chromium and returns its contents. - - Args: - deps_content: The contents of the DEPS file as text. - Returns: - A dictionary of the contents of DEPS at the specified revision - """ - - class FromImpl(object): - """Used to implement the From syntax.""" - - def __init__(self, module_name): - self.module_name = module_name - - def __str__(self): - return 'From("%s")' % self.module_name - - class _VarImpl(object): - - def __init__(self, custom_vars, local_scope): - self._custom_vars = custom_vars - self._local_scope = local_scope - - def Lookup(self, var_name): - """Implements the Var syntax.""" - if var_name in self._custom_vars: - return self._custom_vars[var_name] - elif var_name in self._local_scope.get('vars', {}): - return self._local_scope['vars'][var_name] - raise Exception('Var is not defined: %s' % var_name) - - tmp_locals = {} - var = _VarImpl({}, tmp_locals) - tmp_globals = {'From': FromImpl, 'Var': var.Lookup, 'deps_os': {}} - exec(deps_content) in tmp_globals, tmp_locals # pylint: disable=W0122 - return tmp_locals - - -def _GetProjectMergeInfo(projects, deps_vars): - """Gets the git URL and SHA1 for each project based on DEPS. - - Args: - projects: The list of projects to consider. - deps_vars: The dictionary of dependencies from DEPS. - Returns: - A dictionary from project to git URL and SHA1 - 'path: (url, sha1)' - Raises: - TemporaryMergeError: if a project to be merged is not found in DEPS. - """ - deps_fallback_order = [ - deps_vars['deps'], - deps_vars['deps_os']['unix'], - deps_vars['deps_os']['android'], - ] - result = {} - for path in projects: - for deps in deps_fallback_order: - if path: - upstream_path = os.path.join('src', path) - else: - upstream_path = 'src' - url_plus_sha1 = deps.get(upstream_path) - if url_plus_sha1: - break - else: - raise merge_common.TemporaryMergeError( - 'Could not find DEPS entry for project %s. This probably ' - 'means that the project list in merge_from_chromium.py needs to be ' - 'updated.' % path) - match = re.match('(.*?)@(.*)', url_plus_sha1) - url = match.group(1) - sha1 = match.group(2) - logging.debug(' Got URL %s and SHA1 %s for project %s', url, sha1, path) - result[path] = {'url': url, 'sha1': sha1} - return result - - -def _MergeProjects(version, root_sha1, target, unattended, buildspec_url): - """Merges each required Chromium project into the Android repository. - - DEPS is consulted to determine which revision each project must be merged - at. Only a whitelist of required projects are merged. - - Args: - version: The version to mention in generated commit messages. - root_sha1: The git hash to merge in the root repository. - target: The target branch to merge to. - unattended: Run in unattended mode. - buildspec_url: URL for buildspec repository, when merging a branch. - Returns: - The abbrev sha1 merged. It will be either |root_sha1| itself (when merging - chromium trunk) or the upstream sha1 of the release. - Raises: - TemporaryMergeError: If incompatibly licensed code is left after pruning. - """ - # The logic for this step lives here, in the Android tree, as it makes no - # sense for a Chromium tree to know about this merge. - - if unattended: - branch_create_flag = '-B' - else: - branch_create_flag = '-b' - branch_name = 'merge-from-chromium-%s' % version - - logging.debug('Parsing DEPS ...') - if root_sha1: - deps_content = _ReadGitFile(root_sha1, 'DEPS') - else: - # TODO(primiano): At some point the release branches will use DEPS as well, - # instead of .DEPS.git. Rename below when that day will come. - deps_content = _ReadGitFile('FETCH_HEAD', - 'releases/' + version + '/.DEPS.git', - buildspec_url, - 'master') - - deps_vars = _ParseDEPS(deps_content) - - merge_info = _GetProjectMergeInfo(merge_common.THIRD_PARTY_PROJECTS, - deps_vars) - - for path in merge_info: - # webkit needs special handling as we have a local mirror - local_mirrored = path == 'third_party/WebKit' - url = merge_info[path]['url'] - sha1 = merge_info[path]['sha1'] - dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) - if local_mirrored: - remote = 'history' - else: - remote = 'goog' - merge_common.GetCommandStdout(['git', 'checkout', - branch_create_flag, branch_name, - '-t', remote + '/' + target], - cwd=dest_dir) - if not local_mirrored or not root_sha1: - logging.debug('Fetching project %s at %s ...', path, sha1) - fetch_args = ['git', 'fetch', url, sha1] - merge_common.GetCommandStdout(fetch_args, cwd=dest_dir) - if merge_common.GetCommandStdout(['git', 'rev-list', '-1', 'HEAD..' + sha1], - cwd=dest_dir): - logging.debug('Merging project %s at %s ...', path, sha1) - # Merge conflicts make git merge return 1, so ignore errors - merge_common.GetCommandStdout(['git', 'merge', '--no-commit', sha1], - cwd=dest_dir, ignore_errors=True) - merge_common.CheckNoConflictsAndCommitMerge( - 'Merge %s from %s at %s\n\n%s' % (path, url, sha1, AUTOGEN_MESSAGE), - cwd=dest_dir, unattended=unattended) - else: - logging.debug('No new commits to merge in project %s', path) - - # Handle root repository separately. - merge_common.GetCommandStdout(['git', 'checkout', - branch_create_flag, branch_name, - '-t', 'history/' + target]) - if not root_sha1: - merge_info = _GetProjectMergeInfo([''], deps_vars) - url = merge_info['']['url'] - merged_sha1 = merge_info['']['sha1'] - merge_common.GetCommandStdout(['git', 'fetch', url, merged_sha1]) - merged_sha1 = merge_common.Abbrev(merged_sha1) - merge_msg_version = '%s (%s)' % (version, merged_sha1) - else: - merge_msg_version = root_sha1 - merged_sha1 = root_sha1 - - logging.debug('Merging Chromium at %s ...', merged_sha1) - # Merge conflicts make git merge return 1, so ignore errors - merge_common.GetCommandStdout(['git', 'merge', '--no-commit', merged_sha1], - ignore_errors=True) - merge_common.CheckNoConflictsAndCommitMerge( - 'Merge Chromium at %s\n\n%s' - % (merge_msg_version, AUTOGEN_MESSAGE), unattended=unattended) - - logging.debug('Getting directories to exclude ...') - - # We import this now that we have merged the latest version. - # It imports to a global in order that it can be used to generate NOTICE - # later. We also disable writing bytecode to keep the source tree clean. - sys.path.append(os.path.join(merge_common.REPOSITORY_ROOT, 'android_webview', - 'tools')) - sys.dont_write_bytecode = True - global webview_licenses # pylint: disable=W0602 - import webview_licenses # pylint: disable=W0621,W0612,C6204 - import known_issues # pylint: disable=C6204 - - for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems(): - logging.debug(' %s', '\n '.join(os.path.join(path, x) for x in - exclude_list)) - dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) - merge_common.GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] + - exclude_list, cwd=dest_dir) - if _ModifiedFilesInIndex(dest_dir): - merge_common.GetCommandStdout(['git', 'commit', '-m', - 'Exclude unwanted directories'], - cwd=dest_dir) - assert(root_sha1 is None or root_sha1 == merged_sha1) - return merged_sha1 - - -def _CheckLicenses(): - """Check that no incompatibly licensed directories exist.""" - directories_left_over = webview_licenses.GetIncompatibleDirectories() - if directories_left_over: - raise merge_common.TemporaryMergeError( - 'Incompatibly licensed directories remain: ' + - '\n'.join(directories_left_over)) - - -def _GenerateMakefiles(version, unattended): - """Run gyp to generate the Android build system makefiles. - - Args: - version: The version to mention in generated commit messages. - unattended: Run in unattended mode. - """ - logging.debug('Generating makefiles ...') - - # TODO(torne): come up with a way to deal with hooks from DEPS properly - - # TODO(torne): The .tmp files are generated by - # third_party/WebKit/Source/WebCore/WebCore.gyp/WebCore.gyp into the source - # tree. We should avoid this, or at least use a more specific name to avoid - # accidentally removing or adding other files. - for path in merge_common.ALL_PROJECTS: - dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) - merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', - 'GypAndroid.*.mk', '*.target.*.mk', - '*.host.*.mk', '*.tmp'], cwd=dest_dir) - - try: - merge_common.GetCommandStdout(['android_webview/tools/gyp_webview', 'all']) - except merge_common.MergeError as e: - if not unattended: - raise - else: - for path in merge_common.ALL_PROJECTS: - merge_common.GetCommandStdout( - ['git', 'reset', '--hard'], - cwd=os.path.join(merge_common.REPOSITORY_ROOT, path)) - raise merge_common.TemporaryMergeError('Makefile generation failed: ' + - str(e)) - - for path in merge_common.ALL_PROJECTS: - dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) - # git add doesn't have an --ignore-unmatch so we have to do this instead: - merge_common.GetCommandStdout(['git', 'add', '-f', 'GypAndroid.*.mk'], - ignore_errors=True, cwd=dest_dir) - merge_common.GetCommandStdout(['git', 'add', '-f', '*.target.*.mk'], - ignore_errors=True, cwd=dest_dir) - merge_common.GetCommandStdout(['git', 'add', '-f', '*.host.*.mk'], - ignore_errors=True, cwd=dest_dir) - merge_common.GetCommandStdout(['git', 'add', '-f', '*.tmp'], - ignore_errors=True, cwd=dest_dir) - # Only try to commit the makefiles if something has actually changed. - if _ModifiedFilesInIndex(dest_dir): - merge_common.GetCommandStdout( - ['git', 'commit', '-m', - 'Update makefiles after merge of Chromium at %s\n\n%s' % - (version, AUTOGEN_MESSAGE)], cwd=dest_dir) - - -def _ModifiedFilesInIndex(cwd=merge_common.REPOSITORY_ROOT): - """Returns true if git's index contains any changes.""" - status = merge_common.GetCommandStdout(['git', 'status', '--porcelain'], - cwd=cwd) - return re.search(r'^[MADRC]', status, flags=re.MULTILINE) is not None - - -def _GenerateNoticeFile(version): - """Generates and commits a NOTICE file containing code licenses. - - This covers all third-party code (from Android's perspective) that lives in - the Chromium tree. - - Args: - version: The version to mention in generated commit messages. - """ - logging.debug('Regenerating NOTICE file ...') - - contents = webview_licenses.GenerateNoticeFile() - - with open(os.path.join(merge_common.REPOSITORY_ROOT, 'NOTICE'), 'w') as f: - f.write(contents) - merge_common.GetCommandStdout(['git', 'add', 'NOTICE']) - # Only try to commit the NOTICE update if the file has actually changed. - if _ModifiedFilesInIndex(): - merge_common.GetCommandStdout([ - 'git', 'commit', '-m', - 'Update NOTICE file after merge of Chromium at %s\n\n%s' - % (version, AUTOGEN_MESSAGE)]) - - -def _GenerateLastChange(version, root_sha1): - """Write a build/util/LASTCHANGE file containing the current revision. - - The revision number is compiled into the binary at build time from this file. - - Args: - version: The version to mention in generated commit messages. - root_sha1: The SHA1 of the main project (before the merge). - """ - logging.debug('Updating LASTCHANGE ...') - with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'), - 'w') as f: - f.write('LASTCHANGE=%s\n' % merge_common.Abbrev(root_sha1)) - merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE']) - logging.debug('Updating LASTCHANGE.blink ...') - with open(os.path.join(merge_common.REPOSITORY_ROOT, - 'build/util/LASTCHANGE.blink'), 'w') as f: - f.write('LASTCHANGE=%s\n' % _GetBlinkRevision()) - merge_common.GetCommandStdout(['git', 'add', '-f', - 'build/util/LASTCHANGE.blink']) - if _ModifiedFilesInIndex(): - merge_common.GetCommandStdout([ - 'git', 'commit', '-m', - 'Update LASTCHANGE file after merge of Chromium at %s\n\n%s' - % (version, AUTOGEN_MESSAGE)]) - - -def GetHEAD(): - """Fetch the latest HEAD revision from the Chromium Git mirror. - - Returns: - The latest HEAD revision (A Git abbrev SHA1). - """ - return _GetGitAbbrevSHA1(SRC_GIT_BRANCH, 'HEAD') - - -def _ParseSvnRevisionFromGitCommitMessage(commit_message): - return re.search(r'^git-svn-id: .*@([0-9]+)', commit_message, - flags=re.MULTILINE).group(1) - - -def _GetGitAbbrevSHA1(git_branch, revision): - """Returns an abbrev. SHA for the given revision (or branch, if HEAD).""" - assert revision - logging.debug('Getting Git revision for %s ...', revision) - - upstream = git_branch if revision == 'HEAD' else revision - - # Make sure the remote and the branch exist locally. - try: - merge_common.GetCommandStdout([ - 'git', 'show-ref', '--verify', '--quiet', git_branch]) - except merge_common.CommandError: - raise merge_common.TemporaryMergeError( - 'Cannot find the branch %s. Have you sync\'d master-chromium in this ' - 'checkout?' % git_branch) - - # Make sure the |upstream| Git object has been mirrored. - try: - merge_common.GetCommandStdout([ - 'git', 'merge-base', '--is-ancestor', upstream, git_branch]) - except merge_common.CommandError: - raise merge_common.TemporaryMergeError( - 'Upstream object (%s) not reachable from %s' % (upstream, git_branch)) - - abbrev_sha = merge_common.Abbrev(merge_common.GetCommandStdout( - ['git', 'rev-list', '--max-count=1', upstream]).split()[0]) - return abbrev_sha - - -def _GetBlinkRevision(): - # TODO(primiano): Switch to Git as soon as Blink gets migrated as well. - commit = merge_common.GetCommandStdout( - ['git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b'], - cwd=os.path.join(merge_common.REPOSITORY_ROOT, 'third_party', 'WebKit')) - return _ParseSvnRevisionFromGitCommitMessage(commit) - - -def Snapshot(root_sha1, release, target, unattended, buildspec_url): - """Takes a snapshot of the Chromium tree and merges it into Android. - - Android makefiles and a top-level NOTICE file are generated and committed - after the merge. - - Args: - root_sha1: The abbrev sha1 in the Chromium git mirror to merge from. - release: The Chromium release version to merge from (e.g. "30.0.1599.20"). - Only one of root_sha1 and release should be specified. - target: The target branch to merge to. - unattended: Run in unattended mode. - buildspec_url: URL for buildspec repository, used when merging a release. - - Returns: - True if new commits were merged; False if no new commits were present. - """ - if release: - root_sha1 = None - version = release - else: - root_sha1 = _GetGitAbbrevSHA1(SRC_GIT_BRANCH, root_sha1) - version = root_sha1 - - assert (root_sha1 is not None and len(root_sha1) > 6) or version == release - - if root_sha1 and not merge_common.GetCommandStdout( - ['git', 'rev-list', '-1', 'HEAD..' + root_sha1]): - logging.info('No new commits to merge at %s (%s)', version, root_sha1) - return False - - logging.info('Snapshotting Chromium at %s (%s)', version, root_sha1) - - # 1. Merge, accounting for excluded directories - merged_sha1 = _MergeProjects(version, root_sha1, target, unattended, - buildspec_url) - - # 2. Generate Android makefiles - _GenerateMakefiles(version, unattended) - - # 3. Check for incompatible licenses - _CheckLicenses() - - # 4. Generate Android NOTICE file - _GenerateNoticeFile(version) - - # 5. Generate LASTCHANGE file - _GenerateLastChange(version, merged_sha1) - - return True - - -def Push(version, target): - """Push the finished snapshot to the Android repository.""" - src = 'merge-from-chromium-%s' % version - # Use forced pushes ('+' prefix) for the temporary and archive branches in - # case they already got updated by a previous (possibly failed?) merge, but - # do not force push to the real master-chromium branch as this could erase - # downstream changes. - refspecs = ['%s:%s' % (src, target), - '+%s:refs/archive/chromium-%s' % (src, version)] - if target == 'master-chromium': - refspecs.insert(0, '+%s:master-chromium-merge' % src) - for refspec in refspecs: - logging.debug('Pushing to server (%s) ...', refspec) - for path in merge_common.ALL_PROJECTS: - if path in merge_common.PROJECTS_WITH_FLAT_HISTORY: - remote = 'history' - else: - remote = 'goog' - logging.debug('Pushing %s', path) - dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) - merge_common.GetCommandStdout(['git', 'push', remote, refspec], - cwd=dest_dir) - - -def main(): - parser = optparse.OptionParser(usage='%prog [options]') - parser.epilog = ('Takes a snapshot of the Chromium tree at the specified ' - 'Chromium Git revision and merges it into this repository. ' - 'Paths marked as excluded for license reasons are removed ' - 'as part of the merge. Also generates Android makefiles and ' - 'generates a top-level NOTICE file suitable for use in the ' - 'Android build.') - parser.add_option( - '', '--sha1', - default='HEAD', - help=('Merge to the specified chromium sha1 revision from ' + - SRC_GIT_BRANCH + ' branch. Default is HEAD, to merge from ToT.')) - parser.add_option( - '', '--release', - default=None, - help=('Merge to the specified chromium release buildspec (e.g., "30.0.' - '1599.20"). Only one of --sha1 and --release should be specified')) - parser.add_option( - '', '--buildspec_url', - default=None, - help=('Git URL for buildspec repository.')) - parser.add_option( - '', '--target', - default='master-chromium', metavar='BRANCH', - help=('Target branch to push to. Defaults to master-chromium.')) - parser.add_option( - '', '--push', - default=False, action='store_true', - help=('Push the result of a previous merge to the server. Note ' - '--sha1 must be given.')) - parser.add_option( - '', '--get_head', - default=False, action='store_true', - help=('Just print the current HEAD revision on stdout and exit.')) - parser.add_option( - '', '--unattended', - default=False, action='store_true', - help=('Run in unattended mode.')) - parser.add_option( - '', '--no_changes_exit', - default=0, type='int', - help=('Exit code to use if there are no changes to merge, for scripts.')) - (options, args) = parser.parse_args() - if args: - parser.print_help() - return 1 - - if 'ANDROID_BUILD_TOP' not in os.environ: - print >>sys.stderr, 'You need to run the Android envsetup.sh and lunch.' - return 1 - - if os.environ.get('GYP_DEFINES'): - print >>sys.stderr, ( - 'The environment is defining GYP_DEFINES (=%s). It will affect the ' - ' generated makefiles.' % os.environ['GYP_DEFINES']) - if not options.unattended and raw_input('Continue? [y/N]') != 'y': - return 1 - - logging.basicConfig(format='%(message)s', level=logging.DEBUG, - stream=sys.stdout) - - if options.get_head: - logging.disable(logging.CRITICAL) # Prevent log messages - print GetHEAD() - elif options.push: - if options.release: - Push(options.release, options.target) - elif options.sha1: - Push(options.sha1, options.target) - else: - print >>sys.stderr, 'You need to pass the version to push.' - return 1 - else: - if not Snapshot(options.sha1, options.release, options.target, - options.unattended, options.buildspec_url): - return options.no_changes_exit - - return 0 - -if __name__ == '__main__': - sys.exit(main()) |