diff options
Diffstat (limited to 'tools_libyuv')
| -rwxr-xr-x | tools_libyuv/autoroller/roll_deps.py | 821 | ||||
| -rwxr-xr-x | tools_libyuv/autoroller/unittests/roll_deps_test.py | 148 | ||||
| -rw-r--r-- | tools_libyuv/autoroller/unittests/testdata/DEPS | 21 | ||||
| -rw-r--r-- | tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.new | 13 | ||||
| -rw-r--r-- | tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.old | 13 | ||||
| -rwxr-xr-x | tools_libyuv/get_landmines.py | 39 | ||||
| -rw-r--r-- | tools_libyuv/msan/blacklist.txt | 9 | ||||
| -rw-r--r-- | tools_libyuv/ubsan/blacklist.txt | 15 | ||||
| -rw-r--r-- | tools_libyuv/ubsan/vptr_blacklist.txt | 25 |
9 files changed, 1104 insertions, 0 deletions
diff --git a/tools_libyuv/autoroller/roll_deps.py b/tools_libyuv/autoroller/roll_deps.py new file mode 100755 index 00000000..2b57eb65 --- /dev/null +++ b/tools_libyuv/autoroller/roll_deps.py @@ -0,0 +1,821 @@ +#!/usr/bin/env vpython3 + +# Copyright (c) 2017 The LibYUV project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. +"""Script to automatically roll dependencies in the LibYUV DEPS file.""" + + +import argparse +import base64 +import collections +import logging +import os +import re +import subprocess +import sys +import urllib.request + + +def FindSrcDirPath(): + """Returns the abs path to the src/ dir of the project.""" + src_dir = os.path.dirname(os.path.abspath(__file__)) + while os.path.basename(src_dir) != 'src': + src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) + return src_dir + + +# Skip these dependencies (list without solution name prefix). +DONT_AUTOROLL_THESE = [ + 'src/third_party/gflags/src', + 'src/third_party/mockito/src', +] + +# These dependencies are missing in chromium/src/DEPS, either unused or already +# in-tree. For instance, src/base is a part of the Chromium source git repo, +# but we pull it through a subtree mirror, so therefore it isn't listed in +# Chromium's deps but it is in ours. +LIBYUV_ONLY_DEPS = [ + 'src/base', + 'src/build', + 'src/buildtools', + 'src/ios', + 'src/testing', + 'src/third_party', + 'src/third_party/android_support_test_runner', + 'src/third_party/bazel', + 'src/third_party/bouncycastle', + 'src/third_party/errorprone/lib', + 'src/third_party/findbugs', + 'src/third_party/gson', + 'src/third_party/gtest-parallel', + 'src/third_party/guava', + 'src/third_party/intellij', + 'src/third_party/jsr-305/src', + 'src/third_party/ow2_asm', + 'src/third_party/proguard', + 'src/third_party/ub-uiautomator/lib', + 'src/tools', + 'src/tools/clang/dsymutil', +] + +LIBYUV_URL = 'https://chromium.googlesource.com/libyuv/libyuv' +CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src' +CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s' +CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s' +CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s' + +COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$') +CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'([-0-9a-z]+)\'$') +ROLL_BRANCH_NAME = 'roll_chromium_revision' + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +CHECKOUT_SRC_DIR = FindSrcDirPath() +CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir)) + +# Copied from tools/android/roll/android_deps/.../BuildConfigGenerator.groovy. +ANDROID_DEPS_START = r'=== ANDROID_DEPS Generated Code Start ===' +ANDROID_DEPS_END = r'=== ANDROID_DEPS Generated Code End ===' +# Location of automically gathered android deps. +ANDROID_DEPS_PATH = 'src/third_party/android_deps/' + +sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) +import find_depot_tools + +find_depot_tools.add_depot_tools_to_path() + +CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' +CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', + 'clang', 'scripts', 'update.py') + +DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') +ChangedDep = collections.namedtuple('ChangedDep', + 'path url current_rev new_rev') +CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages') +VersionEntry = collections.namedtuple('VersionEntry', 'version') +ChangedCipdPackage = collections.namedtuple( + 'ChangedCipdPackage', 'path package current_version new_version') +ChangedVersionEntry = collections.namedtuple( + 'ChangedVersionEntry', 'path current_version new_version') + +ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate', + ('current_chromium_rev ' + 'new_chromium_rev ')) + + +class RollError(Exception): + pass + + +def StrExpansion(): + return lambda str_value: str_value + + +def VarLookup(local_scope): + return lambda var_name: local_scope['vars'][var_name] + + +def ParseDepsDict(deps_content): + local_scope = {} + global_scope = { + 'Str': StrExpansion(), + 'Var': VarLookup(local_scope), + 'deps_os': {}, + } + exec(deps_content, global_scope, local_scope) + return local_scope + + +def ParseLocalDepsFile(filename): + with open(filename, 'rb') as f: + deps_content = f.read().decode('utf-8') + return ParseDepsDict(deps_content) + + +def ParseCommitPosition(commit_message): + for line in reversed(commit_message.splitlines()): + m = COMMIT_POSITION_RE.match(line.strip()) + if m: + return int(m.group(1)) + logging.error('Failed to parse commit position id from:\n%s\n', + commit_message) + sys.exit(-1) + + +def _RunCommand(command, + working_dir=None, + ignore_exit_code=False, + extra_env=None, + input_data=None): + """Runs a command and returns the output from that command. + + If the command fails (exit code != 0), the function will exit the process. + + Returns: + A tuple containing the stdout and stderr outputs as strings. + """ + working_dir = working_dir or CHECKOUT_SRC_DIR + logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) + env = os.environ.copy() + if extra_env: + assert all(isinstance(value, str) for value in extra_env.values()) + logging.debug('extra env: %s', extra_env) + env.update(extra_env) + p = subprocess.Popen(command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + cwd=working_dir, + universal_newlines=True) + std_output, err_output = p.communicate(input_data) + p.stdout.close() + p.stderr.close() + if not ignore_exit_code and p.returncode != 0: + logging.error('Command failed: %s\n' + 'stdout:\n%s\n' + 'stderr:\n%s\n', ' '.join(command), std_output, err_output) + sys.exit(p.returncode) + return std_output, err_output + + +def _GetBranches(): + """Returns a tuple of active,branches. + + The 'active' is the name of the currently active branch and 'branches' is a + list of all branches. + """ + lines = _RunCommand(['git', 'branch'])[0].split('\n') + branches = [] + active = '' + for line in lines: + if '*' in line: + # The assumption is that the first char will always be the '*'. + active = line[1:].strip() + branches.append(active) + else: + branch = line.strip() + if branch: + branches.append(branch) + return active, branches + + +def _ReadGitilesContent(url): + # Download and decode BASE64 content until + # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. + base64_content = ReadUrlContent(url + '?format=TEXT') + return base64.b64decode(base64_content[0]).decode('utf-8') + + +def ReadRemoteCrFile(path_below_src, revision): + """Reads a remote Chromium file of a specific revision. + + Args: + path_below_src: A path to the target file relative to src dir. + revision: Revision to read. + Returns: + A string with file content. + """ + return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % + (revision, path_below_src)) + + +def ReadRemoteCrCommit(revision): + """Reads a remote Chromium commit message. Returns a string.""" + return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) + + +def ReadUrlContent(url): + """Connect to a remote host and read the contents. + + Args: + url: URL to connect to. + Returns: + A list of lines. + """ + conn = urllib.request.urlopen(url) + try: + return conn.readlines() + except IOError as e: + logging.exception('Error connecting to %s. Error: %s', url, e) + raise + finally: + conn.close() + + +def GetMatchingDepsEntries(depsentry_dict, dir_path): + """Gets all deps entries matching the provided path. + + This list may contain more than one DepsEntry object. + Example: dir_path='src/testing' would give results containing both + 'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's + DEPS. + Example 2: dir_path='src/build' should return 'src/build' but not + 'src/buildtools'. + + Returns: + A list of DepsEntry objects. + """ + result = [] + for path, depsentry in depsentry_dict.items(): + if path == dir_path: + result.append(depsentry) + else: + parts = path.split('/') + if all(part == parts[i] for i, part in enumerate(dir_path.split('/'))): + result.append(depsentry) + return result + + +def BuildDepsentryDict(deps_dict): + """Builds a dict of paths to DepsEntry objects from a raw deps dict.""" + result = {} + + def AddDepsEntries(deps_subdict): + for path, dep in deps_subdict.items(): + if path in result: + continue + if not isinstance(dep, dict): + dep = {'url': dep} + if dep.get('dep_type') == 'cipd': + result[path] = CipdDepsEntry(path, dep['packages']) + else: + if '@' not in dep['url']: + continue + url, revision = dep['url'].split('@') + result[path] = DepsEntry(path, url, revision) + + def AddVersionEntry(vars_subdict): + for key, value in vars_subdict.items(): + if key in result: + continue + if not key.endswith('_version'): + continue + key = re.sub('_version$', '', key) + result[key] = VersionEntry(value) + + AddDepsEntries(deps_dict['deps']) + for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: + AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) + AddVersionEntry(deps_dict.get('vars', {})) + return result + + +def _FindChangedCipdPackages(path, old_pkgs, new_pkgs): + old_pkgs_names = {p['package'] for p in old_pkgs} + new_pkgs_names = {p['package'] for p in new_pkgs} + pkgs_equal = (old_pkgs_names == new_pkgs_names) + added_pkgs = [p for p in new_pkgs_names if p not in old_pkgs_names] + removed_pkgs = [p for p in old_pkgs_names if p not in new_pkgs_names] + + assert pkgs_equal, ('Old: %s\n New: %s.\nYou need to do a manual roll ' + 'and remove/add entries in DEPS so the old and new ' + 'list match.\nMost likely, you should add \"%s\" and ' + 'remove \"%s\"' % + (old_pkgs, new_pkgs, added_pkgs, removed_pkgs)) + + for old_pkg in old_pkgs: + for new_pkg in new_pkgs: + old_version = old_pkg['version'] + new_version = new_pkg['version'] + if (old_pkg['package'] == new_pkg['package'] + and old_version != new_version): + logging.debug('Roll dependency %s to %s', path, new_version) + yield ChangedCipdPackage(path, old_pkg['package'], old_version, + new_version) + + +def _FindChangedVars(name, old_version, new_version): + if old_version != new_version: + logging.debug('Roll dependency %s to %s', name, new_version) + yield ChangedVersionEntry(name, old_version, new_version) + + +def _FindNewDeps(old, new): + """ Gather dependencies only in `new` and return corresponding paths. """ + old_entries = set(BuildDepsentryDict(old)) + new_entries = set(BuildDepsentryDict(new)) + return [ + path for path in new_entries - old_entries + if path not in DONT_AUTOROLL_THESE + ] + + +def FindAddedDeps(libyuv_deps, new_cr_deps): + """ + Calculate new deps entries of interest. + + Ideally, that would mean: only appearing in chromium DEPS + but transitively used in LibYUV. + + Since it's hard to compute, we restrict ourselves to a well defined subset: + deps sitting in `ANDROID_DEPS_PATH`. + Otherwise, assumes that's a Chromium-only dependency. + + Args: + libyuv_deps: dict of deps as defined in the LibYUV DEPS file. + new_cr_deps: dict of deps as defined in the chromium DEPS file. + + Caveat: Doesn't detect a new package in existing dep. + + Returns: + A tuple consisting of: + A list of paths added dependencies sitting in `ANDROID_DEPS_PATH`. + A list of paths for other added dependencies. + """ + all_added_deps = _FindNewDeps(libyuv_deps, new_cr_deps) + generated_android_deps = [ + path for path in all_added_deps if path.startswith(ANDROID_DEPS_PATH) + ] + other_deps = [ + path for path in all_added_deps if path not in generated_android_deps + ] + return generated_android_deps, other_deps + + +def FindRemovedDeps(libyuv_deps, new_cr_deps): + """ + Calculate obsolete deps entries. + + Ideally, that would mean: no more appearing in chromium DEPS + and not used in LibYUV. + + Since it's hard to compute: + 1/ We restrict ourselves to a well defined subset: + deps sitting in `ANDROID_DEPS_PATH`. + 2/ We rely on existing behavior of CalculateChangeDeps. + I.e. Assumes non-CIPD dependencies are LibYUV-only, don't remove them. + + Args: + libyuv_deps: dict of deps as defined in the LibYUV DEPS file. + new_cr_deps: dict of deps as defined in the chromium DEPS file. + + Caveat: Doesn't detect a deleted package in existing dep. + + Returns: + A tuple consisting of: + A list of paths of dependencies removed from `ANDROID_DEPS_PATH`. + A list of paths of unexpected disappearing dependencies. + """ + all_removed_deps = _FindNewDeps(new_cr_deps, libyuv_deps) + generated_android_deps = sorted( + [path for path in all_removed_deps if path.startswith(ANDROID_DEPS_PATH)]) + # Webrtc-only dependencies are handled in CalculateChangedDeps. + other_deps = sorted([ + path for path in all_removed_deps + if path not in generated_android_deps and path not in LIBYUV_ONLY_DEPS + ]) + return generated_android_deps, other_deps + + +def CalculateChangedDeps(libyuv_deps, new_cr_deps): + """ + Calculate changed deps entries based on entries defined in the LibYUV DEPS + file: + - If a shared dependency with the Chromium DEPS file: roll it to the same + revision as Chromium (i.e. entry in the new_cr_deps dict) + - If it's a Chromium sub-directory, roll it to the HEAD revision (notice + this means it may be ahead of the chromium_revision, but generally these + should be close). + - If it's another DEPS entry (not shared with Chromium), roll it to HEAD + unless it's configured to be skipped. + + Returns: + A list of ChangedDep objects representing the changed deps. + """ + result = [] + libyuv_entries = BuildDepsentryDict(libyuv_deps) + new_cr_entries = BuildDepsentryDict(new_cr_deps) + for path, libyuv_deps_entry in libyuv_entries.items(): + if path in DONT_AUTOROLL_THESE: + continue + cr_deps_entry = new_cr_entries.get(path) + if cr_deps_entry: + assert type(cr_deps_entry) is type(libyuv_deps_entry) + + if isinstance(cr_deps_entry, CipdDepsEntry): + result.extend( + _FindChangedCipdPackages(path, libyuv_deps_entry.packages, + cr_deps_entry.packages)) + continue + + if isinstance(cr_deps_entry, VersionEntry): + result.extend( + _FindChangedVars(path, libyuv_deps_entry.version, + cr_deps_entry.version)) + continue + + # Use the revision from Chromium's DEPS file. + new_rev = cr_deps_entry.revision + assert libyuv_deps_entry.url == cr_deps_entry.url, ( + 'LibYUV DEPS entry %s has a different URL %s than Chromium %s.' % + (path, libyuv_deps_entry.url, cr_deps_entry.url)) + else: + if isinstance(libyuv_deps_entry, DepsEntry): + # Use the HEAD of the deps repo. + stdout, _ = _RunCommand( + ['git', 'ls-remote', libyuv_deps_entry.url, 'HEAD']) + new_rev = stdout.strip().split('\t')[0] + else: + # The dependency has been removed from chromium. + # This is handled by FindRemovedDeps. + continue + + # Check if an update is necessary. + if libyuv_deps_entry.revision != new_rev: + logging.debug('Roll dependency %s to %s', path, new_rev) + result.append( + ChangedDep(path, libyuv_deps_entry.url, libyuv_deps_entry.revision, + new_rev)) + return sorted(result) + + +def CalculateChangedClang(new_cr_rev): + + def GetClangRev(lines): + for line in lines: + match = CLANG_REVISION_RE.match(line) + if match: + return match.group(1) + raise RollError('Could not parse Clang revision!') + + with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'r') as f: + current_lines = f.readlines() + current_rev = GetClangRev(current_lines) + + new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, + new_cr_rev).splitlines() + new_rev = GetClangRev(new_clang_update_py) + return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) + + +def GenerateCommitMessage( + rev_update, + current_commit_pos, + new_commit_pos, + changed_deps_list, + added_deps_paths=None, + removed_deps_paths=None, + clang_change=None, +): + current_cr_rev = rev_update.current_chromium_rev[0:10] + new_cr_rev = rev_update.new_chromium_rev[0:10] + rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) + git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) + + commit_msg = [ + 'Roll chromium_revision %s (%s)\n' % (rev_interval, git_number_interval), + 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval), + 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval) + ] + + def Section(adjective, deps): + noun = 'dependency' if len(deps) == 1 else 'dependencies' + commit_msg.append('%s %s' % (adjective, noun)) + + if changed_deps_list: + Section('Changed', changed_deps_list) + + for c in changed_deps_list: + if isinstance(c, ChangedCipdPackage): + commit_msg.append('* %s: %s..%s' % + (c.path, c.current_version, c.new_version)) + elif isinstance(c, ChangedVersionEntry): + commit_msg.append('* %s_vesion: %s..%s' % + (c.path, c.current_version, c.new_version)) + else: + commit_msg.append('* %s: %s/+log/%s..%s' % + (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10])) + + if added_deps_paths: + Section('Added', added_deps_paths) + commit_msg.extend('* %s' % p for p in added_deps_paths) + + if removed_deps_paths: + Section('Removed', removed_deps_paths) + commit_msg.extend('* %s' % p for p in removed_deps_paths) + + if any([changed_deps_list, added_deps_paths, removed_deps_paths]): + change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') + commit_msg.append('DEPS diff: %s\n' % change_url) + else: + commit_msg.append('No dependencies changed.') + + if clang_change and clang_change.current_rev != clang_change.new_rev: + commit_msg.append('Clang version changed %s:%s' % + (clang_change.current_rev, clang_change.new_rev)) + change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, + CLANG_UPDATE_SCRIPT_URL_PATH) + commit_msg.append('Details: %s\n' % change_url) + else: + commit_msg.append('No update to Clang.\n') + + commit_msg.append('BUG=None') + return '\n'.join(commit_msg) + + +def UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content): + """Update the DEPS file with the new revision.""" + + with open(deps_filename, 'rb') as deps_file: + deps_content = deps_file.read().decode('utf-8') + + # Update the chromium_revision variable. + deps_content = deps_content.replace(rev_update.current_chromium_rev, + rev_update.new_chromium_rev) + + # Add and remove dependencies. For now: only generated android deps. + # Since gclient cannot add or remove deps, we on the fact that + # these android deps are located in one place we can copy/paste. + deps_re = re.compile(ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL) + new_deps = deps_re.search(new_cr_content) + old_deps = deps_re.search(deps_content) + if not new_deps or not old_deps: + faulty = 'Chromium' if not new_deps else 'LibYUV' + raise RollError('Was expecting to find "%s" and "%s"\n' + 'in %s DEPS' % + (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty)) + deps_content = deps_re.sub(new_deps.group(0), deps_content) + + for dep in changed_deps: + if isinstance(dep, ChangedVersionEntry): + deps_content = deps_content.replace(dep.current_version, dep.new_version) + + with open(deps_filename, 'wb') as deps_file: + deps_file.write(deps_content.encode('utf-8')) + + # Update each individual DEPS entry. + for dep in changed_deps: + # ChangedVersionEntry types are already been processed. + if isinstance(dep, ChangedVersionEntry): + continue + local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) + if not os.path.isdir(local_dep_dir): + raise RollError( + 'Cannot find local directory %s. Either run\n' + 'gclient sync --deps=all\n' + 'or make sure the .gclient file for your solution contains all ' + 'platforms in the target_os list, i.e.\n' + 'target_os = ["android", "unix", "mac", "ios", "win"];\n' + 'Then run "gclient sync" again.' % local_dep_dir) + if isinstance(dep, ChangedCipdPackage): + package = dep.package.format() # Eliminate double curly brackets + update = '%s:%s@%s' % (dep.path, package, dep.new_version) + else: + update = '%s@%s' % (dep.path, dep.new_rev) + _RunCommand(['gclient', 'setdep', '--revision', update], + working_dir=CHECKOUT_SRC_DIR) + + +def _IsTreeClean(): + stdout, _ = _RunCommand(['git', 'status', '--porcelain']) + if len(stdout) == 0: + return True + + logging.error('Dirty/unversioned files:\n%s', stdout) + return False + + +def _EnsureUpdatedMainBranch(dry_run): + current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', + 'HEAD'])[0].splitlines()[0] + if current_branch != 'main': + logging.error('Please checkout the main branch and re-run this script.') + if not dry_run: + sys.exit(-1) + + logging.info('Updating main branch...') + _RunCommand(['git', 'pull']) + + +def _CreateRollBranch(dry_run): + logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) + if not dry_run: + _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) + + +def _RemovePreviousRollBranch(dry_run): + active_branch, branches = _GetBranches() + if active_branch == ROLL_BRANCH_NAME: + active_branch = 'main' + if ROLL_BRANCH_NAME in branches: + logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) + if not dry_run: + _RunCommand(['git', 'checkout', active_branch]) + _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) + + +def _LocalCommit(commit_msg, dry_run): + logging.info('Committing changes locally.') + if not dry_run: + _RunCommand(['git', 'add', '--update', '.']) + _RunCommand(['git', 'commit', '-m', commit_msg]) + + +def ChooseCQMode(skip_cq, cq_over, current_commit_pos, new_commit_pos): + if skip_cq: + return 0 + if (new_commit_pos - current_commit_pos) < cq_over: + return 1 + return 2 + + +def _GetCcRecipients(changed_deps_list): + """Returns a list of emails to notify based on the changed deps list. + """ + cc_recipients = [] + for c in changed_deps_list: + pass + return cc_recipients + + +def _UploadCL(commit_queue_mode, add_cc=None): + """Upload the committed changes as a changelist to Gerrit. + + commit_queue_mode: + - 2: Submit to commit queue. + - 1: Run trybots but do not submit to CQ. + - 0: Skip CQ, upload only. + + add_cc: A list of email addresses to add as CC recipients. + """ + cc_recipients = [] + if add_cc: + cc_recipients.extend(add_cc) + cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks'] + if commit_queue_mode >= 2: + logging.info('Sending the CL to the CQ...') + cmd.extend(['-o', 'label=Bot-Commit+1']) + cmd.extend(['-o', 'label=Commit-Queue+2']) + cmd.extend(['--send-mail', '--cc', ','.join(cc_recipients)]) + elif commit_queue_mode >= 1: + logging.info('Starting CQ dry run...') + cmd.extend(['-o', 'label=Commit-Queue+1']) + extra_env = { + 'EDITOR': 'true', + 'SKIP_GCE_AUTH_FOR_GIT': '1', + } + stdout, stderr = _RunCommand(cmd, extra_env=extra_env) + logging.debug('Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s', + stdout, stderr) + + +def GetRollRevisionRanges(opts, libyuv_deps): + current_cr_rev = libyuv_deps['vars']['chromium_revision'] + new_cr_rev = opts.revision + if not new_cr_rev: + stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) + head_rev = stdout.strip().split('\t')[0] + logging.info('No revision specified. Using HEAD: %s', head_rev) + new_cr_rev = head_rev + + return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('--clean', + action='store_true', + default=False, + help='Removes any previous local roll branch.') + p.add_argument('-r', + '--revision', + help=('Chromium Git revision to roll to. Defaults to the ' + 'Chromium HEAD revision if omitted.')) + p.add_argument('--dry-run', + action='store_true', + default=False, + help=('Calculate changes and modify DEPS, but don\'t create ' + 'any local branch, commit, upload CL or send any ' + 'tryjobs.')) + p.add_argument('-i', + '--ignore-unclean-workdir', + action='store_true', + default=False, + help=('Ignore if the current branch is not main or if there ' + 'are uncommitted changes (default: %(default)s).')) + grp = p.add_mutually_exclusive_group() + grp.add_argument('--skip-cq', + action='store_true', + default=False, + help='Skip sending the CL to the CQ (default: %(default)s)') + grp.add_argument('--cq-over', + type=int, + default=1, + help=('Commit queue dry run if the revision difference ' + 'is below this number (default: %(default)s)')) + p.add_argument('-v', + '--verbose', + action='store_true', + default=False, + help='Be extra verbose in printing of log messages.') + opts = p.parse_args() + + if opts.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + if not opts.ignore_unclean_workdir and not _IsTreeClean(): + logging.error('Please clean your local checkout first.') + return 1 + + if opts.clean: + _RemovePreviousRollBranch(opts.dry_run) + + if not opts.ignore_unclean_workdir: + _EnsureUpdatedMainBranch(opts.dry_run) + + deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') + libyuv_deps = ParseLocalDepsFile(deps_filename) + + rev_update = GetRollRevisionRanges(opts, libyuv_deps) + + current_commit_pos = ParseCommitPosition( + ReadRemoteCrCommit(rev_update.current_chromium_rev)) + new_commit_pos = ParseCommitPosition( + ReadRemoteCrCommit(rev_update.new_chromium_rev)) + + new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev) + new_cr_deps = ParseDepsDict(new_cr_content) + changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) + # Discard other deps, assumed to be chromium-only dependencies. + new_generated_android_deps, _ = FindAddedDeps(libyuv_deps, new_cr_deps) + removed_generated_android_deps, other_deps = FindRemovedDeps( + libyuv_deps, new_cr_deps) + if other_deps: + raise RollError('LibYUV DEPS entries are missing from Chromium: %s.\n' + 'Remove them or add them to either ' + 'LIBYUV_ONLY_DEPS or DONT_AUTOROLL_THESE.' % other_deps) + clang_change = CalculateChangedClang(rev_update.new_chromium_rev) + commit_msg = GenerateCommitMessage( + rev_update, + current_commit_pos, + new_commit_pos, + changed_deps, + added_deps_paths=new_generated_android_deps, + removed_deps_paths=removed_generated_android_deps, + clang_change=clang_change) + logging.debug('Commit message:\n%s', commit_msg) + + _CreateRollBranch(opts.dry_run) + if not opts.dry_run: + UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content) + if _IsTreeClean(): + logging.info("No DEPS changes detected, skipping CL creation.") + else: + _LocalCommit(commit_msg, opts.dry_run) + commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over, + current_commit_pos, new_commit_pos) + logging.info('Uploading CL...') + if not opts.dry_run: + _UploadCL(commit_queue_mode, _GetCcRecipients(changed_deps)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools_libyuv/autoroller/unittests/roll_deps_test.py b/tools_libyuv/autoroller/unittests/roll_deps_test.py new file mode 100755 index 00000000..af86bdd5 --- /dev/null +++ b/tools_libyuv/autoroller/unittests/roll_deps_test.py @@ -0,0 +1,148 @@ +#!/usr/bin/env vpython3 + +# Copyright 2017 The LibYuv Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import glob +import os +import shutil +import sys +import tempfile +import unittest + +import roll_deps +from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, \ + ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir) +sys.path.append(PARENT_DIR) + +TEST_DATA_VARS = { + 'chromium_git': 'https://chromium.googlesource.com', + 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d', +} + +DEPS_ENTRIES = { + 'src/build': 'https://build.com', + 'src/buildtools': 'https://buildtools.com', + 'src/testing/gtest': 'https://gtest.com', + 'src/testing/gmock': 'https://gmock.com', +} + +BUILD_OLD_REV = '52f7afeca991d96d68cf0507e20dbdd5b845691f' +BUILD_NEW_REV = 'HEAD' +BUILDTOOLS_OLD_REV = '64e38f0cebdde27aa0cfb405f330063582f9ac76' +BUILDTOOLS_NEW_REV = '55ad626b08ef971fd82a62b7abb325359542952b' + + +class TestError(Exception): + pass + + +class FakeCmd(): + def __init__(self): + self.expectations = [] + + def add_expectation(self, *args, **kwargs): + returns = kwargs.pop('_returns', None) + self.expectations.append((args, kwargs, returns)) + + def __call__(self, *args, **kwargs): + if not self.expectations: + raise TestError('Got unexpected\n%s\n%s' % (args, kwargs)) + exp_args, exp_kwargs, exp_returns = self.expectations.pop(0) + if args != exp_args or kwargs != exp_kwargs: + message = 'Expected:\n args: %s\n kwargs: %s\n' % (exp_args, exp_kwargs) + message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs) + raise TestError(message) + return exp_returns + + +class TestRollChromiumRevision(unittest.TestCase): + def setUp(self): + self._output_dir = tempfile.mkdtemp() + for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')): + shutil.copy(test_file, self._output_dir) + self._libyuv_depsfile = os.path.join(self._output_dir, 'DEPS') + self._old_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.old') + self._new_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.new') + + self.fake = FakeCmd() + self.old_RunCommand = getattr(roll_deps, '_RunCommand') + setattr(roll_deps, '_RunCommand', self.fake) + + def tearDown(self): + shutil.rmtree(self._output_dir, ignore_errors=True) + self.assertEqual(self.fake.expectations, []) + setattr(roll_deps, '_RunCommand', self.old_RunCommand) + + def testVarLookup(self): + local_scope = {'foo': 'wrong', 'vars': {'foo': 'bar'}} + lookup = roll_deps.VarLookup(local_scope) + self.assertEqual(lookup('foo'), 'bar') + + def testUpdateDepsFile(self): + new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111' + + current_rev = TEST_DATA_VARS['chromium_revision'] + UpdateDepsFile(self._libyuv_depsfile, current_rev, new_rev, []) + with open(self._libyuv_depsfile, 'r') as deps_file: + deps_contents = deps_file.read() + self.assertTrue(new_rev in deps_contents, + 'Failed to find %s in\n%s' % (new_rev, deps_contents)) + + def testParseDepsDict(self): + with open(self._libyuv_depsfile, 'r') as deps_file: + deps_contents = deps_file.read() + local_scope = ParseDepsDict(deps_contents) + vars_dict = local_scope['vars'] + + def assertVar(variable_name): + self.assertEqual(vars_dict[variable_name], TEST_DATA_VARS[variable_name]) + assertVar('chromium_git') + assertVar('chromium_revision') + self.assertEqual(len(local_scope['deps']), 3) + + def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self): + entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest') + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0], DEPS_ENTRIES['src/testing/gtest']) + + def testGetMatchingDepsEntriesHandlesSimilarStartingPaths(self): + entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing') + self.assertEqual(len(entries), 2) + + def testGetMatchingDepsEntriesHandlesTwoPathsWithIdenticalFirstParts(self): + entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/build') + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0], DEPS_ENTRIES['src/build']) + + def testCalculateChangedDeps(self): + _SetupGitLsRemoteCall(self.fake, + 'https://chromium.googlesource.com/chromium/src/build', BUILD_NEW_REV) + libyuv_deps = ParseLocalDepsFile(self._libyuv_depsfile) + new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile) + changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) + self.assertEqual(len(changed_deps), 2) + self.assertEqual(changed_deps[0].path, 'src/build') + self.assertEqual(changed_deps[0].current_rev, BUILD_OLD_REV) + self.assertEqual(changed_deps[0].new_rev, BUILD_NEW_REV) + + self.assertEqual(changed_deps[1].path, 'src/buildtools') + self.assertEqual(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV) + self.assertEqual(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV) + + +def _SetupGitLsRemoteCall(cmd_fake, url, revision): + cmd = ['git', 'ls-remote', url, revision] + cmd_fake.add_expectation(cmd, _returns=(revision, None)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools_libyuv/autoroller/unittests/testdata/DEPS b/tools_libyuv/autoroller/unittests/testdata/DEPS new file mode 100644 index 00000000..4f45860c --- /dev/null +++ b/tools_libyuv/autoroller/unittests/testdata/DEPS @@ -0,0 +1,21 @@ +# DEPS file for unit tests. + +vars = { + 'chromium_git': 'https://chromium.googlesource.com', + 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d', + 'ignored_str': Str(''), +} + +deps = { + # Entry that is a directory in Chromium, so we're using a Git subtree mirror for it. + 'src/build': + Var('chromium_git') + '/chromium/src/build' + '@' + '52f7afeca991d96d68cf0507e20dbdd5b845691f', + + # Entry that's also a DEPS entry in the Chromium DEPS file. + 'src/buildtools': + Var('chromium_git') + '/chromium/buildtools.git' + '@' + '64e38f0cebdde27aa0cfb405f330063582f9ac76', + + # Entry only present in libyuv, not Chromium. + 'src/third_party/gflags/src': + Var('chromium_git') + '/external/github.com/gflags/gflags@03bebcb065c83beff83d50ae025a55a4bf94dfca', +} diff --git a/tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.new b/tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.new new file mode 100644 index 00000000..d53083ce --- /dev/null +++ b/tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.new @@ -0,0 +1,13 @@ +# DEPS file for unit tests. + +vars = { + 'chromium_git': 'https://chromium.googlesource.com', + + # This is updated compared to the DEPS.chromium.old file. + 'buildtools_revision': '55ad626b08ef971fd82a62b7abb325359542952b', +} + +deps = { + 'src/buildtools': + Var('chromium_git') + '/chromium/buildtools.git' + '@' + Var('buildtools_revision'), +} diff --git a/tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.old b/tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.old new file mode 100644 index 00000000..dd6ddaec --- /dev/null +++ b/tools_libyuv/autoroller/unittests/testdata/DEPS.chromium.old @@ -0,0 +1,13 @@ +# DEPS file for unit tests. + +vars = { + 'chromium_git': 'https://chromium.googlesource.com', + + # This is and older revision than DEPS.chromium.new file. + 'buildtools_revision': '64e38f0cebdde27aa0cfb405f330063582f9ac76', +} + +deps = { + 'src/buildtools': + Var('chromium_git') + '/chromium/buildtools.git' + '@' + Var('buildtools_revision'), +} diff --git a/tools_libyuv/get_landmines.py b/tools_libyuv/get_landmines.py new file mode 100755 index 00000000..8b33483e --- /dev/null +++ b/tools_libyuv/get_landmines.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +# Copyright 2016 The LibYuv Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +""" +This file emits the list of reasons why a particular build needs to be clobbered +(or a list of 'landmines'). +""" + +import sys + + +def print_landmines(): + """ + ALL LANDMINES ARE EMITTED FROM HERE. + """ + # DO NOT add landmines as part of a regular CL. Landmines are a last-effort + # bandaid fix if a CL that got landed has a build dependency bug and all bots + # need to be cleaned up. If you're writing a new CL that causes build + # dependency problems, fix the dependency problems instead of adding a + # landmine. + # See the Chromium version in src/build/get_landmines.py for usage examples. + print('Clobber to remove GYP artifacts after switching bots to GN.') + print('Another try to remove GYP artifacts after switching bots to GN.') + + +def main(): + print_landmines() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools_libyuv/msan/blacklist.txt b/tools_libyuv/msan/blacklist.txt new file mode 100644 index 00000000..8b5e42a7 --- /dev/null +++ b/tools_libyuv/msan/blacklist.txt @@ -0,0 +1,9 @@ +# The rules in this file are only applied at compile time. +# Because the Chrome buildsystem does not automatically touch the files +# mentioned here, changing this file requires clobbering all MSan bots. +# +# Please think twice before you add or remove these rules. + +# This is a stripped down copy of Chromium's blacklist.txt, to enable +# adding libyuv-specific blacklist entries. + diff --git a/tools_libyuv/ubsan/blacklist.txt b/tools_libyuv/ubsan/blacklist.txt new file mode 100644 index 00000000..8bcb2907 --- /dev/null +++ b/tools_libyuv/ubsan/blacklist.txt @@ -0,0 +1,15 @@ +############################################################################# +# UBSan blacklist. +# Please think twice before you add or remove these rules. + +# This is a stripped down copy of Chromium's blacklist.txt, to enable +# adding WebRTC-specific blacklist entries. + +############################################################################# +# YASM does some funny things that UBsan doesn't like. +# https://crbug.com/489901 +src:*/third_party/yasm/* + +############################################################################# +# Ignore system libraries. +src:*/usr/* diff --git a/tools_libyuv/ubsan/vptr_blacklist.txt b/tools_libyuv/ubsan/vptr_blacklist.txt new file mode 100644 index 00000000..23cfca53 --- /dev/null +++ b/tools_libyuv/ubsan/vptr_blacklist.txt @@ -0,0 +1,25 @@ +############################################################################# +# UBSan vptr blacklist. +# Function and type based blacklisting use a mangled name, and it is especially +# tricky to represent C++ types. For now, any possible changes by name manglings +# are simply represented as wildcard expressions of regexp, and thus it might be +# over-blacklisted. +# +# Please think twice before you add or remove these rules. +# +# This is a stripped down copy of Chromium's vptr_blacklist.txt, to enable +# adding libyuv-specific blacklist entries. + +############################################################################# +# Using raw pointer values. +# +# A raw pointer value (16) is used to infer the field offset by +# GOOGLE_PROTOBUF_GENERATED_MESSAGE_FIELD_OFFSET. + +# Example: +# src:*/third_party/protobuf/src/google/protobuf/compiler/plugin.pb.cc + +############################################################################# +# UBsan goes into an infinite recursion when __dynamic_cast instrumented with +# "vptr". See crbug.com/609786. +src:*/third_party/libc\+\+abi/trunk/src/private_typeinfo.cpp |