Merge changes I7bbf776e,I1b827825

am: fe8b4a6579

* commit 'fe8b4a657979b49e1701bd92f6d5814a99e0b2be': (7237 commits)
  WIP: Changes after merge commit 'cb3f9bd'
  Make the nonlinear beamformer steerable
  Utilize bitrate above codec max to protect video.
  Enable VP9 internal resize by default.
  Filter overlapping RTP header extensions.
  Make VCMEncodedFrameCallback const.
  MediaCodecVideoEncoder: Add number of quality resolution downscales to Encoded callback.
  Remove redudant encoder rate calls.
  Create isolate files for nonparallel tests.
  Register header extensions in RtpRtcpObserver to avoid log spam.
  Make an enum class out of NetEqDecoder, and hide the neteq_decoders_ table
  ACM: Move NACK functionality inside NetEq
  Fix chromium-style warnings in webrtc/sound/.
  Create a 'webrtc_nonparallel_tests' target.
  Update scalability structure data according to updates in the RTP payload profile.
  audio_coding: rename interface -> include
  Rewrote perform_action_on_all_files to be parallell.
  Update reference indices according to updates in the RTP payload profile.
  Disable P2PTransport...TestFailoverControlledSide on Memcheck
  pass clangcl compile options to ignore warnings in gflags.cc
  ...

1 files changed, 456 insertions, 0 deletions

diff --git a/tools/autoroller/roll_chromium_revision.py b/tools/autoroller/roll_chromium_revision.py
new file mode 100755
index 0000000000..917ecd6570
--- /dev/null
+++ b/tools/autoroller/roll_chromium_revision.py
@@ -0,0 +1,456 @@
+#!/usr/bin/env python
+# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+"""Script to roll chromium_revision in the WebRTC DEPS file."""
+
+import argparse
+import base64
+import collections
+import logging
+import os
+import re
+import subprocess
+import sys
+import urllib
+
+
+CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src'
+CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s'
+CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s'
+CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s'
+
+# Run these CQ trybots in addition to the default ones in infra/config/cq.cfg.
+EXTRA_TRYBOTS = 'tryserver.webrtc:win_baremetal,mac_baremetal,linux_baremetal'
+
+COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$')
+CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION=(\d+)$')
+ROLL_BRANCH_NAME = 'roll_chromium_revision'
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
+                                                  os.pardir))
+sys.path.append(CHECKOUT_ROOT_DIR)
+import setup_links
+
+sys.path.append(os.path.join(CHECKOUT_ROOT_DIR, 'tools'))
+import find_depot_tools
+find_depot_tools.add_depot_tools_to_path()
+from gclient import GClientKeywords
+
+CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.sh'
+CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join('tools', 'clang', 'scripts',
+                                              'update.sh')
+
+DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
+ChangedDep = collections.namedtuple('ChangedDep',
+                                    'path url current_rev new_rev')
+
+
+def ParseDepsDict(deps_content):
+  local_scope = {}
+  var = GClientKeywords.VarImpl({}, local_scope)
+  global_scope = {
+    'File': GClientKeywords.FileImpl,
+    'From': GClientKeywords.FromImpl,
+    'Var': var.Lookup,
+    'deps_os': {},
+  }
+  exec(deps_content, global_scope, local_scope)
+  return local_scope
+
+
+def ParseLocalDepsFile(filename):
+  with open(filename, 'rb') as f:
+    deps_content = f.read()
+  return ParseDepsDict(deps_content)
+
+
+def ParseRemoteCrDepsFile(revision):
+  deps_content = ReadRemoteCrFile('DEPS', revision)
+  return ParseDepsDict(deps_content)
+
+
+def ParseCommitPosition(commit_message):
+  for line in reversed(commit_message.splitlines()):
+    m = COMMIT_POSITION_RE.match(line.strip())
+    if m:
+      return m.group(1)
+  logging.error('Failed to parse commit position id from:\n%s\n',
+                commit_message)
+  sys.exit(-1)
+
+
+def _RunCommand(command, working_dir=None, ignore_exit_code=False,
+                extra_env=None):
+  """Runs a command and returns the output from that command.
+
+  If the command fails (exit code != 0), the function will exit the process.
+
+  Returns:
+    A tuple containing the stdout and stderr outputs as strings.
+  """
+  working_dir = working_dir or CHECKOUT_ROOT_DIR
+  logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
+  env = os.environ.copy()
+  if extra_env:
+    assert all(type(value) == str for value in extra_env.values())
+    logging.debug('extra env: %s', extra_env)
+    env.update(extra_env)
+  p = subprocess.Popen(command, stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE, env=env,
+                       cwd=working_dir, universal_newlines=True)
+  std_output = p.stdout.read()
+  err_output = p.stderr.read()
+  p.wait()
+  p.stdout.close()
+  p.stderr.close()
+  if not ignore_exit_code and p.returncode != 0:
+    logging.error('Command failed: %s\n'
+                  'stdout:\n%s\n'
+                  'stderr:\n%s\n', ' '.join(command), std_output, err_output)
+    sys.exit(p.returncode)
+  return std_output, err_output
+
+
+def _GetBranches():
+  """Returns a tuple of active,branches.
+
+  The 'active' is the name of the currently active branch and 'branches' is a
+  list of all branches.
+  """
+  lines = _RunCommand(['git', 'branch'])[0].split('\n')
+  branches = []
+  active = ''
+  for line in lines:
+    if '*' in line:
+      # The assumption is that the first char will always be the '*'.
+      active = line[1:].strip()
+      branches.append(active)
+    else:
+      branch = line.strip()
+      if branch:
+        branches.append(branch)
+  return active, branches
+
+
+def _ReadGitilesContent(url):
+  # Download and decode BASE64 content until
+  # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed.
+  base64_content = ReadUrlContent(url + '?format=TEXT')
+  return base64.b64decode(base64_content[0])
+
+
+def ReadRemoteCrFile(path_below_src, revision):
+  """Reads a remote Chromium file of a specific revision. Returns a string."""
+  return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision,
+                                                       path_below_src))
+
+
+def ReadRemoteCrCommit(revision):
+  """Reads a remote Chromium commit message. Returns a string."""
+  return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision)
+
+
+def ReadUrlContent(url):
+  """Connect to a remote host and read the contents. Returns a list of lines."""
+  conn = urllib.urlopen(url)
+  try:
+    return conn.readlines()
+  except IOError as e:
+    logging.exception('Error connecting to %s. Error: %s', url, e)
+    raise
+  finally:
+    conn.close()
+
+
+def GetMatchingDepsEntries(depsentry_dict, dir_path):
+  """Gets all deps entries matching the provided path.
+
+  This list may contain more than one DepsEntry object.
+  Example: dir_path='src/testing' would give results containing both
+  'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS.
+  Example 2: dir_path='src/build' should return 'src/build' but not
+  'src/buildtools'.
+
+  Returns:
+    A list of DepsEntry objects.
+  """
+  result = []
+  for path, depsentry in depsentry_dict.iteritems():
+    if path == dir_path:
+      result.append(depsentry)
+    else:
+      parts = path.split('/')
+      if all(part == parts[i]
+             for i, part in enumerate(dir_path.split('/'))):
+        result.append(depsentry)
+  return result
+
+
+def BuildDepsentryDict(deps_dict):
+  """Builds a dict of DepsEntry object from a raw parsed deps dict."""
+  result = {}
+  def AddDepsEntries(deps_subdict):
+    for path, deps_url in deps_subdict.iteritems():
+      if not result.has_key(path):
+        url, revision = deps_url.split('@') if deps_url else (None, None)
+        result[path] = DepsEntry(path, url, revision)
+
+  AddDepsEntries(deps_dict['deps'])
+  for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
+    AddDepsEntries(deps_dict['deps_os'].get(deps_os, {}))
+  return result
+
+
+def CalculateChangedDeps(current_deps, new_deps):
+  result = []
+  current_entries = BuildDepsentryDict(current_deps)
+  new_entries = BuildDepsentryDict(new_deps)
+
+  all_deps_dirs = setup_links.DIRECTORIES
+  for deps_dir in all_deps_dirs:
+    # All deps have 'src' prepended to the path in the Chromium DEPS file.
+    dir_path = 'src/%s' % deps_dir
+
+    for entry in GetMatchingDepsEntries(current_entries, dir_path):
+      new_matching_entries = GetMatchingDepsEntries(new_entries, entry.path)
+      assert len(new_matching_entries) <= 1, (
+          'Should never find more than one entry matching %s in %s, found %d' %
+          (entry.path, new_entries, len(new_matching_entries)))
+      if not new_matching_entries:
+        result.append(ChangedDep(entry.path, entry.url, entry.revision, 'None'))
+      elif entry != new_matching_entries[0]:
+        result.append(ChangedDep(entry.path, entry.url, entry.revision,
+                                 new_matching_entries[0].revision))
+  return result
+
+
+def CalculateChangedClang(new_cr_rev):
+  def GetClangRev(lines):
+    for line in lines:
+      match = CLANG_REVISION_RE.match(line)
+      if match:
+        return match.group(1)
+    return None
+
+  chromium_src_path = os.path.join(CHECKOUT_ROOT_DIR, 'chromium', 'src',
+                                   CLANG_UPDATE_SCRIPT_LOCAL_PATH)
+  with open(chromium_src_path, 'rb') as f:
+    current_lines = f.readlines()
+  current_rev = GetClangRev(current_lines)
+
+  new_clang_update_sh = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH,
+                                         new_cr_rev).splitlines()
+  new_rev = GetClangRev(new_clang_update_sh)
+  return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev)
+
+
+def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos,
+                          new_commit_pos, changed_deps_list, clang_change):
+  current_cr_rev = current_cr_rev[0:7]
+  new_cr_rev = new_cr_rev[0:7]
+  rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev)
+  git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos)
+
+  commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval,
+                                                    git_number_interval)]
+  commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval))
+  commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE %
+                                         rev_interval))
+  # TBR field will be empty unless in some custom cases, where some engineers
+  # are added.
+  tbr_authors = ''
+  if changed_deps_list:
+    commit_msg.append('Changed dependencies:')
+
+    for c in changed_deps_list:
+      commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url,
+                                                  c.current_rev[0:7],
+                                                  c.new_rev[0:7]))
+      if 'libvpx' in c.path:
+        tbr_authors += 'marpan@webrtc.org, stefan@webrtc.org, '
+
+    change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS')
+    commit_msg.append('DEPS diff: %s\n' % change_url)
+  else:
+    commit_msg.append('No dependencies changed.')
+
+  if clang_change.current_rev != clang_change.new_rev:
+    commit_msg.append('Clang version changed %s:%s' %
+                      (clang_change.current_rev, clang_change.new_rev))
+    change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval,
+                                           CLANG_UPDATE_SCRIPT_URL_PATH)
+    commit_msg.append('Details: %s\n' % change_url)
+    tbr_authors += 'pbos@webrtc.org'
+  else:
+    commit_msg.append('No update to Clang.\n')
+
+  commit_msg.append('TBR=%s' % tbr_authors)
+  commit_msg.append('CQ_EXTRA_TRYBOTS=%s' % EXTRA_TRYBOTS)
+  return '\n'.join(commit_msg)
+
+
+def UpdateDeps(deps_filename, old_cr_revision, new_cr_revision):
+  """Update the DEPS file with the new revision."""
+  with open(deps_filename, 'rb') as deps_file:
+    deps_content = deps_file.read()
+  deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
+  with open(deps_filename, 'wb') as deps_file:
+    deps_file.write(deps_content)
+
+def _IsTreeClean():
+  stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
+  if len(stdout) == 0:
+    return True
+
+  logging.error('Dirty/unversioned files:\n%s', stdout)
+  return False
+
+
+def _EnsureUpdatedMasterBranch(dry_run):
+  current_branch = _RunCommand(
+      ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0]
+  if current_branch != 'master':
+    logging.error('Please checkout the master branch and re-run this script.')
+    if not dry_run:
+      sys.exit(-1)
+
+  logging.info('Updating master branch...')
+  _RunCommand(['git', 'pull'])
+
+
+def _CreateRollBranch(dry_run):
+  logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME)
+  if not dry_run:
+    _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
+
+
+def _RemovePreviousRollBranch(dry_run):
+  active_branch, branches = _GetBranches()
+  if active_branch == ROLL_BRANCH_NAME:
+    active_branch = 'master'
+  if ROLL_BRANCH_NAME in branches:
+    logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME)
+    if not dry_run:
+      _RunCommand(['git', 'checkout', active_branch])
+      _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
+
+
+def _LocalCommit(commit_msg, dry_run):
+  logging.info('Committing changes locally.')
+  if not dry_run:
+    _RunCommand(['git', 'add', '--update', '.'])
+    _RunCommand(['git', 'commit', '-m', commit_msg])
+
+
+def _UploadCL(dry_run, rietveld_email=None):
+  logging.info('Uploading CL...')
+  if not dry_run:
+    cmd = ['git', 'cl', 'upload', '-f']
+    if rietveld_email:
+      cmd.append('--email=%s' % rietveld_email)
+    _RunCommand(cmd, extra_env={'EDITOR': 'true'})
+
+
+def _LaunchTrybots(dry_run, skip_try):
+  logging.info('Sending tryjobs...')
+  if not dry_run and not skip_try:
+    _RunCommand(['git', 'cl', 'try'])
+
+
+def _SendToCQ(dry_run, skip_cq):
+  logging.info('Sending the CL to the CQ...')
+  if not dry_run and not skip_cq:
+    _RunCommand(['git', 'cl', 'set_commit'])
+    logging.info('Sent the CL to the CQ.')
+
+
+def main():
+  p = argparse.ArgumentParser()
+  p.add_argument('--clean', action='store_true', default=False,
+                 help='Removes any previous local roll branch.')
+  p.add_argument('-r', '--revision',
+                 help=('Chromium Git revision to roll to. Defaults to the '
+                       'Chromium HEAD revision if omitted.'))
+  p.add_argument('-u', '--rietveld-email',
+                 help=('E-mail address to use for creating the CL at Rietveld'
+                       'If omitted a previously cached one will be used or an '
+                       'error will be thrown during upload.'))
+  p.add_argument('--dry-run', action='store_true', default=False,
+                 help=('Calculate changes and modify DEPS, but don\'t create '
+                       'any local branch, commit, upload CL or send any '
+                       'tryjobs.'))
+  p.add_argument('--allow-reverse', action='store_true', default=False,
+                 help=('Allow rolling back in time (disabled by default but '
+                       'may be useful to be able do to manually).'))
+  p.add_argument('-s', '--skip-try', action='store_true', default=False,
+                 help='Skip sending tryjobs (default: %(default)s)')
+  p.add_argument('--skip-cq', action='store_true', default=False,
+                 help='Skip sending the CL to the CQ (default: %(default)s)')
+  p.add_argument('-v', '--verbose', action='store_true', default=False,
+                 help='Be extra verbose in printing of log messages.')
+  opts = p.parse_args()
+
+  if opts.verbose:
+    logging.basicConfig(level=logging.DEBUG)
+  else:
+    logging.basicConfig(level=logging.INFO)
+
+  if not _IsTreeClean():
+    logging.error('Please clean your local checkout first.')
+    return 1
+
+  if opts.clean:
+    _RemovePreviousRollBranch(opts.dry_run)
+
+  _EnsureUpdatedMasterBranch(opts.dry_run)
+
+  new_cr_rev = opts.revision
+  if not new_cr_rev:
+    stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
+    head_rev = stdout.strip().split('\t')[0]
+    logging.info('No revision specified. Using HEAD: %s', head_rev)
+    new_cr_rev = head_rev
+
+  deps_filename = os.path.join(CHECKOUT_ROOT_DIR, 'DEPS')
+  local_deps = ParseLocalDepsFile(deps_filename)
+  current_cr_rev = local_deps['vars']['chromium_revision']
+
+  current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
+  new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
+
+  current_cr_deps = ParseRemoteCrDepsFile(current_cr_rev)
+  new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev)
+
+  if new_commit_pos > current_commit_pos or opts.allow_reverse:
+    changed_deps = sorted(CalculateChangedDeps(current_cr_deps, new_cr_deps))
+    clang_change = CalculateChangedClang(new_cr_rev)
+    commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
+                                       current_commit_pos, new_commit_pos,
+                                       changed_deps, clang_change)
+    logging.debug('Commit message:\n%s', commit_msg)
+  else:
+    logging.info('Currently pinned chromium_revision: %s (#%s) is newer than '
+                 '%s (#%s). To roll to older revisions, you must pass the '
+                 '--allow-reverse flag.\n'
+                 'Aborting without action.', current_cr_rev, current_commit_pos,
+                 new_cr_rev, new_commit_pos)
+    return 0
+
+  _CreateRollBranch(opts.dry_run)
+  UpdateDeps(deps_filename, current_cr_rev, new_cr_rev)
+  _LocalCommit(commit_msg, opts.dry_run)
+  _UploadCL(opts.dry_run, opts.rietveld_email)
+  _LaunchTrybots(opts.dry_run, opts.skip_try)
+  _SendToCQ(opts.dry_run, opts.skip_cq)
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())