Change structure of deps file and tool for adding chromium dep.

Change structure of chromium owned dependencies file to JSON to
simplify work with it in different tools. Also add tool to check in
new chromium owned dep with single command like this:
./tools_webrtc/autoroller/checkin_chromium_dep.py -d <dep name>

Introduce separate file with list of webrtc owned dependencies.



Bug: webrtc:8366
Change-Id: I30a828af34cd105ce7e6bc76d6b5889e6bf7574d
Reviewed-on: https://webrtc-review.googlesource.com/76840
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Reviewed-by: Patrik Höglund <phoglund@webrtc.org>
Commit-Queue: Artem Titov <titovartem@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23270}

13 files changed, 399 insertions, 7 deletions

diff --git a/tools_webrtc/autoroller/checkin_chromium_dep.py b/tools_webrtc/autoroller/checkin_chromium_dep.py
new file mode 100755
index 0000000000..1a6799317f
--- /dev/null
+++ b/tools_webrtc/autoroller/checkin_chromium_dep.py
@@ -0,0 +1,246 @@
+#!/usr/bin/env python
+# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+
+"""
+Script to automatically add specified chromium dependency to WebRTC repo.
+
+If you want to add new chromium owned dependency foo to the WebRTC repo
+you have to run this tool like this:
+./checkin_chromium_deps.py -d foo
+
+It will check in dependency into third_party directory and will add it into
+git index. Also it will update chromium dependencies list with new dependency
+to insure that it will be correctly auto updated in future.
+"""
+
+import argparse
+import errno
+import json
+import logging
+import os.path
+import shutil
+import subprocess
+import sys
+import tempfile
+
+REMOTE_URL = 'https://chromium.googlesource.com/chromium/src/third_party'
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
+                                                 os.pardir))
+
+
+class DependencyAlreadyCheckedIn(Exception):
+  pass
+
+
+class DependencyNotFound(Exception):
+  pass
+
+
+class Config(object):
+
+  def __init__(self, src_root, remote_url, temp_dir):
+    self.src_root = src_root
+    self.dependencies_file = os.path.join(self.src_root,
+                                          'THIRD_PARTY_CHROMIUM_DEPS.json')
+    self.deps_file = os.path.join(self.src_root, 'DEPS')
+    self.third_party_dir = os.path.join(self.src_root, 'third_party')
+    self.remote_url = remote_url
+    self.temp_dir = temp_dir
+
+
+def VarLookup(local_scope):
+  return lambda var_name: local_scope['vars'][var_name]
+
+
+def ParseDepsDict(deps_content):
+  local_scope = {}
+  global_scope = {
+    'Var': VarLookup(local_scope),
+    'deps_os': {},
+  }
+  exec (deps_content, global_scope, local_scope)
+  return local_scope
+
+
+def ParseLocalDepsFile(filename):
+  with open(filename, 'rb') as f:
+    deps_content = f.read()
+  return ParseDepsDict(deps_content)
+
+
+def RunCommand(command, working_dir=None, ignore_exit_code=False,
+    extra_env=None, input_data=None):
+  """Runs a command and returns the output from that command.
+
+  If the command fails (exit code != 0), the function will exit the process.
+
+  Returns:
+    A tuple containing the stdout and stderr outputs as strings.
+  """
+  working_dir = working_dir or CHECKOUT_SRC_DIR
+  logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
+  env = os.environ.copy()
+  if extra_env:
+    assert all(type(value) == str for value in extra_env.values())
+    logging.debug('extra env: %s', extra_env)
+    env.update(extra_env)
+  p = subprocess.Popen(command,
+                       stdin=subprocess.PIPE,
+                       stdout=subprocess.PIPE,
+                       stderr=subprocess.PIPE, env=env,
+                       cwd=working_dir, universal_newlines=True)
+  std_output, err_output = p.communicate(input_data)
+  p.stdout.close()
+  p.stderr.close()
+  if not ignore_exit_code and p.returncode != 0:
+    logging.error('Command failed: %s\n'
+                  'stdout:\n%s\n'
+                  'stderr:\n%s\n', ' '.join(command), std_output, err_output)
+    sys.exit(p.returncode)
+  return std_output, err_output
+
+
+def LoadThirdPartyRevision(deps_file):
+  logging.debug('Loading chromium third_party revision from %s', deps_file)
+  webrtc_deps = ParseLocalDepsFile(deps_file)
+  return webrtc_deps['vars']['chromium_third_party_revision']
+
+
+def CheckoutRequiredDependency(dep_name, config):
+  third_party_revision = LoadThirdPartyRevision(config.deps_file)
+
+  logging.debug('Initializing git repo in %s...', config.temp_dir)
+  RunCommand(['git', 'init'], working_dir=config.temp_dir)
+
+  logging.debug('Adding remote to %s. It may take some time...',
+                config.remote_url)
+  RunCommand(['git', 'remote', 'add', '-f', 'origin', config.remote_url],
+             working_dir=config.temp_dir)
+
+  logging.debug('Configuring sparse checkout...')
+  RunCommand(['git', 'config', 'core.sparseCheckout', 'true'],
+             working_dir=config.temp_dir)
+  sparse_checkout_config_path = os.path.join(config.temp_dir, '.git', 'info',
+                                             'sparse-checkout')
+  with open(sparse_checkout_config_path, 'wb') as f:
+    f.write(dep_name)
+
+  logging.debug('Pulling changes...')
+  _, stderr = RunCommand(['git', 'pull', 'origin', 'master'],
+                         working_dir=config.temp_dir,
+                         ignore_exit_code=True)
+  if "Sparse checkout leaves no entry on working directory" in stderr:
+    # There are no such dependency in chromium third_party
+    raise DependencyNotFound(
+        "Dependency %s not found in chromium repo" % dep_name)
+
+  logging.debug('Switching to revision %s...', third_party_revision)
+  RunCommand(['git', 'checkout', third_party_revision],
+             working_dir=config.temp_dir)
+  return os.path.join(config.temp_dir, dep_name)
+
+
+def CopyDependency(dep_name, source_path, third_party_dir):
+  dest_path = os.path.join(third_party_dir, dep_name)
+  logging.debug('Copying dependency from %s to %s...', source_path, dest_path)
+  shutil.copytree(source_path, dest_path)
+
+
+def AppendToChromiumOwnedDependenciesList(dep_name, dep_file):
+  with open(dep_file, 'rb') as f:
+    data = json.load(f)
+    dep_list = data.get('dependencies', [])
+    dep_list.append(dep_name)
+    data['dependencies'] = dep_list
+
+  with open(dep_file, 'wb') as f:
+    json.dump(data, f, indent=2, sort_keys=True, separators=(',', ': '))
+
+
+def AddToGitIndex(dep_name, config):
+  logging.debug('Adding required changes to git index and commit set...')
+  dest_path = os.path.join(config.third_party_dir, dep_name)
+  RunCommand(['git', 'add', dest_path], working_dir=config.src_root)
+  RunCommand(['git', 'add', config.dependencies_file],
+             working_dir=config.src_root)
+
+
+def CheckinDependency(dep_name, config):
+  dep_path = CheckoutRequiredDependency(dep_name, config)
+  CopyDependency(dep_name, dep_path, config.third_party_dir)
+  AppendToChromiumOwnedDependenciesList(dep_name, config.dependencies_file)
+  AddToGitIndex(dep_name, config)
+  logging.info('Dependency checked into current working tree and added into\n'
+               'git index. You have to commit generated changes and\n'
+               'file the CL to finish adding the dependency')
+
+
+def DefaultConfig(temp_dir):
+  return Config(CHECKOUT_SRC_DIR, REMOTE_URL, temp_dir)
+
+
+def CheckinDependencyWithNewTempDir(dep_name):
+  temp_dir = tempfile.mkdtemp()
+  try:
+    logging.info('Using temp directory: %s', temp_dir)
+    config = DefaultConfig(temp_dir)
+    CheckinDependency(dep_name, config)
+  finally:
+    shutil.rmtree(temp_dir)
+
+
+def CheckDependencyNotCheckedIn(dep_name):
+  config = Config(CHECKOUT_SRC_DIR, REMOTE_URL, '')
+  with open(config.dependencies_file, 'rb') as f:
+    data = json.load(f)
+    dep_list = data.get('dependencies', [])
+    if dep_name in dep_list:
+      raise DependencyAlreadyCheckedIn("Dependency %s has been already checked "
+                                       "into WebRTC repo" % dep_name)
+  if dep_name in os.listdir(config.third_party_dir):
+    raise DependencyAlreadyCheckedIn("Directory for dependency %s already "
+                                     "exists in third_party" % dep_name)
+
+
+def main():
+  p = argparse.ArgumentParser()
+  p.add_argument('-d', '--dependency', required=True,
+                 help='Name of chromium dependency to check in.')
+  p.add_argument('--temp-dir',
+                 help='Temp working directory to use. By default the one '
+                      'provided via tempfile will be used')
+  p.add_argument('-v', '--verbose', action='store_true', default=False,
+                 help='Be extra verbose in printing of log messages.')
+  args = p.parse_args()
+
+  if args.verbose:
+    logging.basicConfig(level=logging.DEBUG)
+  else:
+    logging.basicConfig(level=logging.INFO)
+
+  CheckDependencyNotCheckedIn(args.dependency)
+
+  if args.temp_dir:
+    if not os.path.exists(args.temp_dir):
+      # Raise system error "No such file or directory"
+      raise OSError(
+          errno.ENOENT, os.strerror(errno.ENOENT), args.temp_dir)
+    config = DefaultConfig(args.temp_dir)
+    CheckinDependency(args.dependency, config)
+  else:
+    CheckinDependencyWithNewTempDir(args.dependency)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())
diff --git a/tools_webrtc/autoroller/roll_deps.py b/tools_webrtc/autoroller/roll_deps.py
index 4dce62eb3c..fd5db0a853 100755
--- a/tools_webrtc/autoroller/roll_deps.py
+++ b/tools_webrtc/autoroller/roll_deps.py
@@ -12,6 +12,7 @@
 import argparse
 import base64
 import collections
+import json
 import logging
 import os
 import re
@@ -414,12 +415,9 @@ def UpdateDepsFile(deps_filename, rev_update, changed_deps):
 
 
 def _LoadThirdPartyDepsAndFiles(filename):
-  third_party_deps = {}
   with open(filename, 'rb') as f:
-    deps_content = f.read()
-    global_scope = {}
-    exec (deps_content, global_scope, third_party_deps)
-  return third_party_deps.get('DEPS', [])
+    data = json.load(f)
+    return data.get('dependencies', [])
 
 
 def UpdateThirdPartyDeps(new_rev, dest_dir, source_dir,
@@ -613,10 +611,12 @@ def main():
   logging.debug('Commit message:\n%s', commit_msg)
 
   _CreateRollBranch(opts.dry_run)
+  third_party_chromium_deps_list = os.path.join(
+      CHECKOUT_SRC_DIR, 'THIRD_PARTY_CHROMIUM_DEPS.json')
   UpdateThirdPartyDeps(rev_update.new_third_party_rev,
                        os.path.join(CHECKOUT_SRC_DIR, 'third_party'),
                        cr_3p_repo,
-                       os.path.join(CHECKOUT_SRC_DIR, 'THIRD_PARTY_DEPS'))
+                       third_party_chromium_deps_list)
   UpdateDepsFile(deps_filename, rev_update, changed_deps)
   if _IsTreeClean():
     logging.info("No DEPS changes detected, skipping CL creation.")
diff --git a/tools_webrtc/autoroller/unittests/checkin_chromium_dep_test.py b/tools_webrtc/autoroller/unittests/checkin_chromium_dep_test.py
new file mode 100755
index 0000000000..98147fb916
--- /dev/null
+++ b/tools_webrtc/autoroller/unittests/checkin_chromium_dep_test.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python
+# Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+
+import json
+import os.path
+import shutil
+import sys
+import tempfile
+import unittest
+import distutils.dir_util
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir)
+sys.path.append(PARENT_DIR)
+from checkin_chromium_dep import Config, CheckinDependency, RunCommand, \
+  DependencyNotFound
+
+CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
+                                                 os.pardir))
+FAKE_REMOTE_TEMPLATE_ROOT = os.path.join(SCRIPT_DIR, 'testdata',
+                                         'checkin_chromium_dep', 'remote_root')
+FAKE_SOURCE_TEMPLATE_ROOT = os.path.join(SCRIPT_DIR, 'testdata',
+                                         'checkin_chromium_dep', 'src_root')
+
+
+class TestCheckInChromiumDep(unittest.TestCase):
+  def setUp(self):
+    self._temp_dir = tempfile.mkdtemp(prefix='webrtc_test_')
+    self._fake_chromium_repo = tempfile.mkdtemp(prefix='webrtc_test_')
+    self._fake_source_repo = tempfile.mkdtemp(prefix='webrtc_test_')
+
+    print("Temp dir: %s\n"
+          "Chromium third_party fake repo: %s\n"
+          "WebRTC source fake repo: %s" % (
+            self._temp_dir, self._fake_chromium_repo,
+            self._fake_source_repo))
+
+    self._fake_chromium_revision = TestCheckInChromiumDep._InitFakeChromiumRepo(
+        self._fake_chromium_repo)
+    TestCheckInChromiumDep._InitFakeSourceRepo(self._fake_source_repo,
+                                               self._fake_chromium_revision)
+
+  @staticmethod
+  def _InitFakeChromiumRepo(repo_dir):
+    RunCommand(['git', 'init'], working_dir=repo_dir)
+    distutils.dir_util.copy_tree(FAKE_REMOTE_TEMPLATE_ROOT, repo_dir)
+    RunCommand(['git', 'add', '-A', '.'], working_dir=repo_dir)
+    RunCommand(['git', 'commit', '-m', 'Init'],
+               working_dir=repo_dir)
+    stdout, _ = RunCommand(['git', 'rev-parse', 'HEAD'], working_dir=repo_dir)
+    return stdout.strip()
+
+  @staticmethod
+  def _InitFakeSourceRepo(repo_dir, chromium_third_party_revision):
+    RunCommand(['git', 'init'], working_dir=repo_dir)
+    # Copy repo template
+    distutils.dir_util.copy_tree(FAKE_SOURCE_TEMPLATE_ROOT, repo_dir)
+    # Set right chromium third_party revision in DEPS file
+    with open(os.path.join(repo_dir, 'DEPS'), 'rb') as f:
+      deps_content = f.read()
+    deps_content = deps_content % chromium_third_party_revision
+    with open(os.path.join(repo_dir, 'DEPS'), 'wb') as f:
+      f.write(deps_content)
+    # Commit all repo content
+    RunCommand(['git', 'add', '-A', '.'], working_dir=repo_dir)
+    RunCommand(['git', 'commit', '-m', 'Init'],
+               working_dir=repo_dir)
+
+  def tearDown(self):
+    shutil.rmtree(self._temp_dir)
+    shutil.rmtree(self._fake_chromium_repo)
+    shutil.rmtree(self._fake_source_repo)
+
+  def testCheckIn(self):
+    third_party_dir = os.path.join(self._fake_source_repo, 'third_party')
+
+    CheckinDependency('dep_bar',
+                      Config(
+                          self._fake_source_repo,
+                          'file://%s' % self._fake_chromium_repo,
+                          self._temp_dir))
+    third_party_deps_list_file = os.path.join(self._fake_source_repo,
+                                              'THIRD_PARTY_CHROMIUM_DEPS.json')
+    with open(third_party_deps_list_file, 'rb') as f:
+      deps_list = json.load(f).get('dependencies', [])
+
+    # New dependency appended to deps list file
+    self.assertIn('dep_foo', deps_list)
+    self.assertIn('dep_bar', deps_list)
+    # Only new dependency was appended
+    self.assertNotIn('dep_buzz', deps_list)
+    # New dependency was copied into source tree
+    self.assertIn('dep_bar', os.listdir(third_party_dir))
+    self.assertIn(
+        'source_file.js', os.listdir(os.path.join(third_party_dir, 'dep_bar')))
+    # Only new dependency was copied into source tree
+    self.assertNotIn('dep_buzz', os.listdir(third_party_dir))
+
+  def testCheckInNotExistingDep(self):
+    self.assertRaises(DependencyNotFound,
+                      CheckinDependency,
+                      'dep_missing',
+                      Config(self._fake_source_repo,
+                             'file://%s' % self._fake_chromium_repo,
+                             self._temp_dir))
+
+
+if __name__ == '__main__':
+  unittest.main()
diff --git a/tools_webrtc/autoroller/unittests/roll_deps_test.py b/tools_webrtc/autoroller/unittests/roll_deps_test.py
index 17535087fe..79485ca92f 100755
--- a/tools_webrtc/autoroller/unittests/roll_deps_test.py
+++ b/tools_webrtc/autoroller/unittests/roll_deps_test.py
@@ -68,7 +68,8 @@ class FakeCmd(object):
 class TestRollChromiumRevision(unittest.TestCase):
   def setUp(self):
     self._output_dir = tempfile.mkdtemp()
-    for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')):
+    test_data_dir = os.path.join(SCRIPT_DIR, 'testdata', 'roll_deps')
+    for test_file in glob.glob(os.path.join(test_data_dir, '*')):
       shutil.copy(test_file, self._output_dir)
     self._webrtc_depsfile = os.path.join(self._output_dir, 'DEPS')
     self._new_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.new')
diff --git a/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/remote_root/dep_bar/source_file.js b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/remote_root/dep_bar/source_file.js
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/remote_root/dep_bar/source_file.js
diff --git a/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/remote_root/dep_buzz/source_file.js b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/remote_root/dep_buzz/source_file.js
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/remote_root/dep_buzz/source_file.js
diff --git a/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/remote_root/dep_foo/source_file.js b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/remote_root/dep_foo/source_file.js
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/remote_root/dep_foo/source_file.js
diff --git a/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/src_root/DEPS b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/src_root/DEPS
new file mode 100644
index 0000000000..1da2499d88
--- /dev/null
+++ b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/src_root/DEPS
@@ -0,0 +1,6 @@
+# DEPS file for unit tests.
+
+vars = {
+  'chromium_third_party_revision': '%s',
+}
+
diff --git a/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/src_root/THIRD_PARTY_CHROMIUM_DEPS.json b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/src_root/THIRD_PARTY_CHROMIUM_DEPS.json
new file mode 100644
index 0000000000..9aad1b2c5b
--- /dev/null
+++ b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/src_root/THIRD_PARTY_CHROMIUM_DEPS.json
@@ -0,0 +1,14 @@
+{
+  "comment": [
+    "This file determines which parts of Chromium's third_party/ we copy.",
+    "",
+    "If you want to add new chromium owned dependency, run",
+    "./tools_webrtc/autoroller/checkin_chromium_dep.py -d <dep name>",
+    "",
+    "It will add specified dependency to third_party directory and will add it to this list",
+    "properly."
+  ],
+  "dependencies": [
+    "dep_foo"
+  ]
+}
diff --git a/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/src_root/THIRD_PARTY_WEBRTC_DEPS.json b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/src_root/THIRD_PARTY_WEBRTC_DEPS.json
new file mode 100644
index 0000000000..2f2190f910
--- /dev/null
+++ b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/src_root/THIRD_PARTY_WEBRTC_DEPS.json
@@ -0,0 +1,9 @@
+{
+  "comment": [
+    "This file contains list of third party dependencies owned by WebRTC directly,",
+    "e.g. they're not from Chromium's third party (those are tracked in ",
+    "THIRD_PARTY_CHROMIUM_DEPS.json)."
+  ],
+  "dependencies": [
+  ]
+}
\ No newline at end of file
diff --git a/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/src_root/third_party/dep_foo/source_file.js b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/src_root/third_party/dep_foo/source_file.js
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/tools_webrtc/autoroller/unittests/testdata/checkin_chromium_dep/src_root/third_party/dep_foo/source_file.js
diff --git a/tools_webrtc/autoroller/unittests/testdata/DEPS b/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS
index fcd65a5c9f..fcd65a5c9f 100644
--- a/tools_webrtc/autoroller/unittests/testdata/DEPS
+++ b/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS
diff --git a/tools_webrtc/autoroller/unittests/testdata/DEPS.chromium.new b/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS.chromium.new
index 41526c7b3f..41526c7b3f 100644
--- a/tools_webrtc/autoroller/unittests/testdata/DEPS.chromium.new
+++ b/tools_webrtc/autoroller/unittests/testdata/roll_deps/DEPS.chromium.new