Download full Chromium checkouts by default

This changes sync_chromium.py to download a full Chromium
checkout instead of one with no history. It has been noticed
that the download of the no-history checkout is very slow, even
when on high-speed internet connections, due to current limitations
in the Git backend serving these clones.
Switching to a full checkout is faster, but requires more bandwidth
and disk space.

To keep the old behavior, users must set the CHROMIUM_NO_HISTORY
environment variable to 1.

Using a full checkout also enables the use of the Chromium
infrastructure teams' Git cache functionality, that speeds up
the initial download and also heavily reduces the traffic when
setting up multiple checkouts on the same machine.
This is not enabled by default, but is supported if the user is
setting the cache_dir variable in his checkout's .gclient file to
point at a directory on local disk.

BUG=3882
TESTED=
* Ran gclient sync and verified chromium/src now contained a Git
repo with full history.
* Tested rolling chromium_revision in DEPS forward + sync.
* Tested rolling it back again + sync.
* Tested with an existing no-history checkout:
  CHROMIUM_NO_HISTORY=1 gclient sync
  No change was performed.
* Tested with a .gclient that had cache_dir configured.
* Verified error message is displayed when .gclient has cache_dir
  configured and CHROMIUM_NO_HISTORY=1.

R=iannucci@chromium.org

Review URL: https://webrtc-codereview.appspot.com/22869004

git-svn-id: http://webrtc.googlecode.com/svn/trunk@7506 4adac7df-926f-26a2-2b94-8c16560cd09d

1 files changed, 53 insertions, 15 deletions

diff --git a/sync_chromium.py b/sync_chromium.py
index afcc281b40..9d2c05b9d9 100755
--- a/sync_chromium.py
+++ b/sync_chromium.py
@@ -7,6 +7,21 @@
 # in the file PATENTS.  All contributing project authors may
 # be found in the AUTHORS file in the root of the source tree.
 
+"""Script to download a Chromium checkout into the workspace.
+
+The script downloads a full Chromium Git clone and its DEPS.
+
+The following environment variable can be used to alter the behavior:
+* CHROMIUM_NO_HISTORY - If set to 1, a Git checkout with no history will be
+  downloaded. This is consumes less bandwidth and disk space but is known to be
+  slower in general if you have a high-speed connection.
+
+After a successful sync has completed, a .last_sync_chromium file is written to
+the chromium directory. While it exists, no more gclient sync operations will be
+performed until the --target-revision changes or the SCRIPT_VERSION constant is
+incremented. The file can be removed manually to force a new sync.
+"""
+
 import argparse
 import os
 import subprocess
@@ -14,20 +29,28 @@ import sys
 
 # Bump this whenever the algorithm changes and you need bots/devs to re-sync,
 # ignoring the .last_sync_chromium file
-SCRIPT_VERSION = 2
+SCRIPT_VERSION = 3
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+CHROMIUM_NO_HISTORY = 'CHROMIUM_NO_HISTORY'
 
-
-def get_target_os_list():
+def _parse_gclient_dict():
+  gclient_dict = {}
   try:
     main_gclient = os.path.join(os.path.dirname(ROOT_DIR), '.gclient')
-    config_dict = {}
     with open(main_gclient, 'rb') as deps_content:
-      exec(deps_content, config_dict)
-    return ','.join(config_dict.get('target_os', []))
+      exec(deps_content, gclient_dict)
   except Exception as e:
-    print >> sys.stderr, "error while parsing .gclient:", e
+    print >> sys.stderr, 'error while parsing .gclient:', e
+  return gclient_dict
+
+
+def get_cache_dir():
+  return _parse_gclient_dict().get('cache_dir')
+
+
+def get_target_os_list():
+  return ','.join(_parse_gclient_dict().get('target_os', []))
 
 
 def main():
@@ -55,7 +78,7 @@ def main():
   if os.path.exists(flag_file):
     with open(flag_file, 'r') as f:
       if f.read() == flag_file_content:
-        print "Chromium already up to date:", opts.target_revision
+        print 'Chromium already up to date: ', opts.target_revision
         return 0
     os.unlink(flag_file)
 
@@ -67,6 +90,7 @@ def main():
   ]
 
   if os.environ.get('CHROME_HEADLESS') == '1':
+    # Running on a buildbot.
     args.append('-vvv')
 
     if sys.platform.startswith('win'):
@@ -74,23 +98,37 @@ def main():
                                 'b', 'git-cache')
     else:
       cache_path = '/b/git-cache'
-
+  else:
+    # Support developers setting the cache_dir in .gclient.
+    cache_path = get_cache_dir()
+
+  # Allow for users with poor internet connections to download a Git clone
+  # without history (saves several gigs but is generally slower and doesn't work
+  # with the Git cache).
+  if os.environ.get(CHROMIUM_NO_HISTORY) == '1':
+    if cache_path:
+      print >> sys.stderr, (
+          'You cannot use "no-history" mode for syncing Chrome (i.e. set the '
+          '%s environment variable to 1) when you have cache_dir configured in '
+          'your .gclient.' % CHROMIUM_NO_HISTORY)
+      return 1
+    args.append('--no-history')
+    gclient_entries_file = os.path.join(opts.chromium_dir, '.gclient_entries')
+  else:
+    # Write a temporary .gclient file that has the cache_dir variable added.
     gclientfile = os.path.join(opts.chromium_dir, '.gclient')
     with open(gclientfile, 'rb') as spec:
       spec = spec.read().splitlines()
       spec[-1] = 'cache_dir = %r' % (cache_path,)
-    with open(gclientfile + '.bot', 'wb') as f:
+    with open(gclientfile + '.tmp', 'wb') as f:
       f.write('\n'.join(spec))
 
     args += [
-      '--gclientfile', '.gclient.bot',
+      '--gclientfile', '.gclient.tmp',
       '--delete_unversioned_trees', '--reset', '--upstream'
     ]
     gclient_entries_file = os.path.join(opts.chromium_dir,
-                                        '.gclient.bot_entries')
-  else:
-    args.append('--no-history')
-    gclient_entries_file = os.path.join(opts.chromium_dir, '.gclient_entries')
+                                        '.gclient.tmp_entries')
 
   # To avoid gclient sync problems when DEPS entries have been removed we must
   # wipe the gclient's entries file that contains cached URLs for all DEPS.