diff options
Diffstat (limited to 'utils/git-sync-deps')
| -rwxr-xr-x | utils/git-sync-deps | 71 |
1 files changed, 55 insertions, 16 deletions
diff --git a/utils/git-sync-deps b/utils/git-sync-deps index 7a7e606f..43548fe4 100755 --- a/utils/git-sync-deps +++ b/utils/git-sync-deps @@ -30,6 +30,13 @@ """Parse a DEPS file and git checkout all of the dependencies. Args: + --treeless Clone repos without trees. This is the fast option, useful + when you only need a single commit, like on a build machine. + Defers getting objects until checkout time. + Otherwise clones without blobs. + Requires git 2.20 or later. + https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ + An optional list of deps_os values. Environment Variables: @@ -59,12 +66,14 @@ import sys import threading from builtins import bytes - def git_executable(): """Find the git executable. Returns: - A string suitable for passing to subprocess functions, or None. + A triple: + A string suitable for passing to subprocess functions, or None. + The major version number + The minor version number """ envgit = os.environ.get('GIT_EXECUTABLE') searchlist = ['git', 'git.exe', 'git.bat'] @@ -72,12 +81,21 @@ def git_executable(): searchlist.insert(0, envgit) with open(os.devnull, 'w') as devnull: for git in searchlist: + major=None + minor=None try: - subprocess.call([git, '--version'], stdout=devnull) + version_info = subprocess.check_output([git, '--version']).decode('utf-8') + match = re.search("^git version (\d+)\.(\d+)",version_info) + print("Using {}".format(version_info)) + if match: + major = int(match.group(1)) + minor = int(match.group(2)) + else: + continue except (OSError,): continue - return git - return None + return (git,major,minor) + return (None,0,0) DEFAULT_DEPS_PATH = os.path.normpath( @@ -97,6 +115,9 @@ def usage(deps_file_path = None): sys.stderr.write(__doc__) +def looks_like_raw_commit(commit): + return re.match('^[a-f0-9]{40}$', commit) is not None + def git_repository_sync_is_disabled(git, directory): try: disable = subprocess.check_output( @@ -125,14 +146,14 @@ def is_git_toplevel(git, directory): def status(directory, checkoutable): def truncate(s, length): - return s if len(s) <= length else s[:(length - 3)] + '...' + return s if len(s) <= length else '...' + s[-(length - 3):] dlen = 36 directory = truncate(directory, dlen) checkoutable = truncate(checkoutable, 40) sys.stdout.write('%-*s @ %s\n' % (dlen, directory, checkoutable)) -def git_checkout_to_directory(git, repo, checkoutable, directory, verbose): +def git_checkout_to_directory(git, repo, checkoutable, directory, verbose, treeless): """Checkout (and clone if needed) a Git repository. Args: @@ -147,13 +168,22 @@ def git_checkout_to_directory(git, repo, checkoutable, directory, verbose): directory (string) the path into which the repository should be checked out. - verbose (boolean) + verbose (boolean): emit status info to stdout + + treeless (boolean): when true, clone without any trees. Raises an exception if any calls to git fail. """ if not os.path.isdir(directory): + # Use blobless or treeless checkouts for faster downloads. + # This defers some work to checkout time. + # https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/ + filter = ['--filter=tree:0'] if treeless else ['--filter=blob:none'] + # If the thing to check out looks like a tag (and not like a commit), + # then limit the checkout to that branch. + branch = [] if looks_like_raw_commit(checkoutable) else ['--branch={}'.format(checkoutable)] subprocess.check_call( - [git, 'clone', '--quiet', repo, directory]) + [git, 'clone', '--quiet', '--single-branch'] + filter + branch + [repo, directory]) if not is_git_toplevel(git, directory): # if the directory exists, but isn't a git repo, you will modify @@ -200,7 +230,7 @@ def parse_file_to_dict(path): return dictionary -def git_sync_deps(deps_file_path, command_line_os_requests, verbose): +def git_sync_deps(deps_file_path, command_line_os_requests, verbose, treeless): """Grab dependencies, with optional platform support. Args: @@ -210,11 +240,20 @@ def git_sync_deps(deps_file_path, command_line_os_requests, verbose): List of strings that should each be a key in the deps_os dictionary in the DEPS file. + verbose (boolean): emit status info to stdout + + treeless (boolean): when true, clone as treeless instead of blobless + Raises git Exceptions. """ - git = git_executable() + (git,git_major,git_minor) = git_executable() assert git + # --filter=tree:0 is available in git 2.20 and later + if (git_major,git_minor) < (2,20): + print("disabling --treeless: git is older than v2.20") + treeless = False + deps_file_directory = os.path.dirname(deps_file_path) deps_file = parse_file_to_dict(deps_file_path) dependencies = deps_file['deps'].copy() @@ -232,6 +271,7 @@ def git_sync_deps(deps_file_path, command_line_os_requests, verbose): if directory.startswith(other_dir + '/'): raise Exception('%r is parent of %r' % (other_dir, directory)) list_of_arg_lists = [] + print("deps {}".format(dependencies)) for directory in sorted(dependencies): if '@' in dependencies[directory]: repo, checkoutable = dependencies[directory].split('@', 1) @@ -241,7 +281,7 @@ def git_sync_deps(deps_file_path, command_line_os_requests, verbose): relative_directory = os.path.join(deps_file_directory, directory) list_of_arg_lists.append( - (git, repo, checkoutable, relative_directory, verbose)) + (git, repo, checkoutable, relative_directory, verbose, treeless)) multithread(git_checkout_to_directory, list_of_arg_lists) @@ -266,15 +306,14 @@ def multithread(function, list_of_arg_lists): def main(argv): deps_file_path = os.environ.get('GIT_SYNC_DEPS_PATH', DEFAULT_DEPS_PATH) verbose = not bool(os.environ.get('GIT_SYNC_DEPS_QUIET', False)) + treeless = bool("--treeless" in argv) + argv = [x for x in argv if x != "--treeless"] if '--help' in argv or '-h' in argv: usage(deps_file_path) return 1 - git_sync_deps(deps_file_path, argv, verbose) - # subprocess.check_call( - # [sys.executable, - # os.path.join(os.path.dirname(deps_file_path), 'bin', 'fetch-gn')]) + git_sync_deps(deps_file_path, argv, verbose, treeless) return 0 |