diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/buildtools/README.jcef | 5 | ||||
| -rw-r--r-- | tools/buildtools/download_from_google_storage.py | 86 | ||||
| -rw-r--r-- | tools/buildtools/gsutil.py | 72 | ||||
| -rw-r--r-- | tools/buildtools/linux64/clang-format.sha1 | 2 | ||||
| -rw-r--r-- | tools/buildtools/mac/clang-format.arm64.sha1 | 1 | ||||
| -rw-r--r-- | tools/buildtools/mac/clang-format.sha1 | 1 | ||||
| -rw-r--r-- | tools/buildtools/mac/clang-format.x64.sha1 | 1 | ||||
| -rw-r--r-- | tools/buildtools/subprocess2.py | 37 | ||||
| -rw-r--r-- | tools/buildtools/win/clang-format.exe.sha1 | 2 | ||||
| -rwxr-xr-x | tools/fix_style.sh | 2 | ||||
| -rw-r--r-- | tools/make_distrib.bat | 4 | ||||
| -rwxr-xr-x | tools/make_distrib.sh | 4 |
12 files changed, 141 insertions, 76 deletions
diff --git a/tools/buildtools/README.jcef b/tools/buildtools/README.jcef index 18e2ad0..298b17d 100644 --- a/tools/buildtools/README.jcef +++ b/tools/buildtools/README.jcef @@ -1,8 +1,8 @@ Name: depot_tools Short Name: depot_tools URL: https://chromium.googlesource.com/chromium/tools/depot_tools.git -Date: 11 Jan 2020 -Revision: 7a8bf9489 +Date: 21 Sep 2022 +Revision: 08bb5c487f License: BSD License File: LICENSE @@ -11,4 +11,3 @@ Select tools extracted from depot_tools. Local Modifications: - Remove dependency on vpython. -- Update gsutil version to 4.68. diff --git a/tools/buildtools/download_from_google_storage.py b/tools/buildtools/download_from_google_storage.py index b627017..fb3fb98 100644 --- a/tools/buildtools/download_from_google_storage.py +++ b/tools/buildtools/download_from_google_storage.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. - """Download files from Google Storage based on SHA1 sums.""" from __future__ import print_function @@ -45,9 +44,10 @@ PLATFORM_MAPPING = { 'aix7': 'aix', } - -class FileNotFoundError(IOError): - pass +if sys.version_info.major == 2: + # pylint: disable=redefined-builtin + class FileNotFoundError(IOError): + pass class InvalidFileError(IOError): @@ -78,12 +78,11 @@ class Gsutil(object): RETRY_BASE_DELAY = 5.0 RETRY_DELAY_MULTIPLE = 1.3 - def __init__(self, path, boto_path=None, version='4.68'): + def __init__(self, path, boto_path=None): if not os.path.exists(path): raise FileNotFoundError('GSUtil not found in %s' % path) self.path = path self.boto_path = boto_path - self.version = version def get_sub_env(self): env = os.environ.copy() @@ -100,12 +99,12 @@ class Gsutil(object): return env def call(self, *args): - cmd = [sys.executable, self.path, '--force-version', self.version] + cmd = [sys.executable, self.path] cmd.extend(args) return subprocess2.call(cmd, env=self.get_sub_env()) def check_call(self, *args): - cmd = [sys.executable, self.path, '--force-version', self.version] + cmd = [sys.executable, self.path] cmd.extend(args) ((out, err), code) = subprocess2.communicate( cmd, @@ -129,7 +128,7 @@ class Gsutil(object): def check_call_with_retries(self, *args): delay = self.RETRY_BASE_DELAY - for i in xrange(self.MAX_TRIES): + for i in range(self.MAX_TRIES): code, out, err = self.check_call(*args) if not code or i == self.MAX_TRIES - 1: break @@ -254,40 +253,59 @@ def _downloader_worker_thread(thread_num, q, force, base_url, continue extract_dir = output_filename[:-len('.tar.gz')] if os.path.exists(output_filename) and not force: - if not extract or os.path.exists(extract_dir): - if get_sha1(output_filename) == input_sha1_sum: - continue - # Check if file exists. + skip = get_sha1(output_filename) == input_sha1_sum + if extract: + # Additional condition for extract: + # 1) extract_dir must exist + # 2) .tmp flag file mustn't exist + if not os.path.exists(extract_dir): + out_q.put('%d> Extract dir %s does not exist, re-downloading...' % + (thread_num, extract_dir)) + skip = False + # .tmp file is created just before extraction and removed just after + # extraction. If such file exists, it means the process was terminated + # mid-extraction and therefore needs to be extracted again. + elif os.path.exists(extract_dir + '.tmp'): + out_q.put('%d> Detected tmp flag file for %s, ' + 're-downloading...' % (thread_num, output_filename)) + skip = False + if skip: + continue + file_url = '%s/%s' % (base_url, input_sha1_sum) - (code, _, err) = gsutil.check_call('ls', file_url) + + try: + if delete: + os.remove(output_filename) # Delete the file if it exists already. + except OSError: + if os.path.exists(output_filename): + out_q.put('%d> Warning: deleting %s failed.' % ( + thread_num, output_filename)) + if verbose: + out_q.put('%d> Downloading %s@%s...' % ( + thread_num, output_filename, input_sha1_sum)) + code, _, err = gsutil.check_call('cp', file_url, output_filename) if code != 0: if code == 404: out_q.put('%d> File %s for %s does not exist, skipping.' % ( thread_num, file_url, output_filename)) ret_codes.put((1, 'File %s for %s does not exist.' % ( file_url, output_filename))) + elif code == 401: + out_q.put( + """%d> Failed to fetch file %s for %s due to unauthorized access, + skipping. Try running `gsutil.py config` and pass 0 if you don't + know your project id.""" % (thread_num, file_url, output_filename)) + ret_codes.put( + (1, 'Failed to fetch file %s for %s due to unauthorized access.' % + (file_url, output_filename))) else: # Other error, probably auth related (bad ~/.boto, etc). out_q.put('%d> Failed to fetch file %s for %s, skipping. [Err: %s]' % (thread_num, file_url, output_filename, err)) - ret_codes.put((1, 'Failed to fetch file %s for %s. [Err: %s]' % + ret_codes.put((code, 'Failed to fetch file %s for %s. [Err: %s]' % (file_url, output_filename, err))) continue - # Fetch the file. - if verbose: - out_q.put('%d> Downloading %s...' % (thread_num, output_filename)) - try: - if delete: - os.remove(output_filename) # Delete the file if it exists already. - except OSError: - if os.path.exists(output_filename): - out_q.put('%d> Warning: deleting %s failed.' % ( - thread_num, output_filename)) - code, _, err = gsutil.check_call('cp', file_url, output_filename) - if code != 0: - out_q.put('%d> %s' % (thread_num, err)) - ret_codes.put((code, err)) - continue remote_sha1 = get_sha1(output_filename) if remote_sha1 != input_sha1_sum: @@ -327,7 +345,9 @@ def _downloader_worker_thread(thread_num, q, force, base_url, out_q.put('%d> Extracting %d entries from %s to %s' % (thread_num, len(tar.getmembers()),output_filename, extract_dir)) - tar.extractall(path=dirname) + with open(extract_dir + '.tmp', 'a'): + tar.extractall(path=dirname) + os.remove(extract_dir + '.tmp') # Set executable bit. if sys.platform == 'cygwin': # Under cygwin, mark all files as executable. The executable flag in @@ -373,7 +393,7 @@ def _data_exists(input_sha1_sum, output_filename, extract): input_sha1_sum: Expected sha1 stored on disk. output_filename: The file to potentially download later. Its sha1 will be compared to input_sha1_sum. - extract: Wheather or not a downloaded file should be extracted. If the file + extract: Whether or not a downloaded file should be extracted. If the file is not extracted, this just compares the sha1 of the file. If the file is to be extracted, this only compares the sha1 of the target archive if the target directory already exists. The content of the target directory diff --git a/tools/buildtools/gsutil.py b/tools/buildtools/gsutil.py index 63847d2..cff91fa 100644 --- a/tools/buildtools/gsutil.py +++ b/tools/buildtools/gsutil.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -31,11 +31,11 @@ API_URL = 'https://www.googleapis.com/storage/v1/b/pub/o/' THIS_DIR = os.path.dirname(os.path.abspath(__file__)) DEFAULT_BIN_DIR = os.path.join(THIS_DIR, 'external_bin', 'gsutil') -DEFAULT_FALLBACK_GSUTIL = os.path.join( - THIS_DIR, 'third_party', 'gsutil', 'gsutil') IS_WINDOWS = os.name == 'nt' +VERSION = '4.68' + class InvalidGsutilError(Exception): pass @@ -79,7 +79,7 @@ def download_gsutil(version, target_dir): @contextlib.contextmanager def temporary_directory(base): - tmpdir = tempfile.mkdtemp(prefix='gsutil_py', dir=base) + tmpdir = tempfile.mkdtemp(prefix='t', dir=base) try: yield tmpdir finally: @@ -98,7 +98,20 @@ def ensure_gsutil(version, target, clean): return gsutil_bin if not os.path.exists(target): - os.makedirs(target) + try: + os.makedirs(target) + except FileExistsError: + # Another process is prepping workspace, so let's check if gsutil_bin is + # present. If after several checks it's still not, continue with + # downloading gsutil. + delay = 2 # base delay, in seconds + for _ in range(3): # make N attempts + # sleep first as it's not expected to have file ready just yet. + time.sleep(delay) + delay *= 1.5 # next delay increased by that factor + if os.path.isfile(gsutil_bin): + return gsutil_bin + with temporary_directory(target) as instance_dir: # Clean up if we're redownloading a corrupted gsutil. cleanup_path = os.path.join(instance_dir, 'clean') @@ -109,16 +122,12 @@ def ensure_gsutil(version, target, clean): if cleanup_path: shutil.rmtree(cleanup_path) - download_dir = os.path.join(instance_dir, 'download') + download_dir = os.path.join(instance_dir, 'd') target_zip_filename = download_gsutil(version, instance_dir) with zipfile.ZipFile(target_zip_filename, 'r') as target_zip: target_zip.extractall(download_dir) - try: - os.rename(download_dir, bin_dir) - except (OSError, IOError): - # Something else did this in parallel. - pass + shutil.move(download_dir, bin_dir) # Final check that the gsutil bin exists. This should never fail. if not os.path.isfile(gsutil_bin): raise InvalidGsutilError() @@ -129,29 +138,45 @@ def ensure_gsutil(version, target, clean): return gsutil_bin -def run_gsutil(force_version, fallback, target, args, clean=False): - if force_version: - gsutil_bin = ensure_gsutil(force_version, target, clean) - else: - gsutil_bin = fallback - disable_update = ['-o', 'GSUtil:software_update_check_period=0'] - cmd = [sys.executable, gsutil_bin] + disable_update + args - return subprocess.call(cmd) +def run_gsutil(target, args, clean=False): + gsutil_bin = ensure_gsutil(VERSION, target, clean) + args_opt = ['-o', 'GSUtil:software_update_check_period=0'] + if sys.platform == 'darwin': + # We are experiencing problems with multiprocessing on MacOS where gsutil.py + # may hang. + # This behavior is documented in gsutil codebase, and recommendation is to + # set GSUtil:parallel_process_count=1. + # https://github.com/GoogleCloudPlatform/gsutil/blob/06efc9dc23719fab4fd5fadb506d252bbd3fe0dd/gslib/command.py#L1331 + # https://github.com/GoogleCloudPlatform/gsutil/issues/1100 + args_opt.extend(['-o', 'GSUtil:parallel_process_count=1']) + assert sys.platform != 'cygwin' + cmd = [ + sys.executable, + gsutil_bin + ] + args_opt + args + return subprocess.call(cmd, shell=IS_WINDOWS) def parse_args(): bin_dir = os.environ.get('DEPOT_TOOLS_GSUTIL_BIN_DIR', DEFAULT_BIN_DIR) - parser = argparse.ArgumentParser() - parser.add_argument('--force-version', default='4.30') + # Help is disabled as it conflicts with gsutil -h, which controls headers. + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('--clean', action='store_true', help='Clear any existing gsutil package, forcing a new download.') - parser.add_argument('--fallback', default=DEFAULT_FALLBACK_GSUTIL) parser.add_argument('--target', default=bin_dir, help='The target directory to download/store a gsutil version in. ' '(default is %(default)s).') + + # These two args exist for backwards-compatibility but are no-ops. + parser.add_argument('--force-version', default=VERSION, + help='(deprecated, this flag has no effect)') + parser.add_argument('--fallback', + help='(deprecated, this flag has no effect)') + parser.add_argument('args', nargs=argparse.REMAINDER) args, extras = parser.parse_known_args() @@ -164,8 +189,7 @@ def parse_args(): def main(): args = parse_args() - return run_gsutil(args.force_version, args.fallback, args.target, args.args, - clean=args.clean) + return run_gsutil(args.target, args.args, clean=args.clean) if __name__ == '__main__': diff --git a/tools/buildtools/linux64/clang-format.sha1 b/tools/buildtools/linux64/clang-format.sha1 index e2b3199..6f3349c 100644 --- a/tools/buildtools/linux64/clang-format.sha1 +++ b/tools/buildtools/linux64/clang-format.sha1 @@ -1 +1 @@ -942fc8b1789144b8071d3fc03ff0fcbe1cf81ac8
\ No newline at end of file +dd736afb28430c9782750fc0fd5f0ed497399263
\ No newline at end of file diff --git a/tools/buildtools/mac/clang-format.arm64.sha1 b/tools/buildtools/mac/clang-format.arm64.sha1 new file mode 100644 index 0000000..46f9b3c --- /dev/null +++ b/tools/buildtools/mac/clang-format.arm64.sha1 @@ -0,0 +1 @@ +f1424c44ee758922823d6b37de43705955c99d7e
\ No newline at end of file diff --git a/tools/buildtools/mac/clang-format.sha1 b/tools/buildtools/mac/clang-format.sha1 deleted file mode 100644 index d32c626..0000000 --- a/tools/buildtools/mac/clang-format.sha1 +++ /dev/null @@ -1 +0,0 @@ -025ca7c75f37ef4a40f3a67d81ddd11d7d0cdb9b
\ No newline at end of file diff --git a/tools/buildtools/mac/clang-format.x64.sha1 b/tools/buildtools/mac/clang-format.x64.sha1 new file mode 100644 index 0000000..3a23ea2 --- /dev/null +++ b/tools/buildtools/mac/clang-format.x64.sha1 @@ -0,0 +1 @@ +a1b33be85faf2578f3101d7806e443e1c0949498
\ No newline at end of file diff --git a/tools/buildtools/subprocess2.py b/tools/buildtools/subprocess2.py index dea1a2d..7df49da 100644 --- a/tools/buildtools/subprocess2.py +++ b/tools/buildtools/subprocess2.py @@ -21,18 +21,18 @@ import threading if sys.version_info.major == 2: import Queue codecs.lookup('string-escape') + # Sends stdout or stderr to os.devnull. + DEVNULL = open(os.devnull, 'r+') else: import queue as Queue # pylint: disable=redefined-builtin basestring = (str, bytes) + DEVNULL = subprocess.DEVNULL # Constants forwarded from subprocess. PIPE = subprocess.PIPE STDOUT = subprocess.STDOUT -# Sends stdout or stderr to os.devnull. -VOID = open(os.devnull, 'w') -VOID_INPUT = open(os.devnull, 'r') class CalledProcessError(subprocess.CalledProcessError): @@ -48,7 +48,11 @@ class CalledProcessError(subprocess.CalledProcessError): ' '.join(self.cmd), self.returncode) if self.cwd: out += ' in ' + self.cwd - return '\n'.join(filter(None, (out, self.stdout, self.stderr))) + if self.stdout: + out += '\n' + self.stdout.decode('utf-8', 'ignore') + if self.stderr: + out += '\n' + self.stderr.decode('utf-8', 'ignore') + return out class CygwinRebaseError(CalledProcessError): @@ -103,7 +107,7 @@ class Popen(subprocess.Popen): in English. - Sets shell=True on windows by default. You can override this by forcing shell parameter to a value. - - Adds support for VOID to not buffer when not needed. + - Adds support for DEVNULL to not buffer when not needed. - Adds self.start property. Note: Popen() can throw OSError when cwd or args[0] doesn't exist. Translate @@ -121,6 +125,17 @@ class Popen(subprocess.Popen): env = get_english_env(kwargs.get('env')) if env: kwargs['env'] = env + if kwargs.get('env') is not None and sys.version_info.major != 2: + # Subprocess expects environment variables to be strings in Python 3. + def ensure_str(value): + if isinstance(value, bytes): + return value.decode() + return value + + kwargs['env'] = { + ensure_str(k): ensure_str(v) + for k, v in kwargs['env'].items() + } if kwargs.get('shell') is None: # *Sigh*: Windows needs shell=True, or else it won't search %PATH% for # the executable, but shell=True makes subprocess on Linux fail when it's @@ -182,14 +197,16 @@ def communicate(args, **kwargs): def call(args, **kwargs): """Emulates subprocess.call(). - Automatically convert stdout=PIPE or stderr=PIPE to VOID. + Automatically convert stdout=PIPE or stderr=PIPE to DEVNULL. In no case they can be returned since no code path raises subprocess2.CalledProcessError. + + Returns exit code. """ if kwargs.get('stdout') == PIPE: - kwargs['stdout'] = VOID + kwargs['stdout'] = DEVNULL if kwargs.get('stderr') == PIPE: - kwargs['stderr'] = VOID + kwargs['stderr'] = DEVNULL return communicate(args, **kwargs)[1] @@ -219,7 +236,7 @@ def capture(args, **kwargs): - Discards returncode. - Blocks stdin by default if not specified since no output will be visible. """ - kwargs.setdefault('stdin', VOID_INPUT) + kwargs.setdefault('stdin', DEVNULL) # Like check_output, deny the caller from using stdout arg. return communicate(args, stdout=PIPE, **kwargs)[0][0] @@ -235,7 +252,7 @@ def check_output(args, **kwargs): - Blocks stdin by default if not specified since no output will be visible. - As per doc, "The stdout argument is not allowed as it is used internally." """ - kwargs.setdefault('stdin', VOID_INPUT) + kwargs.setdefault('stdin', DEVNULL) if 'stdout' in kwargs: raise ValueError('stdout argument not allowed, it would be overridden.') return check_call_out(args, stdout=PIPE, **kwargs)[0] diff --git a/tools/buildtools/win/clang-format.exe.sha1 b/tools/buildtools/win/clang-format.exe.sha1 index d31c76f..c7efe6f 100644 --- a/tools/buildtools/win/clang-format.exe.sha1 +++ b/tools/buildtools/win/clang-format.exe.sha1 @@ -1 +1 @@ -b5f5d8d5f8a8fcd2edb5b6cae37c0dc3e129c945
\ No newline at end of file +66882fadbf9e99cc00b8677d8c1e7e8b3cfdf4fe
\ No newline at end of file diff --git a/tools/fix_style.sh b/tools/fix_style.sh index c466ab0..042c01f 100755 --- a/tools/fix_style.sh +++ b/tools/fix_style.sh @@ -1,2 +1,2 @@ #!/bin/sh -python tools/fix_style.py $@ +python3 tools/fix_style.py $@ diff --git a/tools/make_distrib.bat b/tools/make_distrib.bat index 5deb8c9..f4863b1 100644 --- a/tools/make_distrib.bat +++ b/tools/make_distrib.bat @@ -75,8 +75,10 @@ copy %OUT_BINARY_PATH%\libGLESv2.dll %DISTRIB_LIB_PATH% copy %OUT_BINARY_PATH%\natives_blob.bin %DISTRIB_LIB_PATH%
copy %OUT_BINARY_PATH%\snapshot_blob.bin %DISTRIB_LIB_PATH%
copy %OUT_BINARY_PATH%\v8_context_snapshot.bin %DISTRIB_LIB_PATH%
+copy %OUT_BINARY_PATH%\vk_swiftshader.dll %DISTRIB_LIB_PATH%
+copy %OUT_BINARY_PATH%\vk_swiftshader_icd.json %DISTRIB_LIB_PATH%
+copy %OUT_BINARY_PATH%\vulkan-1.dll %DISTRIB_LIB_PATH%
xcopy /sfy %OUT_BINARY_PATH%\*.pak %DISTRIB_LIB_PATH%
-xcopy /sfy %OUT_BINARY_PATH%\swiftshader\* %DISTRIB_LIB_PATH%\swiftshader\
:: Copy documentation to the docs directory.
xcopy /sfy %OUT_DOCS_PATH%\* %DISTRIB_DOCS_PATH%\
diff --git a/tools/make_distrib.sh b/tools/make_distrib.sh index 1b66aba..909cf35 100755 --- a/tools/make_distrib.sh +++ b/tools/make_distrib.sh @@ -98,7 +98,9 @@ else cp -f "$OUT_BINARY_PATH"/v8_context_snapshot.bin "$DISTRIB_LIB_PATH" cp -f "$OUT_BINARY_PATH"/*.pak "$DISTRIB_LIB_PATH" cp -rf "$OUT_BINARY_PATH"/locales/ "$DISTRIB_LIB_PATH" - cp -rf "$OUT_BINARY_PATH"/swiftshader/ "$DISTRIB_LIB_PATH" + cp -f "$OUT_BINARY_PATH"/libvk_swiftshader.so "$DISTRIB_LIB_PATH" + cp -f "$OUT_BINARY_PATH"/vk_swiftshader_icd.json "$DISTRIB_LIB_PATH" + cp -f "$OUT_BINARY_PATH"/libvulkan.so.1 "$DISTRIB_LIB_PATH" fi fi |