diff options
Diffstat (limited to 'catapult/common/bin')
| -rwxr-xr-x | catapult/common/bin/update_chrome_reference_binaries | 229 | ||||
| -rwxr-xr-x | catapult/common/bin/update_chrome_reference_binaries.py | 393 |
2 files changed, 393 insertions, 229 deletions
diff --git a/catapult/common/bin/update_chrome_reference_binaries b/catapult/common/bin/update_chrome_reference_binaries deleted file mode 100755 index e148c747..00000000 --- a/catapult/common/bin/update_chrome_reference_binaries +++ /dev/null @@ -1,229 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Updates the Chrome reference builds. - -Usage: - $ /path/to/update_reference_build.py - $ git commit -a - $ git cl upload -""" - -import collections -import logging -import os -import shutil -import subprocess -import sys -import tempfile -import urllib2 -import zipfile - -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'py_utils')) - -from py_utils import cloud_storage -from dependency_manager import base_config - - -def BuildNotFoundError(error_string): - raise ValueError(error_string) - - -_CHROME_BINARIES_CONFIG = os.path.join( - os.path.dirname(os.path.abspath(__file__)), '..', '..', 'common', - 'py_utils', 'py_utils', 'chrome_binaries.json') - -CHROME_GS_BUCKET = 'chrome-unsigned' - - -# Remove a platform name from this list to disable updating it. -# Add one to enable updating it. (Must also update _PLATFORM_MAP.) -_PLATFORMS_TO_UPDATE = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64', - 'android_k_armeabi-v7a', 'android_l_arm64-v8a', - 'android_l_armeabi-v7a', 'android_n_armeabi-v7a', - 'android_n_arm64-v8a'] - -# Remove a channel name from this list to disable updating it. -# Add one to enable updating it. -_CHANNELS_TO_UPDATE = ['stable', 'canary', 'dev'] - - -# Omaha is Chrome's autoupdate server. It reports the current versions used -# by each platform on each channel. -_OMAHA_PLATFORMS = { 'stable': ['mac', 'linux', 'win', 'android'], - 'dev': ['linux'], 'canary': ['mac', 'win']} - - -# All of the information we need to update each platform. -# omaha: name omaha uses for the platforms. -# zip_name: name of the zip file to be retrieved from cloud storage. -# gs_build: name of the Chrome build platform used in cloud storage. -# destination: Name of the folder to download the reference build to. -UpdateInfo = collections.namedtuple('UpdateInfo', - 'omaha, gs_folder, gs_build, zip_name') -_PLATFORM_MAP = {'mac_x86_64': UpdateInfo(omaha='mac', - gs_folder='desktop-*', - gs_build='mac64', - zip_name='chrome-mac.zip'), - 'win_x86': UpdateInfo(omaha='win', - gs_folder='desktop-*', - gs_build='win-clang', - zip_name='chrome-win-clang.zip'), - 'win_AMD64': UpdateInfo(omaha='win', - gs_folder='desktop-*', - gs_build='win64-clang', - zip_name='chrome-win64-clang.zip'), - 'linux_x86_64': UpdateInfo(omaha='linux', - gs_folder='desktop-*', - gs_build='linux64', - zip_name='chrome-linux64.zip'), - 'android_k_armeabi-v7a': UpdateInfo(omaha='android', - gs_folder='android-*', - gs_build='arm', - zip_name='Chrome.apk'), - 'android_l_arm64-v8a': UpdateInfo(omaha='android', - gs_folder='android-*', - gs_build='arm_64', - zip_name='ChromeModern.apk'), - 'android_l_armeabi-v7a': UpdateInfo(omaha='android', - gs_folder='android-*', - gs_build='arm', - zip_name='Chrome.apk'), - 'android_n_armeabi-v7a': UpdateInfo(omaha='android', - gs_folder='android-*', - gs_build='arm', - zip_name='Monochrome.apk'), - 'android_n_arm64-v8a': UpdateInfo(omaha='android', - gs_folder='android-*', - gs_build='arm_64', - zip_name='Monochrome.apk'), - -} - - -def _ChannelVersionsMap(channel): - rows = _OmahaReportVersionInfo(channel) - omaha_versions_map = _OmahaVersionsMap(rows, channel) - channel_versions_map = {} - for platform in _PLATFORMS_TO_UPDATE: - omaha_platform = _PLATFORM_MAP[platform].omaha - if omaha_platform in omaha_versions_map: - channel_versions_map[platform] = omaha_versions_map[omaha_platform] - return channel_versions_map - - -def _OmahaReportVersionInfo(channel): - url ='https://omahaproxy.appspot.com/all?channel=%s' % channel - lines = urllib2.urlopen(url).readlines() - return [l.split(',') for l in lines] - - -def _OmahaVersionsMap(rows, channel): - platforms = _OMAHA_PLATFORMS.get(channel, []) - if (len(rows) < 1 or - not rows[0][0:3] == ['os', 'channel', 'current_version']): - raise ValueError( - 'Omaha report is not in the expected form: %s.' % rows) - versions_map = {} - for row in rows[1:]: - if row[1] != channel: - raise ValueError( - 'Omaha report contains a line with the channel %s' % row[1]) - if row[0] in platforms: - versions_map[row[0]] = row[2] - logging.warn('versions map: %s' % versions_map) - if not all(platform in versions_map for platform in platforms): - raise ValueError( - 'Omaha report did not contain all desired platforms for channel %s' % channel) - return versions_map - - -def _QueuePlatformUpdate(platform, version, config, channel): - """ platform: the name of the platform for the browser to - be downloaded & updated from cloud storage. """ - platform_info = _PLATFORM_MAP[platform] - filename = platform_info.zip_name - # remote_path example: desktop-*/30.0.1595.0/precise32/chrome-precise32.zip - remote_path = '%s/%s/%s/%s' % ( - platform_info.gs_folder, version, platform_info.gs_build, filename) - if not cloud_storage.Exists(CHROME_GS_BUCKET, remote_path): - cloud_storage_path = 'gs://%s/%s' % (CHROME_GS_BUCKET, remote_path) - raise BuildNotFoundError( - 'Failed to find %s build for version %s at path %s.' % ( - platform, version, cloud_storage_path)) - reference_builds_folder = os.path.join( - os.path.dirname(os.path.abspath(__file__)), 'chrome_telemetry_build', - 'reference_builds', channel) - if not os.path.exists(reference_builds_folder): - os.makedirs(reference_builds_folder) - local_dest_path = os.path.join(reference_builds_folder, filename) - cloud_storage.Get(CHROME_GS_BUCKET, remote_path, local_dest_path) - _ModifyBuildIfNeeded(local_dest_path, platform) - config.AddCloudStorageDependencyUpdateJob( - 'chrome_%s' % channel, platform, local_dest_path, version=version, - execute_job=False) - - -def _ModifyBuildIfNeeded(location, platform): - """Hook to modify the build before saving it for Telemetry to use. - - This can be used to remove various utilities that cause noise in a - test environment. Right now, it is just used to remove Keystone, - which is a tool used to autoupdate Chrome. - """ - if platform == 'mac_x86_64': - _RemoveKeystoneFromBuild(location) - return - - if 'mac' in platform: - raise NotImplementedError( - 'Platform <%s> sounds like it is an OSX version. If so, we may need to ' - 'remove Keystone from it per crbug.com/932615. Please edit this script' - ' and teach it what needs to be done :).') - - -def _RemoveKeystoneFromBuild(location): - """Removes the Keystone autoupdate binary from the chrome mac zipfile.""" - logging.info('Removing keystone from mac build at %s' % location) - temp_folder = tempfile.mkdtemp(prefix='RemoveKeystoneFromBuild') - try: - subprocess.check_call(['unzip', '-q', location, '-d', temp_folder]) - keystone_folder = os.path.join( - temp_folder, 'chrome-mac', 'Google Chrome.app', 'Contents', - 'Frameworks', 'Google Chrome Framework.framework', 'Frameworks', - 'KeystoneRegistration.framework') - shutil.rmtree(keystone_folder) - os.remove(location) - subprocess.check_call(['zip', '--quiet', '--recurse-paths', '--symlinks', - location, 'chrome-mac'], - cwd=temp_folder) - finally: - shutil.rmtree(temp_folder) - - -def UpdateBuilds(): - config = base_config.BaseConfig(_CHROME_BINARIES_CONFIG, writable=True) - for channel in _CHANNELS_TO_UPDATE: - channel_versions_map = _ChannelVersionsMap(channel) - for platform in channel_versions_map: - print 'Downloading Chrome (%s channel) on %s' % (channel, platform) - current_version = config.GetVersion('chrome_%s' % channel, platform) - channel_version = channel_versions_map.get(platform) - print 'current: %s, channel: %s' % (current_version, channel_version) - if current_version and current_version == channel_version: - continue - _QueuePlatformUpdate(platform, channel_version, config, channel) - - print 'Updating chrome builds with downloaded binaries' - config.ExecuteUpdateJobs(force=True) - - -def main(): - logging.getLogger().setLevel(logging.DEBUG) - UpdateBuilds() - -if __name__ == '__main__': - main() diff --git a/catapult/common/bin/update_chrome_reference_binaries.py b/catapult/common/bin/update_chrome_reference_binaries.py new file mode 100755 index 00000000..86a1d7fe --- /dev/null +++ b/catapult/common/bin/update_chrome_reference_binaries.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python +# +# Copyright 2013 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Updates the Chrome reference builds. + +Usage: + $ /path/to/update_reference_build.py + $ git commit -a + $ git cl upload +""" + +import argparse +import collections +import logging +import os +import shutil +import subprocess +import sys +import tempfile +import urllib2 +import zipfile + +sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'py_utils')) + +from py_utils import cloud_storage +from dependency_manager import base_config + + +_CHROME_BINARIES_CONFIG = os.path.join( + os.path.dirname(os.path.abspath(__file__)), '..', '..', 'common', + 'py_utils', 'py_utils', 'chrome_binaries.json') + +_CHROME_GS_BUCKET = 'chrome-unsigned' +_CHROMIUM_GS_BUCKET = 'chromium-browser-snapshots' + +# How many commit positions to search below and above omaha branch position to +# find closest chromium build snapshot. The value 10 is chosen because it looks +# more than sufficient from manual inspection of the bucket. +_CHROMIUM_SNAPSHOT_SEARCH_WINDOW = 10 + +# Remove a platform name from this list to disable updating it. +# Add one to enable updating it. (Must also update _PLATFORM_MAP.) +_PLATFORMS_TO_UPDATE = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64', + 'android_k_armeabi-v7a', 'android_l_arm64-v8a', + 'android_l_armeabi-v7a', 'android_n_armeabi-v7a', + 'android_n_arm64-v8a', 'android_n_bundle_armeabi-v7a', + 'android_n_bundle_arm64-v8a'] + +# Add platforms here if you also want to update chromium binary for it. +# Must add chromium_info for it in _PLATFORM_MAP. +_CHROMIUM_PLATFORMS = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64'] + +# Remove a channel name from this list to disable updating it. +# Add one to enable updating it. +_CHANNELS_TO_UPDATE = ['stable', 'canary', 'dev'] + + +# Omaha is Chrome's autoupdate server. It reports the current versions used +# by each platform on each channel. +_OMAHA_PLATFORMS = { 'stable': ['mac', 'linux', 'win', 'android'], + 'dev': ['linux'], 'canary': ['mac', 'win']} + + +# All of the information we need to update each platform. +# omaha: name omaha uses for the platforms. +# zip_name: name of the zip file to be retrieved from cloud storage. +# gs_build: name of the Chrome build platform used in cloud storage. +# chromium_info: information needed to update chromium (optional). +# destination: Name of the folder to download the reference build to. +UpdateInfo = collections.namedtuple('UpdateInfo', + 'omaha, gs_folder, gs_build, chromium_info, zip_name') +# build_dir: name of the build directory in _CHROMIUM_GS_BUCKET. +# zip_name: name of the zip file to be retrieved from cloud storage. +ChromiumInfo = collections.namedtuple('ChromiumInfo', 'build_dir, zip_name') +_PLATFORM_MAP = {'mac_x86_64': UpdateInfo( + omaha='mac', + gs_folder='desktop-*', + gs_build='mac64', + chromium_info=ChromiumInfo( + build_dir='Mac', + zip_name='chrome-mac.zip'), + zip_name='chrome-mac.zip'), + 'win_x86': UpdateInfo( + omaha='win', + gs_folder='desktop-*', + gs_build='win-clang', + chromium_info=ChromiumInfo( + build_dir='Win', + zip_name='chrome-win.zip'), + zip_name='chrome-win-clang.zip'), + 'win_AMD64': UpdateInfo( + omaha='win', + gs_folder='desktop-*', + gs_build='win64-clang', + chromium_info=ChromiumInfo( + build_dir='Win_x64', + zip_name='chrome-win.zip'), + zip_name='chrome-win64-clang.zip'), + 'linux_x86_64': UpdateInfo( + omaha='linux', + gs_folder='desktop-*', + gs_build='linux64', + chromium_info=ChromiumInfo( + build_dir='Linux_x64', + zip_name='chrome-linux.zip'), + zip_name='chrome-linux64.zip'), + 'android_k_armeabi-v7a': UpdateInfo( + omaha='android', + gs_folder='android-*', + gs_build='arm', + chromium_info=None, + zip_name='Chrome.apk'), + 'android_l_arm64-v8a': UpdateInfo( + omaha='android', + gs_folder='android-*', + gs_build='arm_64', + chromium_info=None, + zip_name='ChromeModern.apk'), + 'android_l_armeabi-v7a': UpdateInfo( + omaha='android', + gs_folder='android-*', + gs_build='arm', + chromium_info=None, + zip_name='Chrome.apk'), + 'android_n_armeabi-v7a': UpdateInfo( + omaha='android', + gs_folder='android-*', + gs_build='arm', + chromium_info=None, + zip_name='Monochrome.apk'), + 'android_n_arm64-v8a': UpdateInfo( + omaha='android', + gs_folder='android-*', + gs_build='arm_64', + chromium_info=None, + zip_name='Monochrome.apk'), + 'android_n_bundle_armeabi-v7a': UpdateInfo( + omaha='android', + gs_folder='android-*', + gs_build='arm', + chromium_info=None, + zip_name='Monochrome.apks'), + 'android_n_bundle_arm64-v8a': UpdateInfo( + omaha='android', + gs_folder='android-*', + gs_build='arm_64', + chromium_info=None, + zip_name='Monochrome.apks') + +} + + +VersionInfo = collections.namedtuple('VersionInfo', + 'version, branch_base_position') + + +def _ChannelVersionsMap(channel): + rows = _OmahaReportVersionInfo(channel) + omaha_versions_map = _OmahaVersionsMap(rows, channel) + channel_versions_map = {} + for platform in _PLATFORMS_TO_UPDATE: + omaha_platform = _PLATFORM_MAP[platform].omaha + if omaha_platform in omaha_versions_map: + channel_versions_map[platform] = omaha_versions_map[omaha_platform] + return channel_versions_map + + +def _OmahaReportVersionInfo(channel): + url ='https://omahaproxy.appspot.com/all?channel=%s' % channel + lines = urllib2.urlopen(url).readlines() + return [l.split(',') for l in lines] + + +def _OmahaVersionsMap(rows, channel): + platforms = _OMAHA_PLATFORMS.get(channel, []) + if (len(rows) < 1 or + rows[0][0:3] != ['os', 'channel', 'current_version'] or + rows[0][7] != 'branch_base_position'): + raise ValueError( + 'Omaha report is not in the expected form: %s.' % rows) + versions_map = {} + for row in rows[1:]: + if row[1] != channel: + raise ValueError( + 'Omaha report contains a line with the channel %s' % row[1]) + if row[0] in platforms: + versions_map[row[0]] = VersionInfo(version=row[2], + branch_base_position=int(row[7])) + logging.warn('versions map: %s' % versions_map) + if not all(platform in versions_map for platform in platforms): + raise ValueError( + 'Omaha report did not contain all desired platforms ' + 'for channel %s' % channel) + return versions_map + + +RemotePath = collections.namedtuple('RemotePath', 'bucket, path') + + +def _ResolveChromeRemotePath(platform_info, version_info): + # Path example: desktop-*/30.0.1595.0/precise32/chrome-precise32.zip + return RemotePath(bucket=_CHROME_GS_BUCKET, + path=('%s/%s/%s/%s' % (platform_info.gs_folder, + version_info.version, + platform_info.gs_build, + platform_info.zip_name))) + + +def _FindClosestChromiumSnapshot(base_position, build_dir): + """Returns the closest chromium snapshot available in cloud storage. + + Chromium snapshots are pulled from _CHROMIUM_BUILD_DIR in CHROMIUM_GS_BUCKET. + + Continuous chromium snapshots do not always contain the exact release build. + This function queries the storage bucket and find the closest snapshot within + +/-_CHROMIUM_SNAPSHOT_SEARCH_WINDOW to find the closest build. + """ + min_position = base_position - _CHROMIUM_SNAPSHOT_SEARCH_WINDOW + max_position = base_position + _CHROMIUM_SNAPSHOT_SEARCH_WINDOW + + # Getting the full list of objects in cloud storage bucket is prohibitively + # slow. It's faster to list objects with a prefix. Assuming we're looking at + # +/- 10 commit positions, for commit position 123456, we want to look at + # positions between 123446 an 123466. We do this by getting all snapshots + # with prefix 12344*, 12345*, and 12346*. This may get a few more snapshots + # that we intended, but that's fine since we take the min distance anyways. + min_position_prefix = min_position / 10; + max_position_prefix = max_position / 10; + + available_positions = [] + for position_prefix in range(min_position_prefix, max_position_prefix + 1): + query = '%s/%d*' % (build_dir, position_prefix) + try: + ls_results = cloud_storage.ListDirs(_CHROMIUM_GS_BUCKET, query) + except cloud_storage.NotFoundError: + # It's fine if there is no chromium snapshot available for one prefix. + # We will look at the rest of the prefixes. + continue + + for entry in ls_results: + # entry looks like '/Linux_x64/${commit_position}/'. + position = int(entry.split('/')[2]) + available_positions.append(position) + + if len(available_positions) == 0: + raise ValueError('No chromium build found +/-%d commit positions of %d' % + (_CHROMIUM_SNAPSHOT_SEARCH_WINDOW, base_position)) + + distance_function = lambda position: abs(position - base_position) + min_distance_snapshot = min(available_positions, key=distance_function) + return min_distance_snapshot + + +def _ResolveChromiumRemotePath(channel, platform, version_info): + platform_info = _PLATFORM_MAP[platform] + branch_base_position = version_info.branch_base_position + omaha_version = version_info.version + build_dir = platform_info.chromium_info.build_dir + # Look through chromium-browser-snapshots for closest match. + closest_snapshot = _FindClosestChromiumSnapshot( + branch_base_position, build_dir) + if closest_snapshot != branch_base_position: + print ('Channel %s corresponds to commit position ' % channel + + '%d on %s, ' % (branch_base_position, platform) + + 'but closest chromium snapshot available on ' + + '%s is %d' % (_CHROMIUM_GS_BUCKET, closest_snapshot)) + return RemotePath(bucket=_CHROMIUM_GS_BUCKET, + path = ('%s/%s/%s' % (build_dir, closest_snapshot, + platform_info.chromium_info.zip_name))) + + +def _QueuePlatformUpdate(binary, platform, version_info, config, channel): + """ platform: the name of the platform for the browser to + be downloaded & updated from cloud storage. """ + platform_info = _PLATFORM_MAP[platform] + + if binary == 'chrome': + remote_path = _ResolveChromeRemotePath(platform_info, version_info) + elif binary == 'chromium': + remote_path = _ResolveChromiumRemotePath(channel, platform, version_info) + else: + raise ValueError('binary must be \'chrome\' or \'chromium\'') + + if not cloud_storage.Exists(remote_path.bucket, remote_path.path): + cloud_storage_path = 'gs://%s/%s' % (remote_path.bucket, remote_path.path) + logging.warn('Failed to find %s build for version %s at path %s.' % ( + platform, version_info.version, cloud_storage_path)) + logging.warn('Skipping this update for this platform/channel.') + return + + reference_builds_folder = os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'chrome_telemetry_build', + 'reference_builds', binary, channel) + if not os.path.exists(reference_builds_folder): + os.makedirs(reference_builds_folder) + local_dest_path = os.path.join(reference_builds_folder, + platform, + platform_info.zip_name) + cloud_storage.Get(remote_path.bucket, remote_path.path, local_dest_path) + _ModifyBuildIfNeeded(binary, local_dest_path, platform) + config.AddCloudStorageDependencyUpdateJob('%s_%s' % (binary, channel), + platform, local_dest_path, version=version_info.version, + execute_job=False) + + +def _ModifyBuildIfNeeded(binary, location, platform): + """Hook to modify the build before saving it for Telemetry to use. + + This can be used to remove various utilities that cause noise in a + test environment. Right now, it is just used to remove Keystone, + which is a tool used to autoupdate Chrome. + """ + if binary != 'chrome': + return + + if platform == 'mac_x86_64': + _RemoveKeystoneFromBuild(location) + return + + if 'mac' in platform: + raise NotImplementedError( + 'Platform <%s> sounds like it is an OSX version. If so, we may need to ' + 'remove Keystone from it per crbug.com/932615. Please edit this script' + ' and teach it what needs to be done :).') + + +def _RemoveKeystoneFromBuild(location): + """Removes the Keystone autoupdate binary from the chrome mac zipfile.""" + logging.info('Removing keystone from mac build at %s' % location) + temp_folder = tempfile.mkdtemp(prefix='RemoveKeystoneFromBuild') + try: + subprocess.check_call(['unzip', '-q', location, '-d', temp_folder]) + keystone_folder = os.path.join( + temp_folder, 'chrome-mac', 'Google Chrome.app', 'Contents', + 'Frameworks', 'Google Chrome Framework.framework', 'Frameworks', + 'KeystoneRegistration.framework') + shutil.rmtree(keystone_folder) + os.remove(location) + subprocess.check_call(['zip', '--quiet', '--recurse-paths', '--symlinks', + location, 'chrome-mac'], + cwd=temp_folder) + finally: + shutil.rmtree(temp_folder) + + +def _NeedsUpdate(config, binary, channel, platform, version_info): + channel_version = version_info.version + print 'Checking %s (%s channel) on %s' % (binary, channel, platform) + current_version = config.GetVersion('%s_%s' % (binary, channel), platform) + print 'current: %s, channel: %s' % (current_version, channel_version) + if current_version and current_version == channel_version: + print 'Already up to date.' + return False + return True + + +def UpdateBuilds(args): + config = base_config.BaseConfig(_CHROME_BINARIES_CONFIG, writable=True) + for channel in _CHANNELS_TO_UPDATE: + channel_versions_map = _ChannelVersionsMap(channel) + for platform in channel_versions_map: + version_info = channel_versions_map.get(platform) + if args.update_chrome: + if _NeedsUpdate(config, 'chrome', channel, platform, version_info): + _QueuePlatformUpdate('chrome', platform, version_info, config, + channel) + if args.update_chromium and platform in _CHROMIUM_PLATFORMS: + if _NeedsUpdate(config, 'chromium', channel, platform, version_info): + _QueuePlatformUpdate('chromium', platform, version_info, + config, channel) + + print 'Updating builds with downloaded binaries' + config.ExecuteUpdateJobs(force=True) + + +def main(): + logging.getLogger().setLevel(logging.DEBUG) + parser = argparse.ArgumentParser( + description='Update reference binaries used by perf bots.') + parser.add_argument('--no-update-chrome', action='store_false', + dest='update_chrome', default=True, + help='do not update chrome binaries') + parser.add_argument('--no-update-chromium', action='store_false', + dest='update_chromium', default=True, + help='do not update chromium binaries') + args = parser.parse_args() + UpdateBuilds(args) + +if __name__ == '__main__': + main() |