diff options
Diffstat (limited to 'build/fuchsia/update_product_bundles.py')
| -rwxr-xr-x | build/fuchsia/update_product_bundles.py | 115 |
1 files changed, 53 insertions, 62 deletions
diff --git a/build/fuchsia/update_product_bundles.py b/build/fuchsia/update_product_bundles.py index 1cde6c4bc..79ad39709 100755 --- a/build/fuchsia/update_product_bundles.py +++ b/build/fuchsia/update_product_bundles.py @@ -15,9 +15,12 @@ import subprocess import sys from contextlib import ExitStack + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), + 'test'))) + import common -import ffx_session -import log_manager +import ffx_integration _PRODUCT_BUNDLES = [ 'core.x64-dfv2', @@ -26,6 +29,7 @@ _PRODUCT_BUNDLES = [ 'workstation_eng.chromebook-x64', 'workstation_eng.chromebook-x64-dfv2', 'workstation_eng.qemu-x64', + 'workstation_eng.x64', ] # TODO(crbug/1361089): Remove when the old scripts have been deprecated. @@ -72,27 +76,23 @@ def get_hash_from_sdk(): return json.load(f)['id'] -def remove_repositories(repo_names_to_remove, ffx_runner): +def remove_repositories(repo_names_to_remove): """Removes given repos from repo list. Repo MUST be present in list to succeed. Args: repo_names_to_remove: List of repo names (as strings) to remove. - ffx_runner: ffx_session.FfxRunner instance to run the command. """ for repo_name in repo_names_to_remove: - ffx_runner.run_ffx(('repository', 'remove', repo_name), check=True) + common.run_ffx_command(('repository', 'remove', repo_name), check=True) -def get_repositories(ffx_runner): +def get_repositories(): """Lists repositories that are available on disk. Also prunes repositories that are listed, but do not have an actual packages directory. - Args: - ffx_runner: An FfxRunner instance. - Returns: List of dictionaries containing info about the repositories. They have the following structure: @@ -106,7 +106,9 @@ def get_repositories(ffx_runner): """ repos = json.loads( - ffx_runner.run_ffx(('--machine', 'json', 'repository', 'list')).strip()) + common.run_ffx_command(('--machine', 'json', 'repository', 'list'), + check=True, + capture_output=True).stdout.strip()) to_prune = set() sdk_root_abspath = os.path.abspath(os.path.dirname(common.SDK_ROOT)) for repo in repos: @@ -121,26 +123,23 @@ def get_repositories(ffx_runner): repos = [repo for repo in repos if repo['name'] not in to_prune] - remove_repositories(to_prune, ffx_runner) + remove_repositories(to_prune) return repos -def update_repositories_list(ffx_runner): +def update_repositories_list(): """Used to prune stale repositories.""" - get_repositories(ffx_runner) + get_repositories() -def remove_product_bundle(product_bundle, ffx_runner): +def remove_product_bundle(product_bundle): """Removes product-bundle given.""" - ffx_runner.run_ffx(('product-bundle', 'remove', '-f', product_bundle)) + common.run_ffx_command(('product-bundle', 'remove', '-f', product_bundle)) -def get_product_bundle_urls(ffx_runner): +def get_product_bundle_urls(): """Retrieves URLs of available product-bundles. - Args: - ffx_runner: An FfxRunner instance. - Returns: List of dictionaries of structure, indicating whether the product-bundle has been downloaded. @@ -150,8 +149,8 @@ def get_product_bundle_urls(ffx_runner): } """ # TODO(fxb/115328): Replaces with JSON API when available. - bundles = ffx_runner.run_ffx(('product-bundle', 'list'), check=True) - + bundles = common.run_ffx_command(('product-bundle', 'list'), + capture_output=True).stdout.strip() urls = [ line.strip() for line in bundles.splitlines() if 'gs://fuchsia' in line ] @@ -165,40 +164,34 @@ def get_product_bundle_urls(ffx_runner): return structured_urls -def keep_product_bundles_by_sdk_version(sdk_version, ffx_runner): +def keep_product_bundles_by_sdk_version(sdk_version): """Prunes product bundles not containing the sdk_version given.""" - urls = get_product_bundle_urls(ffx_runner) + urls = get_product_bundle_urls() for url in urls: if url['downloaded'] and sdk_version not in url['url']: - remove_product_bundle(url['url'], ffx_runner) + remove_product_bundle(url['url']) -def get_product_bundles(ffx_runner): +def get_product_bundles(): """Lists all downloaded product-bundles for the given SDK. Cross-references the repositories with downloaded packages and the stated downloaded product-bundles to validate whether or not a product-bundle is present. Prunes invalid product-bundles with each call as well. - Args: - ffx_runner: An FfxRunner instance. - Returns: List of strings of product-bundle names downloaded and that FFX is aware of. """ downloaded_bundles = [] - for url in get_product_bundle_urls(ffx_runner): + for url in get_product_bundle_urls(): if url['downloaded']: # The product is separated by a # product = url['url'].split('#') downloaded_bundles.append(product[1]) - # For each downloaded bundle, need to verify whether ffx repository believes - # it exists. - to_prune_bundles_index = [] - repos = get_repositories(ffx_runner) + repos = get_repositories() # Some repo names do not match product-bundle names due to underscores. # Normalize them both. @@ -211,41 +204,40 @@ def get_product_bundles(ffx_runner): if name.replace('-', '_') in repo_names: return True - remove_product_bundle(name, ffx_runner) + remove_product_bundle(name) return False return list(filter(bundle_is_active, downloaded_bundles)) -def download_product_bundle(product_bundle, ffx_runner): +def download_product_bundle(product_bundle, download_config): """Download product bundles using the SDK.""" # This also updates the repository list, in case it is stale. - update_repositories_list(ffx_runner) + update_repositories_list() try: - ffx_runner.run_ffx(('product-bundle', 'get', product_bundle)) + common.run_ffx_command( + ('product-bundle', 'get', product_bundle, '--force-repo'), + configs=download_config) except subprocess.CalledProcessError as cpe: logging.error('Product bundle download has failed. ' + _PRODUCT_BUNDLE_FIX_INSTRUCTIONS) raise -def get_current_signature(ffx_runner): +def get_current_signature(): """Determines the SDK version of the product-bundles associated with the SDK. Parses this information from the URLs of the product-bundle. - Args: - ffx_runner: An FfxRunner instance. - Returns: An SDK version string, or None if no product-bundle versions are downloaded. """ - product_bundles = get_product_bundles(ffx_runner) + product_bundles = get_product_bundles() if not product_bundles: - logging.warning('No product bundles - signature will default to None') + logging.info('No product bundles - signature will default to None') return None - product_bundle_urls = get_product_bundle_urls(ffx_runner) + product_bundle_urls = get_product_bundle_urls() # Get the numbers, hope they're the same. signatures = set() @@ -277,7 +269,7 @@ def main(): logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO) # Check whether there's Fuchsia support for this platform. - common.GetHostOsFromPlatform() + common.get_host_os() new_product_bundles = convert_to_product_bundle( args.product_bundles.split(',')) @@ -293,11 +285,10 @@ def main(): 'found in the DEPS file.') with ExitStack() as stack: - ffx_runner = ffx_session.FfxRunner(log_manager.LogManager(None)) # Re-set the directory to which product bundles are downloaded so that # these bundles are located inside the Chromium codebase. - ffx_runner.run_ffx( + common.run_ffx_command( ('config', 'set', 'pbms.storage.path', common.IMAGES_ROOT)) logging.debug('Checking for override file') @@ -305,21 +296,20 @@ def main(): # TODO(crbug/1380807): Remove when product bundles can be downloaded # for custom SDKs without editing metadata override_file = os.path.join(os.path.dirname(__file__), 'sdk_override.txt') + pb_metadata = None if os.path.isfile(override_file): with open(override_file) as f: pb_metadata = f.read().strip().split('\n') pb_metadata.append('{sdk.root}/*.json') - stack.enter_context( - ffx_runner.scoped_config('pbms.metadata', json.dumps((pb_metadata)))) logging.debug('Applied overrides') logging.debug('Getting new SDK hash') new_sdk_hash = get_hash_from_sdk() - keep_product_bundles_by_sdk_version(new_sdk_hash, ffx_runner) + keep_product_bundles_by_sdk_version(new_sdk_hash) logging.debug('Checking for current signature') - curr_signature = get_current_signature(ffx_runner) + curr_signature = get_current_signature() - current_images = get_product_bundles(ffx_runner) + current_images = get_product_bundles() # If SDK versions match, remove the product bundles that are no longer # needed and download missing ones. @@ -329,34 +319,35 @@ def main(): for image in current_images: if image not in new_product_bundles: logging.debug('Removing no longer needed Fuchsia image %s' % image) - remove_product_bundle(image, ffx_runner) + remove_product_bundle(image) bundles_to_download = set(new_product_bundles) - \ set(current_images) for bundle in bundles_to_download: logging.debug('Downloading image: %s', image) - download_product_bundle(bundle, ffx_runner) + download_product_bundle(bundle) return 0 # If SDK versions do not match, remove all existing product bundles # and download the ones required. for pb in current_images: - remove_product_bundle(pb, ffx_runner) + remove_product_bundle(pb) logging.debug('Make clean images root') - curr_subdir = [] - if os.path.exists(common.IMAGES_ROOT): - curr_subdir = os.listdir(common.IMAGES_ROOT) - common.MakeCleanDirectory(common.IMAGES_ROOT) + common.make_clean_directory(common.IMAGES_ROOT) + download_config = None + if pb_metadata: + download_config = [ + '{"pbms":{"metadata": %s}}' % json.dumps((pb_metadata)) + ] for pb in new_product_bundles: logging.debug('Downloading bundle: %s', pb) - download_product_bundle(pb, ffx_runner) + download_product_bundle(pb, download_config) - current_pb = get_product_bundles(ffx_runner) + current_pb = get_product_bundles() - diff = set(current_pb) - set(new_product_bundles) assert set(current_pb) == set(new_product_bundles), ( 'Failed to download expected set of product-bundles. ' f'Expected {new_product_bundles}, got {current_pb}') |