diff options
author | George Burgess IV <gbiv@google.com> | 2023-01-26 13:28:47 -0700 |
---|---|---|
committer | Chromeos LUCI <chromeos-scoped@luci-project-accounts.iam.gserviceaccount.com> | 2023-01-26 21:04:04 +0000 |
commit | b4f76a2ac07ed44dcd2c738a45cbc43019ebc9ca (patch) | |
tree | 4f58bb3edcc9bf6a05d3e2f61adbef6b34def568 /rust_tools/copy_rust_bootstrap.py | |
parent | bdfc6c23183409a05f17d60ae021c5d33a02456d (diff) | |
download | toolchain-utils-b4f76a2ac07ed44dcd2c738a45cbc43019ebc9ca.tar.gz |
copy_rust_bootstrap: format
Somehow this avoided the swap from `yapf` to `black`. Format it.
This also fixes a copyright preupload check.
BUG=b:266817148
TEST=None
Change-Id: Ic34a7ba5b8aa2d5a8c745bc237051bab0aaed8cb
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/4195524
Commit-Queue: George Burgess <gbiv@chromium.org>
Reviewed-by: Jordan Abrahams-Whitehead <ajordanr@google.com>
Tested-by: George Burgess <gbiv@chromium.org>
Diffstat (limited to 'rust_tools/copy_rust_bootstrap.py')
-rwxr-xr-x | rust_tools/copy_rust_bootstrap.py | 291 |
1 files changed, 151 insertions, 140 deletions
diff --git a/rust_tools/copy_rust_bootstrap.py b/rust_tools/copy_rust_bootstrap.py index 5da8007f..a61375ab 100755 --- a/rust_tools/copy_rust_bootstrap.py +++ b/rust_tools/copy_rust_bootstrap.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# Copyright 2022 The ChromiumOS Authors. +# Copyright 2022 The ChromiumOS Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -21,172 +21,183 @@ import tempfile from typing import List -_LOCALMIRROR_ROOT = 'gs://chromeos-localmirror/distfiles/' +_LOCALMIRROR_ROOT = "gs://chromeos-localmirror/distfiles/" def _is_in_chroot() -> bool: - return Path('/etc/cros_chroot_version').exists() + return Path("/etc/cros_chroot_version").exists() def _ensure_pbzip2_is_installed(): - if shutil.which('pbzip2'): - return + if shutil.which("pbzip2"): + return - logging.info('Auto-installing pbzip2...') - subprocess.run(['sudo', 'emerge', '-G', 'pbzip2'], check=True) + logging.info("Auto-installing pbzip2...") + subprocess.run(["sudo", "emerge", "-G", "pbzip2"], check=True) def _determine_target_path(sdk_path: str) -> str: - """Determine where `sdk_path` should sit in localmirror.""" - gs_prefix = 'gs://' - if not sdk_path.startswith(gs_prefix): - raise ValueError(f'Invalid GS path: {sdk_path!r}') + """Determine where `sdk_path` should sit in localmirror.""" + gs_prefix = "gs://" + if not sdk_path.startswith(gs_prefix): + raise ValueError(f"Invalid GS path: {sdk_path!r}") - file_name = Path(sdk_path[len(gs_prefix):]).name - return _LOCALMIRROR_ROOT + file_name + file_name = Path(sdk_path[len(gs_prefix) :]).name + return _LOCALMIRROR_ROOT + file_name def _download(remote_path: str, local_file: Path): - """Downloads the given gs:// path to the given local file.""" - logging.info('Downloading %s -> %s', remote_path, local_file) - subprocess.run( - ['gsutil.py', 'cp', remote_path, - str(local_file)], - check=True, - ) - - -def _debinpkgify(binpkg_file: Path) -> Path: - """Converts a binpkg into the files it installs. - - Note that this function makes temporary files in the same directory as - `binpkg_file`. It makes no attempt to clean them up. - """ - logging.info('Converting %s from a binpkg...', binpkg_file) - - # The SDK builder produces binary packages: - # https://wiki.gentoo.org/wiki/Binary_package_guide - # - # Which means that `binpkg_file` is in the XPAK format. We want to split - # that out, and recompress it from zstd (which is the compression format - # that CrOS uses) to bzip2 (which is what we've historically used, and - # which is what our ebuild expects). - tmpdir = binpkg_file.parent - - def _mkstemp(suffix=None) -> str: - fd, file_path = tempfile.mkstemp(dir=tmpdir, suffix=suffix) - os.close(fd) - return Path(file_path) - - # First, split the actual artifacts that land in the chroot out to - # `temp_file`. - artifacts_file = _mkstemp() - logging.info('Extracting artifacts from %s into %s...', binpkg_file, - artifacts_file) - with artifacts_file.open('wb') as f: + """Downloads the given gs:// path to the given local file.""" + logging.info("Downloading %s -> %s", remote_path, local_file) subprocess.run( - [ - 'qtbz2', - '-s', - '-t', - '-O', - str(binpkg_file), - ], + ["gsutil.py", "cp", remote_path, str(local_file)], check=True, - stdout=f, ) - decompressed_artifacts_file = _mkstemp() - decompressed_artifacts_file.unlink() - logging.info('Decompressing artifacts from %s to %s...', artifacts_file, - decompressed_artifacts_file) - subprocess.run( - [ - 'zstd', - '-d', - str(artifacts_file), - '-o', - str(decompressed_artifacts_file), - ], - check=True, - ) - - # Finally, recompress it as a tbz2. - tbz2_file = _mkstemp('.tbz2') - logging.info( - 'Recompressing artifacts from %s to %s (this may take a while)...', - decompressed_artifacts_file, tbz2_file) - with tbz2_file.open('wb') as f: + +def _debinpkgify(binpkg_file: Path) -> Path: + """Converts a binpkg into the files it installs. + + Note that this function makes temporary files in the same directory as + `binpkg_file`. It makes no attempt to clean them up. + """ + logging.info("Converting %s from a binpkg...", binpkg_file) + + # The SDK builder produces binary packages: + # https://wiki.gentoo.org/wiki/Binary_package_guide + # + # Which means that `binpkg_file` is in the XPAK format. We want to split + # that out, and recompress it from zstd (which is the compression format + # that CrOS uses) to bzip2 (which is what we've historically used, and + # which is what our ebuild expects). + tmpdir = binpkg_file.parent + + def _mkstemp(suffix=None) -> str: + fd, file_path = tempfile.mkstemp(dir=tmpdir, suffix=suffix) + os.close(fd) + return Path(file_path) + + # First, split the actual artifacts that land in the chroot out to + # `temp_file`. + artifacts_file = _mkstemp() + logging.info( + "Extracting artifacts from %s into %s...", binpkg_file, artifacts_file + ) + with artifacts_file.open("wb") as f: + subprocess.run( + [ + "qtbz2", + "-s", + "-t", + "-O", + str(binpkg_file), + ], + check=True, + stdout=f, + ) + + decompressed_artifacts_file = _mkstemp() + decompressed_artifacts_file.unlink() + logging.info( + "Decompressing artifacts from %s to %s...", + artifacts_file, + decompressed_artifacts_file, + ) subprocess.run( [ - 'pbzip2', - '-9', - '-c', + "zstd", + "-d", + str(artifacts_file), + "-o", str(decompressed_artifacts_file), ], check=True, - stdout=f, ) - return tbz2_file + + # Finally, recompress it as a tbz2. + tbz2_file = _mkstemp(".tbz2") + logging.info( + "Recompressing artifacts from %s to %s (this may take a while)...", + decompressed_artifacts_file, + tbz2_file, + ) + with tbz2_file.open("wb") as f: + subprocess.run( + [ + "pbzip2", + "-9", + "-c", + str(decompressed_artifacts_file), + ], + check=True, + stdout=f, + ) + return tbz2_file def _upload(local_file: Path, remote_path: str, force: bool): - """Uploads the local file to the given gs:// path.""" - logging.info('Uploading %s -> %s', local_file, remote_path) - cmd_base = ['gsutil.py', 'cp', '-a', 'public-read'] - if not force: - cmd_base.append('-n') - subprocess.run( - cmd_base + [str(local_file), remote_path], - check=True, - stdin=subprocess.DEVNULL, - ) + """Uploads the local file to the given gs:// path.""" + logging.info("Uploading %s -> %s", local_file, remote_path) + cmd_base = ["gsutil.py", "cp", "-a", "public-read"] + if not force: + cmd_base.append("-n") + subprocess.run( + cmd_base + [str(local_file), remote_path], + check=True, + stdin=subprocess.DEVNULL, + ) def main(argv: List[str]): - logging.basicConfig( - format='>> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: ' - '%(message)s', - level=logging.INFO, - ) - - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - parser.add_argument( - 'sdk_artifact', - help='Path to the SDK rust-bootstrap artifact to copy. e.g., ' - 'gs://chromeos-prebuilt/host/amd64/amd64-host/' - 'chroot-2022.07.12.134334/packages/dev-lang/' - 'rust-bootstrap-1.59.0.tbz2.') - parser.add_argument( - '-n', - '--dry-run', - action='store_true', - help='Do everything except actually uploading the artifact.') - parser.add_argument( - '--force', - action='store_true', - help='Upload the artifact even if one exists in localmirror already.') - opts = parser.parse_args(argv) - - if not _is_in_chroot(): - parser.error('Run me from within the chroot.') - _ensure_pbzip2_is_installed() - - target_path = _determine_target_path(opts.sdk_artifact) - with tempfile.TemporaryDirectory() as tempdir: - download_path = Path(tempdir) / 'sdk_artifact' - _download(opts.sdk_artifact, download_path) - file_to_upload = _debinpkgify(download_path) - if opts.dry_run: - logging.info('--dry-run specified; skipping upload of %s to %s', - file_to_upload, target_path) - else: - _upload(file_to_upload, target_path, opts.force) - - -if __name__ == '__main__': - main(sys.argv[1:]) + logging.basicConfig( + format=">> %(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: " + "%(message)s", + level=logging.INFO, + ) + + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + parser.add_argument( + "sdk_artifact", + help="Path to the SDK rust-bootstrap artifact to copy. e.g., " + "gs://chromeos-prebuilt/host/amd64/amd64-host/" + "chroot-2022.07.12.134334/packages/dev-lang/" + "rust-bootstrap-1.59.0.tbz2.", + ) + parser.add_argument( + "-n", + "--dry-run", + action="store_true", + help="Do everything except actually uploading the artifact.", + ) + parser.add_argument( + "--force", + action="store_true", + help="Upload the artifact even if one exists in localmirror already.", + ) + opts = parser.parse_args(argv) + + if not _is_in_chroot(): + parser.error("Run me from within the chroot.") + _ensure_pbzip2_is_installed() + + target_path = _determine_target_path(opts.sdk_artifact) + with tempfile.TemporaryDirectory() as tempdir: + download_path = Path(tempdir) / "sdk_artifact" + _download(opts.sdk_artifact, download_path) + file_to_upload = _debinpkgify(download_path) + if opts.dry_run: + logging.info( + "--dry-run specified; skipping upload of %s to %s", + file_to_upload, + target_path, + ) + else: + _upload(file_to_upload, target_path, opts.force) + + +if __name__ == "__main__": + main(sys.argv[1:]) |