rust_uprev: fetch distfiles from local mirror

This adds a few extra steps to rust_uprev where rust sources and
rust-bootstrap prebuilts are fetched from the local mirror. This
ensures that these files exist and are readable, and that the
local versions (that will be used to compute the digests for the
manifest) match the ones on the mirror.

BUG=None
TEST=Ran the script with existing and non-existing rust versions

Change-Id: I9868b027356cf14fc10d9e56665c74ad26555345
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/3182632
Tested-by: Bob Haarman <inglorion@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>
Commit-Queue: Bob Haarman <inglorion@chromium.org>

2 files changed, 163 insertions, 1 deletions

diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py
index 492729ea..011639df 100755
--- a/rust_tools/rust_uprev.py
+++ b/rust_tools/rust_uprev.py
@@ -49,6 +49,7 @@ from llvm_tools import chroot, git
 
 EQUERY = 'equery'
 GSUTIL = 'gsutil.py'
+MIRROR_PATH = 'gs://chromeos-localmirror/distfiles'
 RUST_PATH = Path(
     '/mnt/host/source/src/third_party/chromiumos-overlay/dev-lang/rust')
 
@@ -58,6 +59,15 @@ def get_command_output(command: List[str], *args, **kwargs) -> str:
                                  **kwargs).strip()
 
 
+def get_command_output_unchecked(command: List[str], *args, **kwargs) -> str:
+  return subprocess.run(command,
+                        check=False,
+                        stdout=subprocess.PIPE,
+                        encoding='utf-8',
+                        *args,
+                        **kwargs).stdout.strip()
+
+
 class RustVersion(NamedTuple):
   """NamedTuple represents a Rust version"""
   major: int
@@ -93,6 +103,14 @@ class RustVersion(NamedTuple):
                        int(m.group('patch')))
 
 
+def compute_rustc_src_name(version: RustVersion) -> str:
+  return f'rustc-{version}-src.tar.gz'
+
+
+def compute_rust_bootstrap_prebuilt_name(version: RustVersion) -> str:
+  return f'rust-bootstrap-{version}.tbz2'
+
+
 def find_ebuild_for_package(name: str) -> os.PathLike:
   """Returns the path to the ebuild for the named package."""
   return get_command_output([EQUERY, 'w', name])
@@ -369,7 +387,96 @@ def ebuild_actions(package: str, actions: List[str],
   subprocess.check_call(cmd)
 
 
+def fetch_distfile_from_mirror(name: str) -> None:
+  """Gets the named file from the local mirror.
+
+  This ensures that the file exists on the mirror, and
+  that we can read it. We overwrite any existing distfile
+  to ensure the checksums that update_manifest() records
+  match the file as it exists on the mirror.
+
+  This function also attempts to verify the ACL for
+  the file (which is expected to have READER permission
+  for allUsers). We can only see the ACL if the user
+  gsutil runs with is the owner of the file. If not,
+  we get an access denied error. We also count this
+  as a success, because it means we were able to fetch
+  the file even though we don't own it.
+  """
+  mirror_file = MIRROR_PATH + '/' + name
+  local_file = Path(get_distdir(), name)
+  cmd = [GSUTIL, 'cp', mirror_file, local_file]
+  logging.info('Running %r', cmd)
+  rc = subprocess.call(cmd)
+  if rc != 0:
+    logging.error(
+        """Could not fetch %s
+
+If the file does not yet exist at %s
+please download the file, verify its integrity
+with something like:
+
+curl -O https://static.rust-lang.org/dist/%s
+gpg --verify %s.asc
+
+You may need to import the signing key first, e.g.:
+
+gpg --recv-keys 85AB96E6FA1BE5FE
+
+Once you have verify the integrity of the file, upload
+it to the local mirror using gsutil cp.
+""", mirror_file, MIRROR_PATH, name, name)
+    raise Exception(f'Could not fetch {mirror_file}')
+  # Check that the ACL allows allUsers READER access.
+  # If we get an AccessDeniedAcception here, that also
+  # counts as a success, because we were able to fetch
+  # the file as a non-owner.
+  cmd = [GSUTIL, 'acl', 'get', mirror_file]
+  logging.info('Running %r', cmd)
+  output = get_command_output_unchecked(cmd, stderr=subprocess.STDOUT)
+  acl_verified = False
+  if 'AccessDeniedException:' in output:
+    acl_verified = True
+  else:
+    acl = json.loads(output)
+    for x in acl:
+      if x['entity'] == 'allUsers' and x['role'] == 'READER':
+        acl_verified = True
+        break
+  if not acl_verified:
+    logging.error('Output from acl get:\n%s', output)
+    raise Exception('Could not verify that allUsers has READER permission')
+
+
+def fetch_bootstrap_distfiles(old_version: RustVersion,
+                              new_version: RustVersion) -> None:
+  """Fetches rust-bootstrap distfiles from the local mirror
+
+  Fetches the distfiles for a rust-bootstrap ebuild to ensure they
+  are available on the mirror and the local copies are the same as
+  the ones on the mirror.
+  """
+  fetch_distfile_from_mirror(compute_rust_bootstrap_prebuilt_name(old_version))
+  fetch_distfile_from_mirror(compute_rustc_src_name(new_version))
+
+
+def fetch_rust_distfiles(version: RustVersion) -> None:
+  """Fetches rust distfiles from the local mirror
+
+  Fetches the distfiles for a rust ebuild to ensure they
+  are available on the mirror and the local copies are
+  the same as the ones on the mirror.
+  """
+  fetch_distfile_from_mirror(compute_rustc_src_name(version))
+
+
+def get_distdir() -> os.PathLike:
+  """Returns portage's distdir."""
+  return get_command_output(['portageq', 'distdir'])
+
+
 def update_manifest(ebuild_file: os.PathLike) -> None:
+  """Updates the MANIFEST for the ebuild at the given path."""
   ebuild = Path(ebuild_file)
   logging.info('Added "mirror" to RESTRICT to %s', ebuild.name)
   flip_mirror_in_ebuild(ebuild, add=True)
@@ -451,7 +558,7 @@ def prepare_uprev_from_json(
 def create_rust_uprev(rust_version: RustVersion,
                       maybe_template_version: Optional[RustVersion],
                       skip_compile: bool, run_step: Callable[[], T]) -> None:
-  template_version, template_ebuild, _old_bootstrap_version = run_step(
+  template_version, template_ebuild, old_bootstrap_version = run_step(
       'prepare uprev',
       lambda: prepare_uprev(rust_version, maybe_template_version),
       result_from_json=prepare_uprev_from_json,
@@ -459,6 +566,14 @@ def create_rust_uprev(rust_version: RustVersion,
   if template_ebuild is None:
     return
 
+  # The fetch steps will fail (on purpose) if the files they check for
+  # are not available on the mirror. To make them pass, fetch the
+  # required files yourself, verify their checksums, then upload them
+  # to the mirror.
+  run_step(
+      'fetch bootstrap distfiles', lambda: fetch_bootstrap_distfiles(
+          old_bootstrap_version, template_version))
+  run_step('fetch rust distfiles', lambda: fetch_rust_distfiles(rust_version))
   run_step('update bootstrap ebuild', lambda: update_bootstrap_ebuild(
       template_version))
   run_step(
diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py
index 18e8d9e8..00761391 100755
--- a/rust_tools/rust_uprev_test.py
+++ b/rust_tools/rust_uprev_test.py
@@ -20,6 +20,53 @@ import rust_uprev
 from rust_uprev import RustVersion
 
 
+def _fail_command(cmd, *_args, **_kwargs):
+  err = subprocess.CalledProcessError(returncode=1, cmd=cmd)
+  err.stderr = b'mock failure'
+  raise err
+
+
+class FetchDistfileTest(unittest.TestCase):
+  """Tests rust_uprev.fetch_distfile_from_mirror()"""
+
+  @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles')
+  @mock.patch.object(subprocess, 'call', side_effect=_fail_command)
+  def test_fetch_difstfile_fail(self, *_args) -> None:
+    with self.assertRaises(subprocess.CalledProcessError):
+      rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz')
+
+  @mock.patch.object(rust_uprev,
+                     'get_command_output_unchecked',
+                     return_value='AccessDeniedException: Access denied.')
+  @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles')
+  @mock.patch.object(subprocess, 'call', return_value=0)
+  def test_fetch_distfile_acl_access_denied(self, *_args) -> None:
+    rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz')
+
+  @mock.patch.object(
+      rust_uprev,
+      'get_command_output_unchecked',
+      return_value='[ { "entity": "allUsers", "role": "READER" } ]')
+  @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles')
+  @mock.patch.object(subprocess, 'call', return_value=0)
+  def test_fetch_distfile_acl_ok(self, *_args) -> None:
+    rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz')
+
+  @mock.patch.object(
+      rust_uprev,
+      'get_command_output_unchecked',
+      return_value='[ { "entity": "___fake@google.com", "role": "OWNER" } ]')
+  @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles')
+  @mock.patch.object(subprocess, 'call', return_value=0)
+  def test_fetch_distfile_acl_wrong(self, *_args) -> None:
+    with self.assertRaisesRegex(Exception, 'allUsers.*READER'):
+      with self.assertLogs(level='ERROR') as log:
+        rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz')
+        self.assertIn(
+            '[ { "entity": "___fake@google.com", "role": "OWNER" } ]',
+            '\n'.join(log.output))
+
+
 class FindEbuildPathTest(unittest.TestCase):
   """Tests for rust_uprev.find_ebuild_path()"""