rust_tools: Fix the tool using real Rust uprev experiences.

Changes include bugs fixes, support running clean before compile
with ebuild command, and upload all tarballs (not just rustc).

BUG=chromium:1112551
TEST=unittest

Change-Id: I83a6845ba01590ea1319c90ebb427fb48c3da773
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2462042
Reviewed-by: Bob Haarman <inglorion@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>
Commit-Queue: Tiancong Wang <tcwang@google.com>
Tested-by: Tiancong Wang <tcwang@google.com>

2 files changed, 71 insertions, 18 deletions

diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py
index 4ff92109..db907149 100755
--- a/rust_tools/rust_uprev.py
+++ b/rust_tools/rust_uprev.py
@@ -306,7 +306,6 @@ def update_ebuild(ebuild_file: str, stage0_info: Tuple[str, str, str]) -> None:
       'Rust ebuild file has STAGE0_DATE, STAGE0_VERSION, STAGE0_VERSION_CARGO '
       'updated to %s, %s, %s respectively', stage0_date, stage0_rustc,
       stage0_cargo)
-  return ebuild_file
 
 
 def flip_mirror_in_ebuild(ebuild_file: str, add: bool) -> None:
@@ -330,18 +329,18 @@ def flip_mirror_in_ebuild(ebuild_file: str, add: bool) -> None:
     f.write(new_contents)
 
 
-def rust_ebuild_command(command: str, sudo: bool = False) -> None:
+def rust_ebuild_actions(actions: List[str], sudo: bool = False) -> None:
   ebuild_path_inchroot = get_command_output(['equery', 'w', 'rust'])
-  cmd = ['ebuild', ebuild_path_inchroot, command]
+  cmd = ['ebuild', ebuild_path_inchroot] + actions
   if sudo:
     cmd = ['sudo'] + cmd
-  subprocess.check_call(cmd, stderr=subprocess.STDOUT)
+  subprocess.check_call(cmd)
 
 
 def update_manifest(ebuild_file: str) -> None:
   logging.info('Added "mirror" to RESTRICT to Rust ebuild')
   flip_mirror_in_ebuild(ebuild_file, add=True)
-  rust_ebuild_command('manifest')
+  rust_ebuild_actions(['manifest'])
   logging.info('Removed "mirror" to RESTRICT from Rust ebuild')
   flip_mirror_in_ebuild(ebuild_file, add=False)
 
@@ -381,10 +380,10 @@ def update_virtual_rust(template_version: RustVersion,
                         cwd=virtual_rust_dir)
 
 
-def upload_to_localmirror(tempdir: str, rust_version: RustVersion) -> None:
-  tarfile_name = f'rustc-{rust_version}-src.tar.gz'
-  rust_src = f'https://static.rust-lang.org/dist/{tarfile_name}'
-  logging.info('Downloading Rust from %s', rust_src)
+def upload_single_tarball(rust_url: str, tarfile_name: str,
+                          tempdir: str) -> None:
+  rust_src = f'{rust_url}/{tarfile_name}'
+  logging.info('Downloading Rust artifact from %s', rust_src)
   gsutil_location = f'gs://chromeos-localmirror/distfiles/{tarfile_name}'
 
   # Download Rust's source
@@ -417,6 +416,36 @@ def upload_to_localmirror(tempdir: str, rust_version: RustVersion) -> None:
       ['gsutil', 'cp', '-n', '-a', 'public-read', rust_file, gsutil_location])
 
 
+def upload_to_localmirror(tempdir: str, rust_version: RustVersion,
+                          stage0_info: Tuple[str, str, str]) -> None:
+  stage0_date, stage0_rustc, stage0_cargo = stage0_info
+  rust_url = 'https://static.rust-lang.org/dist'
+  # Upload rustc source
+  upload_single_tarball(
+      rust_url,
+      f'rustc-{rust_version}-src.tar.gz',
+      tempdir,
+  )
+  # Upload stage0 toolchain
+  upload_single_tarball(
+      f'{rust_url}/{stage0_date}',
+      f'rust-std-{stage0_rustc}-x86_64-unknown-linux-gnu.tar.gz',
+      tempdir,
+  )
+  # Upload stage0 source
+  upload_single_tarball(
+      rust_url,
+      f'rustc-{stage0_rustc}-x86_64-unknown-linux-gnu.tar.gz',
+      tempdir,
+  )
+  # Upload stage0 cargo
+  upload_single_tarball(
+      rust_url,
+      f'cargo-{stage0_cargo}-x86_64-unknown-linux-gnu.tar.gz',
+      tempdir,
+  )
+
+
 def perform_step(state_file: pathlib.Path,
                  tmp_state_file: pathlib.Path,
                  completed_steps: Dict[str, Any],
@@ -463,11 +492,12 @@ def create_rust_uprev(rust_version: RustVersion,
   run_step('update ebuild', lambda: update_ebuild(ebuild_file, stage0_info))
   with tempfile.TemporaryDirectory(dir='/tmp') as tempdir:
     run_step('upload_to_localmirror', lambda: upload_to_localmirror(
-        tempdir, rust_version))
-  run_step('update manifest', lambda: update_manifest(ebuild_file))
+        tempdir, rust_version, stage0_info))
+  run_step('update manifest to add new version', lambda: update_manifest(
+      ebuild_file))
   if not skip_compile:
-    run_step('compile rust', lambda: rust_ebuild_command('compile'))
-    run_step('merge rust', lambda: rust_ebuild_command('merge', sudo=True))
+    run_step('compile rust', lambda: rust_ebuild_actions(['clean', 'compile']))
+    run_step('merge rust', lambda: rust_ebuild_actions(['merge'], sudo=True))
   run_step('insert version into rust packages', lambda: update_rust_packages(
       rust_version, add=True))
   run_step('upgrade virtual/rust', lambda: update_virtual_rust(
@@ -501,7 +531,8 @@ def remove_rust_uprev(rust_version: Optional[RustVersion],
   run_step('remove ebuild', lambda: remove_files(
       f'rust-{delete_version}.ebuild', RUST_PATH))
   ebuild_file = get_command_output(['equery', 'w', 'rust'])
-  run_step('update manifest', lambda: update_manifest(ebuild_file))
+  run_step('update manifest to delete old version', lambda: update_manifest(
+      ebuild_file))
   run_step('remove version from rust packages', lambda: update_rust_packages(
       delete_version, add=False))
   run_step(
diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py
index a28c551e..c2f071f2 100755
--- a/rust_tools/rust_uprev_test.py
+++ b/rust_tools/rust_uprev_test.py
@@ -180,11 +180,11 @@ class UpdateManifestTest(unittest.TestCase):
         expect_write=False)
 
   @mock.patch.object(rust_uprev, 'flip_mirror_in_ebuild')
-  @mock.patch.object(rust_uprev, 'rust_ebuild_command')
+  @mock.patch.object(rust_uprev, 'rust_ebuild_actions')
   def test_update_manifest(self, mock_run, mock_flip):
     ebuild_file = '/path/to/rust/rust-1.1.1.ebuild'
     rust_uprev.update_manifest(ebuild_file)
-    mock_run.assert_called_once_with('manifest')
+    mock_run.assert_called_once_with(['manifest'])
     mock_flip.assert_has_calls(
         [mock.call(ebuild_file, add=True),
          mock.call(ebuild_file, add=False)])
@@ -231,6 +231,7 @@ class UploadToLocalmirrorTests(unittest.TestCase):
   def setUp(self):
     self.tempdir = '/tmp/any/dir'
     self.new_version = rust_uprev.RustVersion(1, 3, 5)
+    self.rust_url = 'https://static.rust-lang.org/dist'
     self.tarfile_name = f'rustc-{self.new_version}-src.tar.gz'
     self.rust_src = f'https://static.rust-lang.org/dist/{self.tarfile_name}'
     self.gsurl = f'gs://chromeos-localmirror/distfiles/{self.tarfile_name}'
@@ -241,7 +242,8 @@ class UploadToLocalmirrorTests(unittest.TestCase):
   @mock.patch.object(subprocess, 'check_output')
   @mock.patch.object(subprocess, 'run')
   def test_pass_without_retry(self, mock_run, mock_output, mock_call):
-    rust_uprev.upload_to_localmirror(self.tempdir, self.new_version)
+    rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name,
+                                     self.tempdir)
     mock_output.assert_called_once_with(
         ['gpg', '--verify', self.sig_file, self.rust_file],
         encoding='utf-8',
@@ -264,7 +266,8 @@ class UploadToLocalmirrorTests(unittest.TestCase):
     mock_check.side_effect = subprocess.CalledProcessError(
         returncode=2, cmd=None, output="gpg: Can't check signature")
     mock_output.return_value = 'some_gpg_keys'
-    rust_uprev.upload_to_localmirror(self.tempdir, self.new_version)
+    rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name,
+                                     self.tempdir)
     mock_check.assert_called_once_with(
         ['gpg', '--verify', self.sig_file, self.rust_file],
         encoding='utf-8',
@@ -285,6 +288,25 @@ class UploadToLocalmirrorTests(unittest.TestCase):
         ])
     ])
 
+  @mock.patch.object(rust_uprev, 'upload_single_tarball')
+  def test_upload_to_mirror(self, mock_upload):
+    stage0_info = '2020-01-01', '1.1.1', '0.1.0'
+    rust_uprev.upload_to_localmirror(self.tempdir, self.new_version,
+                                     stage0_info)
+    mock_upload.assert_has_calls([
+        mock.call(self.rust_url, f'rustc-{self.new_version}-src.tar.gz',
+                  self.tempdir),
+        mock.call(f'{self.rust_url}/{stage0_info[0]}',
+                  f'rust-std-{stage0_info[1]}-x86_64-unknown-linux-gnu.tar.gz',
+                  self.tempdir),
+        mock.call(self.rust_url,
+                  f'rustc-{stage0_info[1]}-x86_64-unknown-linux-gnu.tar.gz',
+                  self.tempdir),
+        mock.call(self.rust_url,
+                  f'cargo-{stage0_info[2]}-x86_64-unknown-linux-gnu.tar.gz',
+                  self.tempdir),
+    ])
+
 
 class RustUprevOtherStagesTests(unittest.TestCase):
   """Tests for other steps in rust_uprev"""