rust_uprev: check for files in gs:// before downloading them

I had to `--reset` a few times; downloading these files just to not
upload them took a bit, and wasn't useful. This CL adds a check so we
don't do that in the future.

BUG=chromium:1136579, chromium:1112551
TEST=Ran the script

Change-Id: I03f9f21cc07a7b16b540a5be5b4b08e105fc81a5
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2462926
Reviewed-by: Tiancong Wang <tcwang@google.com>
Tested-by: George Burgess <gbiv@chromium.org>

2 files changed, 28 insertions, 3 deletions

diff --git a/rust_tools/rust_uprev.py b/rust_tools/rust_uprev.py
index b4fcaf18..1d29556d 100755
--- a/rust_tools/rust_uprev.py
+++ b/rust_tools/rust_uprev.py
@@ -379,9 +379,20 @@ def update_virtual_rust(template_version: RustVersion,
 def upload_single_tarball(rust_url: str, tarfile_name: str,
                           tempdir: str) -> None:
   rust_src = f'{rust_url}/{tarfile_name}'
-  logging.info('Downloading Rust artifact from %s', rust_src)
   gsutil_location = f'gs://chromeos-localmirror/distfiles/{tarfile_name}'
 
+  missing_file = subprocess.call(
+      ['gsutil', 'ls', gsutil_location],
+      stdout=subprocess.DEVNULL,
+      stderr=subprocess.DEVNULL,
+  )
+  if not missing_file:
+    logging.info('Rust artifact at %s already exists; skipping download',
+                 gsutil_location)
+    return
+
+  logging.info('Downloading Rust artifact from %s', rust_src)
+
   # Download Rust's source
   rust_file = os.path.join(tempdir, tarfile_name)
   subprocess.check_call(['curl', '-f', '-o', rust_file, rust_src])
diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py
index d7efdce2..d13ce912 100755
--- a/rust_tools/rust_uprev_test.py
+++ b/rust_tools/rust_uprev_test.py
@@ -236,16 +236,23 @@ class UploadToLocalmirrorTests(unittest.TestCase):
     self.rust_file = os.path.join(self.tempdir, self.tarfile_name)
     self.sig_file = os.path.join(self.tempdir, 'rustc_sig.asc')
 
+  @mock.patch.object(subprocess, 'call', return_value=1)
   @mock.patch.object(subprocess, 'check_call')
   @mock.patch.object(subprocess, 'check_output')
   @mock.patch.object(subprocess, 'run')
-  def test_pass_without_retry(self, mock_run, mock_output, mock_call):
+  def test_pass_without_retry(self, mock_run, mock_output, mock_call,
+                              mock_raw_call):
     rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name,
                                      self.tempdir)
     mock_output.assert_called_once_with(
         ['gpg', '--verify', self.sig_file, self.rust_file],
         encoding='utf-8',
         stderr=subprocess.STDOUT)
+    mock_raw_call.assert_has_calls([
+        mock.call(['gsutil', 'ls', self.gsurl],
+                  stdout=subprocess.DEVNULL,
+                  stderr=subprocess.DEVNULL)
+    ])
     mock_call.assert_has_calls([
         mock.call(['curl', '-f', '-o', self.rust_file, self.rust_src]),
         mock.call(['curl', '-f', '-o', self.sig_file, f'{self.rust_src}.asc']),
@@ -256,11 +263,13 @@ class UploadToLocalmirrorTests(unittest.TestCase):
     ])
     mock_run.assert_not_called()
 
+  @mock.patch.object(subprocess, 'call')
   @mock.patch.object(subprocess, 'check_call')
   @mock.patch.object(subprocess, 'check_output')
   @mock.patch.object(subprocess, 'run')
   @mock.patch.object(rust_uprev, 'get_command_output')
-  def test_pass_with_retry(self, mock_output, mock_run, mock_check, mock_call):
+  def test_pass_with_retry(self, mock_output, mock_run, mock_check, mock_call,
+                           mock_raw_call):
     mock_check.side_effect = subprocess.CalledProcessError(
         returncode=2, cmd=None, output="gpg: Can't check signature")
     mock_output.return_value = 'some_gpg_keys'
@@ -276,6 +285,11 @@ class UploadToLocalmirrorTests(unittest.TestCase):
                                      input='some_gpg_keys',
                                      encoding='utf-8',
                                      check=True)
+    mock_raw_call.assert_has_calls([
+        mock.call(['gsutil', 'ls', self.gsurl],
+                  stdout=subprocess.DEVNULL,
+                  stderr=subprocess.DEVNULL)
+    ])
     mock_call.assert_has_calls([
         mock.call(['curl', '-f', '-o', self.rust_file, self.rust_src]),
         mock.call(['curl', '-f', '-o', self.sig_file, f'{self.rust_src}.asc']),