diff options
Diffstat (limited to 'rust_tools/rust_uprev_test.py')
| -rwxr-xr-x | rust_tools/rust_uprev_test.py | 415 |
1 files changed, 185 insertions, 230 deletions
diff --git a/rust_tools/rust_uprev_test.py b/rust_tools/rust_uprev_test.py index 00761391..fc506004 100755 --- a/rust_tools/rust_uprev_test.py +++ b/rust_tools/rust_uprev_test.py @@ -6,94 +6,16 @@ """Tests for rust_uprev.py""" +# pylint: disable=cros-logging-import import os import shutil import subprocess -import tempfile import unittest -from pathlib import Path from unittest import mock from llvm_tools import git import rust_uprev -from rust_uprev import RustVersion - - -def _fail_command(cmd, *_args, **_kwargs): - err = subprocess.CalledProcessError(returncode=1, cmd=cmd) - err.stderr = b'mock failure' - raise err - - -class FetchDistfileTest(unittest.TestCase): - """Tests rust_uprev.fetch_distfile_from_mirror()""" - - @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') - @mock.patch.object(subprocess, 'call', side_effect=_fail_command) - def test_fetch_difstfile_fail(self, *_args) -> None: - with self.assertRaises(subprocess.CalledProcessError): - rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') - - @mock.patch.object(rust_uprev, - 'get_command_output_unchecked', - return_value='AccessDeniedException: Access denied.') - @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') - @mock.patch.object(subprocess, 'call', return_value=0) - def test_fetch_distfile_acl_access_denied(self, *_args) -> None: - rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') - - @mock.patch.object( - rust_uprev, - 'get_command_output_unchecked', - return_value='[ { "entity": "allUsers", "role": "READER" } ]') - @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') - @mock.patch.object(subprocess, 'call', return_value=0) - def test_fetch_distfile_acl_ok(self, *_args) -> None: - rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') - - @mock.patch.object( - rust_uprev, - 'get_command_output_unchecked', - return_value='[ { "entity": "___fake@google.com", "role": "OWNER" } ]') - @mock.patch.object(rust_uprev, 'get_distdir', return_value='/fake/distfiles') - @mock.patch.object(subprocess, 'call', return_value=0) - def test_fetch_distfile_acl_wrong(self, *_args) -> None: - with self.assertRaisesRegex(Exception, 'allUsers.*READER'): - with self.assertLogs(level='ERROR') as log: - rust_uprev.fetch_distfile_from_mirror('test_distfile.tar.gz') - self.assertIn( - '[ { "entity": "___fake@google.com", "role": "OWNER" } ]', - '\n'.join(log.output)) - - -class FindEbuildPathTest(unittest.TestCase): - """Tests for rust_uprev.find_ebuild_path()""" - - def test_exact_version(self): - with tempfile.TemporaryDirectory() as tmpdir: - ebuild = Path(tmpdir, 'test-1.3.4.ebuild') - ebuild.touch() - Path(tmpdir, 'test-1.2.3.ebuild').touch() - result = rust_uprev.find_ebuild_path(tmpdir, 'test', - rust_uprev.RustVersion(1, 3, 4)) - self.assertEqual(result, ebuild) - - def test_no_version(self): - with tempfile.TemporaryDirectory() as tmpdir: - ebuild = Path(tmpdir, 'test-1.2.3.ebuild') - ebuild.touch() - result = rust_uprev.find_ebuild_path(tmpdir, 'test') - self.assertEqual(result, ebuild) - - def test_patch_version(self): - with tempfile.TemporaryDirectory() as tmpdir: - ebuild = Path(tmpdir, 'test-1.3.4-r3.ebuild') - ebuild.touch() - Path(tmpdir, 'test-1.2.3.ebuild').touch() - result = rust_uprev.find_ebuild_path(tmpdir, 'test', - rust_uprev.RustVersion(1, 3, 4)) - self.assertEqual(result, ebuild) class RustVersionTest(unittest.TestCase): @@ -127,77 +49,58 @@ class PrepareUprevTest(unittest.TestCase): """Tests for prepare_uprev step in rust_uprev""" def setUp(self): - self.bootstrap_version = rust_uprev.RustVersion(1, 1, 0) self.version_old = rust_uprev.RustVersion(1, 2, 3) self.version_new = rust_uprev.RustVersion(1, 3, 5) - @mock.patch.object(rust_uprev, - 'find_ebuild_for_rust_version', - return_value='/path/to/ebuild') - @mock.patch.object(rust_uprev, 'find_ebuild_path') + @mock.patch.object( + rust_uprev, + 'find_ebuild_for_rust_version', + return_value='/path/to/ebuild') @mock.patch.object(rust_uprev, 'get_command_output') - def test_success_with_template(self, mock_command, mock_find_ebuild, - _ebuild_for_version): - bootstrap_ebuild_path = Path( - '/path/to/rust-bootstrap/', - f'rust-bootstrap-{self.bootstrap_version}.ebuild') - mock_find_ebuild.return_value = bootstrap_ebuild_path - expected = (self.version_old, '/path/to/ebuild', self.bootstrap_version) - actual = rust_uprev.prepare_uprev(rust_version=self.version_new, - template=self.version_old) + def test_success_with_template(self, mock_command, mock_find_ebuild): + expected = (self.version_old, '/path/to/ebuild') + actual = rust_uprev.prepare_uprev( + rust_version=self.version_new, template=self.version_old) self.assertEqual(expected, actual) mock_command.assert_not_called() - @mock.patch.object(rust_uprev, - 'find_ebuild_for_rust_version', - return_value='/path/to/ebuild') - @mock.patch.object(rust_uprev, - 'get_rust_bootstrap_version', - return_value=RustVersion(0, 41, 12)) + @mock.patch.object( + rust_uprev, + 'find_ebuild_for_rust_version', + return_value='/path/to/ebuild') @mock.patch.object(rust_uprev, 'get_command_output') def test_return_none_with_template_larger_than_input(self, mock_command, - *_args): - ret = rust_uprev.prepare_uprev(rust_version=self.version_old, - template=self.version_new) + _mock_find_ebuild): + ret = rust_uprev.prepare_uprev( + rust_version=self.version_old, template=self.version_new) self.assertIsNone(ret) mock_command.assert_not_called() - @mock.patch.object(rust_uprev, 'find_ebuild_path') @mock.patch.object(os.path, 'exists') @mock.patch.object(rust_uprev, 'get_command_output') - def test_success_without_template(self, mock_command, mock_exists, - mock_find_ebuild): + def test_success_without_template(self, mock_command, mock_exists): rust_ebuild_path = f'/path/to/rust/rust-{self.version_old}-r3.ebuild' mock_command.return_value = rust_ebuild_path - bootstrap_ebuild_path = Path( - '/path/to/rust-bootstrap', - f'rust-bootstrap-{self.bootstrap_version}.ebuild') - mock_find_ebuild.return_value = bootstrap_ebuild_path - expected = (self.version_old, rust_ebuild_path, self.bootstrap_version) - actual = rust_uprev.prepare_uprev(rust_version=self.version_new, - template=None) + expected = (self.version_old, rust_ebuild_path) + actual = rust_uprev.prepare_uprev( + rust_version=self.version_new, template=None) self.assertEqual(expected, actual) mock_command.assert_called_once_with(['equery', 'w', 'rust']) mock_exists.assert_not_called() - @mock.patch.object(rust_uprev, - 'get_rust_bootstrap_version', - return_value=RustVersion(0, 41, 12)) @mock.patch.object(os.path, 'exists') @mock.patch.object(rust_uprev, 'get_command_output') def test_return_none_with_ebuild_larger_than_input(self, mock_command, - mock_exists, *_args): + mock_exists): mock_command.return_value = f'/path/to/rust/rust-{self.version_new}.ebuild' - ret = rust_uprev.prepare_uprev(rust_version=self.version_old, - template=None) + ret = rust_uprev.prepare_uprev(rust_version=self.version_old, template=None) self.assertIsNone(ret) mock_exists.assert_not_called() def test_prepare_uprev_from_json(self): ebuild_path = '/path/to/the/ebuild' - json_result = (list(self.version_new), ebuild_path, - list(self.bootstrap_version)) - expected = (self.version_new, ebuild_path, self.bootstrap_version) + json_result = (list(self.version_new), ebuild_path) + expected = (self.version_new, ebuild_path) actual = rust_uprev.prepare_uprev_from_json(json_result) self.assertEqual(expected, actual) @@ -205,30 +108,32 @@ class PrepareUprevTest(unittest.TestCase): class UpdateEbuildTest(unittest.TestCase): """Tests for update_ebuild step in rust_uprev""" ebuild_file_before = """ -BOOTSTRAP_VERSION="1.2.0" + STAGE0_DATE="2019-01-01" + STAGE0_VERSION="any.random.(number)" + STAGE0_VERSION_CARGO="0.0.0" """ ebuild_file_after = """ -BOOTSTRAP_VERSION="1.3.6" + STAGE0_DATE="2020-01-01" + STAGE0_VERSION="1.1.1" + STAGE0_VERSION_CARGO="0.1.0" """ def test_success(self): mock_open = mock.mock_open(read_data=self.ebuild_file_before) - # ebuild_file and new bootstrap version are deliberately different ebuild_file = '/path/to/rust/rust-1.3.5.ebuild' with mock.patch('builtins.open', mock_open): - rust_uprev.update_ebuild(ebuild_file, - rust_uprev.RustVersion.parse('1.3.6')) + rust_uprev.update_ebuild(ebuild_file, ('2020-01-01', '1.1.1', '0.1.0')) mock_open.return_value.__enter__().write.assert_called_once_with( self.ebuild_file_after) def test_fail_when_ebuild_misses_a_variable(self): - mock_open = mock.mock_open(read_data='') + ebuild_file = 'STAGE0_DATE="2019-01-01"' + mock_open = mock.mock_open(read_data=ebuild_file) ebuild_file = '/path/to/rust/rust-1.3.5.ebuild' with mock.patch('builtins.open', mock_open): with self.assertRaises(RuntimeError) as context: - rust_uprev.update_ebuild(ebuild_file, - rust_uprev.RustVersion.parse('1.2.0')) - self.assertEqual('BOOTSTRAP_VERSION not found in rust ebuild', + rust_uprev.update_ebuild(ebuild_file, ('2020-01-01', '1.1.1', '0.1.0')) + self.assertEqual('STAGE0_VERSION not found in rust ebuild', str(context.exception)) @@ -245,79 +150,44 @@ class UpdateManifestTest(unittest.TestCase): f'RESTRICT="{after}"') def test_add_mirror_in_ebuild(self): - self._run_test_flip_mirror(before='variable1 variable2', - after='variable1 variable2 mirror', - add=True, - expect_write=True) + self._run_test_flip_mirror( + before='variable1 variable2', + after='variable1 variable2 mirror', + add=True, + expect_write=True) def test_remove_mirror_in_ebuild(self): - self._run_test_flip_mirror(before='variable1 variable2 mirror', - after='variable1 variable2', - add=False, - expect_write=True) + self._run_test_flip_mirror( + before='variable1 variable2 mirror', + after='variable1 variable2', + add=False, + expect_write=True) def test_add_mirror_when_exists(self): - self._run_test_flip_mirror(before='variable1 variable2 mirror', - after='variable1 variable2 mirror', - add=True, - expect_write=False) + self._run_test_flip_mirror( + before='variable1 variable2 mirror', + after='variable1 variable2 mirror', + add=True, + expect_write=False) def test_remove_mirror_when_not_exists(self): - self._run_test_flip_mirror(before='variable1 variable2', - after='variable1 variable2', - add=False, - expect_write=False) + self._run_test_flip_mirror( + before='variable1 variable2', + after='variable1 variable2', + add=False, + expect_write=False) @mock.patch.object(rust_uprev, 'flip_mirror_in_ebuild') - @mock.patch.object(rust_uprev, 'ebuild_actions') + @mock.patch.object(rust_uprev, 'rust_ebuild_actions') def test_update_manifest(self, mock_run, mock_flip): - ebuild_file = Path('/path/to/rust/rust-1.1.1.ebuild') + ebuild_file = '/path/to/rust/rust-1.1.1.ebuild' rust_uprev.update_manifest(ebuild_file) - mock_run.assert_called_once_with('rust', ['manifest']) + mock_run.assert_called_once_with(['manifest']) mock_flip.assert_has_calls( [mock.call(ebuild_file, add=True), mock.call(ebuild_file, add=False)]) -class UpdateBootstrapEbuildTest(unittest.TestCase): - """Tests for rust_uprev.update_bootstrap_ebuild()""" - - def test_update_bootstrap_ebuild(self): - # The update should do two things: - # 1. Create a copy of rust-bootstrap's ebuild with the new version number. - # 2. Add the old PV to RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE. - with tempfile.TemporaryDirectory() as tmpdir_str, \ - mock.patch.object(rust_uprev, 'find_ebuild_path') as mock_find_ebuild: - tmpdir = Path(tmpdir_str) - bootstrapdir = Path.joinpath(tmpdir, 'rust-bootstrap') - bootstrapdir.mkdir() - old_ebuild = bootstrapdir.joinpath('rust-bootstrap-1.45.2.ebuild') - old_ebuild.write_text(encoding='utf-8', - data=""" -some text -RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( -\t1.43.1 -\t1.44.1 -) -some more text -""") - mock_find_ebuild.return_value = old_ebuild - rust_uprev.update_bootstrap_ebuild(rust_uprev.RustVersion(1, 46, 0)) - new_ebuild = bootstrapdir.joinpath('rust-bootstrap-1.46.0.ebuild') - self.assertTrue(new_ebuild.exists()) - text = new_ebuild.read_text() - self.assertEqual( - text, """ -some text -RUSTC_RAW_FULL_BOOTSTRAP_SEQUENCE=( -\t1.43.1 -\t1.44.1 -\t1.45.2 -) -some more text -""") - - class UpdateRustPackagesTests(unittest.TestCase): """Tests for update_rust_packages step.""" @@ -353,6 +223,103 @@ class UpdateRustPackagesTests(unittest.TestCase): package_after) +class UploadToLocalmirrorTests(unittest.TestCase): + """Tests for upload_to_localmirror""" + + def setUp(self): + self.tempdir = '/tmp/any/dir' + self.new_version = rust_uprev.RustVersion(1, 3, 5) + self.rust_url = 'https://static.rust-lang.org/dist' + self.tarfile_name = f'rustc-{self.new_version}-src.tar.gz' + self.rust_src = f'https://static.rust-lang.org/dist/{self.tarfile_name}' + self.gsurl = f'gs://chromeos-localmirror/distfiles/{self.tarfile_name}' + self.rust_file = os.path.join(self.tempdir, self.tarfile_name) + self.sig_file = os.path.join(self.tempdir, 'rustc_sig.asc') + + @mock.patch.object(subprocess, 'call', return_value=1) + @mock.patch.object(subprocess, 'check_call') + @mock.patch.object(subprocess, 'check_output') + @mock.patch.object(subprocess, 'run') + def test_pass_without_retry(self, mock_run, mock_output, mock_call, + mock_raw_call): + rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name, + self.tempdir) + mock_output.assert_called_once_with( + ['gpg', '--verify', self.sig_file, self.rust_file], + encoding='utf-8', + stderr=subprocess.STDOUT) + mock_raw_call.assert_has_calls([ + mock.call(['gsutil', 'ls', self.gsurl], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + ]) + mock_call.assert_has_calls([ + mock.call(['curl', '-f', '-o', self.rust_file, self.rust_src]), + mock.call(['curl', '-f', '-o', self.sig_file, f'{self.rust_src}.asc']), + mock.call([ + 'gsutil', 'cp', '-n', '-a', 'public-read', self.rust_file, + self.gsurl + ]) + ]) + mock_run.assert_not_called() + + @mock.patch.object(subprocess, 'call') + @mock.patch.object(subprocess, 'check_call') + @mock.patch.object(subprocess, 'check_output') + @mock.patch.object(subprocess, 'run') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_pass_with_retry(self, mock_output, mock_run, mock_check, mock_call, + mock_raw_call): + mock_check.side_effect = subprocess.CalledProcessError( + returncode=2, cmd=None, output="gpg: Can't check signature") + mock_output.return_value = 'some_gpg_keys' + rust_uprev.upload_single_tarball(self.rust_url, self.tarfile_name, + self.tempdir) + mock_check.assert_called_once_with( + ['gpg', '--verify', self.sig_file, self.rust_file], + encoding='utf-8', + stderr=subprocess.STDOUT) + mock_output.assert_called_once_with( + ['curl', '-f', 'https://keybase.io/rust/pgp_keys.asc']) + mock_run.assert_called_once_with(['gpg', '--import'], + input='some_gpg_keys', + encoding='utf-8', + check=True) + mock_raw_call.assert_has_calls([ + mock.call(['gsutil', 'ls', self.gsurl], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + ]) + mock_call.assert_has_calls([ + mock.call(['curl', '-f', '-o', self.rust_file, self.rust_src]), + mock.call(['curl', '-f', '-o', self.sig_file, f'{self.rust_src}.asc']), + mock.call(['gpg', '--verify', self.sig_file, self.rust_file]), + mock.call([ + 'gsutil', 'cp', '-n', '-a', 'public-read', self.rust_file, + self.gsurl + ]) + ]) + + @mock.patch.object(rust_uprev, 'upload_single_tarball') + def test_upload_to_mirror(self, mock_upload): + stage0_info = '2020-01-01', '1.1.1', '0.1.0' + rust_uprev.upload_to_localmirror(self.tempdir, self.new_version, + stage0_info) + mock_upload.assert_has_calls([ + mock.call(self.rust_url, f'rustc-{self.new_version}-src.tar.gz', + self.tempdir), + mock.call(f'{self.rust_url}/{stage0_info[0]}', + f'rust-std-{stage0_info[1]}-x86_64-unknown-linux-gnu.tar.gz', + self.tempdir), + mock.call(self.rust_url, + f'rustc-{stage0_info[1]}-x86_64-unknown-linux-gnu.tar.gz', + self.tempdir), + mock.call(self.rust_url, + f'cargo-{stage0_info[2]}-x86_64-unknown-linux-gnu.tar.gz', + self.tempdir), + ]) + + class RustUprevOtherStagesTests(unittest.TestCase): """Tests for other steps in rust_uprev""" @@ -363,6 +330,25 @@ class RustUprevOtherStagesTests(unittest.TestCase): self.ebuild_file = os.path.join(rust_uprev.RUST_PATH, 'rust-{self.new_version}.ebuild') + @mock.patch.object(rust_uprev, 'get_command_output') + def test_parse_stage0_file(self, mock_get): + stage0_file = """ + unrelated stuff before + date: 2020-01-01 + rustc: 1.1.1 + cargo: 0.1.0 + unrelated stuff after + """ + mock_get.return_value = stage0_file + expected = '2020-01-01', '1.1.1', '0.1.0' + rust_version = rust_uprev.RustVersion(1, 2, 3) + actual = rust_uprev.parse_stage0_file(rust_version) + self.assertEqual(expected, actual) + mock_get.assert_called_once_with([ + 'curl', '-f', 'https://raw.githubusercontent.com/rust-lang/rust/' + f'{rust_version}/src/stage0.txt' + ]) + @mock.patch.object(shutil, 'copyfile') @mock.patch.object(os, 'listdir') @mock.patch.object(subprocess, 'check_call') @@ -373,8 +359,7 @@ class RustUprevOtherStagesTests(unittest.TestCase): f'rust-{self.current_version}-patch-1.patch', f'rust-{self.current_version}-patch-2-new.patch' ] - rust_uprev.copy_patches(rust_uprev.RUST_PATH, self.current_version, - self.new_version) + rust_uprev.copy_patches(self.current_version, self.new_version) mock_copy.assert_has_calls([ mock.call( os.path.join(rust_uprev.RUST_PATH, 'files', @@ -389,8 +374,8 @@ class RustUprevOtherStagesTests(unittest.TestCase): f'rust-{self.new_version}-patch-2-new.patch')) ]) mock_call.assert_called_once_with( - ['git', 'add', f'rust-{self.new_version}-*.patch'], - cwd=rust_uprev.RUST_PATH.joinpath('files')) + ['git', 'add', f'files/rust-{self.new_version}-*.patch'], + cwd=rust_uprev.RUST_PATH) @mock.patch.object(shutil, 'copyfile') @mock.patch.object(subprocess, 'check_call') @@ -399,53 +384,23 @@ class RustUprevOtherStagesTests(unittest.TestCase): rust_uprev.create_ebuild(template_ebuild, self.new_version) mock_copy.assert_called_once_with( template_ebuild, - rust_uprev.RUST_PATH.joinpath(f'rust-{self.new_version}.ebuild')) + os.path.join(rust_uprev.RUST_PATH, f'rust-{self.new_version}.ebuild')) mock_call.assert_called_once_with( ['git', 'add', f'rust-{self.new_version}.ebuild'], cwd=rust_uprev.RUST_PATH) - @mock.patch.object(rust_uprev, 'find_ebuild_for_package') - @mock.patch.object(subprocess, 'check_call') - def test_remove_rust_bootstrap_version(self, mock_call, *_args): - bootstrap_path = os.path.join(rust_uprev.RUST_PATH, '..', 'rust-bootstrap') - rust_uprev.remove_rust_bootstrap_version(self.old_version, lambda *x: ()) - mock_call.has_calls([ - [ - 'git', 'rm', - os.path.join(bootstrap_path, 'files', - f'rust-bootstrap-{self.old_version}-*.patch') - ], - [ - 'git', 'rm', - os.path.join(bootstrap_path, - f'rust-bootstrap-{self.old_version}.ebuild') - ], - ]) - - @mock.patch.object(rust_uprev, 'find_ebuild_path') - @mock.patch.object(subprocess, 'check_call') - def test_remove_virtual_rust(self, mock_call, mock_find_ebuild): - ebuild_path = Path( - f'/some/dir/virtual/rust/rust-{self.old_version}.ebuild') - mock_find_ebuild.return_value = Path(ebuild_path) - rust_uprev.remove_virtual_rust(self.old_version) - mock_call.assert_called_once_with( - ['git', 'rm', str(ebuild_path.name)], cwd=ebuild_path.parent) - - @mock.patch.object(rust_uprev, 'find_ebuild_path') + @mock.patch.object(os.path, 'exists', return_value=True) @mock.patch.object(shutil, 'copyfile') @mock.patch.object(subprocess, 'check_call') - def test_update_virtual_rust(self, mock_call, mock_copy, mock_find_ebuild): - ebuild_path = Path( - f'/some/dir/virtual/rust/rust-{self.current_version}.ebuild') - mock_find_ebuild.return_value = Path(ebuild_path) + def test_update_virtual_rust(self, mock_call, mock_copy, mock_exists): + virtual_rust_dir = os.path.join(rust_uprev.RUST_PATH, '../../virtual/rust') rust_uprev.update_virtual_rust(self.current_version, self.new_version) mock_call.assert_called_once_with( - ['git', 'add', f'rust-{self.new_version}.ebuild'], - cwd=ebuild_path.parent) + ['git', 'add', f'rust-{self.new_version}.ebuild'], cwd=virtual_rust_dir) mock_copy.assert_called_once_with( - ebuild_path.parent.joinpath(f'rust-{self.current_version}.ebuild'), - ebuild_path.parent.joinpath(f'rust-{self.new_version}.ebuild')) + os.path.join(virtual_rust_dir, f'rust-{self.current_version}.ebuild'), + os.path.join(virtual_rust_dir, f'rust-{self.new_version}.ebuild')) + mock_exists.assert_called_once_with(virtual_rust_dir) @mock.patch.object(os, 'listdir') def test_find_oldest_rust_version_in_chroot_pass(self, mock_ls): |