llvm_tools: llvm version detection for roll tools

This CL implements version detection for update_chromeos_llvm_hash so
that uprevs will automatically get the correct major version in the llvm
ebuild.

BUG=b:186802799
TEST=Updated unit tests and ran tools manually

Change-Id: Id3a21ff7f708ce38303e27202428fb3bb519bafe
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2910173
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Commit-Queue: Ryan Beltran <ryanbeltran@chromium.org>
Tested-by: Ryan Beltran <ryanbeltran@chromium.org>

1 files changed, 109 insertions, 34 deletions

diff --git a/llvm_tools/update_chromeos_llvm_hash_unittest.py b/llvm_tools/update_chromeos_llvm_hash_unittest.py
index 205feb0c..2e928be9 100755
--- a/llvm_tools/update_chromeos_llvm_hash_unittest.py
+++ b/llvm_tools/update_chromeos_llvm_hash_unittest.py
@@ -18,6 +18,7 @@ import unittest.mock as mock
 
 import chroot
 import failure_modes
+import get_llvm_hash
 import git
 import llvm_patch_management
 import test_helpers
@@ -67,8 +68,9 @@ class UpdateLLVMHashTest(unittest.TestCase):
       # Verify the exception is raised when the ebuild file does not have
       # 'LLVM_HASH'.
       with self.assertRaises(ValueError) as err:
-        update_chromeos_llvm_hash.UpdateEbuildLLVMHash(
-            ebuild_file, llvm_variant, git_hash, svn_version)
+        update_chromeos_llvm_hash.UpdateEbuildLLVMHash(ebuild_file,
+                                                       llvm_variant, git_hash,
+                                                       svn_version)
 
         self.assertEqual(
             str(err.exception), ('Failed to update %s.', 'LLVM_HASH'))
@@ -95,8 +97,9 @@ class UpdateLLVMHashTest(unittest.TestCase):
       # Verify the exception is raised when the ebuild file does not have
       # 'LLVM_NEXT_HASH'.
       with self.assertRaises(ValueError) as err:
-        update_chromeos_llvm_hash.UpdateEbuildLLVMHash(
-            ebuild_file, llvm_variant, git_hash, svn_version)
+        update_chromeos_llvm_hash.UpdateEbuildLLVMHash(ebuild_file,
+                                                       llvm_variant, git_hash,
+                                                       svn_version)
 
         self.assertEqual(
             str(err.exception), ('Failed to update %s.', 'LLVM_NEXT_HASH'))
@@ -179,19 +182,25 @@ class UpdateLLVMHashTest(unittest.TestCase):
 
     mock_stage_commit_command.assert_called_once()
 
+  @mock.patch.object(get_llvm_hash, 'GetLLVMMajorVersion')
   @mock.patch.object(os.path, 'islink', return_value=False)
-  def testFailedToUprevEbuildToVersionForInvalidSymlink(self, mock_islink):
+  def testFailedToUprevEbuildToVersionForInvalidSymlink(self, mock_islink,
+                                                        mock_llvm_version):
     symlink_path = '/path/to/chroot/package/package.ebuild'
     svn_version = 1000
+    git_hash = 'badf00d'
+    mock_llvm_version.return_value = '1234'
 
     # Verify the exception is raised when a invalid symbolic link is passed in.
     with self.assertRaises(ValueError) as err:
-      update_chromeos_llvm_hash.UprevEbuildToVersion(symlink_path, svn_version)
+      update_chromeos_llvm_hash.UprevEbuildToVersion(symlink_path, svn_version,
+                                                     git_hash)
 
     self.assertEqual(
         str(err.exception), 'Invalid symlink provided: %s' % symlink_path)
 
     mock_islink.assert_called_once()
+    mock_llvm_version.assert_not_called()
 
   @mock.patch.object(os.path, 'islink', return_value=False)
   def testFailedToUprevEbuildSymlinkForInvalidSymlink(self, mock_islink):
@@ -206,22 +215,28 @@ class UpdateLLVMHashTest(unittest.TestCase):
 
     mock_islink.assert_called_once()
 
+  @mock.patch.object(get_llvm_hash, 'GetLLVMMajorVersion')
   # Simulate 'os.path.islink' when a symbolic link is passed in.
   @mock.patch.object(os.path, 'islink', return_value=True)
   # Simulate 'os.path.realpath' when a symbolic link is passed in.
   @mock.patch.object(os.path, 'realpath', return_value=True)
-  def testFailedToUprevEbuildToVersion(self, mock_realpath, mock_islink):
+  def testFailedToUprevEbuildToVersion(self, mock_realpath, mock_islink,
+                                       mock_llvm_version):
     symlink_path = '/path/to/chroot/llvm/llvm_pre123_p.ebuild'
     mock_realpath.return_value = '/abs/path/to/llvm/llvm_pre123_p.ebuild'
+    git_hash = 'badf00d'
+    mock_llvm_version.return_value = '1234'
     svn_version = 1000
 
     # Verify the exception is raised when the symlink does not match the
     # expected pattern
     with self.assertRaises(ValueError) as err:
-      update_chromeos_llvm_hash.UprevEbuildToVersion(symlink_path, svn_version)
+      update_chromeos_llvm_hash.UprevEbuildToVersion(symlink_path, svn_version,
+                                                     git_hash)
 
     self.assertEqual(str(err.exception), 'Failed to uprev the ebuild.')
 
+    mock_llvm_version.assert_called_once_with(git_hash)
     mock_islink.assert_called_once_with(symlink_path)
 
   # Simulate 'os.path.islink' when a symbolic link is passed in.
@@ -238,17 +253,24 @@ class UpdateLLVMHashTest(unittest.TestCase):
 
     mock_islink.assert_called_once_with(symlink_path)
 
+  @mock.patch.object(get_llvm_hash, 'GetLLVMMajorVersion')
   @mock.patch.object(os.path, 'islink', return_value=True)
   @mock.patch.object(os.path, 'realpath')
   @mock.patch.object(subprocess, 'check_output', return_value=None)
   def testSuccessfullyUprevEbuildToVersionLLVM(self, mock_command_output,
-                                               mock_realpath, mock_islink):
-    symlink = '/path/to/llvm/llvm_pre3_p2-r10.ebuild'
-    ebuild = '/abs/path/to/llvm/llvm_pre3_p2.ebuild'
+                                               mock_realpath, mock_islink,
+                                               mock_llvm_version):
+    symlink = '/path/to/llvm/llvm-12.0_pre3_p2-r10.ebuild'
+    ebuild = '/abs/path/to/llvm/llvm-12.0_pre3_p2.ebuild'
     mock_realpath.return_value = ebuild
+    git_hash = 'badf00d'
+    mock_llvm_version.return_value = '1234'
     svn_version = 1000
 
-    update_chromeos_llvm_hash.UprevEbuildToVersion(symlink, svn_version)
+    update_chromeos_llvm_hash.UprevEbuildToVersion(symlink, svn_version,
+                                                   git_hash)
+
+    mock_llvm_version.assert_called_once_with(git_hash)
 
     mock_islink.assert_called()
 
@@ -258,12 +280,8 @@ class UpdateLLVMHashTest(unittest.TestCase):
 
     # Verify commands
     symlink_dir = os.path.dirname(symlink)
-    new_ebuild, _is_changed = re.subn(
-        r'pre([0-9]+)_p([0-9]+)',
-        'pre%s_p%s' % (svn_version, \
-            datetime.today().strftime('%Y%m%d')),
-        ebuild,
-        count=1)
+    timestamp = datetime.today().strftime('%Y%m%d')
+    new_ebuild = '/abs/path/to/llvm/llvm-1234.0_pre1000_p%s.ebuild' % timestamp
     new_symlink = new_ebuild[:-len('.ebuild')] + '-r1.ebuild'
 
     expected_cmd = ['git', '-C', symlink_dir, 'mv', ebuild, new_ebuild]
@@ -291,8 +309,10 @@ class UpdateLLVMHashTest(unittest.TestCase):
     ebuild = '/abs/path/to/compiler-rt/compiler-rt_pre3_p2.ebuild'
     mock_realpath.return_value = ebuild
     svn_version = 1000
+    git_hash = '1234'
 
-    update_chromeos_llvm_hash.UprevEbuildToVersion(symlink, svn_version)
+    update_chromeos_llvm_hash.UprevEbuildToVersion(symlink, svn_version,
+                                                   git_hash)
 
     mock_islink.assert_called()
 
@@ -549,6 +569,7 @@ class UpdateLLVMHashTest(unittest.TestCase):
 
     self.assertEqual(mock_stage_patches_for_commit.call_count, 2)
 
+  @mock.patch.object(get_llvm_hash, 'GetLLVMMajorVersion')
   @mock.patch.object(update_chromeos_llvm_hash,
                      'CreatePathDictionaryFromPackages')
   @mock.patch.object(git, 'CreateBranch')
@@ -560,7 +581,7 @@ class UpdateLLVMHashTest(unittest.TestCase):
   def testExceptionRaisedWhenUpdatingPackages(
       self, mock_realpath, mock_delete_repo, mock_upload_changes,
       mock_uprev_symlink, mock_update_llvm_next, mock_create_repo,
-      mock_create_path_dict):
+      mock_create_path_dict, mock_llvm_major_version):
 
     abs_path_to_package = '/some/path/to/chroot/src/path/to/package.ebuild'
 
@@ -569,6 +590,7 @@ class UpdateLLVMHashTest(unittest.TestCase):
 
     path_to_package_dir = '/some/path/to/chroot/src/path/to'
 
+    mock_llvm_major_version.return_value = '1234'
     # Test function to simulate 'CreateBranch' when successfully created the
     # branch on a valid repo path.
     def SuccessfullyCreateBranchForChanges(_repo_path, branch):
@@ -628,9 +650,9 @@ class UpdateLLVMHashTest(unittest.TestCase):
     # the 'try' block by UprevEbuildSymlink function.
     with self.assertRaises(ValueError) as err:
       update_chromeos_llvm_hash.UpdatePackages(
-          packages_to_update, llvm_variant, git_hash, svn_version, chroot_path,
-          patch_metadata_file, failure_modes.FailureModes.FAIL, git_hash_source,
-          extra_commit_msg)
+          packages_to_update, llvm_variant, git_hash, svn_version,
+          chroot_path, patch_metadata_file, failure_modes.FailureModes.FAIL,
+          git_hash_source, extra_commit_msg)
 
     self.assertEqual(str(err.exception), 'Failed to uprev the ebuild.')
 
@@ -639,8 +661,9 @@ class UpdateLLVMHashTest(unittest.TestCase):
 
     mock_create_repo.assert_called_once_with(path_to_package_dir, branch)
 
-    mock_update_llvm_next.assert_called_once_with(
-        abs_path_to_package, llvm_variant, git_hash, svn_version)
+    mock_update_llvm_next.assert_called_once_with(abs_path_to_package,
+                                                  llvm_variant, git_hash,
+                                                  svn_version)
 
     mock_uprev_symlink.assert_called_once_with(symlink_path_to_package)
 
@@ -648,6 +671,8 @@ class UpdateLLVMHashTest(unittest.TestCase):
 
     mock_delete_repo.assert_called_once_with(path_to_package_dir, branch)
 
+  @mock.patch.object(update_chromeos_llvm_hash, 'EnsurePackageMaskContains')
+  @mock.patch.object(get_llvm_hash, 'GetLLVMMajorVersion')
   @mock.patch.object(update_chromeos_llvm_hash,
                      'CreatePathDictionaryFromPackages')
   @mock.patch.object(git, 'CreateBranch')
@@ -658,10 +683,12 @@ class UpdateLLVMHashTest(unittest.TestCase):
   @mock.patch.object(llvm_patch_management, 'UpdatePackagesPatchMetadataFile')
   @mock.patch.object(update_chromeos_llvm_hash,
                      'StagePatchMetadataFileForCommit')
-  def testSuccessfullyUpdatedPackages(
-      self, mock_stage_patch_file, mock_update_package_metadata_file,
-      mock_delete_repo, mock_upload_changes, mock_uprev_symlink,
-      mock_update_llvm_next, mock_create_repo, mock_create_path_dict):
+  def testSuccessfullyUpdatedPackages(self, mock_stage_patch_file,
+                                      mock_update_package_metadata_file,
+                                      mock_delete_repo, mock_upload_changes,
+                                      mock_uprev_symlink, mock_update_llvm_next,
+                                      mock_create_repo, mock_create_path_dict,
+                                      mock_llvm_version, mock_mask_contains):
 
     abs_path_to_package = '/some/path/to/chroot/src/path/to/package.ebuild'
 
@@ -746,6 +773,8 @@ class UpdateLLVMHashTest(unittest.TestCase):
     mock_uprev_symlink.side_effect = SuccessfullyUprevedEbuildSymlink
     mock_update_package_metadata_file.side_effect = RetrievedPatchResults
     mock_upload_changes.side_effect = SuccessfullyUploadedChanges
+    mock_llvm_version.return_value = '1234'
+    mock_mask_contains.reurn_value = None
 
     packages_to_update = ['test-packages/package1']
     llvm_variant = update_chromeos_llvm_hash.LLVMVariant.next
@@ -758,9 +787,10 @@ class UpdateLLVMHashTest(unittest.TestCase):
     extra_commit_msg = '\ncommit-message-end'
 
     change_list = update_chromeos_llvm_hash.UpdatePackages(
-        packages_to_update, llvm_variant, git_hash, svn_version, chroot_path,
-        patch_metadata_file, failure_modes.FailureModes.DISABLE_PATCHES,
-        git_hash_source, extra_commit_msg)
+        packages_to_update, llvm_variant, git_hash, svn_version,
+        chroot_path, patch_metadata_file,
+        failure_modes.FailureModes.DISABLE_PATCHES, git_hash_source,
+        extra_commit_msg)
 
     self.assertEqual(change_list.url,
                      'https://some_name/path/to/commit/+/12345')
@@ -772,11 +802,14 @@ class UpdateLLVMHashTest(unittest.TestCase):
 
     mock_create_repo.assert_called_once_with(path_to_package_dir, branch)
 
-    mock_update_llvm_next.assert_called_once_with(
-        abs_path_to_package, llvm_variant, git_hash, svn_version)
+    mock_update_llvm_next.assert_called_once_with(abs_path_to_package,
+                                                  llvm_variant, git_hash,
+                                                  svn_version)
 
     mock_uprev_symlink.assert_called_once_with(symlink_path_to_package)
 
+    mock_mask_contains.assert_called_once_with(chroot_path, git_hash)
+
     expected_commit_messages = [
         'llvm-next/tot: upgrade to a123testhash5 (r1000)\n',
         'The following packages have been updated:', 'path/to',
@@ -796,6 +829,48 @@ class UpdateLLVMHashTest(unittest.TestCase):
 
     mock_delete_repo.assert_called_once_with(path_to_package_dir, branch)
 
+  @mock.patch.object(subprocess, 'check_output', return_value=None)
+  @mock.patch.object(get_llvm_hash, 'GetLLVMMajorVersion')
+  def testEnsurePackageMaskContainsExisting(self, mock_llvm_version,
+                                            mock_git_add):
+    chroot_path = 'absolute/path/to/chroot'
+    git_hash = 'badf00d'
+    mock_llvm_version.return_value = '1234'
+    with mock.patch(
+        'update_chromeos_llvm_hash.open',
+        mock.mock_open(read_data='\n=sys-devel/llvm-1234.0_pre*\n'),
+        create=True) as mock_file:
+      update_chromeos_llvm_hash.EnsurePackageMaskContains(chroot_path, git_hash)
+      handle = mock_file()
+      handle.write.assert_not_called()
+    mock_llvm_version.assert_called_once_with(git_hash)
+
+    overlay_dir = 'absolute/path/to/chroot/src/third_party/chromiumos-overlay'
+    mask_path = overlay_dir + '/profiles/targets/chromeos/package.mask'
+    mock_git_add.assert_called_once_with(
+        ['git', '-C', overlay_dir, 'add', mask_path])
+
+  @mock.patch.object(subprocess, 'check_output', return_value=None)
+  @mock.patch.object(get_llvm_hash, 'GetLLVMMajorVersion')
+  def testEnsurePackageMaskContainsNotExisting(self, mock_llvm_version,
+                                               mock_git_add):
+    chroot_path = 'absolute/path/to/chroot'
+    git_hash = 'badf00d'
+    mock_llvm_version.return_value = '1234'
+    with mock.patch(
+        'update_chromeos_llvm_hash.open',
+        mock.mock_open(read_data='nothing relevant'),
+        create=True) as mock_file:
+      update_chromeos_llvm_hash.EnsurePackageMaskContains(chroot_path, git_hash)
+      handle = mock_file()
+      handle.write.assert_called_once_with('=sys-devel/llvm-1234.0_pre*\n')
+    mock_llvm_version.assert_called_once_with(git_hash)
+
+    overlay_dir = 'absolute/path/to/chroot/src/third_party/chromiumos-overlay'
+    mask_path = overlay_dir + '/profiles/targets/chromeos/package.mask'
+    mock_git_add.assert_called_once_with(
+        ['git', '-C', overlay_dir, 'add', mask_path])
+
 
 if __name__ == '__main__':
   unittest.main()