LLVM tools: Added unittest for llvm_bisection.py

BUG=None
TEST='./llvm_bisection_unittest.py' passes

Change-Id: I2689deff7ede41bcfcf5707d4419bd3c816bba39
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1798739
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Tested-by: Salud Lemus <saludlemus@google.com>

2 files changed, 609 insertions, 7 deletions

diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py
index 04fbdd55..b45be0aa 100755
--- a/llvm_tools/llvm_bisection.py
+++ b/llvm_tools/llvm_bisection.py
@@ -371,6 +371,19 @@ def _NoteCompletedBisection(last_tested, src_path, end):
       'The bad revision is %d and its commit hash is %s' % (end, bad_llvm_hash))
 
 
+def LoadStatusFile(last_tested, start, end):
+  """Loads the status file for bisection."""
+
+  try:
+    with open(last_tested) as f:
+      return json.load(f)
+  except IOError as err:
+    if err.errno != errno.ENOENT:
+      raise
+
+  return {'start': start, 'end': end, 'jobs': []}
+
+
 def main(args_output):
   """Bisects LLVM based off of a .JSON file.
 
@@ -390,13 +403,7 @@ def main(args_output):
   start = args_output.start_rev
   end = args_output.end_rev
 
-  try:
-    with open(args_output.last_tested) as f:
-      bisect_contents = json.load(f)
-  except IOError as err:
-    if err.errno != errno.ENOENT:
-      raise
-    bisect_contents = {'start': start, 'end': end, 'jobs': []}
+  bisect_contents = LoadStatusFile(args_output.last_tested, start, end)
 
   _ValidateStartAndEndAgainstJSONStartAndEnd(
       start, end, bisect_contents['start'], bisect_contents['end'])
diff --git a/llvm_tools/llvm_bisection_unittest.py b/llvm_tools/llvm_bisection_unittest.py
new file mode 100755
index 00000000..946a56ff
--- /dev/null
+++ b/llvm_tools/llvm_bisection_unittest.py
@@ -0,0 +1,595 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests for LLVM bisection."""
+
+from __future__ import print_function
+
+import json
+import unittest
+import unittest.mock as mock
+
+from get_llvm_hash import LLVMHash
+from test_helpers import ArgsOutputTest
+from test_helpers import CallCountsToMockFunctions
+from test_helpers import CreateTemporaryJsonFile
+from test_helpers import WritePrettyJsonFile
+import llvm_bisection
+
+
+class LLVMBisectionTest(unittest.TestCase):
+  """Unittests for LLVM bisection."""
+
+  def testStartAndEndDoNotMatchJsonStartAndEnd(self):
+    start = 100
+    end = 150
+
+    json_start = 110
+    json_end = 150
+
+    # Verify the exception is raised when the start and end revision for LLVM
+    # bisection do not match the .JSON's 'start' and 'end' values.
+    with self.assertRaises(ValueError) as err:
+      llvm_bisection._ValidateStartAndEndAgainstJSONStartAndEnd(
+          start, end, json_start, json_end)
+
+    expected_error_message = ('The start %d or the end %d version provided is '
+                              'different than "start" %d or "end" %d in the '
+                              '.JSON file' % (start, end, json_start, json_end))
+
+    self.assertEqual(str(err.exception), expected_error_message)
+
+  def testStartAndEndMatchJsonStartAndEnd(self):
+    start = 100
+    end = 150
+
+    json_start = 100
+    json_end = 150
+
+    llvm_bisection._ValidateStartAndEndAgainstJSONStartAndEnd(
+        start, end, json_start, json_end)
+
+  def testTryjobStatusIsMissing(self):
+    start = 100
+    end = 150
+
+    test_tryjobs = [{
+        'rev': 105,
+        'status': 'good',
+        'link': 'https://some_tryjob_1_url.com'
+    }, {
+        'rev': 120,
+        'status': None,
+        'link': 'https://some_tryjob_2_url.com'
+    }, {
+        'rev': 140,
+        'status': 'bad',
+        'link': 'https://some_tryjob_3_url.com'
+    }]
+
+    # Verify the exception is raised when a tryjob does not have a value for
+    # the 'status' key or the 'status' key is missing.
+    with self.assertRaises(ValueError) as err:
+      llvm_bisection.GetStartAndEndRevision(start, end, test_tryjobs)
+
+    expected_error_message = (
+        '"status" is missing or has no value, please '
+        'go to %s and update it' % test_tryjobs[1]['link'])
+
+    self.assertEqual(str(err.exception), expected_error_message)
+
+  def testGoodRevisionGreaterThanBadRevision(self):
+    start = 100
+    end = 150
+
+    test_tryjobs = [{
+        'rev': 110,
+        'status': 'bad',
+        'link': 'https://some_tryjob_1_url.com'
+    }, {
+        'rev': 125,
+        'status': 'skip',
+        'link': 'https://some_tryjob_2_url.com'
+    }, {
+        'rev': 140,
+        'status': 'good',
+        'link': 'https://some_tryjob_3_url.com'
+    }]
+
+    # Verify the exception is raised when the new 'start' revision is greater
+    # than the new 'bad' revision for bisection (i.e. bisection is broken).
+    with self.assertRaises(AssertionError) as err:
+      llvm_bisection.GetStartAndEndRevision(start, end, test_tryjobs)
+
+    expected_error_message = (
+        'Bisection is broken because %d (good) is >= '
+        '%d (bad)' % (test_tryjobs[2]['rev'], test_tryjobs[0]['rev']))
+
+    self.assertEqual(str(err.exception), expected_error_message)
+
+  def testSuccessfullyGetNewStartAndNewEndRevision(self):
+    start = 100
+    end = 150
+
+    test_tryjobs = [{
+        'rev': 110,
+        'status': 'good',
+        'link': 'https://some_tryjob_1_url.com'
+    }, {
+        'rev': 120,
+        'status': 'good',
+        'link': 'https://some_tryjob_2_url.com'
+    }, {
+        'rev': 130,
+        'status': 'pending',
+        'link': 'https://some_tryjob_3_url.com'
+    }, {
+        'rev': 135,
+        'status': 'skip',
+        'link': 'https://some_tryjob_4_url.com'
+    }, {
+        'rev': 140,
+        'status': 'bad',
+        'link': 'https://some_tryjob_5_url.com'
+    }]
+
+    # Tuple consists of the new good revision, the new bad revision, a set of
+    # 'pending' revisions, and a set of 'skip' revisions.
+    expected_revisions_tuple = 120, 140, {130}, {135}
+
+    self.assertTupleEqual(
+        llvm_bisection.GetStartAndEndRevision(start, end, test_tryjobs),
+        expected_revisions_tuple)
+
+  @mock.patch.object(LLVMHash, 'GetGitHashForVersion')
+  def testNoRevisionsBetweenStartAndEnd(self, mock_get_git_hash):
+    start = 100
+    end = 110
+
+    test_pending_revisions = {107}
+    test_skip_revisions = {101, 102, 103, 104, 108, 109}
+
+    # Simulate behavior of `GetGitHashForVersion()` when the revision does not
+    # exist in the LLVM source tree.
+    def MockGetGitHashForRevisionRaiseException(src_path, revision):
+      raise ValueError('Revision does not exist')
+
+    mock_get_git_hash.side_effect = MockGetGitHashForRevisionRaiseException
+
+    parallel = 3
+
+    abs_path_to_src = '/abs/path/to/src'
+
+    self.assertListEqual(
+        llvm_bisection.GetRevisionsBetweenBisection(
+            start, end, parallel, abs_path_to_src, test_pending_revisions,
+            test_skip_revisions), [])
+
+  @mock.patch.object(LLVMHash, 'GetGitHashForVersion')
+  def testSuccessfullyRetrievedRevisionsBetweenStartAndEnd(
+      self, mock_get_git_hash):
+
+    start = 100
+    end = 110
+
+    test_pending_revisions = set()
+    test_skip_revisions = {101, 102, 103, 104, 106, 108, 109}
+
+    parallel = 3
+
+    abs_path_to_src = '/abs/path/to/src'
+
+    # Valid revision that exist in the LLVM source tree between 'start' and
+    # 'end' and were not in the 'pending' set or 'skip' set.
+    expected_revisions_between_start_and_end = [105, 107]
+
+    self.assertListEqual(
+        llvm_bisection.GetRevisionsBetweenBisection(
+            start, end, parallel, abs_path_to_src, test_pending_revisions,
+            test_skip_revisions), expected_revisions_between_start_and_end)
+
+    self.assertEqual(mock_get_git_hash.call_count, 2)
+
+  # Simulate behavior of `GetGitHashForVersion()` when successfully retrieved
+  # a list git hashes for each revision in the revisions list.
+  @mock.patch.object(LLVMHash, 'GetGitHashForVersion')
+  # Simulate behavior of `GetRevisionsBetweenBisection()` when successfully
+  # retrieved a list of valid revisions between 'start' and 'end'.
+  @mock.patch.object(llvm_bisection, 'GetRevisionsBetweenBisection')
+  # Simulate behavior of `CreatTempLLVMRepo()` when successfully created a
+  # worktree when a source path was not provided.
+  @mock.patch.object(llvm_bisection, 'CreateTempLLVMRepo')
+  def testSuccessfullyGetRevisionsListAndHashList(
+      self, mock_create_temp_llvm_repo, mock_get_revisions_between_bisection,
+      mock_get_git_hash):
+
+    expected_revisions_and_hash_tuple = ([102, 105, 108], [
+        'a123testhash1', 'a123testhash2', 'a123testhash3'
+    ])
+
+    @CallCountsToMockFunctions
+    def MockGetGitHashForRevision(call_count, src_path, rev):
+      # Simulate retrieving the git hash for the revision.
+      if call_count < 3:
+        return expected_revisions_and_hash_tuple[1][call_count]
+
+      assert False, 'Called `GetGitHashForVersion()` more than expected.'
+
+    temp_worktree = '/abs/path/to/tmpDir'
+
+    mock_create_temp_llvm_repo.return_value.__enter__.return_value.name = \
+        temp_worktree
+
+    # Simulate the valid revisions list.
+    mock_get_revisions_between_bisection.return_value = \
+        expected_revisions_and_hash_tuple[0]
+
+    # Simulate behavior of `GetGitHashForVersion()` by using the testing
+    # function.
+    mock_get_git_hash.side_effect = MockGetGitHashForRevision
+
+    start = 100
+    end = 110
+    parallel = 3
+    src_path = None
+    pending_revisions = {103, 104}
+    skip_revisions = {101, 106, 107, 109}
+
+    self.assertTupleEqual(
+        llvm_bisection.GetRevisionsListAndHashList(
+            start, end, parallel, src_path, pending_revisions, skip_revisions),
+        expected_revisions_and_hash_tuple)
+
+    mock_get_revisions_between_bisection.assert_called_once()
+
+    self.assertEqual(mock_get_git_hash.call_count, 3)
+
+  def testSuccessfullyDieWithNoRevisionsError(self):
+    start = 100
+    end = 110
+
+    pending_revisions = {105, 108}
+    skip_revisions = {101, 102, 103, 104, 106, 107, 109}
+
+    expected_no_revisions_message = ('No revisions between start %d and end '
+                                     '%d to create tryjobs' % (start, end))
+
+    expected_no_revisions_message += '\nThe following tryjobs are pending:\n' \
+        + '\n'.join(str(rev) for rev in pending_revisions)
+
+    expected_no_revisions_message += '\nThe following tryjobs were skipped:\n' \
+        + '\n'.join(str(rev) for rev in skip_revisions)
+
+    # Verify that an exception is raised when there are no revisions to launch
+    # tryjobs for between 'start' and 'end' and some tryjobs are 'pending'.
+    with self.assertRaises(ValueError) as err:
+      llvm_bisection.DieWithNoRevisionsError(start, end, skip_revisions,
+                                             pending_revisions)
+
+    self.assertEqual(str(err.exception), expected_no_revisions_message)
+
+  # Simulate behavior of `FindTryjobIndex()` when the index of the tryjob was
+  # found.
+  @mock.patch.object(llvm_bisection, 'FindTryjobIndex', return_value=0)
+  def testTryjobExistsInRevisionsToLaunch(self, mock_find_tryjob_index):
+    test_existing_jobs = [{'rev': 102, 'status': 'good'}]
+
+    revision_to_launch = [102]
+
+    expected_revision_that_exists = 102
+
+    with self.assertRaises(ValueError) as err:
+      llvm_bisection.CheckForExistingTryjobsInRevisionsToLaunch(
+          revision_to_launch, test_existing_jobs)
+
+    expected_found_tryjob_index_error_message = (
+        'Revision %d exists already '
+        'in "jobs"' % expected_revision_that_exists)
+
+    self.assertEqual(
+        str(err.exception), expected_found_tryjob_index_error_message)
+
+    mock_find_tryjob_index.assert_called_once()
+
+  @mock.patch.object(llvm_bisection, 'AddTryjob')
+  def testSuccessfullyUpdatedStatusFileWhenExceptionIsRaised(
+      self, mock_add_tryjob):
+
+    git_hash_list = ['a123testhash1', 'a123testhash2', 'a123testhash3']
+    revisions_list = [102, 104, 106]
+
+    # Simulate behavior of `AddTryjob()` when successfully launched a tryjob for
+    # the updated packages.
+    @CallCountsToMockFunctions
+    def MockAddTryjob(call_count, packages, git_hash, revision, chroot_path,
+                      patch_file, extra_cls, options, builder, verbose,
+                      svn_revision):
+
+      if call_count < 2:
+        return {'rev': revisions_list[call_count], 'status': 'pending'}
+
+      # Simulate an exception happened along the way when updating the
+      # packages' `LLVM_NEXT_HASH`.
+      if call_count == 2:
+        raise ValueError('Unable to launch tryjob')
+
+      assert False, 'Called `AddTryjob()` more than expected.'
+
+    # Use the test function to simulate `AddTryjob()`.
+    mock_add_tryjob.side_effect = MockAddTryjob
+
+    start = 100
+    end = 110
+
+    bisection_contents = {'start': start, 'end': end, 'jobs': []}
+
+    args_output = ArgsOutputTest()
+
+    packages = ['sys-devel/llvm']
+    patch_file = '/abs/path/to/PATCHES.json'
+
+    # Create a temporary .JSON file to simulate a status file for bisection.
+    with CreateTemporaryJsonFile() as temp_json_file:
+      with open(temp_json_file, 'w') as f:
+        WritePrettyJsonFile(bisection_contents, f)
+
+      # Verify that the status file is updated when an exception happened when
+      # attempting to launch a revision (i.e. progress is not lost).
+      with self.assertRaises(ValueError) as err:
+        llvm_bisection.UpdateBisection(
+            revisions_list, git_hash_list, bisection_contents, temp_json_file,
+            packages, args_output.chroot_path, patch_file,
+            args_output.extra_change_lists, args_output.options,
+            args_output.builders, args_output.verbose)
+
+      expected_bisection_contents = {
+          'start':
+              start,
+          'end':
+              end,
+          'jobs': [{
+              'rev': revisions_list[0],
+              'status': 'pending'
+          }, {
+              'rev': revisions_list[1],
+              'status': 'pending'
+          }]
+      }
+
+      # Verify that the launched tryjobs were added to the status file when
+      # an exception happened.
+      with open(temp_json_file) as f:
+        json_contents = json.load(f)
+
+        self.assertDictEqual(json_contents, expected_bisection_contents)
+
+    self.assertEqual(str(err.exception), 'Unable to launch tryjob')
+
+    self.assertEqual(mock_add_tryjob.call_count, 3)
+
+  # Simulate behavior of `GetGitHashForVersion()` when successfully retrieved
+  # the git hash of the bad revision.
+  @mock.patch.object(
+      LLVMHash, 'GetGitHashForVersion', return_value='a123testhash4')
+  def testCompletedBisectionWhenProvidedSrcPath(self, mock_get_git_hash):
+    last_tested = '/some/last/tested_file.json'
+
+    src_path = '/abs/path/to/src/path'
+
+    # The bad revision.
+    end = 150
+
+    llvm_bisection._NoteCompletedBisection(last_tested, src_path, end)
+
+    mock_get_git_hash.assert_called_once()
+
+  # Simulate behavior of `GetLLVMHash()` when successfully retrieved
+  # the git hash of the bad revision.
+  @mock.patch.object(LLVMHash, 'GetLLVMHash', return_value='a123testhash5')
+  def testCompletedBisectionWhenNotProvidedSrcPath(self, mock_get_git_hash):
+    last_tested = '/some/last/tested_file.json'
+
+    src_path = None
+
+    # The bad revision.
+    end = 200
+
+    llvm_bisection._NoteCompletedBisection(last_tested, src_path, end)
+
+    mock_get_git_hash.assert_called_once()
+
+  def testSuccessfullyLoadedStatusFile(self):
+    start = 100
+    end = 150
+
+    test_bisect_contents = {'start': start, 'end': end, 'jobs': []}
+
+    # Simulate that the status file exists.
+    with CreateTemporaryJsonFile() as temp_json_file:
+      with open(temp_json_file, 'w') as f:
+        WritePrettyJsonFile(test_bisect_contents, f)
+
+      self.assertDictEqual(
+          llvm_bisection.LoadStatusFile(temp_json_file, start, end),
+          test_bisect_contents)
+
+  def testLoadedStatusFileThatDoesNotExist(self):
+    start = 200
+    end = 250
+
+    expected_bisect_contents = {'start': start, 'end': end, 'jobs': []}
+
+    last_tested = '/abs/path/to/file_that_does_not_exist.json'
+
+    self.assertDictEqual(
+        llvm_bisection.LoadStatusFile(last_tested, start, end),
+        expected_bisect_contents)
+
+  # Simulate behavior of `_NoteCompletedBisection()` when there are no more
+  # tryjobs to launch between start and end, so bisection is complete.
+  @mock.patch.object(llvm_bisection, '_NoteCompletedBisection')
+  @mock.patch.object(llvm_bisection, 'GetRevisionsListAndHashList')
+  @mock.patch.object(llvm_bisection, 'GetStartAndEndRevision')
+  # Simulate behavior of `_ValidateStartAndEndAgainstJSONStartAndEnd()` when
+  # both start and end revisions match.
+  @mock.patch.object(llvm_bisection,
+                     '_ValidateStartAndEndAgainstJSONStartAndEnd')
+  @mock.patch.object(llvm_bisection, 'LoadStatusFile')
+  # Simulate behavior of `VerifyOutsideChroot()` when successfully invoked the
+  # script outside of the chroot.
+  @mock.patch.object(llvm_bisection, 'VerifyOutsideChroot', return_value=True)
+  def testSuccessfullyBisectedLLVM(
+      self, mock_outside_chroot, mock_load_status_file,
+      mock_validate_start_and_end, mock_get_start_and_end_revision,
+      mock_get_revision_and_hash_list, mock_note_completed_bisection):
+
+    start = 500
+    end = 502
+
+    bisect_contents = {
+        'start': start,
+        'end': end,
+        'jobs': [{
+            'rev': 501,
+            'status': 'skip'
+        }]
+    }
+
+    skip_revisions = {501}
+    pending_revisions = {}
+
+    # Simulate behavior of `LoadStatusFile()` when successfully loaded the
+    # status file.
+    mock_load_status_file.return_value = bisect_contents
+
+    # Simulate behavior of `GetStartAndEndRevision()` when successfully found
+    # the new start and end revision of the bisection.
+    #
+    # Returns new start revision, new end revision, a set of pending revisions,
+    # and a set of skip revisions.
+    mock_get_start_and_end_revision.return_value = (start, end,
+                                                    pending_revisions,
+                                                    skip_revisions)
+
+    # Simulate behavior of `GetRevisionsListAndHashList()` when successfully
+    # retrieved valid revisions (along with their git hashes) between start and
+    # end (in this case, none).
+    mock_get_revision_and_hash_list.return_value = [], []
+
+    args_output = ArgsOutputTest()
+    args_output.start_rev = start
+    args_output.end_rev = end
+    args_output.parallel = 3
+    args_output.src_path = None
+
+    self.assertEqual(
+        llvm_bisection.main(args_output),
+        llvm_bisection.BisectionExitStatus.BISECTION_COMPLETE.value)
+
+    mock_outside_chroot.assert_called_once()
+
+    mock_load_status_file.assert_called_once()
+
+    mock_validate_start_and_end.assert_called_once()
+
+    mock_get_start_and_end_revision.assert_called_once()
+
+    mock_get_revision_and_hash_list.assert_called_once()
+
+    mock_note_completed_bisection.assert_called_once()
+
+  @mock.patch.object(llvm_bisection, 'DieWithNoRevisionsError')
+  # Simulate behavior of `_NoteCompletedBisection()` when there are no more
+  # tryjobs to launch between start and end, so bisection is complete.
+  @mock.patch.object(llvm_bisection, 'GetRevisionsListAndHashList')
+  @mock.patch.object(llvm_bisection, 'GetStartAndEndRevision')
+  # Simulate behavior of `_ValidateStartAndEndAgainstJSONStartAndEnd()` when
+  # both start and end revisions match.
+  @mock.patch.object(llvm_bisection,
+                     '_ValidateStartAndEndAgainstJSONStartAndEnd')
+  @mock.patch.object(llvm_bisection, 'LoadStatusFile')
+  # Simulate behavior of `VerifyOutsideChroot()` when successfully invoked the
+  # script outside of the chroot.
+  @mock.patch.object(llvm_bisection, 'VerifyOutsideChroot', return_value=True)
+  def testNoMoreTryjobsToLaunch(
+      self, mock_outside_chroot, mock_load_status_file,
+      mock_validate_start_and_end, mock_get_start_and_end_revision,
+      mock_get_revision_and_hash_list, mock_die_with_no_revisions_error):
+
+    start = 500
+    end = 502
+
+    bisect_contents = {
+        'start': start,
+        'end': end,
+        'jobs': [{
+            'rev': 501,
+            'status': 'pending'
+        }]
+    }
+
+    skip_revisions = {}
+    pending_revisions = {501}
+
+    no_revisions_error_message = ('No more tryjobs to launch between %d and '
+                                  '%d' % (start, end))
+
+    def MockNoRevisionsErrorException(start, end, skip, pending):
+      raise ValueError(no_revisions_error_message)
+
+    # Simulate behavior of `LoadStatusFile()` when successfully loaded the
+    # status file.
+    mock_load_status_file.return_value = bisect_contents
+
+    # Simulate behavior of `GetStartAndEndRevision()` when successfully found
+    # the new start and end revision of the bisection.
+    #
+    # Returns new start revision, new end revision, a set of pending revisions,
+    # and a set of skip revisions.
+    mock_get_start_and_end_revision.return_value = (start, end,
+                                                    pending_revisions,
+                                                    skip_revisions)
+
+    # Simulate behavior of `GetRevisionsListAndHashList()` when successfully
+    # retrieved valid revisions (along with their git hashes) between start and
+    # end (in this case, none).
+    mock_get_revision_and_hash_list.return_value = [], []
+
+    # Use the test function to simulate `DieWithNoRevisionsWithError()`
+    # behavior.
+    mock_die_with_no_revisions_error.side_effect = MockNoRevisionsErrorException
+
+    # Simulate behavior of arguments passed into the command line.
+    args_output = ArgsOutputTest()
+    args_output.start_rev = start
+    args_output.end_rev = end
+    args_output.parallel = 3
+    args_output.src_path = None
+
+    # Verify the exception is raised when there are no more tryjobs to launch
+    # between start and end when there are tryjobs that are 'pending', so
+    # the actual bad revision can change when those tryjobs's 'status' are
+    # updated.
+    with self.assertRaises(ValueError) as err:
+      llvm_bisection.main(args_output)
+
+    self.assertEqual(str(err.exception), no_revisions_error_message)
+
+    mock_outside_chroot.assert_called_once()
+
+    mock_load_status_file.assert_called_once()
+
+    mock_validate_start_and_end.assert_called_once()
+
+    mock_get_start_and_end_revision.assert_called_once()
+
+    mock_get_revision_and_hash_list.assert_called_once()
+
+    mock_die_with_no_revisions_error.assert_called_once()
+
+
+if __name__ == '__main__':
+  unittest.main()