LLVM tools: Unittests for update_tryjob_status.py

BUG=None
TEST='./update_tryjob_status_unittest.py' passes

Change-Id: Ia70a31c7e6f69c818c719986d831d3dbb300a1cd
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1793909
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>
Tested-by: Salud Lemus <saludlemus@google.com>

1 files changed, 617 insertions, 0 deletions

diff --git a/llvm_tools/update_tryjob_status_unittest.py b/llvm_tools/update_tryjob_status_unittest.py
new file mode 100755
index 00000000..b5e6556c
--- /dev/null
+++ b/llvm_tools/update_tryjob_status_unittest.py
@@ -0,0 +1,617 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests when updating a tryjob's status."""
+
+from __future__ import print_function
+
+import json
+import os
+import subprocess
+import unittest
+import unittest.mock as mock
+
+from test_helpers import CreateTemporaryJsonFile
+from test_helpers import WritePrettyJsonFile
+from update_tryjob_status import TryjobStatus
+from update_tryjob_status import CustomScriptStatus
+import update_tryjob_status
+
+
+class UpdateTryjobStatusTest(unittest.TestCase):
+  """Unittests for updating a tryjob's 'status'."""
+
+  def testFoundTryjobIndex(self):
+    test_tryjobs = [{
+        'rev': 123,
+        'url': 'https://some_url_to_CL.com',
+        'cl': 'https://some_link_to_tryjob.com',
+        'status': 'good',
+        'buildbucket_id': 91835
+    },
+                    {
+                        'rev': 1000,
+                        'url': 'https://some_url_to_CL.com',
+                        'cl': 'https://some_link_to_tryjob.com',
+                        'status': 'pending',
+                        'buildbucket_id': 10931
+                    }]
+
+    expected_index = 0
+
+    revision_to_find = 123
+
+    self.assertEqual(
+        update_tryjob_status.FindTryjobIndex(revision_to_find, test_tryjobs),
+        expected_index)
+
+  def testNotFindTryjobIndex(self):
+    test_tryjobs = [{
+        'rev': 500,
+        'url': 'https://some_url_to_CL.com',
+        'cl': 'https://some_link_to_tryjob.com',
+        'status': 'bad',
+        'buildbucket_id': 390
+    },
+                    {
+                        'rev': 10,
+                        'url': 'https://some_url_to_CL.com',
+                        'cl': 'https://some_link_to_tryjob.com',
+                        'status': 'skip',
+                        'buildbucket_id': 10
+                    }]
+
+    revision_to_find = 250
+
+    self.assertIsNone(
+        update_tryjob_status.FindTryjobIndex(revision_to_find, test_tryjobs))
+
+  # Simulate the behavior of `ChrootRunCommand()` when executing a command
+  # inside the chroot.
+  @mock.patch.object(update_tryjob_status, 'ChrootRunCommand')
+  def testGetStatusFromCrosBuildResult(self, mock_chroot_command):
+    tryjob_contents = {
+        '192': {
+            'status': 'good',
+            'CleanUpChroot': 'pass',
+            'artifacts_url': None
+        }
+    }
+
+    # Use the test function to simulate 'ChrootRunCommand()' behavior.
+    mock_chroot_command.return_value = json.dumps(tryjob_contents)
+
+    buildbucket_id = 192
+
+    chroot_path = '/some/path/to/chroot'
+
+    self.assertEqual(
+        update_tryjob_status.GetStatusFromCrosBuildResult(
+            chroot_path, buildbucket_id), 'good')
+
+    expected_cmd = [
+        'cros', 'buildresult', '--buildbucket-id',
+        str(buildbucket_id), '--report', 'json'
+    ]
+
+    mock_chroot_command.assert_called_once_with(chroot_path, expected_cmd)
+
+  # Simulate the behavior of `GetStatusFromCrosBuildResult()` when `cros
+  # buildresult` returned a string that is not in the mapping.
+  @mock.patch.object(
+      update_tryjob_status,
+      'GetStatusFromCrosBuildResult',
+      return_value='querying')
+  def testInvalidCrosBuildResultValue(self, mock_cros_buildresult):
+    chroot_path = '/some/path/to/chroot'
+    buildbucket_id = 50
+
+    # Verify the exception is raised when the return value of `cros buildresult`
+    # is not in the `builder_status_mapping`.
+    with self.assertRaises(ValueError) as err:
+      update_tryjob_status.GetAutoResult(chroot_path, buildbucket_id)
+
+    self.assertEqual(
+        str(err.exception),
+        '"cros buildresult" return value is invalid: querying')
+
+    mock_cros_buildresult.assert_called_once_with(chroot_path, buildbucket_id)
+
+  # Simulate the behavior of `GetStatusFromCrosBuildResult()` when `cros
+  # buildresult` returned a string that is in the mapping.
+  @mock.patch.object(
+      update_tryjob_status,
+      'GetStatusFromCrosBuildResult',
+      return_value=update_tryjob_status.BuilderStatus.PASS.value)
+  def testValidCrosBuildResultValue(self, mock_cros_buildresult):
+    chroot_path = '/some/path/to/chroot'
+    buildbucket_id = 100
+
+    self.assertEqual(
+        update_tryjob_status.GetAutoResult(chroot_path, buildbucket_id),
+        TryjobStatus.GOOD.value)
+
+    mock_cros_buildresult.assert_called_once_with(chroot_path, buildbucket_id)
+
+  @mock.patch.object(subprocess, 'Popen')
+  # Simulate the behavior of `os.rename()` when successfully renamed a file.
+  @mock.patch.object(os, 'rename', return_value=None)
+  # Simulate the behavior of `os.path.basename()` when successfully retrieved
+  # the basename of the temp .JSON file.
+  @mock.patch.object(os.path, 'basename', return_value='tmpFile.json')
+  def testInvalidExitCodeByCustomScript(self, mock_basename, mock_rename_file,
+                                        mock_exec_custom_script):
+
+    error_message_by_custom_script = 'Failed to parse .JSON file'
+
+    # Simulate the behavior of 'subprocess.Popen()' when executing the custom
+    # script.
+    #
+    # `Popen.communicate()` returns a tuple of `stdout` and `stderr`.
+    mock_exec_custom_script.return_value.communicate.return_value = (
+        None, error_message_by_custom_script)
+
+    # Exit code of 1 is not in the mapping, so an exception will be raised.
+    custom_script_exit_code = 1
+
+    mock_exec_custom_script.return_value.returncode = custom_script_exit_code
+
+    tryjob_contents = {
+        'status': 'good',
+        'rev': 1234,
+        'url': 'https://some_url_to_CL.com',
+        'link': 'https://some_url_to_tryjob.com'
+    }
+
+    custom_script_path = '/abs/path/to/script.py'
+    status_file_path = '/abs/path/to/status_file.json'
+
+    name_json_file = os.path.join(
+        os.path.dirname(status_file_path), 'tmpFile.json')
+
+    expected_error_message = (
+        'Custom script %s exit code %d did not match '
+        'any of the expected exit codes: %s for "good", '
+        '%d for "bad", or %d for "skip".\nPlease check '
+        '%s for information about the tryjob: %s' %
+        (custom_script_path, custom_script_exit_code,
+         CustomScriptStatus.GOOD.value, CustomScriptStatus.BAD.value,
+         CustomScriptStatus.SKIP.value, name_json_file,
+         error_message_by_custom_script))
+
+    # Verify the exception is raised when the exit code by the custom script
+    # does not match any of the exit codes in the mapping of
+    # `custom_script_exit_value_mapping`.
+    with self.assertRaises(ValueError) as err:
+      update_tryjob_status.GetCustomScriptResult(
+          custom_script_path, status_file_path, tryjob_contents)
+
+    self.assertEqual(str(err.exception), expected_error_message)
+
+    mock_exec_custom_script.assert_called_once()
+
+    mock_rename_file.assert_called_once()
+
+    mock_basename.assert_called_once()
+
+  @mock.patch.object(subprocess, 'Popen')
+  # Simulate the behavior of `os.rename()` when successfully renamed a file.
+  @mock.patch.object(os, 'rename', return_value=None)
+  # Simulate the behavior of `os.path.basename()` when successfully retrieved
+  # the basename of the temp .JSON file.
+  @mock.patch.object(os.path, 'basename', return_value='tmpFile.json')
+  def testValidExitCodeByCustomScript(self, mock_basename, mock_rename_file,
+                                      mock_exec_custom_script):
+
+    # Simulate the behavior of 'subprocess.Popen()' when executing the custom
+    # script.
+    #
+    # `Popen.communicate()` returns a tuple of `stdout` and `stderr`.
+    mock_exec_custom_script.return_value.communicate.return_value = (None, None)
+
+    mock_exec_custom_script.return_value.returncode = \
+        CustomScriptStatus.GOOD.value
+
+    tryjob_contents = {
+        'status': 'good',
+        'rev': 1234,
+        'url': 'https://some_url_to_CL.com',
+        'link': 'https://some_url_to_tryjob.com'
+    }
+
+    custom_script_path = '/abs/path/to/script.py'
+    status_file_path = '/abs/path/to/status_file.json'
+
+    self.assertEqual(
+        update_tryjob_status.GetCustomScriptResult(
+            custom_script_path, status_file_path, tryjob_contents),
+        TryjobStatus.GOOD.value)
+
+    mock_exec_custom_script.assert_called_once()
+
+    mock_rename_file.assert_not_called()
+
+    mock_basename.assert_not_called()
+
+  def testNoTryjobsInStatusFileWhenUpdatingTryjobStatus(self):
+    bisect_test_contents = {'start': 369410, 'end': 369420, 'jobs': []}
+
+    # Create a temporary .JSON file to simulate a .JSON file that has bisection
+    # contents.
+    with CreateTemporaryJsonFile() as temp_json_file:
+      with open(temp_json_file, 'w') as f:
+        WritePrettyJsonFile(bisect_test_contents, f)
+
+      revision_to_update = 369412
+
+      chroot_path = '/abs/path/to/chroot'
+
+      custom_script = None
+
+      # Verify the exception is raised when the `status_file` does not have any
+      # `jobs` (empty).
+      with self.assertRaises(SystemExit) as err:
+        update_tryjob_status.UpdateTryjobStatus(
+            revision_to_update, TryjobStatus.GOOD, temp_json_file, chroot_path,
+            custom_script)
+
+      self.assertEqual(str(err.exception), 'No tryjobs in %s' % temp_json_file)
+
+  # Simulate the behavior of `FindTryjobIndex()` when the tryjob does not exist
+  # in the status file.
+  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=None)
+  def testNotFindTryjobIndexWhenUpdatingTryjobStatus(self,
+                                                     mock_find_tryjob_index):
+
+    bisect_test_contents = {
+        'start': 369410,
+        'end': 369420,
+        'jobs': [{
+            'rev': 369411,
+            'status': 'pending'
+        }]
+    }
+
+    # Create a temporary .JSON file to simulate a .JSON file that has bisection
+    # contents.
+    with CreateTemporaryJsonFile() as temp_json_file:
+      with open(temp_json_file, 'w') as f:
+        WritePrettyJsonFile(bisect_test_contents, f)
+
+      revision_to_update = 369416
+
+      chroot_path = '/abs/path/to/chroot'
+
+      custom_script = None
+
+      # Verify the exception is raised when the `status_file` does not have any
+      # `jobs` (empty).
+      with self.assertRaises(ValueError) as err:
+        update_tryjob_status.UpdateTryjobStatus(
+            revision_to_update, TryjobStatus.SKIP, temp_json_file, chroot_path,
+            custom_script)
+
+      self.assertEqual(
+          str(err.exception), 'Unable to find tryjob for %d in %s' %
+          (revision_to_update, temp_json_file))
+
+    mock_find_tryjob_index.assert_called_once()
+
+  # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the
+  # status file.
+  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
+  def testSuccessfullyUpdatedTryjobStatusToGood(self, mock_find_tryjob_index):
+    bisect_test_contents = {
+        'start': 369410,
+        'end': 369420,
+        'jobs': [{
+            'rev': 369411,
+            'status': 'pending'
+        }]
+    }
+
+    # Create a temporary .JSON file to simulate a .JSON file that has bisection
+    # contents.
+    with CreateTemporaryJsonFile() as temp_json_file:
+      with open(temp_json_file, 'w') as f:
+        WritePrettyJsonFile(bisect_test_contents, f)
+
+      revision_to_update = 369411
+
+      # Index of the tryjob that is going to have its 'status' value updated.
+      tryjob_index = 0
+
+      chroot_path = '/abs/path/to/chroot'
+
+      custom_script = None
+
+      update_tryjob_status.UpdateTryjobStatus(revision_to_update,
+                                              TryjobStatus.GOOD, temp_json_file,
+                                              chroot_path, custom_script)
+
+      # Verify that the tryjob's 'status' has been updated in the status file.
+      with open(temp_json_file) as status_file:
+        bisect_contents = json.load(status_file)
+
+        self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'],
+                         TryjobStatus.GOOD.value)
+
+    mock_find_tryjob_index.assert_called_once()
+
+  # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the
+  # status file.
+  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
+  def testSuccessfullyUpdatedTryjobStatusToBad(self, mock_find_tryjob_index):
+    bisect_test_contents = {
+        'start': 369410,
+        'end': 369420,
+        'jobs': [{
+            'rev': 369411,
+            'status': 'pending'
+        }]
+    }
+
+    # Create a temporary .JSON file to simulate a .JSON file that has bisection
+    # contents.
+    with CreateTemporaryJsonFile() as temp_json_file:
+      with open(temp_json_file, 'w') as f:
+        WritePrettyJsonFile(bisect_test_contents, f)
+
+      revision_to_update = 369411
+
+      # Index of the tryjob that is going to have its 'status' value updated.
+      tryjob_index = 0
+
+      chroot_path = '/abs/path/to/chroot'
+
+      custom_script = None
+
+      update_tryjob_status.UpdateTryjobStatus(revision_to_update,
+                                              TryjobStatus.BAD, temp_json_file,
+                                              chroot_path, custom_script)
+
+      # Verify that the tryjob's 'status' has been updated in the status file.
+      with open(temp_json_file) as status_file:
+        bisect_contents = json.load(status_file)
+
+        self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'],
+                         TryjobStatus.BAD.value)
+
+    mock_find_tryjob_index.assert_called_once()
+
+  # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the
+  # status file.
+  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
+  def testSuccessfullyUpdatedTryjobStatusToPending(self,
+                                                   mock_find_tryjob_index):
+    bisect_test_contents = {
+        'start': 369410,
+        'end': 369420,
+        'jobs': [{
+            'rev': 369411,
+            'status': 'skip'
+        }]
+    }
+
+    # Create a temporary .JSON file to simulate a .JSON file that has bisection
+    # contents.
+    with CreateTemporaryJsonFile() as temp_json_file:
+      with open(temp_json_file, 'w') as f:
+        WritePrettyJsonFile(bisect_test_contents, f)
+
+      revision_to_update = 369411
+
+      # Index of the tryjob that is going to have its 'status' value updated.
+      tryjob_index = 0
+
+      chroot_path = '/abs/path/to/chroot'
+
+      custom_script = None
+
+      update_tryjob_status.UpdateTryjobStatus(
+          revision_to_update, update_tryjob_status.TryjobStatus.SKIP,
+          temp_json_file, chroot_path, custom_script)
+
+      # Verify that the tryjob's 'status' has been updated in the status file.
+      with open(temp_json_file) as status_file:
+        bisect_contents = json.load(status_file)
+
+        self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'],
+                         update_tryjob_status.TryjobStatus.SKIP.value)
+
+    mock_find_tryjob_index.assert_called_once()
+
+  # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the
+  # status file.
+  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
+  def testSuccessfullyUpdatedTryjobStatusToSkip(self, mock_find_tryjob_index):
+    bisect_test_contents = {
+        'start': 369410,
+        'end': 369420,
+        'jobs': [{
+            'rev': 369411,
+            'status': 'pending',
+        }]
+    }
+
+    # Create a temporary .JSON file to simulate a .JSON file that has bisection
+    # contents.
+    with CreateTemporaryJsonFile() as temp_json_file:
+      with open(temp_json_file, 'w') as f:
+        WritePrettyJsonFile(bisect_test_contents, f)
+
+      revision_to_update = 369411
+
+      # Index of the tryjob that is going to have its 'status' value updated.
+      tryjob_index = 0
+
+      chroot_path = '/abs/path/to/chroot'
+
+      custom_script = None
+
+      update_tryjob_status.UpdateTryjobStatus(
+          revision_to_update, update_tryjob_status.TryjobStatus.PENDING,
+          temp_json_file, chroot_path, custom_script)
+
+      # Verify that the tryjob's 'status' has been updated in the status file.
+      with open(temp_json_file) as status_file:
+        bisect_contents = json.load(status_file)
+
+        self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'],
+                         update_tryjob_status.TryjobStatus.PENDING.value)
+
+    mock_find_tryjob_index.assert_called_once()
+
+  # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the
+  # status file.
+  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
+  # Simulate the behavior of `GetAutoResult()` when `cros buildresult` returns
+  # a value that is in the mapping.
+  @mock.patch.object(
+      update_tryjob_status,
+      'GetAutoResult',
+      return_value=TryjobStatus.GOOD.value)
+  def testSuccessfullyUpdatedTryjobStatusToAuto(self, mock_get_auto_result,
+                                                mock_find_tryjob_index):
+    bisect_test_contents = {
+        'start': 369410,
+        'end': 369420,
+        'jobs': [{
+            'rev': 369411,
+            'status': 'pending',
+            'buildbucket_id': 1200
+        }]
+    }
+
+    # Create a temporary .JSON file to simulate a .JSON file that has bisection
+    # contents.
+    with CreateTemporaryJsonFile() as temp_json_file:
+      with open(temp_json_file, 'w') as f:
+        WritePrettyJsonFile(bisect_test_contents, f)
+
+      revision_to_update = 369411
+
+      # Index of the tryjob that is going to have its 'status' value updated.
+      tryjob_index = 0
+
+      path_to_chroot = '/abs/path/to/chroot'
+
+      custom_script = None
+
+      update_tryjob_status.UpdateTryjobStatus(
+          revision_to_update, update_tryjob_status.TryjobStatus.AUTO,
+          temp_json_file, path_to_chroot, custom_script)
+
+      # Verify that the tryjob's 'status' has been updated in the status file.
+      with open(temp_json_file) as status_file:
+        bisect_contents = json.load(status_file)
+
+        self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'],
+                         update_tryjob_status.TryjobStatus.GOOD.value)
+
+    mock_get_auto_result.assert_called_once_with(
+        path_to_chroot,
+        bisect_test_contents['jobs'][tryjob_index]['buildbucket_id'])
+
+    mock_find_tryjob_index.assert_called_once()
+
+  # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the
+  # status file.
+  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
+  # Simulate the behavior of `GetCustomScriptResult()` when the custom script
+  # exit code is in the mapping.
+  @mock.patch.object(
+      update_tryjob_status,
+      'GetCustomScriptResult',
+      return_value=TryjobStatus.SKIP.value)
+  def testSuccessfullyUpdatedTryjobStatusToAuto(
+      self, mock_get_custom_script_result, mock_find_tryjob_index):
+    bisect_test_contents = {
+        'start': 369410,
+        'end': 369420,
+        'jobs': [{
+            'rev': 369411,
+            'status': 'pending',
+            'buildbucket_id': 1200
+        }]
+    }
+
+    # Create a temporary .JSON file to simulate a .JSON file that has bisection
+    # contents.
+    with CreateTemporaryJsonFile() as temp_json_file:
+      with open(temp_json_file, 'w') as f:
+        WritePrettyJsonFile(bisect_test_contents, f)
+
+      revision_to_update = 369411
+
+      # Index of the tryjob that is going to have its 'status' value updated.
+      tryjob_index = 0
+
+      path_to_chroot = '/abs/path/to/chroot'
+
+      custom_script_path = '/abs/path/to/custom_script.py'
+
+      update_tryjob_status.UpdateTryjobStatus(
+          revision_to_update, update_tryjob_status.TryjobStatus.CUSTOM_SCRIPT,
+          temp_json_file, path_to_chroot, custom_script_path)
+
+      # Verify that the tryjob's 'status' has been updated in the status file.
+      with open(temp_json_file) as status_file:
+        bisect_contents = json.load(status_file)
+
+        self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'],
+                         update_tryjob_status.TryjobStatus.SKIP.value)
+
+    mock_get_custom_script_result.assert_called_once()
+
+    mock_find_tryjob_index.assert_called_once()
+
+  # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the
+  # status file.
+  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
+  def testSetStatusDoesNotExistWhenUpdatingTryjobStatus(self,
+                                                        mock_find_tryjob_index):
+
+    bisect_test_contents = {
+        'start': 369410,
+        'end': 369420,
+        'jobs': [{
+            'rev': 369411,
+            'status': 'pending',
+            'buildbucket_id': 1200
+        }]
+    }
+
+    # Create a temporary .JSON file to simulate a .JSON file that has bisection
+    # contents.
+    with CreateTemporaryJsonFile() as temp_json_file:
+      with open(temp_json_file, 'w') as f:
+        WritePrettyJsonFile(bisect_test_contents, f)
+
+      revision_to_update = 369411
+
+      path_to_chroot = '/abs/path/to/chroot'
+
+      nonexistent_update_status = 'revert_status'
+
+      custom_script = None
+
+      # Verify the exception is raised when the `set_status` command line
+      # argument does not exist in the mapping.
+      with self.assertRaises(ValueError) as err:
+        update_tryjob_status.UpdateTryjobStatus(
+            revision_to_update, nonexistent_update_status, temp_json_file,
+            path_to_chroot, custom_script)
+
+      self.assertEqual(
+          str(err.exception),
+          'Invalid "set_status" option provided: revert_status')
+
+    mock_find_tryjob_index.assert_called_once()
+
+
+if __name__ == '__main__':
+  unittest.main()