diff options
author | Salud Lemus <saludlemus@google.com> | 2019-09-09 16:27:04 -0700 |
---|---|---|
committer | Salud Lemus <saludlemus@google.com> | 2019-09-12 00:05:49 +0000 |
commit | 3a15cc55c2a3b9de276603e6eda18ccfa521efbf (patch) | |
tree | d8b750bcdb93ca9ec7120be7531ce22b7696305e /llvm_tools/update_tryjob_status_unittest.py | |
parent | 34226d1317a8c447e9712de1a41ffec0f444ff2c (diff) | |
download | toolchain-utils-3a15cc55c2a3b9de276603e6eda18ccfa521efbf.tar.gz |
LLVM tools: Unittests for update_tryjob_status.py
BUG=None
TEST='./update_tryjob_status_unittest.py' passes
Change-Id: Ia70a31c7e6f69c818c719986d831d3dbb300a1cd
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1793909
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>
Tested-by: Salud Lemus <saludlemus@google.com>
Diffstat (limited to 'llvm_tools/update_tryjob_status_unittest.py')
-rwxr-xr-x | llvm_tools/update_tryjob_status_unittest.py | 617 |
1 files changed, 617 insertions, 0 deletions
diff --git a/llvm_tools/update_tryjob_status_unittest.py b/llvm_tools/update_tryjob_status_unittest.py new file mode 100755 index 00000000..b5e6556c --- /dev/null +++ b/llvm_tools/update_tryjob_status_unittest.py @@ -0,0 +1,617 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2019 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests when updating a tryjob's status.""" + +from __future__ import print_function + +import json +import os +import subprocess +import unittest +import unittest.mock as mock + +from test_helpers import CreateTemporaryJsonFile +from test_helpers import WritePrettyJsonFile +from update_tryjob_status import TryjobStatus +from update_tryjob_status import CustomScriptStatus +import update_tryjob_status + + +class UpdateTryjobStatusTest(unittest.TestCase): + """Unittests for updating a tryjob's 'status'.""" + + def testFoundTryjobIndex(self): + test_tryjobs = [{ + 'rev': 123, + 'url': 'https://some_url_to_CL.com', + 'cl': 'https://some_link_to_tryjob.com', + 'status': 'good', + 'buildbucket_id': 91835 + }, + { + 'rev': 1000, + 'url': 'https://some_url_to_CL.com', + 'cl': 'https://some_link_to_tryjob.com', + 'status': 'pending', + 'buildbucket_id': 10931 + }] + + expected_index = 0 + + revision_to_find = 123 + + self.assertEqual( + update_tryjob_status.FindTryjobIndex(revision_to_find, test_tryjobs), + expected_index) + + def testNotFindTryjobIndex(self): + test_tryjobs = [{ + 'rev': 500, + 'url': 'https://some_url_to_CL.com', + 'cl': 'https://some_link_to_tryjob.com', + 'status': 'bad', + 'buildbucket_id': 390 + }, + { + 'rev': 10, + 'url': 'https://some_url_to_CL.com', + 'cl': 'https://some_link_to_tryjob.com', + 'status': 'skip', + 'buildbucket_id': 10 + }] + + revision_to_find = 250 + + self.assertIsNone( + update_tryjob_status.FindTryjobIndex(revision_to_find, test_tryjobs)) + + # Simulate the behavior of `ChrootRunCommand()` when executing a command + # inside the chroot. + @mock.patch.object(update_tryjob_status, 'ChrootRunCommand') + def testGetStatusFromCrosBuildResult(self, mock_chroot_command): + tryjob_contents = { + '192': { + 'status': 'good', + 'CleanUpChroot': 'pass', + 'artifacts_url': None + } + } + + # Use the test function to simulate 'ChrootRunCommand()' behavior. + mock_chroot_command.return_value = json.dumps(tryjob_contents) + + buildbucket_id = 192 + + chroot_path = '/some/path/to/chroot' + + self.assertEqual( + update_tryjob_status.GetStatusFromCrosBuildResult( + chroot_path, buildbucket_id), 'good') + + expected_cmd = [ + 'cros', 'buildresult', '--buildbucket-id', + str(buildbucket_id), '--report', 'json' + ] + + mock_chroot_command.assert_called_once_with(chroot_path, expected_cmd) + + # Simulate the behavior of `GetStatusFromCrosBuildResult()` when `cros + # buildresult` returned a string that is not in the mapping. + @mock.patch.object( + update_tryjob_status, + 'GetStatusFromCrosBuildResult', + return_value='querying') + def testInvalidCrosBuildResultValue(self, mock_cros_buildresult): + chroot_path = '/some/path/to/chroot' + buildbucket_id = 50 + + # Verify the exception is raised when the return value of `cros buildresult` + # is not in the `builder_status_mapping`. + with self.assertRaises(ValueError) as err: + update_tryjob_status.GetAutoResult(chroot_path, buildbucket_id) + + self.assertEqual( + str(err.exception), + '"cros buildresult" return value is invalid: querying') + + mock_cros_buildresult.assert_called_once_with(chroot_path, buildbucket_id) + + # Simulate the behavior of `GetStatusFromCrosBuildResult()` when `cros + # buildresult` returned a string that is in the mapping. + @mock.patch.object( + update_tryjob_status, + 'GetStatusFromCrosBuildResult', + return_value=update_tryjob_status.BuilderStatus.PASS.value) + def testValidCrosBuildResultValue(self, mock_cros_buildresult): + chroot_path = '/some/path/to/chroot' + buildbucket_id = 100 + + self.assertEqual( + update_tryjob_status.GetAutoResult(chroot_path, buildbucket_id), + TryjobStatus.GOOD.value) + + mock_cros_buildresult.assert_called_once_with(chroot_path, buildbucket_id) + + @mock.patch.object(subprocess, 'Popen') + # Simulate the behavior of `os.rename()` when successfully renamed a file. + @mock.patch.object(os, 'rename', return_value=None) + # Simulate the behavior of `os.path.basename()` when successfully retrieved + # the basename of the temp .JSON file. + @mock.patch.object(os.path, 'basename', return_value='tmpFile.json') + def testInvalidExitCodeByCustomScript(self, mock_basename, mock_rename_file, + mock_exec_custom_script): + + error_message_by_custom_script = 'Failed to parse .JSON file' + + # Simulate the behavior of 'subprocess.Popen()' when executing the custom + # script. + # + # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. + mock_exec_custom_script.return_value.communicate.return_value = ( + None, error_message_by_custom_script) + + # Exit code of 1 is not in the mapping, so an exception will be raised. + custom_script_exit_code = 1 + + mock_exec_custom_script.return_value.returncode = custom_script_exit_code + + tryjob_contents = { + 'status': 'good', + 'rev': 1234, + 'url': 'https://some_url_to_CL.com', + 'link': 'https://some_url_to_tryjob.com' + } + + custom_script_path = '/abs/path/to/script.py' + status_file_path = '/abs/path/to/status_file.json' + + name_json_file = os.path.join( + os.path.dirname(status_file_path), 'tmpFile.json') + + expected_error_message = ( + 'Custom script %s exit code %d did not match ' + 'any of the expected exit codes: %s for "good", ' + '%d for "bad", or %d for "skip".\nPlease check ' + '%s for information about the tryjob: %s' % + (custom_script_path, custom_script_exit_code, + CustomScriptStatus.GOOD.value, CustomScriptStatus.BAD.value, + CustomScriptStatus.SKIP.value, name_json_file, + error_message_by_custom_script)) + + # Verify the exception is raised when the exit code by the custom script + # does not match any of the exit codes in the mapping of + # `custom_script_exit_value_mapping`. + with self.assertRaises(ValueError) as err: + update_tryjob_status.GetCustomScriptResult( + custom_script_path, status_file_path, tryjob_contents) + + self.assertEqual(str(err.exception), expected_error_message) + + mock_exec_custom_script.assert_called_once() + + mock_rename_file.assert_called_once() + + mock_basename.assert_called_once() + + @mock.patch.object(subprocess, 'Popen') + # Simulate the behavior of `os.rename()` when successfully renamed a file. + @mock.patch.object(os, 'rename', return_value=None) + # Simulate the behavior of `os.path.basename()` when successfully retrieved + # the basename of the temp .JSON file. + @mock.patch.object(os.path, 'basename', return_value='tmpFile.json') + def testValidExitCodeByCustomScript(self, mock_basename, mock_rename_file, + mock_exec_custom_script): + + # Simulate the behavior of 'subprocess.Popen()' when executing the custom + # script. + # + # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. + mock_exec_custom_script.return_value.communicate.return_value = (None, None) + + mock_exec_custom_script.return_value.returncode = \ + CustomScriptStatus.GOOD.value + + tryjob_contents = { + 'status': 'good', + 'rev': 1234, + 'url': 'https://some_url_to_CL.com', + 'link': 'https://some_url_to_tryjob.com' + } + + custom_script_path = '/abs/path/to/script.py' + status_file_path = '/abs/path/to/status_file.json' + + self.assertEqual( + update_tryjob_status.GetCustomScriptResult( + custom_script_path, status_file_path, tryjob_contents), + TryjobStatus.GOOD.value) + + mock_exec_custom_script.assert_called_once() + + mock_rename_file.assert_not_called() + + mock_basename.assert_not_called() + + def testNoTryjobsInStatusFileWhenUpdatingTryjobStatus(self): + bisect_test_contents = {'start': 369410, 'end': 369420, 'jobs': []} + + # Create a temporary .JSON file to simulate a .JSON file that has bisection + # contents. + with CreateTemporaryJsonFile() as temp_json_file: + with open(temp_json_file, 'w') as f: + WritePrettyJsonFile(bisect_test_contents, f) + + revision_to_update = 369412 + + chroot_path = '/abs/path/to/chroot' + + custom_script = None + + # Verify the exception is raised when the `status_file` does not have any + # `jobs` (empty). + with self.assertRaises(SystemExit) as err: + update_tryjob_status.UpdateTryjobStatus( + revision_to_update, TryjobStatus.GOOD, temp_json_file, chroot_path, + custom_script) + + self.assertEqual(str(err.exception), 'No tryjobs in %s' % temp_json_file) + + # Simulate the behavior of `FindTryjobIndex()` when the tryjob does not exist + # in the status file. + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=None) + def testNotFindTryjobIndexWhenUpdatingTryjobStatus(self, + mock_find_tryjob_index): + + bisect_test_contents = { + 'start': 369410, + 'end': 369420, + 'jobs': [{ + 'rev': 369411, + 'status': 'pending' + }] + } + + # Create a temporary .JSON file to simulate a .JSON file that has bisection + # contents. + with CreateTemporaryJsonFile() as temp_json_file: + with open(temp_json_file, 'w') as f: + WritePrettyJsonFile(bisect_test_contents, f) + + revision_to_update = 369416 + + chroot_path = '/abs/path/to/chroot' + + custom_script = None + + # Verify the exception is raised when the `status_file` does not have any + # `jobs` (empty). + with self.assertRaises(ValueError) as err: + update_tryjob_status.UpdateTryjobStatus( + revision_to_update, TryjobStatus.SKIP, temp_json_file, chroot_path, + custom_script) + + self.assertEqual( + str(err.exception), 'Unable to find tryjob for %d in %s' % + (revision_to_update, temp_json_file)) + + mock_find_tryjob_index.assert_called_once() + + # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the + # status file. + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) + def testSuccessfullyUpdatedTryjobStatusToGood(self, mock_find_tryjob_index): + bisect_test_contents = { + 'start': 369410, + 'end': 369420, + 'jobs': [{ + 'rev': 369411, + 'status': 'pending' + }] + } + + # Create a temporary .JSON file to simulate a .JSON file that has bisection + # contents. + with CreateTemporaryJsonFile() as temp_json_file: + with open(temp_json_file, 'w') as f: + WritePrettyJsonFile(bisect_test_contents, f) + + revision_to_update = 369411 + + # Index of the tryjob that is going to have its 'status' value updated. + tryjob_index = 0 + + chroot_path = '/abs/path/to/chroot' + + custom_script = None + + update_tryjob_status.UpdateTryjobStatus(revision_to_update, + TryjobStatus.GOOD, temp_json_file, + chroot_path, custom_script) + + # Verify that the tryjob's 'status' has been updated in the status file. + with open(temp_json_file) as status_file: + bisect_contents = json.load(status_file) + + self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], + TryjobStatus.GOOD.value) + + mock_find_tryjob_index.assert_called_once() + + # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the + # status file. + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) + def testSuccessfullyUpdatedTryjobStatusToBad(self, mock_find_tryjob_index): + bisect_test_contents = { + 'start': 369410, + 'end': 369420, + 'jobs': [{ + 'rev': 369411, + 'status': 'pending' + }] + } + + # Create a temporary .JSON file to simulate a .JSON file that has bisection + # contents. + with CreateTemporaryJsonFile() as temp_json_file: + with open(temp_json_file, 'w') as f: + WritePrettyJsonFile(bisect_test_contents, f) + + revision_to_update = 369411 + + # Index of the tryjob that is going to have its 'status' value updated. + tryjob_index = 0 + + chroot_path = '/abs/path/to/chroot' + + custom_script = None + + update_tryjob_status.UpdateTryjobStatus(revision_to_update, + TryjobStatus.BAD, temp_json_file, + chroot_path, custom_script) + + # Verify that the tryjob's 'status' has been updated in the status file. + with open(temp_json_file) as status_file: + bisect_contents = json.load(status_file) + + self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], + TryjobStatus.BAD.value) + + mock_find_tryjob_index.assert_called_once() + + # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the + # status file. + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) + def testSuccessfullyUpdatedTryjobStatusToPending(self, + mock_find_tryjob_index): + bisect_test_contents = { + 'start': 369410, + 'end': 369420, + 'jobs': [{ + 'rev': 369411, + 'status': 'skip' + }] + } + + # Create a temporary .JSON file to simulate a .JSON file that has bisection + # contents. + with CreateTemporaryJsonFile() as temp_json_file: + with open(temp_json_file, 'w') as f: + WritePrettyJsonFile(bisect_test_contents, f) + + revision_to_update = 369411 + + # Index of the tryjob that is going to have its 'status' value updated. + tryjob_index = 0 + + chroot_path = '/abs/path/to/chroot' + + custom_script = None + + update_tryjob_status.UpdateTryjobStatus( + revision_to_update, update_tryjob_status.TryjobStatus.SKIP, + temp_json_file, chroot_path, custom_script) + + # Verify that the tryjob's 'status' has been updated in the status file. + with open(temp_json_file) as status_file: + bisect_contents = json.load(status_file) + + self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], + update_tryjob_status.TryjobStatus.SKIP.value) + + mock_find_tryjob_index.assert_called_once() + + # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the + # status file. + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) + def testSuccessfullyUpdatedTryjobStatusToSkip(self, mock_find_tryjob_index): + bisect_test_contents = { + 'start': 369410, + 'end': 369420, + 'jobs': [{ + 'rev': 369411, + 'status': 'pending', + }] + } + + # Create a temporary .JSON file to simulate a .JSON file that has bisection + # contents. + with CreateTemporaryJsonFile() as temp_json_file: + with open(temp_json_file, 'w') as f: + WritePrettyJsonFile(bisect_test_contents, f) + + revision_to_update = 369411 + + # Index of the tryjob that is going to have its 'status' value updated. + tryjob_index = 0 + + chroot_path = '/abs/path/to/chroot' + + custom_script = None + + update_tryjob_status.UpdateTryjobStatus( + revision_to_update, update_tryjob_status.TryjobStatus.PENDING, + temp_json_file, chroot_path, custom_script) + + # Verify that the tryjob's 'status' has been updated in the status file. + with open(temp_json_file) as status_file: + bisect_contents = json.load(status_file) + + self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], + update_tryjob_status.TryjobStatus.PENDING.value) + + mock_find_tryjob_index.assert_called_once() + + # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the + # status file. + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) + # Simulate the behavior of `GetAutoResult()` when `cros buildresult` returns + # a value that is in the mapping. + @mock.patch.object( + update_tryjob_status, + 'GetAutoResult', + return_value=TryjobStatus.GOOD.value) + def testSuccessfullyUpdatedTryjobStatusToAuto(self, mock_get_auto_result, + mock_find_tryjob_index): + bisect_test_contents = { + 'start': 369410, + 'end': 369420, + 'jobs': [{ + 'rev': 369411, + 'status': 'pending', + 'buildbucket_id': 1200 + }] + } + + # Create a temporary .JSON file to simulate a .JSON file that has bisection + # contents. + with CreateTemporaryJsonFile() as temp_json_file: + with open(temp_json_file, 'w') as f: + WritePrettyJsonFile(bisect_test_contents, f) + + revision_to_update = 369411 + + # Index of the tryjob that is going to have its 'status' value updated. + tryjob_index = 0 + + path_to_chroot = '/abs/path/to/chroot' + + custom_script = None + + update_tryjob_status.UpdateTryjobStatus( + revision_to_update, update_tryjob_status.TryjobStatus.AUTO, + temp_json_file, path_to_chroot, custom_script) + + # Verify that the tryjob's 'status' has been updated in the status file. + with open(temp_json_file) as status_file: + bisect_contents = json.load(status_file) + + self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], + update_tryjob_status.TryjobStatus.GOOD.value) + + mock_get_auto_result.assert_called_once_with( + path_to_chroot, + bisect_test_contents['jobs'][tryjob_index]['buildbucket_id']) + + mock_find_tryjob_index.assert_called_once() + + # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the + # status file. + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) + # Simulate the behavior of `GetCustomScriptResult()` when the custom script + # exit code is in the mapping. + @mock.patch.object( + update_tryjob_status, + 'GetCustomScriptResult', + return_value=TryjobStatus.SKIP.value) + def testSuccessfullyUpdatedTryjobStatusToAuto( + self, mock_get_custom_script_result, mock_find_tryjob_index): + bisect_test_contents = { + 'start': 369410, + 'end': 369420, + 'jobs': [{ + 'rev': 369411, + 'status': 'pending', + 'buildbucket_id': 1200 + }] + } + + # Create a temporary .JSON file to simulate a .JSON file that has bisection + # contents. + with CreateTemporaryJsonFile() as temp_json_file: + with open(temp_json_file, 'w') as f: + WritePrettyJsonFile(bisect_test_contents, f) + + revision_to_update = 369411 + + # Index of the tryjob that is going to have its 'status' value updated. + tryjob_index = 0 + + path_to_chroot = '/abs/path/to/chroot' + + custom_script_path = '/abs/path/to/custom_script.py' + + update_tryjob_status.UpdateTryjobStatus( + revision_to_update, update_tryjob_status.TryjobStatus.CUSTOM_SCRIPT, + temp_json_file, path_to_chroot, custom_script_path) + + # Verify that the tryjob's 'status' has been updated in the status file. + with open(temp_json_file) as status_file: + bisect_contents = json.load(status_file) + + self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'], + update_tryjob_status.TryjobStatus.SKIP.value) + + mock_get_custom_script_result.assert_called_once() + + mock_find_tryjob_index.assert_called_once() + + # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the + # status file. + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) + def testSetStatusDoesNotExistWhenUpdatingTryjobStatus(self, + mock_find_tryjob_index): + + bisect_test_contents = { + 'start': 369410, + 'end': 369420, + 'jobs': [{ + 'rev': 369411, + 'status': 'pending', + 'buildbucket_id': 1200 + }] + } + + # Create a temporary .JSON file to simulate a .JSON file that has bisection + # contents. + with CreateTemporaryJsonFile() as temp_json_file: + with open(temp_json_file, 'w') as f: + WritePrettyJsonFile(bisect_test_contents, f) + + revision_to_update = 369411 + + path_to_chroot = '/abs/path/to/chroot' + + nonexistent_update_status = 'revert_status' + + custom_script = None + + # Verify the exception is raised when the `set_status` command line + # argument does not exist in the mapping. + with self.assertRaises(ValueError) as err: + update_tryjob_status.UpdateTryjobStatus( + revision_to_update, nonexistent_update_status, temp_json_file, + path_to_chroot, custom_script) + + self.assertEqual( + str(err.exception), + 'Invalid "set_status" option provided: revert_status') + + mock_find_tryjob_index.assert_called_once() + + +if __name__ == '__main__': + unittest.main() |