path: root/llvm_tools/update_tryjob_status_unittest.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Copyright 2019 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Tests when updating a tryjob's status."""

from __future__ import print_function

import json
import os
import subprocess
import unittest
import unittest.mock as mock

from test_helpers import CreateTemporaryJsonFile
from test_helpers import WritePrettyJsonFile
from update_tryjob_status import TryjobStatus
from update_tryjob_status import CustomScriptStatus
import update_tryjob_status


class UpdateTryjobStatusTest(unittest.TestCase):
  """Unittests for updating a tryjob's 'status'."""

  def testFoundTryjobIndex(self):
    test_tryjobs = [{
        'rev': 123,
        'url': 'https://some_url_to_CL.com',
        'cl': 'https://some_link_to_tryjob.com',
        'status': 'good',
        'buildbucket_id': 91835
    }, {
        'rev': 1000,
        'url': 'https://some_url_to_CL.com',
        'cl': 'https://some_link_to_tryjob.com',
        'status': 'pending',
        'buildbucket_id': 10931
    }]

    expected_index = 0

    revision_to_find = 123

    self.assertEqual(
        update_tryjob_status.FindTryjobIndex(revision_to_find, test_tryjobs),
        expected_index)

  def testNotFindTryjobIndex(self):
    test_tryjobs = [{
        'rev': 500,
        'url': 'https://some_url_to_CL.com',
        'cl': 'https://some_link_to_tryjob.com',
        'status': 'bad',
        'buildbucket_id': 390
    }, {
        'rev': 10,
        'url': 'https://some_url_to_CL.com',
        'cl': 'https://some_link_to_tryjob.com',
        'status': 'skip',
        'buildbucket_id': 10
    }]

    revision_to_find = 250

    self.assertIsNone(
        update_tryjob_status.FindTryjobIndex(revision_to_find, test_tryjobs))

  @mock.patch.object(subprocess, 'Popen')
  # Simulate the behavior of `os.rename()` when successfully renamed a file.
  @mock.patch.object(os, 'rename', return_value=None)
  # Simulate the behavior of `os.path.basename()` when successfully retrieved
  # the basename of the temp .JSON file.
  @mock.patch.object(os.path, 'basename', return_value='tmpFile.json')
  def testInvalidExitCodeByCustomScript(self, mock_basename, mock_rename_file,
                                        mock_exec_custom_script):

    error_message_by_custom_script = 'Failed to parse .JSON file'

    # Simulate the behavior of 'subprocess.Popen()' when executing the custom
    # script.
    #
    # `Popen.communicate()` returns a tuple of `stdout` and `stderr`.
    mock_exec_custom_script.return_value.communicate.return_value = (
        None, error_message_by_custom_script)

    # Exit code of 1 is not in the mapping, so an exception will be raised.
    custom_script_exit_code = 1

    mock_exec_custom_script.return_value.returncode = custom_script_exit_code

    tryjob_contents = {
        'status': 'good',
        'rev': 1234,
        'url': 'https://some_url_to_CL.com',
        'link': 'https://some_url_to_tryjob.com'
    }

    custom_script_path = '/abs/path/to/script.py'
    status_file_path = '/abs/path/to/status_file.json'

    name_json_file = os.path.join(
        os.path.dirname(status_file_path), 'tmpFile.json')

    expected_error_message = (
        'Custom script %s exit code %d did not match '
        'any of the expected exit codes: %s for "good", '
        '%d for "bad", or %d for "skip".\nPlease check '
        '%s for information about the tryjob: %s' %
        (custom_script_path, custom_script_exit_code,
         CustomScriptStatus.GOOD.value, CustomScriptStatus.BAD.value,
         CustomScriptStatus.SKIP.value, name_json_file,
         error_message_by_custom_script))

    # Verify the exception is raised when the exit code by the custom script
    # does not match any of the exit codes in the mapping of
    # `custom_script_exit_value_mapping`.
    with self.assertRaises(ValueError) as err:
      update_tryjob_status.GetCustomScriptResult(custom_script_path,
                                                 status_file_path,
                                                 tryjob_contents)

    self.assertEqual(str(err.exception), expected_error_message)

    mock_exec_custom_script.assert_called_once()

    mock_rename_file.assert_called_once()

    mock_basename.assert_called_once()

  @mock.patch.object(subprocess, 'Popen')
  # Simulate the behavior of `os.rename()` when successfully renamed a file.
  @mock.patch.object(os, 'rename', return_value=None)
  # Simulate the behavior of `os.path.basename()` when successfully retrieved
  # the basename of the temp .JSON file.
  @mock.patch.object(os.path, 'basename', return_value='tmpFile.json')
  def testValidExitCodeByCustomScript(self, mock_basename, mock_rename_file,
                                      mock_exec_custom_script):

    # Simulate the behavior of 'subprocess.Popen()' when executing the custom
    # script.
    #
    # `Popen.communicate()` returns a tuple of `stdout` and `stderr`.
    mock_exec_custom_script.return_value.communicate.return_value = (None, None)

    mock_exec_custom_script.return_value.returncode = (
        CustomScriptStatus.GOOD.value)

    tryjob_contents = {
        'status': 'good',
        'rev': 1234,
        'url': 'https://some_url_to_CL.com',
        'link': 'https://some_url_to_tryjob.com'
    }

    custom_script_path = '/abs/path/to/script.py'
    status_file_path = '/abs/path/to/status_file.json'

    self.assertEqual(
        update_tryjob_status.GetCustomScriptResult(custom_script_path,
                                                   status_file_path,
                                                   tryjob_contents),
        TryjobStatus.GOOD.value)

    mock_exec_custom_script.assert_called_once()

    mock_rename_file.assert_not_called()

    mock_basename.assert_not_called()

  def testNoTryjobsInStatusFileWhenUpdatingTryjobStatus(self):
    bisect_test_contents = {'start': 369410, 'end': 369420, 'jobs': []}

    # Create a temporary .JSON file to simulate a .JSON file that has bisection
    # contents.
    with CreateTemporaryJsonFile() as temp_json_file:
      with open(temp_json_file, 'w') as f:
        WritePrettyJsonFile(bisect_test_contents, f)

      revision_to_update = 369412

      custom_script = None

      # Verify the exception is raised when the `status_file` does not have any
      # `jobs` (empty).
      with self.assertRaises(SystemExit) as err:
        update_tryjob_status.UpdateTryjobStatus(revision_to_update,
                                                TryjobStatus.GOOD,
                                                temp_json_file, custom_script)

      self.assertEqual(str(err.exception), 'No tryjobs in %s' % temp_json_file)

  # Simulate the behavior of `FindTryjobIndex()` when the tryjob does not exist
  # in the status file.
  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=None)
  def testNotFindTryjobIndexWhenUpdatingTryjobStatus(self,
                                                     mock_find_tryjob_index):

    bisect_test_contents = {
        'start': 369410,
        'end': 369420,
        'jobs': [{
            'rev': 369411,
            'status': 'pending'
        }]
    }

    # Create a temporary .JSON file to simulate a .JSON file that has bisection
    # contents.
    with CreateTemporaryJsonFile() as temp_json_file:
      with open(temp_json_file, 'w') as f:
        WritePrettyJsonFile(bisect_test_contents, f)

      revision_to_update = 369416

      custom_script = None

      # Verify the exception is raised when the `status_file` does not have any
      # `jobs` (empty).
      with self.assertRaises(ValueError) as err:
        update_tryjob_status.UpdateTryjobStatus(revision_to_update,
                                                TryjobStatus.SKIP,
                                                temp_json_file, custom_script)

      self.assertEqual(
          str(err.exception), 'Unable to find tryjob for %d in %s' %
          (revision_to_update, temp_json_file))

    mock_find_tryjob_index.assert_called_once()

  # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the
  # status file.
  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
  def testSuccessfullyUpdatedTryjobStatusToGood(self, mock_find_tryjob_index):
    bisect_test_contents = {
        'start': 369410,
        'end': 369420,
        'jobs': [{
            'rev': 369411,
            'status': 'pending'
        }]
    }

    # Create a temporary .JSON file to simulate a .JSON file that has bisection
    # contents.
    with CreateTemporaryJsonFile() as temp_json_file:
      with open(temp_json_file, 'w') as f:
        WritePrettyJsonFile(bisect_test_contents, f)

      revision_to_update = 369411

      # Index of the tryjob that is going to have its 'status' value updated.
      tryjob_index = 0

      custom_script = None

      update_tryjob_status.UpdateTryjobStatus(revision_to_update,
                                              TryjobStatus.GOOD, temp_json_file,
                                              custom_script)

      # Verify that the tryjob's 'status' has been updated in the status file.
      with open(temp_json_file) as status_file:
        bisect_contents = json.load(status_file)

        self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'],
                         TryjobStatus.GOOD.value)

    mock_find_tryjob_index.assert_called_once()

  # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the
  # status file.
  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
  def testSuccessfullyUpdatedTryjobStatusToBad(self, mock_find_tryjob_index):
    bisect_test_contents = {
        'start': 369410,
        'end': 369420,
        'jobs': [{
            'rev': 369411,
            'status': 'pending'
        }]
    }

    # Create a temporary .JSON file to simulate a .JSON file that has bisection
    # contents.
    with CreateTemporaryJsonFile() as temp_json_file:
      with open(temp_json_file, 'w') as f:
        WritePrettyJsonFile(bisect_test_contents, f)

      revision_to_update = 369411

      # Index of the tryjob that is going to have its 'status' value updated.
      tryjob_index = 0

      custom_script = None

      update_tryjob_status.UpdateTryjobStatus(revision_to_update,
                                              TryjobStatus.BAD, temp_json_file,
                                              custom_script)

      # Verify that the tryjob's 'status' has been updated in the status file.
      with open(temp_json_file) as status_file:
        bisect_contents = json.load(status_file)

        self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'],
                         TryjobStatus.BAD.value)

    mock_find_tryjob_index.assert_called_once()

  # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the
  # status file.
  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
  def testSuccessfullyUpdatedTryjobStatusToPending(self,
                                                   mock_find_tryjob_index):
    bisect_test_contents = {
        'start': 369410,
        'end': 369420,
        'jobs': [{
            'rev': 369411,
            'status': 'skip'
        }]
    }

    # Create a temporary .JSON file to simulate a .JSON file that has bisection
    # contents.
    with CreateTemporaryJsonFile() as temp_json_file:
      with open(temp_json_file, 'w') as f:
        WritePrettyJsonFile(bisect_test_contents, f)

      revision_to_update = 369411

      # Index of the tryjob that is going to have its 'status' value updated.
      tryjob_index = 0

      custom_script = None

      update_tryjob_status.UpdateTryjobStatus(
          revision_to_update, update_tryjob_status.TryjobStatus.SKIP,
          temp_json_file, custom_script)

      # Verify that the tryjob's 'status' has been updated in the status file.
      with open(temp_json_file) as status_file:
        bisect_contents = json.load(status_file)

        self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'],
                         update_tryjob_status.TryjobStatus.SKIP.value)

    mock_find_tryjob_index.assert_called_once()

  # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the
  # status file.
  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
  def testSuccessfullyUpdatedTryjobStatusToSkip(self, mock_find_tryjob_index):
    bisect_test_contents = {
        'start': 369410,
        'end': 369420,
        'jobs': [{
            'rev': 369411,
            'status': 'pending',
        }]
    }

    # Create a temporary .JSON file to simulate a .JSON file that has bisection
    # contents.
    with CreateTemporaryJsonFile() as temp_json_file:
      with open(temp_json_file, 'w') as f:
        WritePrettyJsonFile(bisect_test_contents, f)

      revision_to_update = 369411

      # Index of the tryjob that is going to have its 'status' value updated.
      tryjob_index = 0

      custom_script = None

      update_tryjob_status.UpdateTryjobStatus(
          revision_to_update, update_tryjob_status.TryjobStatus.PENDING,
          temp_json_file, custom_script)

      # Verify that the tryjob's 'status' has been updated in the status file.
      with open(temp_json_file) as status_file:
        bisect_contents = json.load(status_file)

        self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'],
                         update_tryjob_status.TryjobStatus.PENDING.value)

    mock_find_tryjob_index.assert_called_once()

  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
  @mock.patch.object(
      update_tryjob_status,
      'GetCustomScriptResult',
      return_value=TryjobStatus.SKIP.value)
  def testUpdatedTryjobStatusToAutoPassedWithCustomScript(
      self, mock_get_custom_script_result, mock_find_tryjob_index):
    bisect_test_contents = {
        'start': 369410,
        'end': 369420,
        'jobs': [{
            'rev': 369411,
            'status': 'pending',
            'buildbucket_id': 1200
        }]
    }

    # Create a temporary .JSON file to simulate a .JSON file that has bisection
    # contents.
    with CreateTemporaryJsonFile() as temp_json_file:
      with open(temp_json_file, 'w') as f:
        WritePrettyJsonFile(bisect_test_contents, f)

      revision_to_update = 369411

      # Index of the tryjob that is going to have its 'status' value updated.
      tryjob_index = 0

      custom_script_path = '/abs/path/to/custom_script.py'

      update_tryjob_status.UpdateTryjobStatus(
          revision_to_update, update_tryjob_status.TryjobStatus.CUSTOM_SCRIPT,
          temp_json_file, custom_script_path)

      # Verify that the tryjob's 'status' has been updated in the status file.
      with open(temp_json_file) as status_file:
        bisect_contents = json.load(status_file)

        self.assertEqual(bisect_contents['jobs'][tryjob_index]['status'],
                         update_tryjob_status.TryjobStatus.SKIP.value)

    mock_get_custom_script_result.assert_called_once()

    mock_find_tryjob_index.assert_called_once()

  # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the
  # status file.
  @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0)
  def testSetStatusDoesNotExistWhenUpdatingTryjobStatus(self,
                                                        mock_find_tryjob_index):

    bisect_test_contents = {
        'start': 369410,
        'end': 369420,
        'jobs': [{
            'rev': 369411,
            'status': 'pending',
            'buildbucket_id': 1200
        }]
    }

    # Create a temporary .JSON file to simulate a .JSON file that has bisection
    # contents.
    with CreateTemporaryJsonFile() as temp_json_file:
      with open(temp_json_file, 'w') as f:
        WritePrettyJsonFile(bisect_test_contents, f)

      revision_to_update = 369411

      nonexistent_update_status = 'revert_status'

      custom_script = None

      # Verify the exception is raised when the `set_status` command line
      # argument does not exist in the mapping.
      with self.assertRaises(ValueError) as err:
        update_tryjob_status.UpdateTryjobStatus(revision_to_update,
                                                nonexistent_update_status,
                                                temp_json_file, custom_script)

      self.assertEqual(
          str(err.exception),
          'Invalid "set_status" option provided: revert_status')

    mock_find_tryjob_index.assert_called_once()


if __name__ == '__main__':
  unittest.main()