diff options
-rw-r--r-- | llvm_tools/assert_not_in_chroot.py | 24 | ||||
-rwxr-xr-x | llvm_tools/auto_llvm_bisection.py | 6 | ||||
-rwxr-xr-x | llvm_tools/auto_llvm_bisection_unittest.py | 37 | ||||
-rwxr-xr-x | llvm_tools/chroot.py | 96 | ||||
-rwxr-xr-x | llvm_tools/chroot_unittest.py | 66 | ||||
-rwxr-xr-x | llvm_tools/git.py | 136 | ||||
-rwxr-xr-x | llvm_tools/git_unittest.py | 173 | ||||
-rwxr-xr-x | llvm_tools/llvm_bisection.py | 56 | ||||
-rwxr-xr-x | llvm_tools/llvm_bisection_unittest.py | 58 | ||||
-rwxr-xr-x | llvm_tools/llvm_patch_management.py | 26 | ||||
-rwxr-xr-x | llvm_tools/llvm_patch_management_unittest.py | 19 | ||||
-rwxr-xr-x | llvm_tools/modify_a_tryjob.py | 32 | ||||
-rwxr-xr-x | llvm_tools/modify_a_tryjob_unittest.py | 73 | ||||
-rwxr-xr-x | llvm_tools/update_all_tryjobs_with_auto.py | 13 | ||||
-rwxr-xr-x | llvm_tools/update_chromeos_llvm_hash.py | 246 | ||||
-rwxr-xr-x | llvm_tools/update_chromeos_llvm_hash_unittest.py | 353 | ||||
-rwxr-xr-x | llvm_tools/update_packages_and_run_tests.py | 32 | ||||
-rwxr-xr-x | llvm_tools/update_packages_and_run_tests_unittest.py | 52 | ||||
-rwxr-xr-x | llvm_tools/update_tryjob_status.py | 4 |
19 files changed, 719 insertions, 783 deletions
diff --git a/llvm_tools/assert_not_in_chroot.py b/llvm_tools/assert_not_in_chroot.py deleted file mode 100644 index 6b78d95c..00000000 --- a/llvm_tools/assert_not_in_chroot.py +++ /dev/null @@ -1,24 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2019 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""Helper module to determine whether a script is executed inside the chroot.""" - -from __future__ import print_function - -import os - - -def VerifyOutsideChroot(): - """Checks whether the script invoked was executed in the chroot. - - Raises: - AssertionError: The script was run inside the chroot. - """ - - chroot_only_path = '/mnt/host/depot_tools' - - in_chroot_err_message = 'Script should be run outside the chroot.' - - assert not os.path.isdir(chroot_only_path), in_chroot_err_message diff --git a/llvm_tools/auto_llvm_bisection.py b/llvm_tools/auto_llvm_bisection.py index cd3d70b6..dd29cf4f 100755 --- a/llvm_tools/auto_llvm_bisection.py +++ b/llvm_tools/auto_llvm_bisection.py @@ -14,10 +14,10 @@ import sys import time import traceback -from assert_not_in_chroot import VerifyOutsideChroot -from update_all_tryjobs_with_auto import GetPathToUpdateAllTryjobsWithAutoScript +import chroot from llvm_bisection import BisectionExitStatus import llvm_bisection +from update_all_tryjobs_with_auto import GetPathToUpdateAllTryjobsWithAutoScript # Used to re-try for 'llvm_bisection.py' to attempt to launch more tryjobs. BISECTION_RETRY_TIME_SECS = 10 * 60 @@ -46,7 +46,7 @@ def main(): AssertionError: The script was run inside the chroot. """ - VerifyOutsideChroot() + chroot.VerifyOutsideChroot() args_output = llvm_bisection.GetCommandLineArgs() diff --git a/llvm_tools/auto_llvm_bisection_unittest.py b/llvm_tools/auto_llvm_bisection_unittest.py index 3e6e3a3e..56b556e6 100755 --- a/llvm_tools/auto_llvm_bisection_unittest.py +++ b/llvm_tools/auto_llvm_bisection_unittest.py @@ -15,10 +15,10 @@ import traceback import unittest import unittest.mock as mock -from test_helpers import ArgsOutputTest -from test_helpers import CallCountsToMockFunctions import auto_llvm_bisection +import chroot import llvm_bisection +import test_helpers class AutoLLVMBisectionTest(unittest.TestCase): @@ -26,8 +26,7 @@ class AutoLLVMBisectionTest(unittest.TestCase): # Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking # the script outside of the chroot. - @mock.patch.object( - auto_llvm_bisection, 'VerifyOutsideChroot', return_value=True) + @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) # Simulate behavior of `time.sleep()` when waiting for errors to settle caused # by `llvm_bisection.main()` (e.g. network issue, etc.). @mock.patch.object(time, 'sleep') @@ -49,12 +48,14 @@ class AutoLLVMBisectionTest(unittest.TestCase): # Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line # arguments required by the bisection script. @mock.patch.object( - llvm_bisection, 'GetCommandLineArgs', return_value=ArgsOutputTest()) + llvm_bisection, + 'GetCommandLineArgs', + return_value=test_helpers.ArgsOutputTest()) def testFailedToStartBisection( self, mock_get_args, mock_get_auto_script, mock_is_file, mock_llvm_bisection, mock_traceback, mock_sleep, mock_outside_chroot): - def MockLLVMBisectionRaisesException(args_output): + def MockLLVMBisectionRaisesException(_args_output): raise ValueError('Failed to launch more tryjobs.') # Use the test function to simulate the behavior of an exception happening @@ -82,8 +83,7 @@ class AutoLLVMBisectionTest(unittest.TestCase): @mock.patch.object(subprocess, 'call', return_value=0) # Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking # the script outside of the chroot. - @mock.patch.object( - auto_llvm_bisection, 'VerifyOutsideChroot', return_value=True) + @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) # Simulate behavior of `time.sleep()` when waiting for errors to settle caused # by `llvm_bisection.main()` (e.g. network issue, etc.). @mock.patch.object(time, 'sleep') @@ -105,7 +105,9 @@ class AutoLLVMBisectionTest(unittest.TestCase): # Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line # arguments required by the bisection script. @mock.patch.object( - llvm_bisection, 'GetCommandLineArgs', return_value=ArgsOutputTest()) + llvm_bisection, + 'GetCommandLineArgs', + return_value=test_helpers.ArgsOutputTest()) def testSuccessfullyBisectedLLVMRevision( self, mock_get_args, mock_get_auto_script, mock_is_file, mock_llvm_bisection, mock_traceback, mock_sleep, mock_outside_chroot, @@ -113,8 +115,8 @@ class AutoLLVMBisectionTest(unittest.TestCase): # Simulate the behavior of `os.path.isfile()` when checking whether the # status file provided exists. - @CallCountsToMockFunctions - def MockStatusFileCheck(call_count, last_tested): + @test_helpers.CallCountsToMockFunctions + def MockStatusFileCheck(call_count, _last_tested): # Simulate that the status file does not exist, so the LLVM bisection # script would create the status file and launch tryjobs. if call_count < 2: @@ -130,8 +132,8 @@ class AutoLLVMBisectionTest(unittest.TestCase): # Simulate behavior of `llvm_bisection.main()` when successfully bisected # between the good and bad LLVM revision. - @CallCountsToMockFunctions - def MockLLVMBisectionReturnValue(call_count, args_output): + @test_helpers.CallCountsToMockFunctions + def MockLLVMBisectionReturnValue(call_count, _args_output): # Simulate that successfully launched more tryjobs. if call_count == 0: return 0 @@ -176,8 +178,7 @@ class AutoLLVMBisectionTest(unittest.TestCase): @mock.patch.object(time, 'time') # Simulate the behavior of `VerifyOutsideChroot()` when successfully invoking # the script outside of the chroot. - @mock.patch.object( - auto_llvm_bisection, 'VerifyOutsideChroot', return_value=True) + @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) # Simulate behavior of `time.sleep()` when waiting for errors to settle caused # by `llvm_bisection.main()` (e.g. network issue, etc.). @mock.patch.object(time, 'sleep') @@ -193,13 +194,15 @@ class AutoLLVMBisectionTest(unittest.TestCase): # Simulate `llvm_bisection.GetCommandLineArgs()` when parsing the command line # arguments required by the bisection script. @mock.patch.object( - llvm_bisection, 'GetCommandLineArgs', return_value=ArgsOutputTest()) + llvm_bisection, + 'GetCommandLineArgs', + return_value=test_helpers.ArgsOutputTest()) def testFailedToUpdatePendingTryJobs( self, mock_get_args, mock_get_auto_script, mock_is_file, mock_sleep, mock_outside_chroot, mock_time, mock_update_tryjobs): # Simulate behavior of `time.time()` for time passed. - @CallCountsToMockFunctions + @test_helpers.CallCountsToMockFunctions def MockTimePassed(call_count): if call_count < 3: return call_count diff --git a/llvm_tools/chroot.py b/llvm_tools/chroot.py new file mode 100755 index 00000000..1c3eb025 --- /dev/null +++ b/llvm_tools/chroot.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Chroot helper functions.""" + +from __future__ import print_function + +import os +import subprocess +import collections + +CommitContents = collections.namedtuple('CommitContents', ['url', 'cl_number']) + + +def InChroot(): + """Returns True if currently in the chroot.""" + return 'CROS_WORKON_SRCROOT' in os.environ + + +def VerifyOutsideChroot(): + """Checks whether the script invoked was executed in the chroot. + + Raises: + AssertionError: The script was run inside the chroot. + """ + + assert not InChroot(), 'Script should be run outside the chroot.' + + +def GetChrootEbuildPaths(chromeos_root, packages): + """Gets the chroot path(s) of the package(s). + + Args: + chromeos_root: The absolute path to the chroot to + use for executing chroot commands. + packages: A list of a package/packages to + be used to find their chroot path. + + Returns: + A list of chroot paths of the packages' ebuild files. + + Raises: + ValueError: Failed to get the chroot path of a package. + """ + + chroot_paths = [] + + # Find the chroot path for each package's ebuild. + for package in sorted(set(packages)): + chroot_path = subprocess.check_output( + ['cros_sdk', '--', 'equery', 'w', package], + cwd=chromeos_root, + encoding='utf-8') + chroot_paths.append(chroot_path.strip()) + + return chroot_paths + + +def ConvertChrootPathsToAbsolutePaths(chromeos_root, chroot_paths): + """Converts the chroot path(s) to absolute symlink path(s). + + Args: + chromeos_root: The absolute path to the chroot. + chroot_paths: A list of chroot paths to convert to absolute paths. + + Returns: + A list of absolute path(s). + + Raises: + ValueError: Invalid prefix for the chroot path or + invalid chroot paths were provided. + """ + + abs_paths = [] + + chroot_prefix = '/mnt/host/source/' + + # Iterate through the chroot paths. + # + # For each chroot file path, remove '/mnt/host/source/' prefix + # and combine the chroot path with the result and add it to the list. + for chroot_path in chroot_paths: + if not chroot_path.startswith(chroot_prefix): + raise ValueError('Invalid prefix for the chroot path: %s' % chroot_path) + + rel_path = chroot_path[len(chroot_prefix):] + + # combine the chromeos root path + '/src/...' + abs_path = os.path.join(chromeos_root, rel_path) + + abs_paths.append(abs_path) + + return abs_paths diff --git a/llvm_tools/chroot_unittest.py b/llvm_tools/chroot_unittest.py new file mode 100755 index 00000000..5eec5675 --- /dev/null +++ b/llvm_tools/chroot_unittest.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unit tests for chroot helper functions.""" + +from __future__ import print_function + +import subprocess +import unittest +import unittest.mock as mock + +import chroot + +# These are unittests; protected access is OK to a point. +# pylint: disable=protected-access + + +class HelperFunctionsTest(unittest.TestCase): + """Test class for updating LLVM hashes of packages.""" + + @mock.patch.object(subprocess, 'check_output') + def testSucceedsToGetChrootEbuildPathForPackage(self, mock_chroot_command): + package_chroot_path = '/chroot/path/to/package.ebuild' + + # Emulate ChrootRunCommandWOutput behavior when a chroot path is found for + # a valid package. + mock_chroot_command.return_value = package_chroot_path + + chroot_path = '/test/chroot/path' + package_list = ['new-test/package'] + + self.assertEqual( + chroot.GetChrootEbuildPaths(chroot_path, package_list), + [package_chroot_path]) + + mock_chroot_command.assert_called_once() + + def testFailedToConvertChrootPathWithInvalidPrefix(self): + chroot_path = '/path/to/chroot' + chroot_file_path = '/src/package.ebuild' + + # Verify the exception is raised when a chroot path does not have the prefix + # '/mnt/host/source/'. + with self.assertRaises(ValueError) as err: + chroot.ConvertChrootPathsToAbsolutePaths(chroot_path, [chroot_file_path]) + + self.assertEqual( + str(err.exception), 'Invalid prefix for the chroot path: ' + '%s' % chroot_file_path) + + def testSucceedsToConvertChrootPathToAbsolutePath(self): + chroot_path = '/path/to/chroot' + chroot_file_paths = ['/mnt/host/source/src/package.ebuild'] + + expected_abs_path = '/path/to/chroot/src/package.ebuild' + + self.assertEqual( + chroot.ConvertChrootPathsToAbsolutePaths( + chroot_path, chroot_file_paths), [expected_abs_path]) + + +if __name__ == '__main__': + unittest.main() diff --git a/llvm_tools/git.py b/llvm_tools/git.py new file mode 100755 index 00000000..778d9695 --- /dev/null +++ b/llvm_tools/git.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Git helper functions.""" + +from __future__ import print_function + +import collections +import os +import re +import subprocess +import tempfile + +CommitContents = collections.namedtuple('CommitContents', ['url', 'cl_number']) + + +def InChroot(): + """Returns True if currently in the chroot.""" + return 'CROS_WORKON_SRCROOT' in os.environ + + +def VerifyOutsideChroot(): + """Checks whether the script invoked was executed in the chroot. + + Raises: + AssertionError: The script was run inside the chroot. + """ + + assert not InChroot(), 'Script should be run outside the chroot.' + + +def CreateBranch(repo, branch): + """Creates a branch in the given repo. + + Args: + repo: The absolute path to the repo. + branch: The name of the branch to create. + + Raises: + ValueError: Failed to create a repo in that directory. + """ + + if not os.path.isdir(repo): + raise ValueError('Invalid directory path provided: %s' % repo) + + subprocess.check_output(['git', '-C', repo, 'reset', 'HEAD', '--hard'], + encoding='utf-8') + + subprocess.check_output(['repo', 'start', branch], cwd=repo, encoding='utf-8') + + +def DeleteBranch(repo, branch): + """Deletes a branch in the given repo. + + Args: + repo: The absolute path of the repo. + branch: The name of the branch to delete. + + Raises: + ValueError: Failed to delete the repo in that directory. + """ + + if not os.path.isdir(repo): + raise ValueError('Invalid directory path provided: %s' % repo) + + subprocess.check_output(['git', '-C', repo, 'checkout', 'cros/master'], + encoding='utf-8') + + subprocess.check_output(['git', '-C', repo, 'reset', 'HEAD', '--hard'], + encoding='utf-8') + + subprocess.check_output(['git', '-C', repo, 'branch', '-D', branch], + encoding='utf-8') + + +def UploadChanges(repo, branch, commit_messages): + """Uploads the changes in the specifed branch of the given repo for review. + + Args: + repo: The absolute path to the repo where changes were made. + branch: The name of the branch to upload. + commit_messages: A string of commit message(s) (i.e. '[message]' + of the changes made. + + Returns: + A nametuple that has two (key, value) pairs, where the first pair is the + Gerrit commit URL and the second pair is the change list number. + + Raises: + ValueError: Failed to create a commit or failed to upload the + changes for review. + """ + + if not os.path.isdir(repo): + raise ValueError('Invalid path provided: %s' % repo) + + # Create a git commit. + with tempfile.NamedTemporaryFile(mode='w+t') as f: + f.write('\n'.join(commit_messages)) + f.flush() + + subprocess.check_output(['git', 'commit', '-F', f.name], + cwd=repo, + encoding='utf-8') + + # Upload the changes for review. + # Pylint currently doesn't lint things in py3 mode, and py2 didn't allow + # users to specify `encoding`s for Popen. Hence, pylint is "wrong" here. + # pylint: disable=unexpected-keyword-arg + # The CL URL is sent to 'stderr', so need to redirect 'stderr' to 'stdout'. + upload_changes_obj = subprocess.Popen( + ['repo', 'upload', '--yes', '--ne', '--no-verify', + '--br=%s' % branch], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding='utf-8') + + out, _ = upload_changes_obj.communicate() + + if upload_changes_obj.returncode: # Failed to upload changes. + print(out) + raise ValueError('Failed to upload changes for review') + + found_url = re.search( + r'https://chromium-review.googlesource.com/c/' + r'chromiumos/overlays/chromiumos-overlay/\+/([0-9]+)', out.rstrip()) + + if not found_url: + raise ValueError('Failed to find change list URL.') + + return CommitContents( + url=found_url.group(0), cl_number=int(found_url.group(1))) diff --git a/llvm_tools/git_unittest.py b/llvm_tools/git_unittest.py new file mode 100755 index 00000000..39316276 --- /dev/null +++ b/llvm_tools/git_unittest.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2020 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Unit tests for git helper functions.""" + +from __future__ import print_function + +import os +import subprocess +import unittest +import unittest.mock as mock + +import git + +# These are unittests; protected access is OK to a point. +# pylint: disable=protected-access + + +class HelperFunctionsTest(unittest.TestCase): + """Test class for updating LLVM hashes of packages.""" + + @mock.patch.object(os.path, 'isdir', return_value=False) + def testFailedToCreateBranchForInvalidDirectoryPath(self, mock_isdir): + path_to_repo = '/invalid/path/to/repo' + branch = 'branch-name' + + # Verify the exception is raised when provided an invalid directory path. + with self.assertRaises(ValueError) as err: + git.CreateBranch(path_to_repo, branch) + + self.assertEqual( + str(err.exception), + 'Invalid directory path provided: %s' % path_to_repo) + + mock_isdir.assert_called_once() + + @mock.patch.object(os.path, 'isdir', return_value=True) + @mock.patch.object(subprocess, 'check_output', return_value=None) + def testSuccessfullyCreatedBranch(self, mock_command_output, mock_isdir): + path_to_repo = '/path/to/repo' + branch = 'branch-name' + + git.CreateBranch(path_to_repo, branch) + + mock_isdir.assert_called_once_with(path_to_repo) + + self.assertEqual(mock_command_output.call_count, 2) + + @mock.patch.object(os.path, 'isdir', return_value=False) + def testFailedToDeleteBranchForInvalidDirectoryPath(self, mock_isdir): + path_to_repo = '/invalid/path/to/repo' + branch = 'branch-name' + + # Verify the exception is raised on an invalid repo path. + with self.assertRaises(ValueError) as err: + git.DeleteBranch(path_to_repo, branch) + + self.assertEqual( + str(err.exception), + 'Invalid directory path provided: %s' % path_to_repo) + + mock_isdir.assert_called_once() + + @mock.patch.object(os.path, 'isdir', return_value=True) + @mock.patch.object(subprocess, 'check_output', return_value=None) + def testSuccessfullyDeletedBranch(self, mock_command_output, mock_isdir): + path_to_repo = '/valid/path/to/repo' + branch = 'branch-name' + + git.DeleteBranch(path_to_repo, branch) + + mock_isdir.assert_called_once_with(path_to_repo) + + self.assertEqual(mock_command_output.call_count, 3) + + @mock.patch.object(os.path, 'isdir', return_value=False) + def testFailedToUploadChangesForInvalidDirectoryPath(self, mock_isdir): + path_to_repo = '/some/path/to/repo' + branch = 'update-LLVM_NEXT_HASH-a123testhash3' + commit_messages = ['Test message'] + + # Verify exception is raised when on an invalid repo path. + with self.assertRaises(ValueError) as err: + git.UploadChanges(path_to_repo, branch, commit_messages) + + self.assertEqual( + str(err.exception), 'Invalid path provided: %s' % path_to_repo) + + mock_isdir.assert_called_once() + + @mock.patch.object(os.path, 'isdir', return_value=True) + @mock.patch.object(subprocess, 'check_output', return_value=None) + @mock.patch.object(subprocess, 'Popen') + def testFailedToUploadChangesForReview(self, mock_repo_upload, + mock_command_output, mock_isdir): + + # Simulate the behavior of 'subprocess.Popen()' when uploading the changes + # for review + # + # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. + mock_repo_upload.return_value.communicate.return_value = ( + None, 'Branch does not exist.') + + # Exit code of 1 means failed to upload changes for review. + mock_repo_upload.return_value.returncode = 1 + + path_to_repo = '/some/path/to/repo' + branch = 'invalid-branch-name' + commit_messages = ['Test message'] + + # Verify exception is raised when failed to upload the changes for review. + with self.assertRaises(ValueError) as err: + git.UploadChanges(path_to_repo, branch, commit_messages) + + self.assertEqual(str(err.exception), 'Failed to upload changes for review') + + mock_isdir.assert_called_once_with(path_to_repo) + + mock_command_output.assert_called_once() + mock_command_output_args = mock_command_output.call_args_list[0][0][0] + expected_mock_command_output_prefix = ['git', 'commit', '-F'] + self.assertEqual( + mock_command_output_args[:len(expected_mock_command_output_prefix)], + expected_mock_command_output_prefix) + + mock_repo_upload.assert_called_once() + + @mock.patch.object(os.path, 'isdir', return_value=True) + @mock.patch.object(subprocess, 'check_output', return_value=None) + @mock.patch.object(subprocess, 'Popen') + def testSuccessfullyUploadedChangesForReview(self, mock_repo_upload, + mock_command_output, mock_isdir): + + # A test CL generated by `repo upload`. + repo_upload_contents = ('remote: https://chromium-review.googlesource.' + 'com/c/chromiumos/overlays/chromiumos-overlay/' + '+/193147 Fix stdout') + + # Simulate the behavior of 'subprocess.Popen()' when uploading the changes + # for review + # + # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. + mock_repo_upload.return_value.communicate.return_value = ( + repo_upload_contents, None) + + # Exit code of 0 means successfully uploaded changes for review. + mock_repo_upload.return_value.returncode = 0 + + path_to_repo = '/some/path/to/repo' + branch = 'branch-name' + commit_messages = ['Test message'] + + change_list = git.UploadChanges(path_to_repo, branch, commit_messages) + + self.assertEqual( + change_list.url, + 'https://chromium-review.googlesource.com/c/chromiumos/overlays/' + 'chromiumos-overlay/+/193147') + + self.assertEqual(change_list.cl_number, 193147) + + mock_isdir.assert_called_once_with(path_to_repo) + + mock_command_output.assert_called_once() + + mock_repo_upload.assert_called_once() + + +if __name__ == '__main__': + unittest.main() diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py index 4ff921d2..2772ca48 100755 --- a/llvm_tools/llvm_bisection.py +++ b/llvm_tools/llvm_bisection.py @@ -11,17 +11,14 @@ from __future__ import print_function import argparse import enum import errno -import get_llvm_hash import json import os import sys -from assert_not_in_chroot import VerifyOutsideChroot -from get_llvm_hash import CreateTempLLVMRepo -from get_llvm_hash import LLVMHash -from modify_a_tryjob import AddTryjob -from update_tryjob_status import FindTryjobIndex -from update_tryjob_status import TryjobStatus +import chroot +import get_llvm_hash +import modify_a_tryjob +import update_tryjob_status class BisectionExitStatus(enum.Enum): @@ -182,18 +179,20 @@ def GetStartAndEndRevision(start, end, tryjobs): 'go to %s and update it' % cur_tryjob_dict['link']) all_bad_revisions = [end] - all_bad_revisions.extend(cur_tryjob['rev'] - for cur_tryjob in tryjobs - if cur_tryjob['status'] == TryjobStatus.BAD.value) + all_bad_revisions.extend( + cur_tryjob['rev'] + for cur_tryjob in tryjobs + if cur_tryjob['status'] == update_tryjob_status.TryjobStatus.BAD.value) # The minimum value for the 'bad' field in the tryjobs is the new end # version. bad_rev = min(all_bad_revisions) all_good_revisions = [start] - all_good_revisions.extend(cur_tryjob['rev'] - for cur_tryjob in tryjobs - if cur_tryjob['status'] == TryjobStatus.GOOD.value) + all_good_revisions.extend( + cur_tryjob['rev'] + for cur_tryjob in tryjobs + if cur_tryjob['status'] == update_tryjob_status.TryjobStatus.GOOD.value) # The maximum value for the 'good' field in the tryjobs is the new start # version. @@ -212,7 +211,7 @@ def GetStartAndEndRevision(start, end, tryjobs): pending_revisions = { tryjob['rev'] for tryjob in tryjobs - if tryjob['status'] == TryjobStatus.PENDING.value and + if tryjob['status'] == update_tryjob_status.TryjobStatus.PENDING.value and good_rev < tryjob['rev'] < bad_rev } @@ -224,7 +223,7 @@ def GetStartAndEndRevision(start, end, tryjobs): skip_revisions = { tryjob['rev'] for tryjob in tryjobs - if tryjob['status'] == TryjobStatus.SKIP.value and + if tryjob['status'] == update_tryjob_status.TryjobStatus.SKIP.value and good_rev < tryjob['rev'] < bad_rev } @@ -254,8 +253,6 @@ def GetRevisionsBetweenBisection(start, end, parallel, src_path, A list of revisions between 'start' and 'end'. """ - new_llvm = LLVMHash() - valid_revisions = [] # Start at ('start' + 1) because 'start' is the good revision. @@ -292,10 +289,10 @@ def GetRevisionsListAndHashList(start, end, parallel, src_path, pending_revisions, skip_revisions): """Determines the revisions between start and end.""" - new_llvm = LLVMHash() + new_llvm = get_llvm_hash.LLVMHash() with new_llvm.CreateTempDirectory() as temp_dir: - with CreateTempLLVMRepo(temp_dir) as new_repo: + with get_llvm_hash.CreateTempLLVMRepo(temp_dir) as new_repo: if not src_path: src_path = new_repo @@ -303,7 +300,9 @@ def GetRevisionsListAndHashList(start, end, parallel, src_path, revisions = GetRevisionsBetweenBisection( start, end, parallel, src_path, pending_revisions, skip_revisions) - git_hashes = [get_llvm_hash.GetGitHashFrom(src_path, rev) for rev in revisions] + git_hashes = [ + get_llvm_hash.GetGitHashFrom(src_path, rev) for rev in revisions + ] return revisions, git_hashes @@ -329,7 +328,7 @@ def CheckForExistingTryjobsInRevisionsToLaunch(revisions, jobs): """Checks if a revision in 'revisions' exists in 'jobs' list.""" for rev in revisions: - if FindTryjobIndex(rev, jobs) is not None: + if update_tryjob_status.FindTryjobIndex(rev, jobs) is not None: raise ValueError('Revision %d exists already in "jobs"' % rev) @@ -340,10 +339,10 @@ def UpdateBisection(revisions, git_hashes, bisect_contents, last_tested, try: for svn_revision, git_hash in zip(revisions, git_hashes): - tryjob_dict = AddTryjob(update_packages, git_hash, svn_revision, - chroot_path, patch_metadata_file, - extra_change_lists, options, builder, verbose, - svn_revision) + tryjob_dict = modify_a_tryjob.AddTryjob( + update_packages, git_hash, svn_revision, chroot_path, + patch_metadata_file, extra_change_lists, options, builder, verbose, + svn_revision) bisect_contents['jobs'].append(tryjob_dict) finally: @@ -364,7 +363,7 @@ def _NoteCompletedBisection(last_tested, src_path, end): if src_path: bad_llvm_hash = get_llvm_hash.GetGitHashFrom(src_path, end) else: - bad_llvm_hash = LLVMHash().GetLLVMHash(end) + bad_llvm_hash = get_llvm_hash.LLVMHash().GetLLVMHash(end) print( 'The bad revision is %d and its commit hash is %s' % (end, bad_llvm_hash)) @@ -390,7 +389,7 @@ def main(args_output): AssertionError: The script was run inside the chroot. """ - VerifyOutsideChroot() + chroot.VerifyOutsideChroot() update_packages = [ 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx', @@ -458,5 +457,4 @@ def main(args_output): if __name__ == '__main__': - args_output = GetCommandLineArgs() - sys.exit(main(args_output)) + sys.exit(main(GetCommandLineArgs())) diff --git a/llvm_tools/llvm_bisection_unittest.py b/llvm_tools/llvm_bisection_unittest.py index ccef649c..e730293b 100755 --- a/llvm_tools/llvm_bisection_unittest.py +++ b/llvm_tools/llvm_bisection_unittest.py @@ -4,21 +4,22 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +# pylint: disable=protected-access + """Tests for LLVM bisection.""" from __future__ import print_function -import get_llvm_hash import json -import llvm_bisection import unittest import unittest.mock as mock -from get_llvm_hash import LLVMHash -from test_helpers import ArgsOutputTest -from test_helpers import CallCountsToMockFunctions -from test_helpers import CreateTemporaryJsonFile -from test_helpers import WritePrettyJsonFile +import chroot +import get_llvm_hash +import llvm_bisection +import modify_a_tryjob +import test_helpers +import update_tryjob_status class LLVMBisectionTest(unittest.TestCase): @@ -155,7 +156,7 @@ class LLVMBisectionTest(unittest.TestCase): # Simulate behavior of `GetGitHashFrom()` when the revision does not # exist in the LLVM source tree. - def MockGetGitHashForRevisionRaiseException(src_path, revision): + def MockGetGitHashForRevisionRaiseException(_src_path, _revision): raise ValueError('Revision does not exist') mock_get_git_hash.side_effect = MockGetGitHashForRevisionRaiseException @@ -204,7 +205,7 @@ class LLVMBisectionTest(unittest.TestCase): @mock.patch.object(llvm_bisection, 'GetRevisionsBetweenBisection') # Simulate behavior of `CreatTempLLVMRepo()` when successfully created a # worktree when a source path was not provided. - @mock.patch.object(llvm_bisection, 'CreateTempLLVMRepo') + @mock.patch.object(get_llvm_hash, 'CreateTempLLVMRepo') def testSuccessfullyGetRevisionsListAndHashList( self, mock_create_temp_llvm_repo, mock_get_revisions_between_bisection, mock_get_git_hash): @@ -213,8 +214,8 @@ class LLVMBisectionTest(unittest.TestCase): 'a123testhash1', 'a123testhash2', 'a123testhash3' ]) - @CallCountsToMockFunctions - def MockGetGitHashForRevision(call_count, src_path, rev): + @test_helpers.CallCountsToMockFunctions + def MockGetGitHashForRevision(call_count, _src_path, _rev): # Simulate retrieving the git hash for the revision. if call_count < 3: return expected_revisions_and_hash_tuple[1][call_count] @@ -276,7 +277,7 @@ class LLVMBisectionTest(unittest.TestCase): # Simulate behavior of `FindTryjobIndex()` when the index of the tryjob was # found. - @mock.patch.object(llvm_bisection, 'FindTryjobIndex', return_value=0) + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) def testTryjobExistsInRevisionsToLaunch(self, mock_find_tryjob_index): test_existing_jobs = [{'rev': 102, 'status': 'good'}] @@ -297,7 +298,7 @@ class LLVMBisectionTest(unittest.TestCase): mock_find_tryjob_index.assert_called_once() - @mock.patch.object(llvm_bisection, 'AddTryjob') + @mock.patch.object(modify_a_tryjob, 'AddTryjob') def testSuccessfullyUpdatedStatusFileWhenExceptionIsRaised( self, mock_add_tryjob): @@ -306,10 +307,10 @@ class LLVMBisectionTest(unittest.TestCase): # Simulate behavior of `AddTryjob()` when successfully launched a tryjob for # the updated packages. - @CallCountsToMockFunctions - def MockAddTryjob(call_count, packages, git_hash, revision, chroot_path, - patch_file, extra_cls, options, builder, verbose, - svn_revision): + @test_helpers.CallCountsToMockFunctions + def MockAddTryjob(call_count, _packages, _git_hash, _revision, _chroot_path, + _patch_file, _extra_cls, _options, _builder, _verbose, + _svn_revision): if call_count < 2: return {'rev': revisions_list[call_count], 'status': 'pending'} @@ -329,15 +330,15 @@ class LLVMBisectionTest(unittest.TestCase): bisection_contents = {'start': start, 'end': end, 'jobs': []} - args_output = ArgsOutputTest() + args_output = test_helpers.ArgsOutputTest() packages = ['sys-devel/llvm'] patch_file = '/abs/path/to/PATCHES.json' # Create a temporary .JSON file to simulate a status file for bisection. - with CreateTemporaryJsonFile() as temp_json_file: + with test_helpers.CreateTemporaryJsonFile() as temp_json_file: with open(temp_json_file, 'w') as f: - WritePrettyJsonFile(bisection_contents, f) + test_helpers.WritePrettyJsonFile(bisection_contents, f) # Verify that the status file is updated when an exception happened when # attempting to launch a revision (i.e. progress is not lost). @@ -392,7 +393,8 @@ class LLVMBisectionTest(unittest.TestCase): # Simulate behavior of `GetLLVMHash()` when successfully retrieved # the git hash of the bad revision. - @mock.patch.object(LLVMHash, 'GetLLVMHash', return_value='a123testhash5') + @mock.patch.object( + get_llvm_hash.LLVMHash, 'GetLLVMHash', return_value='a123testhash5') def testCompletedBisectionWhenNotProvidedSrcPath(self, mock_get_git_hash): last_tested = '/some/last/tested_file.json' @@ -412,9 +414,9 @@ class LLVMBisectionTest(unittest.TestCase): test_bisect_contents = {'start': start, 'end': end, 'jobs': []} # Simulate that the status file exists. - with CreateTemporaryJsonFile() as temp_json_file: + with test_helpers.CreateTemporaryJsonFile() as temp_json_file: with open(temp_json_file, 'w') as f: - WritePrettyJsonFile(test_bisect_contents, f) + test_helpers.WritePrettyJsonFile(test_bisect_contents, f) self.assertDictEqual( llvm_bisection.LoadStatusFile(temp_json_file, start, end), @@ -444,7 +446,7 @@ class LLVMBisectionTest(unittest.TestCase): @mock.patch.object(llvm_bisection, 'LoadStatusFile') # Simulate behavior of `VerifyOutsideChroot()` when successfully invoked the # script outside of the chroot. - @mock.patch.object(llvm_bisection, 'VerifyOutsideChroot', return_value=True) + @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) def testSuccessfullyBisectedLLVM( self, mock_outside_chroot, mock_load_status_file, mock_validate_start_and_end, mock_get_start_and_end_revision, @@ -483,7 +485,7 @@ class LLVMBisectionTest(unittest.TestCase): # end (in this case, none). mock_get_revision_and_hash_list.return_value = [], [] - args_output = ArgsOutputTest() + args_output = test_helpers.ArgsOutputTest() args_output.start_rev = start args_output.end_rev = end args_output.parallel = 3 @@ -517,7 +519,7 @@ class LLVMBisectionTest(unittest.TestCase): @mock.patch.object(llvm_bisection, 'LoadStatusFile') # Simulate behavior of `VerifyOutsideChroot()` when successfully invoked the # script outside of the chroot. - @mock.patch.object(llvm_bisection, 'VerifyOutsideChroot', return_value=True) + @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) def testNoMoreTryjobsToLaunch( self, mock_outside_chroot, mock_load_status_file, mock_validate_start_and_end, mock_get_start_and_end_revision, @@ -541,7 +543,7 @@ class LLVMBisectionTest(unittest.TestCase): no_revisions_error_message = ('No more tryjobs to launch between %d and ' '%d' % (start, end)) - def MockNoRevisionsErrorException(start, end, skip, pending): + def MockNoRevisionsErrorException(_start, _end, _skip, _pending): raise ValueError(no_revisions_error_message) # Simulate behavior of `LoadStatusFile()` when successfully loaded the @@ -567,7 +569,7 @@ class LLVMBisectionTest(unittest.TestCase): mock_die_with_no_revisions_error.side_effect = MockNoRevisionsErrorException # Simulate behavior of arguments passed into the command line. - args_output = ArgsOutputTest() + args_output = test_helpers.ArgsOutputTest() args_output.start_rev = start args_output.end_rev = end args_output.parallel = 3 diff --git a/llvm_tools/llvm_patch_management.py b/llvm_tools/llvm_patch_management.py index ef8b65c8..90f9a5c0 100755 --- a/llvm_tools/llvm_patch_management.py +++ b/llvm_tools/llvm_patch_management.py @@ -3,23 +3,21 @@ # Copyright 2019 The Chromium OS Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +# +# pylint: disable=global-statement """Creates the arguments for the patch manager for LLVM.""" from __future__ import print_function import argparse -import get_llvm_hash import os -import patch_manager -from assert_not_in_chroot import VerifyOutsideChroot from failure_modes import FailureModes -from get_llvm_hash import CreateTempLLVMRepo -from get_llvm_hash import GetGoogle3LLVMVersion -from get_llvm_hash import LLVMHash -from subprocess_helpers import ChrootRunCommand -from subprocess_helpers import ExecCommandAndCaptureOutput +import chroot +import get_llvm_hash +import patch_manager +import subprocess_helpers # If set to `True`, then the contents of `stdout` after executing a command will # be displayed to the terminal. @@ -123,7 +121,7 @@ def GetPathToFilesDirectory(chroot_path, package): raise ValueError('Invalid chroot provided: %s' % chroot_path) # Get the absolute chroot path to the ebuild. - chroot_ebuild_path = ChrootRunCommand( + chroot_ebuild_path = subprocess_helpers.ChrootRunCommand( chroot_path, ['equery', 'w', package], verbose=verbose) # Get the absolute chroot path to $FILESDIR's parent directory. @@ -180,7 +178,7 @@ def _MoveSrcTreeHEADToGitHash(src_path, git_hash): move_head_cmd = ['git', '-C', src_path, 'checkout', git_hash] - ExecCommandAndCaptureOutput(move_head_cmd, verbose=verbose) + subprocess_helpers.ExecCommandAndCaptureOutput(move_head_cmd, verbose=verbose) def UpdatePackagesPatchMetadataFile(chroot_path, svn_version, @@ -206,10 +204,10 @@ def UpdatePackagesPatchMetadataFile(chroot_path, svn_version, # that has information on the patches. package_info = {} - llvm_hash = LLVMHash() + llvm_hash = get_llvm_hash.LLVMHash() with llvm_hash.CreateTempDirectory() as temp_dir: - with CreateTempLLVMRepo(temp_dir) as src_path: + with get_llvm_hash.CreateTempLLVMRepo(temp_dir) as src_path: # Ensure that 'svn_version' exists in the chromiumum mirror of LLVM by # finding its corresponding git hash. git_hash = get_llvm_hash.GetGitHashFrom(src_path, svn_version) @@ -247,14 +245,14 @@ def main(): AssertionError: The script was run inside the chroot. """ - VerifyOutsideChroot() + chroot.VerifyOutsideChroot() args_output = GetCommandLineArgs() # Get the google3 LLVM version if a LLVM version was not provided. llvm_version = args_output.llvm_version if llvm_version in ('', 'google3', 'google3-unstable'): - llvm_version = GetGoogle3LLVMVersion( + llvm_version = get_llvm_hash.GetGoogle3LLVMVersion( stable=llvm_version != 'google3-unstable') UpdatePackagesPatchMetadataFile(args_output.chroot_path, llvm_version, diff --git a/llvm_tools/llvm_patch_management_unittest.py b/llvm_tools/llvm_patch_management_unittest.py index 75f003e9..968a816a 100755 --- a/llvm_tools/llvm_patch_management_unittest.py +++ b/llvm_tools/llvm_patch_management_unittest.py @@ -4,20 +4,21 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +# pylint: disable=protected-access + """Unit tests when creating the arguments for the patch manager.""" from __future__ import print_function from collections import namedtuple -from failure_modes import FailureModes - -import get_llvm_hash -import llvm_patch_management import os -import patch_manager -import subprocess import unittest import unittest.mock as mock +from failure_modes import FailureModes +import get_llvm_hash +import llvm_patch_management +import patch_manager +import subprocess_helpers class LlvmPatchManagementTest(unittest.TestCase): @@ -43,7 +44,7 @@ class LlvmPatchManagementTest(unittest.TestCase): # Simulate the behavior of 'os.path.isdir()' when a valid chroot path is # passed in. @mock.patch.object(os.path, 'isdir', return_value=True) - @mock.patch.object(llvm_patch_management, 'ChrootRunCommand') + @mock.patch.object(subprocess_helpers, 'ChrootRunCommand') @mock.patch.object(llvm_patch_management, '_GetRelativePathOfChrootPath') def testSuccessfullyGetPathToFilesDir( self, mock_get_relative_path_of_chroot_path, mock_chroot_cmd, mock_isdir): @@ -149,7 +150,7 @@ class LlvmPatchManagementTest(unittest.TestCase): get_llvm_hash, 'GetGitHashFrom', return_value='a123testhash1') # Simulate `CreateTempLLVMRepo()` when successfully created a work tree from # the LLVM repo copy in `llvm_tools` directory. - @mock.patch.object(llvm_patch_management, 'CreateTempLLVMRepo') + @mock.patch.object(get_llvm_hash, 'CreateTempLLVMRepo') # Simulate behavior of `_MoveSrcTreeHEADToGitHash()` when successfully moved # the head pointer to the git hash of the revision. @mock.patch.object(llvm_patch_management, '_MoveSrcTreeHEADToGitHash') @@ -222,7 +223,7 @@ class LlvmPatchManagementTest(unittest.TestCase): get_llvm_hash, 'GetGitHashFrom', return_value='a123testhash1') # Simulate `CreateTempLLVMRepo()` when successfully created a work tree from # the LLVM repo copy in `llvm_tools` directory. - @mock.patch.object(llvm_patch_management, 'CreateTempLLVMRepo') + @mock.patch.object(get_llvm_hash, 'CreateTempLLVMRepo') # Simulate behavior of `_MoveSrcTreeHEADToGitHash()` when successfully moved # the head pointer to the git hash of the revision. @mock.patch.object(llvm_patch_management, '_MoveSrcTreeHEADToGitHash') diff --git a/llvm_tools/modify_a_tryjob.py b/llvm_tools/modify_a_tryjob.py index 41337257..0d6b070f 100755 --- a/llvm_tools/modify_a_tryjob.py +++ b/llvm_tools/modify_a_tryjob.py @@ -14,12 +14,11 @@ import json import os import sys -from assert_not_in_chroot import VerifyOutsideChroot -from failure_modes import FailureModes -from get_llvm_hash import GetLLVMHashAndVersionFromSVNOption -from update_packages_and_run_tests import RunTryJobs -from update_tryjob_status import FindTryjobIndex -from update_tryjob_status import TryjobStatus +import chroot +import failure_modes +import get_llvm_hash +import update_packages_and_run_tests +import update_tryjob_status import update_chromeos_llvm_hash @@ -124,7 +123,7 @@ def GetCLAfterUpdatingPackages(packages, git_hash, svn_version, chroot_path, svn_version, chroot_path, patch_metadata_file, - FailureModes.DISABLE_PATCHES, + failure_modes.FailureModes.DISABLE_PATCHES, svn_option, extra_commit_msg=None) @@ -150,15 +149,15 @@ def CreateNewTryjobEntryForBisection(cl, extra_cls, options, builder, # 'builder' : [BUILDER_AS_A_LIST] # } # ] - tryjob_results = RunTryJobs(cl, extra_cls, options, [builder], chroot_path, - verbose) + tryjob_results = update_packages_and_run_tests.RunTryJobs( + cl, extra_cls, options, [builder], chroot_path, verbose) print('\nTryjob:') print(tryjob_results[0]) # Add necessary information about the tryjob. tryjob_results[0]['url'] = cl_url tryjob_results[0]['rev'] = revision - tryjob_results[0]['status'] = TryjobStatus.PENDING.value + tryjob_results[0]['status'] = update_tryjob_status.TryjobStatus.PENDING.value tryjob_results[0]['cl'] = cl return tryjob_results[0] @@ -215,7 +214,8 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, if not bisect_contents['jobs'] and modify_tryjob != ModifyTryjob.ADD: sys.exit('No tryjobs in %s' % status_file) - tryjob_index = FindTryjobIndex(revision, bisect_contents['jobs']) + tryjob_index = update_tryjob_status.FindTryjobIndex(revision, + bisect_contents['jobs']) # 'FindTryjobIndex()' returns None if the tryjob was not found. if tryjob_index is None and modify_tryjob != ModifyTryjob.ADD: @@ -229,13 +229,14 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, print('Successfully deleted the tryjob of revision %d' % revision) elif modify_tryjob == ModifyTryjob.RELAUNCH: # Need to update the tryjob link and buildbucket ID. - tryjob_results = RunTryJobs( + tryjob_results = update_packages_and_run_tests.RunTryJobs( bisect_contents['jobs'][tryjob_index]['cl'], bisect_contents['jobs'][tryjob_index]['extra_cls'], bisect_contents['jobs'][tryjob_index]['options'], bisect_contents['jobs'][tryjob_index]['builder'], chroot_path, verbose) - bisect_contents['jobs'][tryjob_index]['status'] = TryjobStatus.PENDING.value + bisect_contents['jobs'][tryjob_index][ + 'status'] = update_tryjob_status.TryjobStatus.PENDING.value bisect_contents['jobs'][tryjob_index]['link'] = tryjob_results[0]['link'] bisect_contents['jobs'][tryjob_index]['buildbucket_id'] = tryjob_results[0][ 'buildbucket_id'] @@ -258,7 +259,8 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, patch_metadata_file = 'PATCHES.json' - git_hash, revision = GetLLVMHashAndVersionFromSVNOption(revision) + git_hash, revision = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption( + revision) tryjob_dict = AddTryjob(update_packages, git_hash, revision, chroot_path, patch_metadata_file, extra_cls, options, builder, @@ -280,7 +282,7 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls, def main(): """Removes, relaunches, or adds a tryjob.""" - VerifyOutsideChroot() + chroot.VerifyOutsideChroot() args_output = GetCommandLineArgs() diff --git a/llvm_tools/modify_a_tryjob_unittest.py b/llvm_tools/modify_a_tryjob_unittest.py index 2b492f1d..e3c62972 100755 --- a/llvm_tools/modify_a_tryjob_unittest.py +++ b/llvm_tools/modify_a_tryjob_unittest.py @@ -12,10 +12,11 @@ import json import unittest import unittest.mock as mock -from test_helpers import ArgsOutputTest -from test_helpers import CreateTemporaryJsonFile -from test_helpers import WritePrettyJsonFile +import get_llvm_hash import modify_a_tryjob +import test_helpers +import update_packages_and_run_tests +import update_tryjob_status class ModifyATryjobTest(unittest.TestCase): @@ -26,13 +27,13 @@ class ModifyATryjobTest(unittest.TestCase): # Create a temporary .JSON file to simulate a .JSON file that has bisection # contents. - with CreateTemporaryJsonFile() as temp_json_file: + with test_helpers.CreateTemporaryJsonFile() as temp_json_file: with open(temp_json_file, 'w') as f: - WritePrettyJsonFile(bisect_test_contents, f) + test_helpers.WritePrettyJsonFile(bisect_test_contents, f) revision_to_modify = 369411 - args_output = ArgsOutputTest() + args_output = test_helpers.ArgsOutputTest() args_output.builders = None args_output.options = None @@ -48,7 +49,7 @@ class ModifyATryjobTest(unittest.TestCase): # Simulate the behavior of `FindTryjobIndex()` when the index of the tryjob # was not found. - @mock.patch.object(modify_a_tryjob, 'FindTryjobIndex', return_value=None) + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=None) def testNoTryjobIndexFound(self, mock_find_tryjob_index): bisect_test_contents = { 'start': 369410, @@ -62,13 +63,13 @@ class ModifyATryjobTest(unittest.TestCase): # Create a temporary .JSON file to simulate a .JSON file that has bisection # contents. - with CreateTemporaryJsonFile() as temp_json_file: + with test_helpers.CreateTemporaryJsonFile() as temp_json_file: with open(temp_json_file, 'w') as f: - WritePrettyJsonFile(bisect_test_contents, f) + test_helpers.WritePrettyJsonFile(bisect_test_contents, f) revision_to_modify = 369412 - args_output = ArgsOutputTest() + args_output = test_helpers.ArgsOutputTest() args_output.builders = None args_output.options = None @@ -88,7 +89,7 @@ class ModifyATryjobTest(unittest.TestCase): # Simulate the behavior of `FindTryjobIndex()` when the index of the tryjob # was found. - @mock.patch.object(modify_a_tryjob, 'FindTryjobIndex', return_value=0) + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) def testSuccessfullyRemovedTryjobInStatusFile(self, mock_find_tryjob_index): bisect_test_contents = { 'start': 369410, @@ -102,13 +103,13 @@ class ModifyATryjobTest(unittest.TestCase): # Create a temporary .JSON file to simulate a .JSON file that has bisection # contents. - with CreateTemporaryJsonFile() as temp_json_file: + with test_helpers.CreateTemporaryJsonFile() as temp_json_file: with open(temp_json_file, 'w') as f: - WritePrettyJsonFile(bisect_test_contents, f) + test_helpers.WritePrettyJsonFile(bisect_test_contents, f) revision_to_modify = 369414 - args_output = ArgsOutputTest() + args_output = test_helpers.ArgsOutputTest() args_output.builders = None args_output.options = None @@ -129,10 +130,10 @@ class ModifyATryjobTest(unittest.TestCase): # Simulate the behavior of `RunTryJobs()` when successfully submitted a # tryjob. - @mock.patch.object(modify_a_tryjob, 'RunTryJobs') + @mock.patch.object(update_packages_and_run_tests, 'RunTryJobs') # Simulate the behavior of `FindTryjobIndex()` when the index of the tryjob # was found. - @mock.patch.object(modify_a_tryjob, 'FindTryjobIndex', return_value=0) + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) def testSuccessfullyRelaunchedTryjob(self, mock_find_tryjob_index, mock_run_tryjob): @@ -162,13 +163,13 @@ class ModifyATryjobTest(unittest.TestCase): # Create a temporary .JSON file to simulate a .JSON file that has bisection # contents. - with CreateTemporaryJsonFile() as temp_json_file: + with test_helpers.CreateTemporaryJsonFile() as temp_json_file: with open(temp_json_file, 'w') as f: - WritePrettyJsonFile(bisect_test_contents, f) + test_helpers.WritePrettyJsonFile(bisect_test_contents, f) revision_to_modify = 369411 - args_output = ArgsOutputTest() + args_output = test_helpers.ArgsOutputTest() args_output.builders = None args_output.options = None @@ -207,7 +208,7 @@ class ModifyATryjobTest(unittest.TestCase): # Simulate the behavior of `FindTryjobIndex()` when the index of the tryjob # was found. - @mock.patch.object(modify_a_tryjob, 'FindTryjobIndex', return_value=0) + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) def testAddingTryjobThatAlreadyExists(self, mock_find_tryjob_index): bisect_test_contents = { 'start': 369410, @@ -221,16 +222,16 @@ class ModifyATryjobTest(unittest.TestCase): # Create a temporary .JSON file to simulate a .JSON file that has bisection # contents. - with CreateTemporaryJsonFile() as temp_json_file: + with test_helpers.CreateTemporaryJsonFile() as temp_json_file: with open(temp_json_file, 'w') as f: - WritePrettyJsonFile(bisect_test_contents, f) + test_helpers.WritePrettyJsonFile(bisect_test_contents, f) revision_to_add = 369411 # Index of the tryjob in 'jobs' list. tryjob_index = 0 - args_output = ArgsOutputTest() + args_output = test_helpers.ArgsOutputTest() args_output.options = None # Verify the exception is raised when the tryjob that is going to added @@ -248,7 +249,7 @@ class ModifyATryjobTest(unittest.TestCase): mock_find_tryjob_index.assert_called_once() # Simulate the behavior of `FindTryjobIndex()` when the tryjob was not found. - @mock.patch.object(modify_a_tryjob, 'FindTryjobIndex', return_value=None) + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=None) def testSuccessfullyDidNotAddTryjobOutsideOfBisectionBounds( self, mock_find_tryjob_index): @@ -263,14 +264,14 @@ class ModifyATryjobTest(unittest.TestCase): # Create a temporary .JSON file to simulate a .JSON file that has bisection # contents. - with CreateTemporaryJsonFile() as temp_json_file: + with test_helpers.CreateTemporaryJsonFile() as temp_json_file: with open(temp_json_file, 'w') as f: - WritePrettyJsonFile(bisect_test_contents, f) + test_helpers.WritePrettyJsonFile(bisect_test_contents, f) # Add a revision that is outside of 'start' and 'end'. revision_to_add = 369450 - args_output = ArgsOutputTest() + args_output = test_helpers.ArgsOutputTest() args_output.options = None # Verify the exception is raised when adding a tryjob that does not exist @@ -292,11 +293,11 @@ class ModifyATryjobTest(unittest.TestCase): # Simulate the behavior of `GetLLVMHashAndVersionFromSVNOption()` when # successfully retrieved the git hash of the revision to launch a tryjob for. @mock.patch.object( - modify_a_tryjob, + get_llvm_hash, 'GetLLVMHashAndVersionFromSVNOption', return_value=('a123testhash1', 369418)) # Simulate the behavior of `FindTryjobIndex()` when the tryjob was not found. - @mock.patch.object(modify_a_tryjob, 'FindTryjobIndex', return_value=None) + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=None) def testSuccessfullyAddedTryjob(self, mock_find_tryjob_index, mock_get_llvm_hash, mock_add_tryjob): @@ -311,14 +312,14 @@ class ModifyATryjobTest(unittest.TestCase): # Create a temporary .JSON file to simulate a .JSON file that has bisection # contents. - with CreateTemporaryJsonFile() as temp_json_file: + with test_helpers.CreateTemporaryJsonFile() as temp_json_file: with open(temp_json_file, 'w') as f: - WritePrettyJsonFile(bisect_test_contents, f) + test_helpers.WritePrettyJsonFile(bisect_test_contents, f) # Add a revision that is outside of 'start' and 'end'. revision_to_add = 369418 - args_output = ArgsOutputTest() + args_output = test_helpers.ArgsOutputTest() args_output.options = None new_tryjob_info = { @@ -358,7 +359,7 @@ class ModifyATryjobTest(unittest.TestCase): mock_add_tryjob.assert_called_once() # Simulate the behavior of `FindTryjobIndex()` when the tryjob was found. - @mock.patch.object(modify_a_tryjob, 'FindTryjobIndex', return_value=0) + @mock.patch.object(update_tryjob_status, 'FindTryjobIndex', return_value=0) def testModifyATryjobOptionDoesNotExist(self, mock_find_tryjob_index): bisect_test_contents = { 'start': 369410, @@ -371,14 +372,14 @@ class ModifyATryjobTest(unittest.TestCase): # Create a temporary .JSON file to simulate a .JSON file that has bisection # contents. - with CreateTemporaryJsonFile() as temp_json_file: + with test_helpers.CreateTemporaryJsonFile() as temp_json_file: with open(temp_json_file, 'w') as f: - WritePrettyJsonFile(bisect_test_contents, f) + test_helpers.WritePrettyJsonFile(bisect_test_contents, f) # Add a revision that is outside of 'start' and 'end'. revision_to_modify = 369414 - args_output = ArgsOutputTest() + args_output = test_helpers.ArgsOutputTest() args_output.builders = None args_output.options = None diff --git a/llvm_tools/update_all_tryjobs_with_auto.py b/llvm_tools/update_all_tryjobs_with_auto.py index 511bfffa..11e67ed4 100755 --- a/llvm_tools/update_all_tryjobs_with_auto.py +++ b/llvm_tools/update_all_tryjobs_with_auto.py @@ -12,9 +12,8 @@ import argparse import json import os -from assert_not_in_chroot import VerifyOutsideChroot -from update_tryjob_status import GetAutoResult -from update_tryjob_status import TryjobStatus +import chroot +import update_tryjob_status def GetPathToUpdateAllTryjobsWithAutoScript(): @@ -59,7 +58,7 @@ def GetCommandLineArgs(): def main(): """Updates the status of a tryjob.""" - VerifyOutsideChroot() + chroot.VerifyOutsideChroot() args_output = GetCommandLineArgs() @@ -67,9 +66,9 @@ def main(): bisect_contents = json.load(tryjobs) for tryjob in bisect_contents['jobs']: - if tryjob['status'] == TryjobStatus.PENDING.value: - tryjob['status'] = GetAutoResult(args_output.chroot_path, - tryjob['buildbucket_id']) + if tryjob['status'] == update_tryjob_status.TryjobStatus.PENDING.value: + tryjob['status'] = update_tryjob_status.GetAutoResult( + args_output.chroot_path, tryjob['buildbucket_id']) new_file = '%s.new' % args_output.last_tested with open(new_file, 'w') as update_tryjobs: diff --git a/llvm_tools/update_chromeos_llvm_hash.py b/llvm_tools/update_chromeos_llvm_hash.py index 4f6e9936..f8cb04e4 100755 --- a/llvm_tools/update_chromeos_llvm_hash.py +++ b/llvm_tools/update_chromeos_llvm_hash.py @@ -13,21 +13,18 @@ for review. """ from __future__ import print_function -from collections import namedtuple from datetime import datetime from enum import Enum import argparse import os import re -import subprocess -import tempfile -from assert_not_in_chroot import VerifyOutsideChroot from failure_modes import FailureModes -from subprocess_helpers import ChrootRunCommand, ExecCommandAndCaptureOutput -from get_llvm_hash import GetLLVMHashAndVersionFromSVNOption, is_svn_option +from subprocess_helpers import ExecCommandAndCaptureOutput +import chroot import get_llvm_hash +import git import llvm_patch_management @@ -43,8 +40,6 @@ class LLVMVariant(Enum): # be displayed to the terminal. verbose = False -CommitContents = namedtuple('CommitContents', ['url', 'cl_number']) - def GetCommandLineArgs(): """Parses the command line for the optional command line arguments. @@ -96,7 +91,7 @@ def GetCommandLineArgs(): # Add argument for the LLVM version to use. parser.add_argument( '--llvm_version', - type=is_svn_option, + type=get_llvm_hash.is_svn_option, required=True, help='which git hash to use. Either a svn revision, or one ' 'of %s' % sorted(get_llvm_hash.KNOWN_HASH_SOURCES)) @@ -131,76 +126,6 @@ def GetCommandLineArgs(): return args_output -def GetChrootBuildPaths(chromeos_root, package_list): - """Gets the chroot path(s) of the package(s). - - Args: - chromeos_root: The absolute path to the chroot to - use for executing chroot commands. - package_list: A list of a package/packages to - be used to find their chroot path. - - Returns: - A list of a chroot path/chroot paths of the package's ebuild file. - - Raises: - ValueError: Failed to get the chroot path of a package. - """ - - chroot_paths = [] - - # Find the chroot path for each package's ebuild. - for cur_package in sorted(set(package_list)): - # Cmd to find the chroot path for the package. - equery_cmd = ['equery', 'w', cur_package] - - chroot_path = ChrootRunCommand(chromeos_root, equery_cmd, verbose=verbose) - - chroot_paths.append(chroot_path.strip()) - - return chroot_paths - - -def _ConvertChrootPathsToSymLinkPaths(chromeos_root, chroot_file_paths): - """Converts the chroot path(s) to absolute symlink path(s). - - Args: - chromeos_root: The absolute path to the chroot. - chroot_file_paths: A list of a chroot path/chroot paths to convert to - a absolute symlink path/symlink paths. - - Returns: - A list of absolute path(s) which are symlinks that point to - the ebuild of the package(s). - - Raises: - ValueError: Invalid prefix for the chroot path or - invalid chroot path(s) were provided. - """ - - symlink_file_paths = [] - - chroot_prefix = '/mnt/host/source/' - - # Iterate through the chroot paths. - # - # For each chroot file path, remove '/mnt/host/source/' prefix - # and combine the chroot path with the result and add it to the list. - for cur_chroot_file_path in chroot_file_paths: - if not cur_chroot_file_path.startswith(chroot_prefix): - raise ValueError( - 'Invalid prefix for the chroot path: %s' % cur_chroot_file_path) - - rel_path = cur_chroot_file_path[len(chroot_prefix):] - - # combine the chromeos root path + '/src/...' - absolute_symlink_path = os.path.join(chromeos_root, rel_path) - - symlink_file_paths.append(absolute_symlink_path) - - return symlink_file_paths - - def GetEbuildPathsFromSymLinkPaths(symlinks): """Reads the symlink(s) to get the ebuild path(s) to the package(s). @@ -405,154 +330,6 @@ def UprevEbuildToVersion(symlink, svn_version): ExecCommandAndCaptureOutput(cmd, verbose=verbose) -def _CreateRepo(path_to_repo_dir, branch): - """Creates a temporary repo for the changes. - - Args: - path_to_repo_dir: The absolute path to the repo. - branch: The name of the branch to create. - llvm_variant: The LLVM hash to update. - git_hash: The new git hash. - - Raises: - ValueError: Failed to create a repo in that directory. - """ - - if not os.path.isdir(path_to_repo_dir): - raise ValueError('Invalid directory path provided: %s' % path_to_repo_dir) - - reset_changes_cmd = [ - 'git', - '-C', - path_to_repo_dir, - 'reset', - 'HEAD', - '--hard', - ] - - ExecCommandAndCaptureOutput(reset_changes_cmd, verbose=verbose) - - create_repo_cmd = ['repo', 'start', branch] - - ExecCommandAndCaptureOutput( - create_repo_cmd, cwd=path_to_repo_dir, verbose=verbose) - - -def _DeleteRepo(path_to_repo_dir, branch): - """Deletes the temporary repo. - - Args: - path_to_repo_dir: The absolute path of the repo. - branch: The name of the branch to delete. - - Raises: - ValueError: Failed to delete the repo in that directory. - """ - - if not os.path.isdir(path_to_repo_dir): - raise ValueError('Invalid directory path provided: %s' % path_to_repo_dir) - - checkout_to_master_cmd = [ - 'git', '-C', path_to_repo_dir, 'checkout', 'cros/master' - ] - - ExecCommandAndCaptureOutput(checkout_to_master_cmd, verbose=verbose) - - reset_head_cmd = ['git', '-C', path_to_repo_dir, 'reset', 'HEAD', '--hard'] - - ExecCommandAndCaptureOutput(reset_head_cmd, verbose=verbose) - - delete_repo_cmd = ['git', '-C', path_to_repo_dir, 'branch', '-D', branch] - - ExecCommandAndCaptureOutput(delete_repo_cmd, verbose=verbose) - - -def GetGerritRepoUploadContents(repo_upload_contents): - """Parses 'repo upload' to get the Gerrit commit URL and CL number. - - Args: - repo_upload_contents: The contents of the 'repo upload' command. - - Returns: - A nametuple that has two (key, value) pairs, where the first pair is the - Gerrit commit URL and the second pair is the change list number. - - Raises: - ValueError: The contents of the 'repo upload' command did not contain a - Gerrit commit URL. - """ - - found_url = re.search( - r'https://chromium-review.googlesource.com/c/' - r'chromiumos/overlays/chromiumos-overlay/\+/([0-9]+)', - repo_upload_contents) - - if not found_url: - raise ValueError('Failed to find change list URL.') - - cl_number = int(found_url.group(1)) - - return CommitContents(url=found_url.group(0), cl_number=cl_number) - - -def UploadChanges(path_to_repo_dir, branch, commit_messages): - """Uploads the changes (updating LLVM next hash and uprev symlink) for review. - - Args: - path_to_repo_dir: The absolute path to the repo where changes were made. - branch: The name of the branch to upload. - commit_messages: A string of commit message(s) (i.e. '[message]' - of the changes made. - - Returns: - A nametuple that has two (key, value) pairs, where the first pair is the - Gerrit commit URL and the second pair is the change list number. - - Raises: - ValueError: Failed to create a commit or failed to upload the - changes for review. - """ - - if not os.path.isdir(path_to_repo_dir): - raise ValueError('Invalid directory path provided: %s' % path_to_repo_dir) - - # Create a git commit. - with tempfile.NamedTemporaryFile(mode='w+t') as commit_msg_file: - commit_msg_file.write('\n'.join(commit_messages)) - commit_msg_file.flush() - - commit_cmd = ['git', 'commit', '-F', commit_msg_file.name] - - ExecCommandAndCaptureOutput( - commit_cmd, cwd=path_to_repo_dir, verbose=verbose) - - # Upload the changes for review. - # Use --ne to avoid sending email notifications. - upload_change_cmd = [ - 'repo', 'upload', '--yes', '--ne', '--no-verify', - '--br=%s' % branch - ] - - # Pylint currently doesn't lint things in py3 mode, and py2 didn't allow - # users to specify `encoding`s for Popen. Hence, pylint is "wrong" here. - # pylint: disable=unexpected-keyword-arg - # The CL URL is sent to 'stderr', so need to redirect 'stderr' to 'stdout'. - upload_changes_obj = subprocess.Popen( - upload_change_cmd, - cwd=path_to_repo_dir, - encoding='UTF-8', - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - - out, _ = upload_changes_obj.communicate() - - if upload_changes_obj.returncode: # Failed to upload changes. - print(out) - raise ValueError('Failed to upload changes for review') - - return GetGerritRepoUploadContents(out.rstrip()) - - def CreatePathDictionaryFromPackages(chroot_path, update_packages): """Creates a symlink and ebuild path pair dictionary from the packages. @@ -567,10 +344,10 @@ def CreatePathDictionaryFromPackages(chroot_path, update_packages): """ # Construct a list containing the chroot file paths of the package(s). - chroot_file_paths = GetChrootBuildPaths(chroot_path, update_packages) + chroot_file_paths = chroot.GetChrootEbuildPaths(chroot_path, update_packages) # Construct a list containing the symlink(s) of the package(s). - symlink_file_paths = _ConvertChrootPathsToSymLinkPaths( + symlink_file_paths = chroot.ConvertChrootPathsToAbsolutePaths( chroot_path, chroot_file_paths) # Create a dictionary where the key is the absolute path of the symlink to @@ -708,7 +485,7 @@ def UpdatePackages(packages, llvm_variant, git_hash, svn_version, chroot_path, branch = 'update-' + llvm_variant.value + '-' + git_hash - _CreateRepo(repo_path, branch) + git.CreateBranch(repo_path, branch) try: commit_message_header = 'llvm' @@ -762,10 +539,10 @@ def UpdatePackages(packages, llvm_variant, git_hash, svn_version, chroot_path, if extra_commit_msg: commit_messages.append(extra_commit_msg) - change_list = UploadChanges(repo_path, branch, commit_messages) + change_list = git.UploadChanges(repo_path, branch, commit_messages) finally: - _DeleteRepo(repo_path, branch) + git.DeleteBranch(repo_path, branch) return change_list @@ -777,7 +554,7 @@ def main(): AssertionError: The script was run inside the chroot. """ - VerifyOutsideChroot() + chroot.VerifyOutsideChroot() args_output = GetCommandLineArgs() @@ -787,7 +564,8 @@ def main(): git_hash_source = args_output.llvm_version - git_hash, svn_version = GetLLVMHashAndVersionFromSVNOption(git_hash_source) + git_hash, svn_version = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption( + git_hash_source) change_list = UpdatePackages( args_output.update_packages, diff --git a/llvm_tools/update_chromeos_llvm_hash_unittest.py b/llvm_tools/update_chromeos_llvm_hash_unittest.py index 2ea483b6..5f86c2b0 100755 --- a/llvm_tools/update_chromeos_llvm_hash_unittest.py +++ b/llvm_tools/update_chromeos_llvm_hash_unittest.py @@ -12,13 +12,14 @@ from collections import namedtuple from datetime import datetime import os import re -import subprocess import unittest import unittest.mock as mock -from failure_modes import FailureModes -from test_helpers import CreateTemporaryJsonFile +import chroot +import failure_modes +import git import llvm_patch_management +import test_helpers import update_chromeos_llvm_hash # These are unittests; protected access is OK to a point. @@ -28,81 +29,6 @@ import update_chromeos_llvm_hash class UpdateLLVMHashTest(unittest.TestCase): """Test class for updating LLVM hashes of packages.""" - @mock.patch.object(update_chromeos_llvm_hash, 'ChrootRunCommand') - def testSucceedsToGetChrootPathForPackage(self, mock_chroot_command): - package_chroot_path = '/chroot/path/to/package.ebuild' - - # Emulate ChrootRunCommandWOutput behavior when a chroot path is found for - # a valid package. - mock_chroot_command.return_value = package_chroot_path - - chroot_path = '/test/chroot/path' - package_list = ['new-test/package'] - - self.assertEqual( - update_chromeos_llvm_hash.GetChrootBuildPaths( - chroot_path, package_list), [package_chroot_path]) - - mock_chroot_command.assert_called_once() - - def testFailedToConvertChrootPathWithInvalidPrefixToSymlinkPath(self): - chroot_path = '/path/to/chroot' - chroot_file_path = '/src/package.ebuild' - - # Verify the exception is raised when a symlink does not have the prefix - # '/mnt/host/source/'. - with self.assertRaises(ValueError) as err: - update_chromeos_llvm_hash._ConvertChrootPathsToSymLinkPaths( - chroot_path, [chroot_file_path]) - - self.assertEqual( - str(err.exception), 'Invalid prefix for the chroot path: ' - '%s' % chroot_file_path) - - def testSucceedsToConvertChrootPathToSymlinkPath(self): - chroot_path = '/path/to/chroot' - chroot_file_paths = ['/mnt/host/source/src/package.ebuild'] - - expected_symlink_path = '/path/to/chroot/src/package.ebuild' - - self.assertEqual( - update_chromeos_llvm_hash._ConvertChrootPathsToSymLinkPaths( - chroot_path, chroot_file_paths), [expected_symlink_path]) - - # Simulate 'os.path.islink' when a path is not a symbolic link. - @mock.patch.object(os.path, 'islink', return_value=False) - def testFailedToGetEbuildPathFromInvalidSymlink(self, mock_islink): - symlink_path = '/symlink/path/src/to/package-r1.ebuild' - - # Verify the exception is raised when the argument is not a symbolic link. - with self.assertRaises(ValueError) as err: - update_chromeos_llvm_hash.GetEbuildPathsFromSymLinkPaths([symlink_path]) - - self.assertEqual( - str(err.exception), 'Invalid symlink provided: %s' % symlink_path) - - mock_islink.assert_called_once_with(symlink_path) - - # Simulate 'os.path.islink' when a path is a symbolic link. - @mock.patch.object(os.path, 'islink', return_value=True) - @mock.patch.object(os.path, 'realpath') - def testSucceedsToGetEbuildPathFromValidSymlink(self, mock_realpath, - mock_islink): - - symlink_path = '/path/to/chroot/src/package-r1.ebuild' - abs_path_to_package = '/abs/path/to/src/package.ebuild' - mock_realpath.return_value = abs_path_to_package - - expected_resolved_paths = {symlink_path: abs_path_to_package} - - self.assertEqual( - update_chromeos_llvm_hash.GetEbuildPathsFromSymLinkPaths( - [symlink_path]), expected_resolved_paths) - - mock_realpath.assert_called_once_with(symlink_path) - - mock_islink.assert_called_once_with(symlink_path) - # Simulate behavior of 'os.path.isfile()' when the ebuild path to a package # does not exist. @mock.patch.object(os.path, 'isfile', return_value=False) @@ -126,7 +52,7 @@ class UpdateLLVMHashTest(unittest.TestCase): @mock.patch.object(os.path, 'isfile', return_value=True) def testFailedToUpdateLLVMHash(self, mock_isfile): # Create a temporary file to simulate an ebuild file of a package. - with CreateTemporaryJsonFile() as ebuild_file: + with test_helpers.CreateTemporaryJsonFile() as ebuild_file: with open(ebuild_file, 'w') as f: f.write('\n'.join([ 'First line in the ebuild', 'Second line in the ebuild', @@ -154,7 +80,7 @@ class UpdateLLVMHashTest(unittest.TestCase): @mock.patch.object(os.path, 'isfile', return_value=True) def testFailedToUpdateLLVMNextHash(self, mock_isfile): # Create a temporary file to simulate an ebuild file of a package. - with CreateTemporaryJsonFile() as ebuild_file: + with test_helpers.CreateTemporaryJsonFile() as ebuild_file: with open(ebuild_file, 'w') as f: f.write('\n'.join([ 'First line in the ebuild', 'Second line in the ebuild', @@ -188,7 +114,7 @@ class UpdateLLVMHashTest(unittest.TestCase): self, mock_stage_commit_command, mock_isfile): # Create a temporary file to simulate an ebuild file of a package. - with CreateTemporaryJsonFile() as ebuild_file: + with test_helpers.CreateTemporaryJsonFile() as ebuild_file: # Updates LLVM_HASH to 'git_hash' and revision to # 'svn_version'. llvm_variant = update_chromeos_llvm_hash.LLVMVariant.current @@ -231,7 +157,7 @@ class UpdateLLVMHashTest(unittest.TestCase): self, mock_stage_commit_command, mock_isfile): # Create a temporary file to simulate an ebuild file of a package. - with CreateTemporaryJsonFile() as ebuild_file: + with test_helpers.CreateTemporaryJsonFile() as ebuild_file: # Updates LLVM_NEXT_HASH to 'git_hash' and revision to # 'svn_version'. llvm_variant = update_chromeos_llvm_hash.LLVMVariant.next @@ -435,213 +361,8 @@ class UpdateLLVMHashTest(unittest.TestCase): # directory. - @mock.patch.object(os.path, 'isdir', return_value=False) - def testFailedToCreateRepoForInvalidDirectoryPath(self, mock_isdir): - path_to_repo = '/path/to/repo' - branch = 'update-LLVM_NEXT_HASH-a123testhash1' - - # Verify the exception is raised when provided an invalid directory path. - with self.assertRaises(ValueError) as err: - update_chromeos_llvm_hash._CreateRepo(path_to_repo, branch) - - self.assertEqual( - str(err.exception), - 'Invalid directory path provided: %s' % path_to_repo) - - mock_isdir.assert_called_once() - - # Simulate 'os.path.isdir' when a valid repo path is provided. - @mock.patch.object(os.path, 'isdir', return_value=True) - # Simulate behavior of 'ExecCommandAndCaptureOutput()' when successfully reset - # changes and created a repo. - @mock.patch.object( - update_chromeos_llvm_hash, - 'ExecCommandAndCaptureOutput', - return_value=None) - def testSuccessfullyCreatedRepo(self, mock_command_output, mock_isdir): - path_to_repo = '/path/to/repo' - branch = 'update-LLVM_NEXT_HASH-a123testhash1' - - update_chromeos_llvm_hash._CreateRepo(path_to_repo, branch) - - mock_isdir.assert_called_once_with(path_to_repo) - - self.assertEqual(mock_command_output.call_count, 2) - - # Simulate behavior of 'os.path.isdir()' when the path to the repo is not a - # directory. - @mock.patch.object(os.path, 'isdir', return_value=False) - def testFailedToDeleteRepoForInvalidDirectoryPath(self, mock_isdir): - path_to_repo = '/some/path/to/repo' - branch = 'update-LLVM_NEXT_HASH-a123testhash2' - - # Verify the exception is raised on an invalid repo path. - with self.assertRaises(ValueError) as err: - update_chromeos_llvm_hash._DeleteRepo(path_to_repo, branch) - - self.assertEqual( - str(err.exception), - 'Invalid directory path provided: %s' % path_to_repo) - - mock_isdir.assert_called_once() - - # Simulate 'os.path.isdir' on valid directory path. - @mock.patch.object(os.path, 'isdir', return_value=True) - # Simulate 'ExecCommandAndCaptureOutput()' when successfully checkout to - # cros/master, reset changes, and deleted the repo. - @mock.patch.object( - update_chromeos_llvm_hash, - 'ExecCommandAndCaptureOutput', - return_value=None) - def testSuccessfullyDeletedRepo(self, mock_command_output, mock_isdir): - path_to_repo = '/some/path/to/repo' - branch = 'update-LLVM_NEXT_HASH-a123testhash2' - - update_chromeos_llvm_hash._DeleteRepo(path_to_repo, branch) - - mock_isdir.assert_called_once_with(path_to_repo) - - self.assertEqual(mock_command_output.call_count, 3) - - def testFailedToFindChangeListURL(self): - repo_upload_contents = 'remote: https://some_url' - - # Verify the exception is raised when failed to find the Gerrit URL when - # parsing the 'repo upload' contents. - with self.assertRaises(ValueError) as err: - update_chromeos_llvm_hash.GetGerritRepoUploadContents( - repo_upload_contents) - - self.assertEqual(str(err.exception), 'Failed to find change list URL.') - - def testSuccessfullyGetGerritRepoUploadContents(self): - repo_upload_contents = ('remote: https://chromium-review.googlesource.com' - '/c/chromiumos/overlays/chromiumos-overlay/+/' - '193147 Some commit header') - - change_list = update_chromeos_llvm_hash.GetGerritRepoUploadContents( - repo_upload_contents) - - self.assertEqual( - change_list.url, - 'https://chromium-review.googlesource.com/c/chromiumos/overlays/' - 'chromiumos-overlay/+/193147') - - self.assertEqual(change_list.cl_number, 193147) - - # Simulate behavior of 'os.path.isdir()' when the path to the repo is not a - # directory. - @mock.patch.object(os.path, 'isdir', return_value=False) - def testFailedToUploadChangesForInvalidPathDirectory(self, mock_isdir): - path_to_repo = '/some/path/to/repo' - branch = 'update-LLVM_NEXT_HASH-a123testhash3' - commit_messages = ['Test message'] - - # Verify exception is raised when on an invalid repo path. - with self.assertRaises(ValueError) as err: - update_chromeos_llvm_hash.UploadChanges(path_to_repo, branch, - commit_messages) - - self.assertEqual( - str(err.exception), - 'Invalid directory path provided: %s' % path_to_repo) - - mock_isdir.assert_called_once() - - # Simulate 'os.path.isdir' on a valid repo path. - @mock.patch.object(os.path, 'isdir', return_value=True) - # Simulate behavior of 'ExecCommandAndCaptureOutput()' when successfully - # committed the changes. - @mock.patch.object( - update_chromeos_llvm_hash, - 'ExecCommandAndCaptureOutput', - return_value=None) - @mock.patch.object(subprocess, 'Popen') - def testFailedToUploadChangesForReview(self, mock_repo_upload, - mock_command_output, mock_isdir): - - # Simulate the behavior of 'subprocess.Popen()' when uploading the changes - # for review - # - # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. - mock_repo_upload.return_value.communicate.return_value = ( - None, 'Branch does not exist.') - - # Exit code of 1 means failed to upload changes for review. - mock_repo_upload.return_value.returncode = 1 - - path_to_repo = '/some/path/to/repo' - branch = 'update-LLVM_NEXT_HASH-a123testhash3' - commit_messages = ['Test message'] - - # Verify exception is raised when failed to upload the changes for review. - with self.assertRaises(ValueError) as err: - update_chromeos_llvm_hash.UploadChanges(path_to_repo, branch, - commit_messages) - - self.assertEqual(str(err.exception), 'Failed to upload changes for review') - - mock_isdir.assert_called_once_with(path_to_repo) - - mock_command_output.assert_called_once() - mock_command_output_args = mock_command_output.call_args_list[0][0][0] - expected_mock_command_output_prefix = ['git', 'commit', '-F'] - self.assertEqual( - mock_command_output_args[:len(expected_mock_command_output_prefix)], - expected_mock_command_output_prefix) - - mock_repo_upload.assert_called_once() - - # Simulate 'os.path.isdir' when a valid repo path is passed in. - @mock.patch.object(os.path, 'isdir', return_value=True) - # Simulate behavior of 'ExecCommandAndCaptureOutput()' when successfully - # committed the changes. - @mock.patch.object( - update_chromeos_llvm_hash, - 'ExecCommandAndCaptureOutput', - return_value=None) - @mock.patch.object(subprocess, 'Popen') - def testSuccessfullyUploadedChangesForReview(self, mock_repo_upload, - mock_command_output, mock_isdir): - - # A test CL generated by `repo upload`. - repo_upload_contents = ('remote: https://chromium-review.googlesource.' - 'com/c/chromiumos/overlays/chromiumos-overlay/' - '+/193147 Fix stdout') - - # Simulate the behavior of 'subprocess.Popen()' when uploading the changes - # for review - # - # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. - mock_repo_upload.return_value.communicate.return_value = ( - repo_upload_contents, None) - - # Exit code of 0 means successfully uploaded changes for review. - mock_repo_upload.return_value.returncode = 0 - - path_to_repo = '/some/path/to/repo' - branch = 'update-LLVM_NEXT_HASH-a123testhash3' - commit_messages = ['Test message'] - - change_list = update_chromeos_llvm_hash.UploadChanges( - path_to_repo, branch, commit_messages) - - self.assertEqual( - change_list.url, - 'https://chromium-review.googlesource.com/c/chromiumos/overlays/' - 'chromiumos-overlay/+/193147') - - self.assertEqual(change_list.cl_number, 193147) - - mock_isdir.assert_called_once_with(path_to_repo) - - mock_command_output.assert_called_once() - - mock_repo_upload.assert_called_once() - - @mock.patch.object(update_chromeos_llvm_hash, 'GetChrootBuildPaths') - @mock.patch.object(update_chromeos_llvm_hash, - '_ConvertChrootPathsToSymLinkPaths') + @mock.patch.object(chroot, 'GetChrootEbuildPaths') + @mock.patch.object(chroot, 'ConvertChrootPathsToAbsolutePaths') def testExceptionRaisedWhenCreatingPathDictionaryFromPackages( self, mock_chroot_paths_to_symlinks, mock_get_chroot_paths): @@ -650,18 +371,18 @@ class UpdateLLVMHashTest(unittest.TestCase): package_name = 'test-pckg/package' package_chroot_path = '/some/chroot/path/to/package-r1.ebuild' - # Test function to simulate '_ConvertChrootPathsToSymLinkPaths' when a + # Test function to simulate 'ConvertChrootPathsToAbsolutePaths' when a # symlink does not start with the prefix '/mnt/host/source'. def BadPrefixChrootPath(_chroot_path, _chroot_file_paths): raise ValueError('Invalid prefix for the chroot path: ' '%s' % package_chroot_path) - # Simulate 'GetChrootBuildPaths' when valid packages are passed in. + # Simulate 'GetChrootEbuildPaths' when valid packages are passed in. # # Returns a list of chroot paths. mock_get_chroot_paths.return_value = [package_chroot_path] - # Use test function to simulate '_ConvertChrootPathsToSymLinkPaths' + # Use test function to simulate 'ConvertChrootPathsToAbsolutePaths' # behavior. mock_chroot_paths_to_symlinks.side_effect = BadPrefixChrootPath @@ -679,9 +400,8 @@ class UpdateLLVMHashTest(unittest.TestCase): mock_chroot_paths_to_symlinks.assert_called_once_with( chroot_path, [package_chroot_path]) - @mock.patch.object(update_chromeos_llvm_hash, 'GetChrootBuildPaths') - @mock.patch.object(update_chromeos_llvm_hash, - '_ConvertChrootPathsToSymLinkPaths') + @mock.patch.object(chroot, 'GetChrootEbuildPaths') + @mock.patch.object(chroot, 'ConvertChrootPathsToAbsolutePaths') @mock.patch.object(update_chromeos_llvm_hash, 'GetEbuildPathsFromSymLinkPaths') def testSuccessfullyCreatedPathDictionaryFromPackages( @@ -690,7 +410,7 @@ class UpdateLLVMHashTest(unittest.TestCase): package_chroot_path = '/mnt/host/source/src/path/to/package-r1.ebuild' - # Simulate 'GetChrootBuildPaths' when returning a chroot path for a valid + # Simulate 'GetChrootEbuildPaths' when returning a chroot path for a valid # package. # # Returns a list of chroot paths. @@ -698,7 +418,7 @@ class UpdateLLVMHashTest(unittest.TestCase): package_symlink_path = '/some/path/to/chroot/src/path/to/package-r1.ebuild' - # Simulate '_ConvertChrootPathsToSymLinkPaths' when returning a symlink to + # Simulate 'ConvertChrootPathsToAbsolutePaths' when returning a symlink to # a chroot path that points to a package. # # Returns a list of symlink file paths. @@ -868,11 +588,11 @@ class UpdateLLVMHashTest(unittest.TestCase): @mock.patch.object(update_chromeos_llvm_hash, 'CreatePathDictionaryFromPackages') - @mock.patch.object(update_chromeos_llvm_hash, '_CreateRepo') + @mock.patch.object(git, 'CreateBranch') @mock.patch.object(update_chromeos_llvm_hash, 'UpdateEbuildLLVMHash') @mock.patch.object(update_chromeos_llvm_hash, 'UprevEbuildSymlink') - @mock.patch.object(update_chromeos_llvm_hash, 'UploadChanges') - @mock.patch.object(update_chromeos_llvm_hash, '_DeleteRepo') + @mock.patch.object(git, 'UploadChanges') + @mock.patch.object(git, 'DeleteBranch') @mock.patch.object(os.path, 'realpath') def testExceptionRaisedWhenUpdatingPackages( self, mock_realpath, mock_delete_repo, mock_upload_changes, @@ -886,9 +606,9 @@ class UpdateLLVMHashTest(unittest.TestCase): path_to_package_dir = '/some/path/to/chroot/src/path/to' - # Test function to simulate '_CreateRepo' when successfully created the - # repo on a valid repo path. - def SuccessfullyCreateRepoForChanges(_repo_path, branch): + # Test function to simulate 'CreateBranch' when successfully created the + # branch on a valid repo path. + def SuccessfullyCreateBranchForChanges(_repo_path, branch): self.assertEqual(branch, 'update-LLVM_NEXT_HASH-a123testhash4') return @@ -925,7 +645,7 @@ class UpdateLLVMHashTest(unittest.TestCase): mock_create_path_dict.return_value = test_package_path_dict # Use test function to simulate behavior. - mock_create_repo.side_effect = SuccessfullyCreateRepoForChanges + mock_create_repo.side_effect = SuccessfullyCreateBranchForChanges mock_update_llvm_next.side_effect = SuccessfullyUpdatedLLVMHash mock_uprev_symlink.side_effect = FailedToUprevEbuildSymlink mock_upload_changes.side_effect = ShouldNotExecuteUploadChanges @@ -946,7 +666,7 @@ class UpdateLLVMHashTest(unittest.TestCase): with self.assertRaises(ValueError) as err: update_chromeos_llvm_hash.UpdatePackages( packages_to_update, llvm_variant, git_hash, svn_version, chroot_path, - patch_metadata_file, FailureModes.FAIL, git_hash_source, + patch_metadata_file, failure_modes.FailureModes.FAIL, git_hash_source, extra_commit_msg) self.assertEqual(str(err.exception), 'Failed to uprev the ebuild.') @@ -967,11 +687,11 @@ class UpdateLLVMHashTest(unittest.TestCase): @mock.patch.object(update_chromeos_llvm_hash, 'CreatePathDictionaryFromPackages') - @mock.patch.object(update_chromeos_llvm_hash, '_CreateRepo') + @mock.patch.object(git, 'CreateBranch') @mock.patch.object(update_chromeos_llvm_hash, 'UpdateEbuildLLVMHash') @mock.patch.object(update_chromeos_llvm_hash, 'UprevEbuildSymlink') - @mock.patch.object(update_chromeos_llvm_hash, 'UploadChanges') - @mock.patch.object(update_chromeos_llvm_hash, '_DeleteRepo') + @mock.patch.object(git, 'UploadChanges') + @mock.patch.object(git, 'DeleteBranch') @mock.patch.object(llvm_patch_management, 'UpdatePackagesPatchMetadataFile') @mock.patch.object(update_chromeos_llvm_hash, 'StagePatchMetadataFileForCommit') @@ -987,9 +707,9 @@ class UpdateLLVMHashTest(unittest.TestCase): path_to_package_dir = '/some/path/to/chroot/src/path/to' - # Test function to simulate '_CreateRepo' when successfully created the repo - # for the changes to be made to the ebuild files. - def SuccessfullyCreateRepoForChanges(_repo_path, branch): + # Test function to simulate 'CreateBranch' when successfully created the + # branch for the changes to be made to the ebuild files. + def SuccessfullyCreateBranchForChanges(_repo_path, branch): self.assertEqual(branch, 'update-LLVM_NEXT_HASH-a123testhash5') return @@ -1019,7 +739,7 @@ class UpdateLLVMHashTest(unittest.TestCase): self.assertEqual(svn_version, 1000) self.assertEqual(patch_metadata_file, 'PATCHES.json') self.assertListEqual(packages, ['path/to']) - self.assertEqual(mode, FailureModes.DISABLE_PATCHES) + self.assertEqual(mode, failure_modes.FailureModes.DISABLE_PATCHES) PatchInfo = namedtuple('PatchInfo', [ 'applied_patches', 'failed_patches', 'non_applicable_patches', @@ -1047,8 +767,7 @@ class UpdateLLVMHashTest(unittest.TestCase): def SuccessfullyUploadedChanges(_repo_path, _branch, _commit_messages): commit_url = 'https://some_name/path/to/commit/+/12345' - return update_chromeos_llvm_hash.CommitContents( - url=commit_url, cl_number=12345) + return git.CommitContents(url=commit_url, cl_number=12345) test_package_path_dict = {symlink_path_to_package: abs_path_to_package} @@ -1059,7 +778,7 @@ class UpdateLLVMHashTest(unittest.TestCase): mock_create_path_dict.return_value = test_package_path_dict # Use test function to simulate behavior. - mock_create_repo.side_effect = SuccessfullyCreateRepoForChanges + mock_create_repo.side_effect = SuccessfullyCreateBranchForChanges mock_update_llvm_next.side_effect = SuccessfullyUpdatedLLVMHash mock_uprev_symlink.side_effect = SuccessfullyUprevedEbuildSymlink mock_update_package_metadata_file.side_effect = RetrievedPatchResults @@ -1077,8 +796,8 @@ class UpdateLLVMHashTest(unittest.TestCase): change_list = update_chromeos_llvm_hash.UpdatePackages( packages_to_update, llvm_variant, git_hash, svn_version, chroot_path, - patch_metadata_file, FailureModes.DISABLE_PATCHES, git_hash_source, - extra_commit_msg) + patch_metadata_file, failure_modes.FailureModes.DISABLE_PATCHES, + git_hash_source, extra_commit_msg) self.assertEqual(change_list.url, 'https://some_name/path/to/commit/+/12345') diff --git a/llvm_tools/update_packages_and_run_tests.py b/llvm_tools/update_packages_and_run_tests.py index ee57ea09..9fe4fe02 100755 --- a/llvm_tools/update_packages_and_run_tests.py +++ b/llvm_tools/update_packages_and_run_tests.py @@ -13,12 +13,10 @@ import datetime import json import os -from assert_not_in_chroot import VerifyOutsideChroot -from failure_modes import FailureModes -from get_llvm_hash import GetLLVMHashAndVersionFromSVNOption -from get_llvm_hash import is_svn_option -from subprocess_helpers import ChrootRunCommand -from subprocess_helpers import ExecCommandAndCaptureOutput +import chroot +import failure_modes +import get_llvm_hash +import subprocess_helpers import update_chromeos_llvm_hash @@ -73,7 +71,7 @@ def GetCommandLineArgs(): # Add argument for the LLVM version to use. parser.add_argument( '--llvm_version', - type=is_svn_option, + type=get_llvm_hash.is_svn_option, required=True, help='which git hash of LLVM to find ' '{google3, ToT, <svn_version>} ' @@ -159,7 +157,7 @@ def AddReviewers(cl, reviewers, chroot_path): for reviewer in reviewers: cmd = [gerrit_abs_path, 'reviewers', str(cl), reviewer] - ExecCommandAndCaptureOutput(cmd) + subprocess_helpers.ExecCommandAndCaptureOutput(cmd) def AddTryjobLinkToCL(tryjobs, cl, chroot_path): @@ -181,7 +179,7 @@ def AddTryjobLinkToCL(tryjobs, cl, chroot_path): str(cl), '\n'.join(tryjob_links) ] - ExecCommandAndCaptureOutput(add_message_cmd) + subprocess_helpers.ExecCommandAndCaptureOutput(add_message_cmd) # Testing with tryjobs @@ -254,7 +252,8 @@ def RunTryJobs(cl_number, extra_change_lists, options, builders, chroot_path, tryjob_cmd = GetTryJobCommand(cl_number, extra_change_lists, options, cur_builder) - out = ChrootRunCommand(chroot_path, tryjob_cmd, verbose=verbose) + out = subprocess_helpers.ChrootRunCommand( + chroot_path, tryjob_cmd, verbose=verbose) tryjob_launch_time = GetCurrentTimeInUTC() @@ -300,7 +299,7 @@ def StartCQDryRun(cl, dependent_cls, chroot_path): for changes in cl_list: cq_dry_run_cmd = [gerrit_abs_path, 'label-cq', str(changes), '1'] - ExecCommandAndCaptureOutput(cq_dry_run_cmd) + subprocess_helpers.ExecCommandAndCaptureOutput(cq_dry_run_cmd) def main(): @@ -310,7 +309,7 @@ def main(): AssertionError: The script was run inside the chroot. """ - VerifyOutsideChroot() + chroot.VerifyOutsideChroot() args_output = GetCommandLineArgs() @@ -323,7 +322,8 @@ def main(): svn_option = args_output.llvm_version - git_hash, svn_version = GetLLVMHashAndVersionFromSVNOption(svn_option) + git_hash, svn_version = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption( + svn_option) # There is no need to run tryjobs when all the key parameters remain unchanged # from last time. @@ -331,8 +331,8 @@ def main(): # If --last_tested is specified, check if the current run has the same # arguments last time --last_tested is used. if args_output.last_tested: - chroot_file_paths = update_chromeos_llvm_hash.GetChrootBuildPaths( - args_output.chroot_path, update_packages) + chroot_file_paths = chroot.GetChrootEbuildPaths(args_output.chroot_path, + update_packages) arg_dict = { 'svn_version': svn_version, 'ebuilds': chroot_file_paths, @@ -361,7 +361,7 @@ def main(): svn_version, args_output.chroot_path, patch_metadata_file, - FailureModes.DISABLE_PATCHES, + failure_modes.FailureModes.DISABLE_PATCHES, svn_option, extra_commit_msg=extra_commit_msg) diff --git a/llvm_tools/update_packages_and_run_tests_unittest.py b/llvm_tools/update_packages_and_run_tests_unittest.py index 721b995f..70ecfffe 100755 --- a/llvm_tools/update_packages_and_run_tests_unittest.py +++ b/llvm_tools/update_packages_and_run_tests_unittest.py @@ -12,9 +12,11 @@ import json import unittest import unittest.mock as mock -from test_helpers import ArgsOutputTest -from test_helpers import CreateTemporaryFile -from update_chromeos_llvm_hash import CommitContents +import chroot +import get_llvm_hash +import git +import subprocess_helpers +import test_helpers import update_chromeos_llvm_hash import update_packages_and_run_tests @@ -28,7 +30,7 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): update_packages_and_run_tests.UnchangedSinceLastRun(None, {}), False) def testEmptyLastTestedFile(self): - with CreateTemporaryFile() as temp_file: + with test_helpers.CreateTemporaryFile() as temp_file: self.assertEqual( update_packages_and_run_tests.UnchangedSinceLastRun(temp_file, {}), False) @@ -42,7 +44,7 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): '/some/file/that/does/not/exist.txt', {}), False) def testMatchedLastTestedFile(self): - with CreateTemporaryFile() as last_tested_file: + with test_helpers.CreateTemporaryFile() as last_tested_file: arg_dict = { 'svn_version': 1234, @@ -112,7 +114,7 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): 'GetCurrentTimeInUTC', return_value='2019-09-09') @mock.patch.object(update_packages_and_run_tests, 'AddTryjobLinkToCL') - @mock.patch.object(update_packages_and_run_tests, 'ChrootRunCommand') + @mock.patch.object(subprocess_helpers, 'ChrootRunCommand') def testSuccessfullySubmittedTryJob( self, mock_chroot_cmd, mock_add_tryjob_link_to_cl, mock_launch_time): @@ -157,9 +159,7 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): mock_add_tryjob_link_to_cl.assert_called_once() @mock.patch.object( - update_packages_and_run_tests, - 'ExecCommandAndCaptureOutput', - return_value=None) + subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None) def testSuccessfullyAddedTryjobLinkToCL(self, mock_exec_cmd): chroot_path = '/abs/path/to/chroot' @@ -181,11 +181,9 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): @mock.patch.object(update_packages_and_run_tests, 'RunTryJobs') @mock.patch.object(update_chromeos_llvm_hash, 'UpdatePackages') @mock.patch.object(update_packages_and_run_tests, 'GetCommandLineArgs') - @mock.patch.object(update_packages_and_run_tests, - 'GetLLVMHashAndVersionFromSVNOption') - @mock.patch.object( - update_packages_and_run_tests, 'VerifyOutsideChroot', return_value=True) - @mock.patch.object(update_chromeos_llvm_hash, 'GetChrootBuildPaths') + @mock.patch.object(get_llvm_hash, 'GetLLVMHashAndVersionFromSVNOption') + @mock.patch.object(chroot, 'VerifyOutsideChroot', return_value=True) + @mock.patch.object(chroot, 'GetChrootEbuildPaths') def testUpdatedLastTestedFileWithNewTestedRevision( self, mock_get_chroot_build_paths, mock_outside_chroot, mock_get_hash_and_version, mock_get_commandline_args, @@ -193,7 +191,7 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): # Create a temporary file to simulate the last tested file that contains a # revision. - with CreateTemporaryFile() as last_tested_file: + with test_helpers.CreateTemporaryFile() as last_tested_file: builders = [ 'kevin-llvm-next-toolchain-tryjob', 'eve-llvm-next-toolchain-tryjob' ] @@ -216,7 +214,7 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): json.dump(arg_dict, f, indent=2) # Call with a changed LLVM svn version - args_output = ArgsOutputTest() + args_output = test_helpers.ArgsOutputTest() args_output.is_llvm_next = True args_output.extra_change_lists = extra_cls args_output.last_tested = last_tested_file @@ -232,7 +230,7 @@ class UpdatePackagesAndRunTryjobsTest(unittest.TestCase): mock_get_hash_and_version.return_value = ('a123testhash2', 200) - mock_update_packages.return_value = CommitContents( + mock_update_packages.return_value = git.CommitContents( url='https://some_cl_url.com', cl_number=12345) mock_run_tryjobs.return_value = [{ @@ -283,9 +281,7 @@ class UpdatePackagesAndRunTestCQTest(unittest.TestCase): # Mock ExecCommandAndCaptureOutput for the gerrit command execution. @mock.patch.object( - update_packages_and_run_tests, - 'ExecCommandAndCaptureOutput', - return_value=None) + subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None) def testStartCQDryRunNoDeps(self, mock_exec_cmd): chroot_path = '/abs/path/to/chroot' test_cl_number = 1000 @@ -304,9 +300,7 @@ class UpdatePackagesAndRunTestCQTest(unittest.TestCase): # Mock ExecCommandAndCaptureOutput for the gerrit command execution. @mock.patch.object( - update_packages_and_run_tests, - 'ExecCommandAndCaptureOutput', - return_value=None) + subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None) # test with a single deps cl. def testStartCQDryRunSingleDep(self, mock_exec_cmd): chroot_path = '/abs/path/to/chroot' @@ -333,9 +327,7 @@ class UpdatePackagesAndRunTestCQTest(unittest.TestCase): # Mock ExecCommandAndCaptureOutput for the gerrit command execution. @mock.patch.object( - update_packages_and_run_tests, - 'ExecCommandAndCaptureOutput', - return_value=None) + subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None) def testStartCQDryRunMultipleDep(self, mock_exec_cmd): chroot_path = '/abs/path/to/chroot' test_cl_number = 1000 @@ -368,9 +360,7 @@ class UpdatePackagesAndRunTestCQTest(unittest.TestCase): # Mock ExecCommandAndCaptureOutput for the gerrit command execution. @mock.patch.object( - update_packages_and_run_tests, - 'ExecCommandAndCaptureOutput', - return_value=None) + subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None) # test with no reviewers. def testAddReviewersNone(self, mock_exec_cmd): chroot_path = '/abs/path/to/chroot' @@ -383,9 +373,7 @@ class UpdatePackagesAndRunTestCQTest(unittest.TestCase): # Mock ExecCommandAndCaptureOutput for the gerrit command execution. @mock.patch.object( - update_packages_and_run_tests, - 'ExecCommandAndCaptureOutput', - return_value=None) + subprocess_helpers, 'ExecCommandAndCaptureOutput', return_value=None) # test with multiple reviewers. def testAddReviewersMultiple(self, mock_exec_cmd): chroot_path = '/abs/path/to/chroot' diff --git a/llvm_tools/update_tryjob_status.py b/llvm_tools/update_tryjob_status.py index 38eab8e5..f1500364 100755 --- a/llvm_tools/update_tryjob_status.py +++ b/llvm_tools/update_tryjob_status.py @@ -15,7 +15,7 @@ import os import subprocess import sys -from assert_not_in_chroot import VerifyOutsideChroot +import chroot from subprocess_helpers import ChrootRunCommand from test_helpers import CreateTemporaryJsonFile @@ -310,7 +310,7 @@ def UpdateTryjobStatus(revision, set_status, status_file, chroot_path, def main(): """Updates the status of a tryjob.""" - VerifyOutsideChroot() + chroot.VerifyOutsideChroot() args_output = GetCommandLineArgs() |