diff options
author | Salud Lemus <saludlemus@google.com> | 2019-08-23 16:49:28 -0700 |
---|---|---|
committer | Salud Lemus <saludlemus@google.com> | 2019-08-26 20:48:09 +0000 |
commit | 42235709e88cbb239b9514d888919beb7c4a2c9e (patch) | |
tree | 522da299d73617637f8f4492955b24796998ad57 /llvm_tools | |
parent | 6277e420d2b9260f9bcc1ac6b22b63ef7ef4b79c (diff) | |
download | toolchain-utils-42235709e88cbb239b9514d888919beb7c4a2c9e.tar.gz |
LLVM tools: Maintain up-to-date LLVM repo in 'llvm_tools'
Keeps an up-to-date LLVM source tree in 'llvm_tools' directory.
Previously, the LLVM repo would be cloned once and then be forgotten.
BUG=None
TEST=Ran the 'llvm_bisection.py' script on a previously created JSON
file (the script created it before). Successfully created tryjobs that
were not in the 'jobs' list. Also, successfully kept the LLVM repo
up-to-date in 'llvm_tools'.
Change-Id: If9038da788eb0fe0fe258274c1942f1d687868b4
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/1769483
Reviewed-by: Manoj Gupta <manojgupta@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>
Tested-by: Salud Lemus <saludlemus@google.com>
Diffstat (limited to 'llvm_tools')
-rwxr-xr-x | llvm_tools/get_llvm_hash.py | 120 | ||||
-rwxr-xr-x | llvm_tools/llvm_bisection.py | 19 | ||||
-rwxr-xr-x | llvm_tools/llvm_patch_management.py | 46 |
3 files changed, 145 insertions, 40 deletions
diff --git a/llvm_tools/get_llvm_hash.py b/llvm_tools/get_llvm_hash.py index dbc743bd..20e00eb5 100755 --- a/llvm_tools/get_llvm_hash.py +++ b/llvm_tools/get_llvm_hash.py @@ -18,6 +18,115 @@ import subprocess import tempfile +@contextmanager +def CreateTempLLVMRepo(temp_dir): + """Adds a LLVM worktree to 'temp_dir'. + + Creating a worktree because the LLVM source tree in + '../toolchain-utils/llvm_tools/llvm-project-copy' should not be modified. + + This is useful for applying patches to a source tree but do not want to modify + the actual LLVM source tree in 'llvm-project-copy'. + + Args: + temp_dir: An absolute path to the temporary directory to put the worktree in + (obtained via 'tempfile.mkdtemp()'). + + Returns: + The absolute path to 'temp_dir'. + + Raises: + subprocess.CalledProcessError: Failed to remove the worktree. + ValueError: Failed to add a worktree. + """ + + abs_path_to_llvm_project_dir = GetAndUpdateLLVMProjectInLLVMTools() + + add_worktree_cmd = [ + 'git', '-C', abs_path_to_llvm_project_dir, 'worktree', 'add', '--detach', + temp_dir, 'master' + ] + + add_worktree_cmd_obj = subprocess.Popen( + add_worktree_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, stderr = add_worktree_cmd_obj.communicate() + + if add_worktree_cmd_obj.returncode: + raise ValueError('Failed to add worktree for %s: %s' % + (abs_path_to_llvm_project_dir, stderr)) + + try: + yield temp_dir + finally: + if os.path.isdir(temp_dir): + subprocess.check_output([ + 'git', '-C', abs_path_to_llvm_project_dir, 'worktree', 'remove', '-f', + temp_dir + ]) + + +def GetAndUpdateLLVMProjectInLLVMTools(): + """Gets the absolute path to 'llvm-project-copy' directory in 'llvm_tools'. + + The intent of this function is to avoid cloning the LLVM repo and then + discarding the contents of the repo. The function will create a directory + in '../toolchain-utils/llvm_tools' called 'llvm-project-copy' if this + directory does not exist yet. If it does not exist, then it will use the + LLVMHash() class to clone the LLVM repo into 'llvm-project-copy'. Otherwise, + it will clean the contents of that directory and then fetch from the chromium + LLVM mirror. In either case, this function will return the absolute path to + 'llvm-project-copy' directory. + + Raises: + ValueError: LLVM repo (in 'llvm-project-copy' dir.) has changes or failed to + checkout to master or failed to fetch from chromium mirror of LLVM. + """ + + abs_path_to_llvm_tools_dir = os.path.dirname(os.path.abspath(__file__)) + + abs_path_to_llvm_project_dir = os.path.join(abs_path_to_llvm_tools_dir, + 'llvm-project-copy') + + if not os.path.isdir(abs_path_to_llvm_project_dir): + os.mkdir(abs_path_to_llvm_project_dir) + + LLVMHash().CloneLLVMRepo(abs_path_to_llvm_project_dir) + else: + # `git status` has a '-s'/'--short' option that shortens the output. + # With the '-s' option, if no changes were made to the LLVM repo, then the + # output (assigned to 'repo_status') would be empty. + repo_status = subprocess.check_output( + ['git', '-C', abs_path_to_llvm_project_dir, 'status', '-s']) + + if repo_status.rstrip(): + raise ValueError('LLVM repo in %s has changes, please remove.' % + abs_path_to_llvm_project_dir) + + checkout_to_master_cmd = [ + 'git', '-C', abs_path_to_llvm_project_dir, 'checkout', 'master' + ] + + checkout_cmd_obj = subprocess.Popen( + checkout_to_master_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, stderr = checkout_cmd_obj.communicate() + + if checkout_cmd_obj.returncode: + raise ValueError('Failed to checkout to master for %s: %s' % + (abs_path_to_llvm_project_dir, stderr)) + + update_master_cmd = ['git', '-C', abs_path_to_llvm_project_dir, 'pull'] + + update_cmd_obj = subprocess.Popen( + update_master_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + _, stderr = update_cmd_obj.communicate() + + if update_cmd_obj.returncode: + raise ValueError('Failed to fetch from chromium mirror of LLVM for %s: %s' + % (abs_path_to_llvm_project_dir, stderr)) + + return abs_path_to_llvm_project_dir + + def GetGoogle3LLVMVersion(): """Gets the latest google3 LLVM version. @@ -99,7 +208,8 @@ def GetLLVMHashAndVersionFromSVNOption(svn_option): else: llvm_version = GetGoogle3LLVMVersion() - llvm_hash = new_llvm_hash.GetLLVMHash(llvm_version) + new_llvm_hash.GetGitHashForVersion(GetAndUpdateLLVMProjectInLLVMTools(), + llvm_version) return llvm_hash, llvm_version @@ -266,12 +376,8 @@ class LLVMHash(object): The hash as a string that corresponds to the LLVM version. """ - with self.CreateTempDirectory() as temp_dir: - # Clone the "llvm-project" repo. - self.CloneLLVMRepo(temp_dir) - - # Find the git hash. - hash_value = self.GetGitHashForVersion(temp_dir, llvm_version) + hash_value = self.GetGitHashForVersion(GetAndUpdateLLVMProjectInLLVMTools(), + llvm_version) return hash_value diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py index 9cf2f447..b4705d49 100755 --- a/llvm_tools/llvm_bisection.py +++ b/llvm_tools/llvm_bisection.py @@ -15,6 +15,7 @@ import json import os from assert_not_in_chroot import VerifyOutsideChroot +from get_llvm_hash import CreateTempLLVMRepo from get_llvm_hash import LLVMHash from modify_a_tryjob import AddTryjob from patch_manager import _ConvertToASCII @@ -272,17 +273,17 @@ def GetRevisionsListAndHashList(start, end, parallel, src_path, new_llvm = LLVMHash() with new_llvm.CreateTempDirectory() as temp_dir: - if not src_path: - new_llvm.CloneLLVMRepo(temp_dir) - src_path = temp_dir + with CreateTempLLVMRepo(temp_dir) as new_repo: + if not src_path: + src_path = new_repo - # Get a list of revisions between start and end. - revisions = GetRevisionsBetweenBisection(start, end, parallel, src_path, - pending_revisions) + # Get a list of revisions between start and end. + revisions = GetRevisionsBetweenBisection(start, end, parallel, src_path, + pending_revisions) - git_hashes = [ - new_llvm.GetGitHashForVersion(src_path, rev) for rev in revisions - ] + git_hashes = [ + new_llvm.GetGitHashForVersion(src_path, rev) for rev in revisions + ] assert revisions, ('No revisions between start %d and end %d to create ' 'tryjobs.' % (start, end)) diff --git a/llvm_tools/llvm_patch_management.py b/llvm_tools/llvm_patch_management.py index 4812627a..90f82d30 100755 --- a/llvm_tools/llvm_patch_management.py +++ b/llvm_tools/llvm_patch_management.py @@ -16,6 +16,7 @@ import patch_manager from assert_not_in_chroot import VerifyOutsideChroot from cros_utils import command_executer from failure_modes import FailureModes +from get_llvm_hash import CreateTempLLVMRepo from get_llvm_hash import GetGoogle3LLVMVersion from get_llvm_hash import LLVMHash @@ -215,37 +216,34 @@ def UpdatePackagesPatchMetadataFile(chroot_path, svn_version, llvm_hash = LLVMHash() - with llvm_hash.CreateTempDirectory() as src_path: - # Clone from the chromium mirror to ensure that the git hashes are - # available. - llvm_hash.CloneLLVMRepo(src_path) + with llvm_hash.CreateTempDirectory() as temp_dir: + with CreateTempLLVMRepo(temp_dir) as src_path: + # Ensure that 'svn_version' exists in the chromiumum mirror of LLVM by + # finding its corresponding git hash. + git_hash = llvm_hash.GetGitHashForVersion(src_path, svn_version) - # Ensure that 'svn_version' exists in the chromiumum mirror of LLVM by - # finding its corresponding git hash. - git_hash = llvm_hash.GetGitHashForVersion(src_path, svn_version) + # Git hash of 'svn_version' exists, so move the source tree's HEAD to + # 'git_hash' via `git checkout`. + _MoveSrcTreeHEADToGitHash(src_path, git_hash) - # Git hash of 'svn_version' exists, so move the source tree's HEAD to - # 'git_hash' via `git checkout`. - _MoveSrcTreeHEADToGitHash(src_path, git_hash) + for cur_package in packages: + # Get the absolute path to $FILESDIR of the package. + filesdir_path = GetPathToFilesDirectory(chroot_path, cur_package) - for cur_package in packages: - # Get the absolute path to $FILESDIR of the package. - filesdir_path = GetPathToFilesDirectory(chroot_path, cur_package) + # Construct the absolute path to the patch metadata file where all the + # patches and their metadata are. + patch_metadata_path = os.path.join(filesdir_path, patch_metadata_file) - # Construct the absolute path to the patch metadata file where all the - # patches and their metadata are. - patch_metadata_path = os.path.join(filesdir_path, patch_metadata_file) + # Make sure the patch metadata path is valid. + _CheckPatchMetadataPath(patch_metadata_path) - # Make sure the patch metadata path is valid. - _CheckPatchMetadataPath(patch_metadata_path) + patch_manager.CleanSrcTree(src_path) - patch_manager.CleanSrcTree(src_path) + # Get the patch results for the current package. + patches_info = patch_manager.HandlePatches( + svn_version, patch_metadata_path, filesdir_path, src_path, mode) - # Get the patch results for the current package. - patches_info = patch_manager.HandlePatches( - svn_version, patch_metadata_path, filesdir_path, src_path, mode) - - package_info[cur_package] = patches_info._asdict() + package_info[cur_package] = patches_info._asdict() return package_info |