aboutsummaryrefslogtreecommitdiff
path: root/llvm_tools
diff options
context:
space:
mode:
Diffstat (limited to 'llvm_tools')
-rw-r--r--llvm_tools/README.md1
-rwxr-xr-xllvm_tools/get_upstream_patch.py249
-rwxr-xr-xllvm_tools/llvm_bisection.py151
-rwxr-xr-xllvm_tools/modify_a_tryjob.py75
-rwxr-xr-xllvm_tools/nightly_revert_checker.py97
-rw-r--r--llvm_tools/patch_sync/.gitignore1
-rw-r--r--llvm_tools/patch_sync/Cargo.lock453
-rw-r--r--llvm_tools/patch_sync/Cargo.toml20
-rw-r--r--llvm_tools/patch_sync/src/main.rs173
-rw-r--r--llvm_tools/patch_sync/src/patch_parsing.rs313
-rw-r--r--llvm_tools/patch_sync/src/version_control.rs238
-rwxr-xr-xllvm_tools/update_chromeos_llvm_hash.py75
-rwxr-xr-xllvm_tools/update_packages_and_run_tests.py100
13 files changed, 1616 insertions, 330 deletions
diff --git a/llvm_tools/README.md b/llvm_tools/README.md
index 43c80ad6..74fad6c9 100644
--- a/llvm_tools/README.md
+++ b/llvm_tools/README.md
@@ -119,6 +119,7 @@ For example, to create a roll CL to the git hash of revision 367622:
$ ./update_chromeos_llvm_hash.py \
--update_packages sys-devel/llvm sys-libs/compiler-rt \
sys-libs/libcxx sys-libs/libcxxabi sys-libs/llvm-libunwind \
+ 'dev-util/lldb-server' \
--llvm_version 367622 \
--failure_mode disable_patches
```
diff --git a/llvm_tools/get_upstream_patch.py b/llvm_tools/get_upstream_patch.py
index 77783d41..7a4be3eb 100755
--- a/llvm_tools/get_upstream_patch.py
+++ b/llvm_tools/get_upstream_patch.py
@@ -6,8 +6,6 @@
"""Get an upstream patch to LLVM's PATCHES.json."""
-from __future__ import print_function
-
import argparse
import json
import logging
@@ -18,12 +16,20 @@ import sys
import typing as t
from datetime import datetime
+import dataclasses
+
import chroot
import get_llvm_hash
import git
import git_llvm_rev
import update_chromeos_llvm_hash
+__DOC_EPILOGUE = """
+Example Usage:
+ get_upstream_patch --chroot_path ~/chromiumos --platform chromiumos \
+--sha 1234567 --sha 890abdc
+"""
+
class CherrypickError(ValueError):
"""A ValueError that highlights the cherry-pick has been seen before"""
@@ -32,7 +38,7 @@ class CherrypickError(ValueError):
def add_patch(patches_json_path: str, patches_dir: str,
relative_patches_dir: str, start_version: git_llvm_rev.Rev,
llvm_dir: str, rev: t.Union[git_llvm_rev.Rev, str], sha: str,
- package: str):
+ package: str, platforms: t.List[str]):
"""Gets the start and end intervals in 'json_file'.
Args:
@@ -47,6 +53,7 @@ def add_patch(patches_json_path: str, patches_dir: str,
revisions, the git sha from the local commit created by 'arc patch'
is used.
package: The LLVM project name this patch applies to.
+ platforms: List of platforms this patch applies to.
Raises:
CherrypickError: A ValueError that highlights the cherry-pick has been
@@ -85,20 +92,29 @@ def add_patch(patches_json_path: str, patches_dir: str,
subprocess.check_call(cmd, stdout=f, cwd=llvm_dir)
commit_subject = subprocess.check_output(
- ['git', 'log', '-n1', '--format=%s', sha], cwd=llvm_dir, encoding='utf-8')
+ ['git', 'log', '-n1', '--format=%s', sha],
+ cwd=llvm_dir,
+ encoding='utf-8')
- patch_metadata = {
- 'comment': commit_subject.strip(),
+ patch_props = {
'rel_patch_path': rel_patch_path,
'start_version': start_version.number,
+ 'metadata': {
+ 'title': commit_subject.strip(),
+ 'info': [],
+ },
+ 'platforms': sorted(platforms),
+ 'end_version': rev.number if isinstance(rev, git_llvm_rev.Rev) else None,
}
- if isinstance(rev, git_llvm_rev.Rev):
- patch_metadata['end_version'] = rev.number
- patches_json.append(patch_metadata)
+ patches_json.append(patch_props)
temp_file = patches_json_path + '.tmp'
with open(temp_file, 'w', encoding='utf-8') as f:
- json.dump(patches_json, f, indent=4, separators=(',', ': '))
+ json.dump(patches_json,
+ f,
+ indent=4,
+ separators=(',', ': '),
+ sort_keys=True)
f.write('\n')
os.rename(temp_file, patches_json_path)
@@ -178,7 +194,7 @@ def get_package_names(sha: str, llvm_dir: str) -> list:
def create_patch_for_packages(packages: t.List[str], symlinks: t.List[str],
start_rev: git_llvm_rev.Rev,
rev: t.Union[git_llvm_rev.Rev, str], sha: str,
- llvm_dir: str):
+ llvm_dir: str, platforms: t.List[str]):
"""Create a patch and add its metadata for each package"""
for package, symlink in zip(packages, symlinks):
symlink_dir = os.path.dirname(symlink)
@@ -186,8 +202,15 @@ def create_patch_for_packages(packages: t.List[str], symlinks: t.List[str],
relative_patches_dir = 'cherry' if package == 'llvm' else ''
patches_dir = os.path.join(symlink_dir, 'files', relative_patches_dir)
logging.info('Getting %s (%s) into %s', rev, sha, package)
- add_patch(patches_json_path, patches_dir, relative_patches_dir, start_rev,
- llvm_dir, rev, sha, package)
+ add_patch(patches_json_path,
+ patches_dir,
+ relative_patches_dir,
+ start_rev,
+ llvm_dir,
+ rev,
+ sha,
+ package,
+ platforms=platforms)
def make_cl(symlinks_to_uprev: t.List[str], llvm_symlink_dir: str, branch: str,
@@ -212,35 +235,35 @@ def resolve_symbolic_sha(start_sha: str, llvm_symlink_dir: str) -> str:
return start_sha
-def find_patches_and_make_cl(chroot_path: str, patches: t.List[str],
- start_rev: git_llvm_rev.Rev,
- llvm_config: git_llvm_rev.LLVMConfig,
- llvm_symlink_dir: str, create_cl: bool,
- reviewers: t.Optional[t.List[str]],
- cc: t.Optional[t.List[str]]):
+def find_patches_and_make_cl(
+ chroot_path: str, patches: t.List[str], start_rev: git_llvm_rev.Rev,
+ llvm_config: git_llvm_rev.LLVMConfig, llvm_symlink_dir: str,
+ create_cl: bool, skip_dependencies: bool,
+ reviewers: t.Optional[t.List[str]], cc: t.Optional[t.List[str]],
+ platforms: t.List[str]):
+
+ converted_patches = [
+ _convert_patch(llvm_config, skip_dependencies, p) for p in patches
+ ]
+ potential_duplicates = _get_duplicate_shas(converted_patches)
+ if potential_duplicates:
+ err_msg = '\n'.join(f'{a.patch} == {b.patch}'
+ for a, b in potential_duplicates)
+ raise RuntimeError(f'Found Duplicate SHAs:\n{err_msg}')
+
+ # CL Related variables, only used if `create_cl`
+ symlinks_to_uprev = []
+ commit_messages = [
+ 'llvm: get patches from upstream\n',
+ ]
+ branch = f'get-upstream-{datetime.now().strftime("%Y%m%d%H%M%S%f")}'
+
if create_cl:
- branch = f'get-upstream-{datetime.now().strftime("%Y%m%d%H%M%S%f")}'
git.CreateBranch(llvm_symlink_dir, branch)
- symlinks_to_uprev = []
- commit_messages = [
- 'llvm: get patches from upstream\n',
- ]
-
- for patch in patches:
- # git hash should only have lower-case letters
- is_differential = patch.startswith('D')
- if is_differential:
- subprocess.check_output(
- ['arc', 'patch', '--nobranch', '--revision', patch],
- cwd=llvm_config.dir,
- )
- sha = resolve_llvm_ref(llvm_config.dir, 'HEAD')
- rev = patch
- else:
- sha = resolve_llvm_ref(llvm_config.dir, patch)
- rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha)
+
+ for parsed_patch in converted_patches:
# Find out the llvm projects changed in this commit
- packages = get_package_names(sha, llvm_config.dir)
+ packages = get_package_names(parsed_patch.sha, llvm_config.dir)
# Find out the ebuild symlinks of the corresponding ChromeOS packages
symlinks = chroot.GetChrootEbuildPaths(chroot_path, [
'sys-devel/llvm' if package == 'llvm' else 'sys-libs/' + package
@@ -248,23 +271,25 @@ def find_patches_and_make_cl(chroot_path: str, patches: t.List[str],
])
symlinks = chroot.ConvertChrootPathsToAbsolutePaths(chroot_path, symlinks)
# Create a local patch for all the affected llvm projects
- create_patch_for_packages(packages, symlinks, start_rev, rev, sha,
- llvm_config.dir)
+ create_patch_for_packages(packages,
+ symlinks,
+ start_rev,
+ parsed_patch.rev,
+ parsed_patch.sha,
+ llvm_config.dir,
+ platforms=platforms)
if create_cl:
symlinks_to_uprev.extend(symlinks)
- if is_differential:
- msg = f'\n\nreviews.llvm.org/{patch}\n'
- else:
- msg = f'\n\nreviews.llvm.org/rG{sha}\n'
commit_messages.extend([
- msg,
- subprocess.check_output(['git', 'log', '-n1', '--oneline', sha],
- cwd=llvm_config.dir,
- encoding='utf-8')
+ parsed_patch.git_msg(),
+ subprocess.check_output(
+ ['git', 'log', '-n1', '--oneline', parsed_patch.sha],
+ cwd=llvm_config.dir,
+ encoding='utf-8')
])
- if is_differential:
+ if parsed_patch.is_differential:
subprocess.check_output(['git', 'reset', '--hard', 'HEAD^'],
cwd=llvm_config.dir)
@@ -273,10 +298,67 @@ def find_patches_and_make_cl(chroot_path: str, patches: t.List[str],
reviewers, cc)
+@dataclasses.dataclass(frozen=True)
+class ParsedPatch:
+ """Class to keep track of bundled patch info."""
+ patch: str
+ sha: str
+ is_differential: bool
+ rev: t.Union[git_llvm_rev.Rev, str]
+
+ def git_msg(self) -> str:
+ if self.is_differential:
+ return f'\n\nreviews.llvm.org/{self.patch}\n'
+ return f'\n\nreviews.llvm.org/rG{self.sha}\n'
+
+
+def _convert_patch(llvm_config: git_llvm_rev.LLVMConfig,
+ skip_dependencies: bool, patch: str) -> ParsedPatch:
+ """Extract git revision info from a patch.
+
+ Args:
+ llvm_config: LLVM configuration object.
+ skip_dependencies: Pass --skip-dependecies for to `arc`
+ patch: A single patch referent string.
+
+ Returns:
+ A [ParsedPatch] object.
+ """
+
+ # git hash should only have lower-case letters
+ is_differential = patch.startswith('D')
+ if is_differential:
+ subprocess.check_output(
+ [
+ 'arc', 'patch', '--nobranch',
+ '--skip-dependencies' if skip_dependencies else '--revision', patch
+ ],
+ cwd=llvm_config.dir,
+ )
+ sha = resolve_llvm_ref(llvm_config.dir, 'HEAD')
+ rev = patch
+ else:
+ sha = resolve_llvm_ref(llvm_config.dir, patch)
+ rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha)
+ return ParsedPatch(patch=patch,
+ sha=sha,
+ rev=rev,
+ is_differential=is_differential)
+
+
+def _get_duplicate_shas(
+ patches: t.List[ParsedPatch]) -> t.List[t.Tuple[ParsedPatch, ParsedPatch]]:
+ """Return a list of Patches which have duplicate SHA's"""
+ return [(left, right) for i, left in enumerate(patches)
+ for right in patches[i + 1:] if left.sha == right.sha]
+
+
def get_from_upstream(chroot_path: str,
create_cl: bool,
start_sha: str,
patches: t.List[str],
+ platforms: t.List[str],
+ skip_dependencies: bool = False,
reviewers: t.List[str] = None,
cc: t.List[str] = None):
llvm_symlink = chroot.ConvertChrootPathsToAbsolutePaths(
@@ -299,15 +381,17 @@ def get_from_upstream(chroot_path: str,
remote='origin', dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools())
start_sha = resolve_llvm_ref(llvm_config.dir, start_sha)
- find_patches_and_make_cl(
- chroot_path=chroot_path,
- patches=patches,
- start_rev=git_llvm_rev.translate_sha_to_rev(llvm_config, start_sha),
- llvm_config=llvm_config,
- llvm_symlink_dir=llvm_symlink_dir,
- create_cl=create_cl,
- reviewers=reviewers,
- cc=cc)
+ find_patches_and_make_cl(chroot_path=chroot_path,
+ patches=patches,
+ platforms=platforms,
+ start_rev=git_llvm_rev.translate_sha_to_rev(
+ llvm_config, start_sha),
+ llvm_config=llvm_config,
+ llvm_symlink_dir=llvm_symlink_dir,
+ create_cl=create_cl,
+ skip_dependencies=skip_dependencies,
+ reviewers=reviewers,
+ cc=cc)
logging.info('Complete.')
@@ -318,41 +402,60 @@ def main():
level=logging.INFO,
)
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument(
- '--chroot_path',
- default=os.path.join(os.path.expanduser('~'), 'chromiumos'),
- help='the path to the chroot (default: %(default)s)')
+ parser = argparse.ArgumentParser(
+ description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ epilog=__DOC_EPILOGUE)
+ parser.add_argument('--chroot_path',
+ default=os.path.join(os.path.expanduser('~'),
+ 'chromiumos'),
+ help='the path to the chroot (default: %(default)s)')
parser.add_argument(
'--start_sha',
default='llvm-next',
help='LLVM SHA that the patch should start applying at. You can specify '
'"llvm" or "llvm-next", as well. Defaults to %(default)s.')
- parser.add_argument(
- '--sha',
- action='append',
- default=[],
- help='The LLVM git SHA to cherry-pick.')
+ parser.add_argument('--sha',
+ action='append',
+ default=[],
+ help='The LLVM git SHA to cherry-pick.')
parser.add_argument(
'--differential',
action='append',
default=[],
help='The LLVM differential revision to apply. Example: D1234')
parser.add_argument(
- '--create_cl',
- default=False,
+ '--platform',
+ action='append',
+ required=True,
+ help='Apply this patch to the give platform. Common options include '
+ '"chromiumos" and "android". Can be specified multiple times to '
+ 'apply to multiple platforms')
+ parser.add_argument('--create_cl',
+ action='store_true',
+ help='Automatically create a CL if specified')
+ parser.add_argument(
+ '--skip_dependencies',
action='store_true',
- help='Automatically create a CL if specified')
+ help="Skips a LLVM differential revision's dependencies. Only valid "
+ 'when --differential appears exactly once.')
args = parser.parse_args()
if not (args.sha or args.differential):
parser.error('--sha or --differential required')
+ if args.skip_dependencies and len(args.differential) != 1:
+ parser.error("--skip_dependencies is only valid when there's exactly one "
+ 'supplied differential')
+
get_from_upstream(
chroot_path=args.chroot_path,
create_cl=args.create_cl,
start_sha=args.start_sha,
- patches=args.sha + args.differential)
+ patches=args.sha + args.differential,
+ skip_dependencies=args.skip_dependencies,
+ platforms=args.platform,
+ )
if __name__ == '__main__':
diff --git a/llvm_tools/llvm_bisection.py b/llvm_tools/llvm_bisection.py
index b1898ea9..0148efd2 100755
--- a/llvm_tools/llvm_bisection.py
+++ b/llvm_tools/llvm_bisection.py
@@ -20,6 +20,7 @@ import chroot
import get_llvm_hash
import git_llvm_rev
import modify_a_tryjob
+import update_chromeos_llvm_hash
import update_tryjob_status
@@ -51,18 +52,16 @@ def GetCommandLineArgs():
'the first bad version (default: %(default)s)')
# Add argument for the good LLVM revision for bisection.
- parser.add_argument(
- '--start_rev',
- required=True,
- type=int,
- help='The good revision for the bisection.')
+ parser.add_argument('--start_rev',
+ required=True,
+ type=int,
+ help='The good revision for the bisection.')
# Add argument for the bad LLVM revision for bisection.
- parser.add_argument(
- '--end_rev',
- required=True,
- type=int,
- help='The bad revision for the bisection.')
+ parser.add_argument('--end_rev',
+ required=True,
+ type=int,
+ help='The bad revision for the bisection.')
# Add argument for the absolute path to the file that contains information on
# the previous tested svn version.
@@ -88,42 +87,38 @@ def GetCommandLineArgs():
'of updating the packages')
# Add argument for custom options for the tryjob.
- parser.add_argument(
- '--options',
- required=False,
- nargs='+',
- help='options to use for the tryjob testing')
+ parser.add_argument('--options',
+ required=False,
+ nargs='+',
+ help='options to use for the tryjob testing')
# Add argument for the builder to use for the tryjob.
- parser.add_argument(
- '--builder', required=True, help='builder to use for the tryjob testing')
+ parser.add_argument('--builder',
+ required=True,
+ help='builder to use for the tryjob testing')
# Add argument for the description of the tryjob.
- parser.add_argument(
- '--description',
- required=False,
- nargs='+',
- help='the description of the tryjob')
+ parser.add_argument('--description',
+ required=False,
+ nargs='+',
+ help='the description of the tryjob')
# Add argument for a specific chroot path.
- parser.add_argument(
- '--chroot_path',
- default=cros_root,
- help='the path to the chroot (default: %(default)s)')
+ parser.add_argument('--chroot_path',
+ default=cros_root,
+ help='the path to the chroot (default: %(default)s)')
# Add argument for whether to display command contents to `stdout`.
- parser.add_argument(
- '--verbose',
- action='store_true',
- help='display contents of a command to the terminal '
- '(default: %(default)s)')
+ parser.add_argument('--verbose',
+ action='store_true',
+ help='display contents of a command to the terminal '
+ '(default: %(default)s)')
# Add argument for whether to display command contents to `stdout`.
- parser.add_argument(
- '--nocleanup',
- action='store_false',
- dest='cleanup',
- help='Abandon CLs created for bisectoin')
+ parser.add_argument('--nocleanup',
+ action='store_false',
+ dest='cleanup',
+ help='Abandon CLs created for bisectoin')
args_output = parser.parse_args()
@@ -174,8 +169,7 @@ def GetRemainingRange(start, end, tryjobs):
all_bad_revisions = [end]
all_bad_revisions.extend(
- cur_tryjob['rev']
- for cur_tryjob in tryjobs
+ cur_tryjob['rev'] for cur_tryjob in tryjobs
if cur_tryjob['status'] == update_tryjob_status.TryjobStatus.BAD.value)
# The minimum value for the 'bad' field in the tryjobs is the new end
@@ -184,8 +178,7 @@ def GetRemainingRange(start, end, tryjobs):
all_good_revisions = [start]
all_good_revisions.extend(
- cur_tryjob['rev']
- for cur_tryjob in tryjobs
+ cur_tryjob['rev'] for cur_tryjob in tryjobs
if cur_tryjob['status'] == update_tryjob_status.TryjobStatus.GOOD.value)
# The maximum value for the 'good' field in the tryjobs is the new start
@@ -205,8 +198,8 @@ def GetRemainingRange(start, end, tryjobs):
pending_revisions = {
tryjob['rev']
for tryjob in tryjobs
- if tryjob['status'] == update_tryjob_status.TryjobStatus.PENDING.value and
- good_rev < tryjob['rev'] < bad_rev
+ if tryjob['status'] == update_tryjob_status.TryjobStatus.PENDING.value
+ and good_rev < tryjob['rev'] < bad_rev
}
# Find all revisions that are to be skipped within 'good_rev' and 'bad_rev'.
@@ -217,8 +210,8 @@ def GetRemainingRange(start, end, tryjobs):
skip_revisions = {
tryjob['rev']
for tryjob in tryjobs
- if tryjob['status'] == update_tryjob_status.TryjobStatus.SKIP.value and
- good_rev < tryjob['rev'] < bad_rev
+ if tryjob['status'] == update_tryjob_status.TryjobStatus.SKIP.value
+ and good_rev < tryjob['rev'] < bad_rev
}
return good_rev, bad_rev, pending_revisions, skip_revisions
@@ -295,66 +288,62 @@ def main(args_output):
"""
chroot.VerifyOutsideChroot()
- update_packages = [
- 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx',
- 'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind'
- ]
patch_metadata_file = 'PATCHES.json'
start = args_output.start_rev
end = args_output.end_rev
bisect_state = LoadStatusFile(args_output.last_tested, start, end)
if start != bisect_state['start'] or end != bisect_state['end']:
- raise ValueError(f'The start {start} or the end {end} version provided is '
- f'different than "start" {bisect_state["start"]} or "end" '
- f'{bisect_state["end"]} in the .JSON file')
+ raise ValueError(
+ f'The start {start} or the end {end} version provided is '
+ f'different than "start" {bisect_state["start"]} or "end" '
+ f'{bisect_state["end"]} in the .JSON file')
- # Pending and skipped revisions are between 'start_revision' and
- # 'end_revision'.
- start_revision, end_revision, pending_revisions, skip_revisions = \
- GetRemainingRange(start, end, bisect_state['jobs'])
+ # Pending and skipped revisions are between 'start_rev' and 'end_rev'.
+ start_rev, end_rev, pending_revs, skip_revs = GetRemainingRange(
+ start, end, bisect_state['jobs'])
- revisions, git_hashes = GetCommitsBetween(start_revision, end_revision,
+ revisions, git_hashes = GetCommitsBetween(start_rev, end_rev,
args_output.parallel,
- args_output.src_path,
- pending_revisions, skip_revisions)
+ args_output.src_path, pending_revs,
+ skip_revs)
- # No more revisions between 'start_revision' and 'end_revision', so
+ # No more revisions between 'start_rev' and 'end_rev', so
# bisection is complete.
#
- # This is determined by finding all valid revisions between 'start_revision'
- # and 'end_revision' and that are NOT in the 'pending' and 'skipped' set.
+ # This is determined by finding all valid revisions between 'start_rev'
+ # and 'end_rev' and that are NOT in the 'pending' and 'skipped' set.
if not revisions:
- if pending_revisions:
+ if pending_revs:
# Some tryjobs are not finished which may change the actual bad
# commit/revision when those tryjobs are finished.
- no_revisions_message = (f'No revisions between start {start_revision} '
- f'and end {end_revision} to create tryjobs\n')
+ no_revisions_message = (f'No revisions between start {start_rev} '
+ f'and end {end_rev} to create tryjobs\n')
- if pending_revisions:
- no_revisions_message += (
- 'The following tryjobs are pending:\n' +
- '\n'.join(str(rev) for rev in pending_revisions) + '\n')
+ if pending_revs:
+ no_revisions_message += ('The following tryjobs are pending:\n' +
+ '\n'.join(str(rev)
+ for rev in pending_revs) + '\n')
- if skip_revisions:
+ if skip_revs:
no_revisions_message += ('The following tryjobs were skipped:\n' +
- '\n'.join(str(rev) for rev in skip_revisions) +
- '\n')
+ '\n'.join(str(rev)
+ for rev in skip_revs) + '\n')
raise ValueError(no_revisions_message)
print(f'Finished bisecting for {args_output.last_tested}')
if args_output.src_path:
bad_llvm_hash = get_llvm_hash.GetGitHashFrom(args_output.src_path,
- end_revision)
+ end_rev)
else:
- bad_llvm_hash = get_llvm_hash.LLVMHash().GetLLVMHash(end_revision)
- print(f'The bad revision is {end_revision} and its commit hash is '
+ bad_llvm_hash = get_llvm_hash.LLVMHash().GetLLVMHash(end_rev)
+ print(f'The bad revision is {end_rev} and its commit hash is '
f'{bad_llvm_hash}')
- if skip_revisions:
- skip_revisions_message = ('\nThe following revisions were skipped:\n' +
- '\n'.join(str(rev) for rev in skip_revisions))
- print(skip_revisions_message)
+ if skip_revs:
+ skip_revs_message = ('\nThe following revisions were skipped:\n' +
+ '\n'.join(str(rev) for rev in skip_revs))
+ print(skip_revs_message)
if args_output.cleanup:
# Abandon all the CLs created for bisection
@@ -378,9 +367,9 @@ def main(args_output):
raise ValueError(f'Revision {rev} exists already in "jobs"')
Bisect(revisions, git_hashes, bisect_state, args_output.last_tested,
- update_packages, args_output.chroot_path, patch_metadata_file,
- args_output.extra_change_lists, args_output.options,
- args_output.builder, args_output.verbose)
+ update_chromeos_llvm_hash.DEFAULT_PACKAGES, args_output.chroot_path,
+ patch_metadata_file, args_output.extra_change_lists,
+ args_output.options, args_output.builder, args_output.verbose)
if __name__ == '__main__':
diff --git a/llvm_tools/modify_a_tryjob.py b/llvm_tools/modify_a_tryjob.py
index 4d41e6b2..519fb51e 100755
--- a/llvm_tools/modify_a_tryjob.py
+++ b/llvm_tools/modify_a_tryjob.py
@@ -17,9 +17,9 @@ import sys
import chroot
import failure_modes
import get_llvm_hash
+import update_chromeos_llvm_hash
import update_packages_and_run_tests
import update_tryjob_status
-import update_chromeos_llvm_hash
class ModifyTryjob(enum.Enum):
@@ -57,11 +57,10 @@ def GetCommandLineArgs():
# Add argument that determines which revision to search for in the list of
# tryjobs.
- parser.add_argument(
- '--revision',
- required=True,
- type=int,
- help='The revision to either remove or relaunch.')
+ parser.add_argument('--revision',
+ required=True,
+ type=int,
+ help='The revision to either remove or relaunch.')
# Add argument for other change lists that want to run alongside the tryjob.
parser.add_argument(
@@ -72,40 +71,38 @@ def GetCommandLineArgs():
'of updating the packages')
# Add argument for custom options for the tryjob.
- parser.add_argument(
- '--options',
- required=False,
- nargs='+',
- help='options to use for the tryjob testing')
+ parser.add_argument('--options',
+ required=False,
+ nargs='+',
+ help='options to use for the tryjob testing')
# Add argument for the builder to use for the tryjob.
- parser.add_argument('--builder', help='builder to use for the tryjob testing')
+ parser.add_argument('--builder',
+ help='builder to use for the tryjob testing')
# Add argument for a specific chroot path.
- parser.add_argument(
- '--chroot_path',
- default=cros_root,
- help='the path to the chroot (default: %(default)s)')
+ parser.add_argument('--chroot_path',
+ default=cros_root,
+ help='the path to the chroot (default: %(default)s)')
# Add argument for whether to display command contents to `stdout`.
- parser.add_argument(
- '--verbose',
- action='store_true',
- help='display contents of a command to the terminal '
- '(default: %(default)s)')
+ parser.add_argument('--verbose',
+ action='store_true',
+ help='display contents of a command to the terminal '
+ '(default: %(default)s)')
args_output = parser.parse_args()
- if not os.path.isfile(args_output.status_file) or \
- not args_output.status_file.endswith('.json'):
+ if (not os.path.isfile(args_output.status_file)
+ or not args_output.status_file.endswith('.json')):
raise ValueError('File does not exist or does not ending in ".json" '
': %s' % args_output.status_file)
- if args_output.modify_tryjob == ModifyTryjob.ADD.value and \
- not args_output.builder:
+ if (args_output.modify_tryjob == ModifyTryjob.ADD.value
+ and not args_output.builder):
raise ValueError('A builder is required for adding a tryjob.')
- elif args_output.modify_tryjob != ModifyTryjob.ADD.value and \
- args_output.builder:
+ elif (args_output.modify_tryjob != ModifyTryjob.ADD.value
+ and args_output.builder):
raise ValueError('Specifying a builder is only available when adding a '
'tryjob.')
@@ -234,13 +231,13 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls,
bisect_contents['jobs'][tryjob_index]['cl'],
bisect_contents['jobs'][tryjob_index]['extra_cls'],
bisect_contents['jobs'][tryjob_index]['options'],
- bisect_contents['jobs'][tryjob_index]['builder'], chroot_path, verbose)
+ bisect_contents['jobs'][tryjob_index]['builder'], chroot_path)
bisect_contents['jobs'][tryjob_index][
'status'] = update_tryjob_status.TryjobStatus.PENDING.value
bisect_contents['jobs'][tryjob_index]['link'] = tryjob_results[0]['link']
- bisect_contents['jobs'][tryjob_index]['buildbucket_id'] = tryjob_results[0][
- 'buildbucket_id']
+ bisect_contents['jobs'][tryjob_index]['buildbucket_id'] = tryjob_results[
+ 0]['buildbucket_id']
print('Successfully relaunched the tryjob for revision %d and updated '
'the tryjob link to %s' % (revision, tryjob_results[0]['link']))
@@ -253,17 +250,14 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls,
# Make sure the revision is within the bounds of the start and end of the
# bisection.
elif bisect_contents['start'] < revision < bisect_contents['end']:
- update_packages = [
- 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx',
- 'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind'
- ]
patch_metadata_file = 'PATCHES.json'
git_hash, revision = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption(
revision)
- tryjob_dict = AddTryjob(update_packages, git_hash, revision, chroot_path,
+ tryjob_dict = AddTryjob(update_chromeos_llvm_hash.DEFAULT_PACKAGES,
+ git_hash, revision, chroot_path,
patch_metadata_file, extra_cls, options, builder,
verbose, revision)
@@ -277,7 +271,10 @@ def PerformTryjobModification(revision, modify_tryjob, status_file, extra_cls,
modify_tryjob)
with open(status_file, 'w') as update_tryjobs:
- json.dump(bisect_contents, update_tryjobs, indent=4, separators=(',', ': '))
+ json.dump(bisect_contents,
+ update_tryjobs,
+ indent=4,
+ separators=(',', ': '))
def main():
@@ -290,9 +287,9 @@ def main():
PerformTryjobModification(args_output.revision,
ModifyTryjob(args_output.modify_tryjob),
args_output.status_file,
- args_output.extra_change_lists, args_output.options,
- args_output.builder, args_output.chroot_path,
- args_output.verbose)
+ args_output.extra_change_lists,
+ args_output.options, args_output.builder,
+ args_output.chroot_path, args_output.verbose)
if __name__ == '__main__':
diff --git a/llvm_tools/nightly_revert_checker.py b/llvm_tools/nightly_revert_checker.py
index 6941f3d6..5e878816 100755
--- a/llvm_tools/nightly_revert_checker.py
+++ b/llvm_tools/nightly_revert_checker.py
@@ -33,7 +33,7 @@ State = t.Any
def _find_interesting_android_shas(android_llvm_toolchain_dir: str
- ) -> t.List[t.Tuple[str, str]]:
+ ) -> t.List[t.Tuple[str, str]]:
llvm_project = os.path.join(android_llvm_toolchain_dir,
'toolchain/llvm-project')
@@ -44,7 +44,7 @@ def _find_interesting_android_shas(android_llvm_toolchain_dir: str
encoding='utf-8',
).strip()
- main_legacy = get_llvm_merge_base('aosp/master-legacy')
+ main_legacy = get_llvm_merge_base('aosp/master-legacy') # nocheck
testing_upstream = get_llvm_merge_base('aosp/testing-upstream')
result = [('main-legacy', main_legacy)]
@@ -55,8 +55,7 @@ def _find_interesting_android_shas(android_llvm_toolchain_dir: str
def _parse_llvm_ebuild_for_shas(ebuild_file: io.TextIOWrapper
- ) -> t.List[t.Tuple[str, str]]:
-
+ ) -> t.List[t.Tuple[str, str]]:
def parse_ebuild_assignment(line: str) -> str:
no_comments = line.split('#')[0]
no_assign = no_comments.split('=', 1)[1].strip()
@@ -84,12 +83,11 @@ def _parse_llvm_ebuild_for_shas(ebuild_file: io.TextIOWrapper
def _find_interesting_chromeos_shas(chromeos_base: str
- ) -> t.List[t.Tuple[str, str]]:
+ ) -> t.List[t.Tuple[str, str]]:
llvm_dir = os.path.join(chromeos_base,
'src/third_party/chromiumos-overlay/sys-devel/llvm')
candidate_ebuilds = [
- os.path.join(llvm_dir, x)
- for x in os.listdir(llvm_dir)
+ os.path.join(llvm_dir, x) for x in os.listdir(llvm_dir)
if '_pre' in x and not os.path.islink(os.path.join(llvm_dir, x))
]
@@ -197,8 +195,10 @@ def find_shas(llvm_dir: str, interesting_shas: t.List[t.Tuple[str, str]],
state: State, new_state: State):
for friendly_name, sha in interesting_shas:
logging.info('Finding reverts across %s (%s)', friendly_name, sha)
- all_reverts = revert_checker.find_reverts(
- llvm_dir, sha, root='origin/' + git_llvm_rev.MAIN_BRANCH)
+ all_reverts = revert_checker.find_reverts(llvm_dir,
+ sha,
+ root='origin/' +
+ git_llvm_rev.MAIN_BRANCH)
logging.info('Detected the following revert(s) across %s:\n%s',
friendly_name, pprint.pformat(all_reverts))
@@ -230,13 +230,12 @@ def do_cherrypick(chroot_path: str, llvm_dir: str,
seen.add(friendly_name)
for sha, reverted_sha in reverts:
try:
- get_upstream_patch.get_from_upstream(
- chroot_path=chroot_path,
- create_cl=True,
- start_sha=reverted_sha,
- patches=[sha],
- reviewers=reviewers,
- cc=cc)
+ get_upstream_patch.get_from_upstream(chroot_path=chroot_path,
+ create_cl=True,
+ start_sha=reverted_sha,
+ patches=[sha],
+ reviewers=reviewers,
+ cc=cc)
except get_upstream_patch.CherrypickError as e:
logging.info('%s, skipping...', str(e))
return new_state
@@ -245,7 +244,6 @@ def do_cherrypick(chroot_path: str, llvm_dir: str,
def do_email(is_dry_run: bool, llvm_dir: str, repository: str,
interesting_shas: t.List[t.Tuple[str, str]], state: State,
recipients: _EmailRecipients) -> State:
-
def prettify_sha(sha: str) -> tiny_render.Piece:
rev = get_llvm_hash.GetVersionFrom(llvm_dir, sha)
@@ -253,8 +251,8 @@ def do_email(is_dry_run: bool, llvm_dir: str, repository: str,
short_sha = sha[:12]
return tiny_render.Switch(
text=f'r{rev} ({short_sha})',
- html=tiny_render.Link(
- href='https://reviews.llvm.org/rG' + sha, inner='r' + str(rev)),
+ html=tiny_render.Link(href='https://reviews.llvm.org/rG' + sha,
+ inner='r' + str(rev)),
)
def get_sha_description(sha: str) -> tiny_render.Piece:
@@ -267,11 +265,12 @@ def do_email(is_dry_run: bool, llvm_dir: str, repository: str,
new_state: State = {}
for friendly_name, sha, new_reverts in find_shas(llvm_dir, interesting_shas,
state, new_state):
- email = _generate_revert_email(repository, friendly_name, sha, prettify_sha,
- get_sha_description, new_reverts)
+ email = _generate_revert_email(repository, friendly_name, sha,
+ prettify_sha, get_sha_description,
+ new_reverts)
if is_dry_run:
- logging.info('Would send email:\nSubject: %s\nBody:\n%s\n', email.subject,
- tiny_render.render_text_pieces(email.body))
+ logging.info('Would send email:\nSubject: %s\nBody:\n%s\n',
+ email.subject, tiny_render.render_text_pieces(email.body))
else:
logging.info('Sending email with subject %r...', email.subject)
_send_revert_email(recipients, email)
@@ -281,16 +280,19 @@ def do_email(is_dry_run: bool, llvm_dir: str, repository: str,
def parse_args(argv: t.List[str]) -> t.Any:
parser = argparse.ArgumentParser(
- description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
'action',
choices=['cherry-pick', 'email', 'dry-run'],
help='Automatically cherry-pick upstream reverts, send an email, or '
'write to stdout.')
- parser.add_argument(
- '--state_file', required=True, help='File to store persistent state in.')
- parser.add_argument(
- '--llvm_dir', required=True, help='Up-to-date LLVM directory to use.')
+ parser.add_argument('--state_file',
+ required=True,
+ help='File to store persistent state in.')
+ parser.add_argument('--llvm_dir',
+ required=True,
+ help='Up-to-date LLVM directory to use.')
parser.add_argument('--debug', action='store_true')
parser.add_argument(
'--reviewers',
@@ -309,8 +311,9 @@ def parse_args(argv: t.List[str]) -> t.Any:
subparsers.required = True
chromeos_subparser = subparsers.add_parser('chromeos')
- chromeos_subparser.add_argument(
- '--chromeos_dir', required=True, help='Up-to-date CrOS directory to use.')
+ chromeos_subparser.add_argument('--chromeos_dir',
+ required=True,
+ help='Up-to-date CrOS directory to use.')
android_subparser = subparsers.add_parser('android')
android_subparser.add_argument(
@@ -322,7 +325,7 @@ def parse_args(argv: t.List[str]) -> t.Any:
def find_chroot(opts: t.Any, reviewers: t.List[str], cc: t.List[str]
- ) -> t.Tuple[str, t.List[t.Tuple[str, str]], _EmailRecipients]:
+ ) -> t.Tuple[str, t.List[t.Tuple[str, str]], _EmailRecipients]:
recipients = reviewers + cc
if opts.repository == 'chromeos':
chroot_path = opts.chromeos_dir
@@ -335,9 +338,9 @@ def find_chroot(opts: t.Any, reviewers: t.List[str], cc: t.List[str]
chroot_path = opts.android_llvm_toolchain_dir
return (chroot_path, _find_interesting_android_shas(chroot_path),
- _EmailRecipients(
- well_known=[],
- direct=['android-llvm-dev@google.com'] + recipients))
+ _EmailRecipients(well_known=[],
+ direct=['android-llvm-dev@google.com'] +
+ recipients))
else:
raise ValueError(f'Unknown repository {opts.repository}')
@@ -366,21 +369,19 @@ def main(argv: t.List[str]) -> int:
# We want to be as free of obvious side-effects as possible in case something
# above breaks. Hence, action as late as possible.
if action == 'cherry-pick':
- new_state = do_cherrypick(
- chroot_path=chroot_path,
- llvm_dir=llvm_dir,
- interesting_shas=interesting_shas,
- state=state,
- reviewers=reviewers,
- cc=cc)
+ new_state = do_cherrypick(chroot_path=chroot_path,
+ llvm_dir=llvm_dir,
+ interesting_shas=interesting_shas,
+ state=state,
+ reviewers=reviewers,
+ cc=cc)
else:
- new_state = do_email(
- is_dry_run=action == 'dry-run',
- llvm_dir=llvm_dir,
- repository=repository,
- interesting_shas=interesting_shas,
- state=state,
- recipients=recipients)
+ new_state = do_email(is_dry_run=action == 'dry-run',
+ llvm_dir=llvm_dir,
+ repository=repository,
+ interesting_shas=interesting_shas,
+ state=state,
+ recipients=recipients)
_write_state(state_file, new_state)
return 0
diff --git a/llvm_tools/patch_sync/.gitignore b/llvm_tools/patch_sync/.gitignore
new file mode 100644
index 00000000..2f7896d1
--- /dev/null
+++ b/llvm_tools/patch_sync/.gitignore
@@ -0,0 +1 @@
+target/
diff --git a/llvm_tools/patch_sync/Cargo.lock b/llvm_tools/patch_sync/Cargo.lock
new file mode 100644
index 00000000..63a9fcf8
--- /dev/null
+++ b/llvm_tools/patch_sync/Cargo.lock
@@ -0,0 +1,453 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "aho-corasick"
+version = "0.7.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "ansi_term"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.51"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b26702f315f53b6071259e15dd9d64528213b44d61de1ec926eca7715d62203"
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "block-buffer"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "clap"
+version = "2.34.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
+dependencies = [
+ "ansi_term",
+ "atty",
+ "bitflags",
+ "strsim",
+ "textwrap",
+ "unicode-width",
+ "vec_map",
+]
+
+[[package]]
+name = "cpufeatures"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "digest"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "generic-array"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
+[[package]]
+name = "heck"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
+dependencies = [
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.112"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
+
+[[package]]
+name = "memchr"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
+
+[[package]]
+name = "opaque-debug"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
+
+[[package]]
+name = "patch_sync"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "rand",
+ "regex",
+ "serde",
+ "serde_json",
+ "sha2",
+ "structopt",
+ "time",
+]
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba"
+
+[[package]]
+name = "proc-macro-error"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f84e92c0f7c9d58328b85a78557813e4bd845130db68d7184635344399423b1"
+dependencies = [
+ "unicode-xid",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rand"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
+dependencies = [
+ "libc",
+ "rand_chacha",
+ "rand_core",
+ "rand_hc",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
+dependencies = [
+ "getrandom",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
+dependencies = [
+ "rand_core",
+]
+
+[[package]]
+name = "regex"
+version = "1.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
+
+[[package]]
+name = "ryu"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
+
+[[package]]
+name = "serde"
+version = "1.0.132"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b9875c23cf305cd1fd7eb77234cbb705f21ea6a72c637a5c6db5fe4b8e7f008"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.132"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ecc0db5cb2556c0e558887d9bbdcf6ac4471e83ff66cf696e5419024d1606276"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bcbd0344bc6533bc7ec56df11d42fb70f1b912351c0825ccb7211b59d8af7cf5"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "sha2"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa"
+dependencies = [
+ "block-buffer",
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+ "opaque-debug",
+]
+
+[[package]]
+name = "strsim"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
+
+[[package]]
+name = "structopt"
+version = "0.3.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40b9788f4202aa75c240ecc9c15c65185e6a39ccdeb0fd5d008b98825464c87c"
+dependencies = [
+ "clap",
+ "lazy_static",
+ "structopt-derive",
+]
+
+[[package]]
+name = "structopt-derive"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0"
+dependencies = [
+ "heck",
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.83"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23a1dfb999630e338648c83e91c59a4e9fb7620f520c3194b6b89e276f2f1959"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
+dependencies = [
+ "unicode-width",
+]
+
+[[package]]
+name = "time"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41effe7cfa8af36f439fac33861b66b049edc6f9a32331e2312660529c1c24ad"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "typenum"
+version = "1.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
+
+[[package]]
+name = "vec_map"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
+
+[[package]]
+name = "version_check"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
+
+[[package]]
+name = "wasi"
+version = "0.10.2+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
diff --git a/llvm_tools/patch_sync/Cargo.toml b/llvm_tools/patch_sync/Cargo.toml
new file mode 100644
index 00000000..43082627
--- /dev/null
+++ b/llvm_tools/patch_sync/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+name = "patch_sync"
+version = "0.1.0"
+authors = ["Jordan R Abrahams-Whitehead <ajordanr@google.com>"]
+edition = "2018"
+
+[profile.release]
+panic = "abort"
+
+[dependencies]
+anyhow = "1.0"
+regex = "1.5"
+serde = {version = "1.0", features = ["derive"]}
+serde_json = "1.0"
+sha2 = "0.9"
+structopt = "0.3"
+time = "0.3"
+
+[dev-dependencies]
+rand = "0.8"
diff --git a/llvm_tools/patch_sync/src/main.rs b/llvm_tools/patch_sync/src/main.rs
new file mode 100644
index 00000000..081ce01a
--- /dev/null
+++ b/llvm_tools/patch_sync/src/main.rs
@@ -0,0 +1,173 @@
+mod patch_parsing;
+mod version_control;
+
+use anyhow::{Context, Result};
+use std::path::PathBuf;
+use structopt::StructOpt;
+
+fn main() -> Result<()> {
+ match Opt::from_args() {
+ Opt::Show {
+ cros_checkout_path,
+ android_checkout_path,
+ sync,
+ } => show_subcmd(cros_checkout_path, android_checkout_path, sync),
+ Opt::Transpose {
+ cros_checkout_path,
+ old_cros_ref,
+ android_checkout_path,
+ old_android_ref,
+ sync,
+ verbose,
+ dry_run,
+ no_commit,
+ } => transpose_subcmd(TransposeOpt {
+ cros_checkout_path,
+ old_cros_ref,
+ android_checkout_path,
+ old_android_ref,
+ sync,
+ verbose,
+ dry_run,
+ no_commit,
+ }),
+ }
+}
+
+fn show_subcmd(
+ cros_checkout_path: PathBuf,
+ android_checkout_path: PathBuf,
+ sync: bool,
+) -> Result<()> {
+ let ctx = version_control::RepoSetupContext {
+ cros_checkout: cros_checkout_path,
+ android_checkout: android_checkout_path,
+ sync_before: sync,
+ };
+ ctx.setup()?;
+ let cros_patches_path = ctx.cros_patches_path();
+ let android_patches_path = ctx.android_patches_path();
+ let cur_cros_collection = patch_parsing::PatchCollection::parse_from_file(&cros_patches_path)
+ .context("could not parse cros PATCHES.json")?;
+ let cur_android_collection =
+ patch_parsing::PatchCollection::parse_from_file(&android_patches_path)
+ .context("could not parse android PATCHES.json")?;
+ let merged = cur_cros_collection.union(&cur_android_collection)?;
+ println!("{}", merged.serialize_patches()?);
+ Ok(())
+}
+
+#[allow(dead_code)]
+struct TransposeOpt {
+ cros_checkout_path: PathBuf,
+ old_cros_ref: String,
+ android_checkout_path: PathBuf,
+ old_android_ref: String,
+ sync: bool,
+ verbose: bool,
+ dry_run: bool,
+ no_commit: bool,
+}
+
+fn transpose_subcmd(args: TransposeOpt) -> Result<()> {
+ let ctx = version_control::RepoSetupContext {
+ cros_checkout: args.cros_checkout_path,
+ android_checkout: args.android_checkout_path,
+ sync_before: args.sync,
+ };
+ ctx.setup()?;
+ let cros_patches_path = ctx.cros_patches_path();
+ let android_patches_path = ctx.android_patches_path();
+
+ // Chromium OS Patches ----------------------------------------------------
+ let mut cur_cros_collection =
+ patch_parsing::PatchCollection::parse_from_file(&cros_patches_path)
+ .context("parsing cros PATCHES.json")?;
+ let new_cros_patches: patch_parsing::PatchCollection = {
+ let cros_old_patches_json = ctx.old_cros_patch_contents(&args.old_cros_ref)?;
+ let old_cros_collection = patch_parsing::PatchCollection::parse_from_str(
+ cros_patches_path.parent().unwrap().to_path_buf(),
+ &cros_old_patches_json,
+ )?;
+ cur_cros_collection.subtract(&old_cros_collection)?
+ };
+
+ // Android Patches -------------------------------------------------------
+ let mut cur_android_collection =
+ patch_parsing::PatchCollection::parse_from_file(&android_patches_path)
+ .context("parsing android PATCHES.json")?;
+ let new_android_patches: patch_parsing::PatchCollection = {
+ let android_old_patches_json = ctx.old_android_patch_contents(&args.old_android_ref)?;
+ let old_android_collection = patch_parsing::PatchCollection::parse_from_str(
+ android_patches_path.parent().unwrap().to_path_buf(),
+ &android_old_patches_json,
+ )?;
+ cur_android_collection.subtract(&old_android_collection)?
+ };
+
+ // Transpose Patches -----------------------------------------------------
+ new_cros_patches.transpose_write(&mut cur_cros_collection)?;
+ new_android_patches.transpose_write(&mut cur_android_collection)?;
+
+ if !args.no_commit {
+ return Ok(());
+ }
+ // Commit and upload for review ------------------------------------------
+ ctx.cros_repo_upload()
+ .context("uploading chromiumos changes")?;
+ ctx.android_repo_upload()
+ .context("uploading android changes")?;
+ Ok(())
+}
+
+#[derive(Debug, structopt::StructOpt)]
+#[structopt(name = "patch_sync", about = "A pipeline for syncing the patch code")]
+enum Opt {
+ /// Show a combined view of the PATCHES.json file, without making any changes.
+ #[allow(dead_code)]
+ Show {
+ #[structopt(parse(from_os_str))]
+ cros_checkout_path: PathBuf,
+ #[structopt(parse(from_os_str))]
+ android_checkout_path: PathBuf,
+ #[structopt(short, long)]
+ sync: bool,
+ },
+ /// Transpose patches from two PATCHES.json files
+ /// to each other.
+ Transpose {
+ /// Path to the ChromiumOS source repo checkout.
+ #[structopt(long = "cros-checkout", parse(from_os_str))]
+ cros_checkout_path: PathBuf,
+
+ /// Git ref (e.g. hash) for the ChromiumOS overlay to use as the base.
+ #[structopt(long = "overlay-base-ref")]
+ old_cros_ref: String,
+
+ /// Path to the Android Open Source Project source repo checkout.
+ #[structopt(long = "aosp-checkout", parse(from_os_str))]
+ android_checkout_path: PathBuf,
+
+ /// Git ref (e.g. hash) for the llvm_android repo to use as the base.
+ #[structopt(long = "aosp-base-ref")]
+ old_android_ref: String,
+
+ /// Run repo sync before transposing.
+ #[structopt(short, long)]
+ sync: bool,
+
+ /// Print information to stdout
+ #[structopt(short, long)]
+ verbose: bool,
+
+ /// Do not change any files. Useful in combination with `--verbose`
+ /// Implies `--no-commit` and `--no-upload`.
+ #[structopt(long)]
+ dry_run: bool,
+
+ /// Do not commit any changes made.
+ /// Implies `--no-upload`.
+ #[structopt(long)]
+ no_commit: bool,
+ },
+}
diff --git a/llvm_tools/patch_sync/src/patch_parsing.rs b/llvm_tools/patch_sync/src/patch_parsing.rs
new file mode 100644
index 00000000..733451ae
--- /dev/null
+++ b/llvm_tools/patch_sync/src/patch_parsing.rs
@@ -0,0 +1,313 @@
+use std::collections::{BTreeMap, BTreeSet};
+use std::fs::{copy, File};
+use std::io::{BufRead, BufReader, Read, Write};
+use std::path::{Path, PathBuf};
+
+use anyhow::{anyhow, Context, Result};
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+
+/// JSON serde struct.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct PatchDictSchema {
+ pub rel_patch_path: String,
+ pub start_version: Option<u64>,
+ pub end_version: Option<u64>,
+ pub platforms: BTreeSet<String>,
+ pub metadata: Option<BTreeMap<String, serde_json::Value>>,
+}
+
+/// Struct to keep track of patches and their relative paths.
+#[derive(Debug, Clone)]
+pub struct PatchCollection {
+ pub patches: Vec<PatchDictSchema>,
+ pub workdir: PathBuf,
+}
+
+impl PatchCollection {
+ /// Create a `PatchCollection` from a PATCHES.
+ pub fn parse_from_file(json_file: &Path) -> Result<Self> {
+ Ok(Self {
+ patches: serde_json::from_reader(File::open(json_file)?)?,
+ workdir: json_file
+ .parent()
+ .ok_or_else(|| anyhow!("failed to get json_file parent"))?
+ .to_path_buf(),
+ })
+ }
+
+ /// Create a `PatchCollection` from a string literal and a workdir.
+ pub fn parse_from_str(workdir: PathBuf, contents: &str) -> Result<Self> {
+ Ok(Self {
+ patches: serde_json::from_str(contents).context("parsing from str")?,
+ workdir,
+ })
+ }
+
+ #[allow(dead_code)]
+ /// Return true if the collection is tracking any patches.
+ pub fn is_empty(&self) -> bool {
+ self.patches.is_empty()
+ }
+
+ /// Compute the set-set subtraction, returning a new `PatchCollection` which
+ /// keeps the minuend's wordir.
+ pub fn subtract(&self, subtrahend: &Self) -> Result<Self> {
+ let mut new_patches = Vec::new();
+ // This is O(n^2) when it could be much faster, but n is always going to be less
+ // than 1k and speed is not important here.
+ for our_patch in &self.patches {
+ let found_in_sub = subtrahend.patches.iter().any(|sub_patch| {
+ let hash1 = subtrahend
+ .hash_from_rel_patch(sub_patch)
+ .expect("getting hash from subtrahend patch");
+ let hash2 = self
+ .hash_from_rel_patch(our_patch)
+ .expect("getting hash from our patch");
+ hash1 == hash2
+ });
+ if !found_in_sub {
+ new_patches.push(our_patch.clone());
+ }
+ }
+ Ok(Self {
+ patches: new_patches,
+ workdir: self.workdir.clone(),
+ })
+ }
+
+ pub fn union(&self, other: &Self) -> Result<Self> {
+ self.union_helper(
+ other,
+ |p| self.hash_from_rel_patch(p),
+ |p| other.hash_from_rel_patch(p),
+ )
+ }
+
+ fn union_helper(
+ &self,
+ other: &Self,
+ our_hash_f: impl Fn(&PatchDictSchema) -> Result<String>,
+ their_hash_f: impl Fn(&PatchDictSchema) -> Result<String>,
+ ) -> Result<Self> {
+ // 1. For all our patches:
+ // a. If there exists a matching patch hash from `other`:
+ // i. Create a new patch with merged platform info,
+ // ii. add the new patch to our new collection.
+ // iii. Mark the other patch as "merged"
+ // b. Otherwise, copy our patch to the new collection
+ // 2. For all unmerged patches from the `other`
+ // a. Copy their patch into the new collection
+ let mut combined_patches = Vec::new();
+ let mut other_merged = vec![false; other.patches.len()];
+
+ // 1.
+ for p in &self.patches {
+ let our_hash = our_hash_f(p)?;
+ let mut found = false;
+ // a.
+ for (idx, merged) in other_merged.iter_mut().enumerate() {
+ if !*merged {
+ let other_p = &other.patches[idx];
+ let their_hash = their_hash_f(other_p)?;
+ if our_hash == their_hash {
+ // i.
+ let new_platforms =
+ p.platforms.union(&other_p.platforms).cloned().collect();
+ // ii.
+ combined_patches.push(PatchDictSchema {
+ rel_patch_path: p.rel_patch_path.clone(),
+ start_version: p.start_version,
+ end_version: p.end_version,
+ platforms: new_platforms,
+ metadata: p.metadata.clone(),
+ });
+ // iii.
+ *merged = true;
+ found = true;
+ break;
+ }
+ }
+ }
+ // b.
+ if !found {
+ combined_patches.push(p.clone());
+ }
+ }
+ // 2.
+ // Add any remaining, other-only patches.
+ for (idx, merged) in other_merged.iter().enumerate() {
+ if !*merged {
+ combined_patches.push(other.patches[idx].clone());
+ }
+ }
+
+ Ok(Self {
+ workdir: self.workdir.clone(),
+ patches: combined_patches,
+ })
+ }
+
+ /// Copy all patches from this collection into another existing collection, and write that
+ /// to the existing collection's file.
+ pub fn transpose_write(&self, existing_collection: &mut Self) -> Result<()> {
+ for p in &self.patches {
+ let original_file_path = self.workdir.join(&p.rel_patch_path);
+ let copy_file_path = existing_collection.workdir.join(&p.rel_patch_path);
+ copy_create_parents(&original_file_path, &copy_file_path)?;
+ existing_collection.patches.push(p.clone());
+ }
+ existing_collection.write_patches_json("PATCHES.json")
+ }
+
+ /// Write out the patch collection contents to a PATCHES.json file.
+ fn write_patches_json(&self, filename: &str) -> Result<()> {
+ let write_path = self.workdir.join(filename);
+ let mut new_patches_file = File::create(&write_path)
+ .with_context(|| format!("writing to {}", write_path.display()))?;
+ new_patches_file.write_all(self.serialize_patches()?.as_bytes())?;
+ Ok(())
+ }
+
+ pub fn serialize_patches(&self) -> Result<String> {
+ let mut serialization_buffer = Vec::<u8>::new();
+ // Four spaces to indent json serialization.
+ let mut serializer = serde_json::Serializer::with_formatter(
+ &mut serialization_buffer,
+ serde_json::ser::PrettyFormatter::with_indent(b" "),
+ );
+ self.patches
+ .serialize(&mut serializer)
+ .context("serializing patches to JSON")?;
+ // Append a newline at the end if not present. This is necessary to get
+ // past some pre-upload hooks.
+ if serialization_buffer.last() != Some(&b'\n') {
+ serialization_buffer.push(b'\n');
+ }
+ Ok(std::str::from_utf8(&serialization_buffer)?.to_string())
+ }
+
+ fn hash_from_rel_patch(&self, patch: &PatchDictSchema) -> Result<String> {
+ hash_from_patch_path(&self.workdir.join(&patch.rel_patch_path))
+ }
+}
+
+/// Get the hash from the patch file contents.
+///
+/// Not every patch file actually contains its own hash,
+/// we must compute the hash ourselves when it's not found.
+fn hash_from_patch(patch_contents: impl Read) -> Result<String> {
+ let mut reader = BufReader::new(patch_contents);
+ let mut buf = String::new();
+ reader.read_line(&mut buf)?;
+ let mut first_line_iter = buf.trim().split(' ').fuse();
+ let (fst_word, snd_word) = (first_line_iter.next(), first_line_iter.next());
+ if let (Some("commit" | "From"), Some(hash_str)) = (fst_word, snd_word) {
+ // If the first line starts with either "commit" or "From", the following
+ // text is almost certainly a commit hash.
+ Ok(hash_str.to_string())
+ } else {
+ // This is an annoying case where the patch isn't actually a commit.
+ // So we'll hash the entire file, and hope that's sufficient.
+ let mut hasher = Sha256::new();
+ hasher.update(&buf); // Have to hash the first line.
+ reader.read_to_string(&mut buf)?;
+ hasher.update(buf); // Hash the rest of the file.
+ let sha = hasher.finalize();
+ Ok(format!("{:x}", &sha))
+ }
+}
+
+fn hash_from_patch_path(patch: &Path) -> Result<String> {
+ let f = File::open(patch)?;
+ hash_from_patch(f)
+}
+
+/// Copy a file from one path to another, and create any parent
+/// directories along the way.
+fn copy_create_parents(from: &Path, to: &Path) -> Result<()> {
+ let to_parent = to
+ .parent()
+ .with_context(|| format!("getting parent of {}", to.display()))?;
+ if !to_parent.exists() {
+ std::fs::create_dir_all(to_parent)?;
+ }
+
+ copy(&from, &to)
+ .with_context(|| format!("copying file from {} to {}", &from.display(), &to.display()))?;
+ Ok(())
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+
+ /// Test we can extract the hash from patch files.
+ #[test]
+ fn test_hash_from_patch() {
+ // Example git patch from Gerrit
+ let desired_hash = "004be4037e1e9c6092323c5c9268acb3ecf9176c";
+ let test_file_contents = "commit 004be4037e1e9c6092323c5c9268acb3ecf9176c\n\
+ Author: An Author <some_email>\n\
+ Date: Thu Aug 6 12:34:16 2020 -0700";
+ assert_eq!(
+ &hash_from_patch(test_file_contents.as_bytes()).unwrap(),
+ desired_hash
+ );
+
+ // Example git patch from upstream
+ let desired_hash = "6f85225ef3791357f9b1aa097b575b0a2b0dff48";
+ let test_file_contents = "From 6f85225ef3791357f9b1aa097b575b0a2b0dff48\n\
+ Mon Sep 17 00:00:00 2001\n\
+ From: Another Author <another_email>\n\
+ Date: Wed, 18 Aug 2021 15:03:03 -0700";
+ assert_eq!(
+ &hash_from_patch(test_file_contents.as_bytes()).unwrap(),
+ desired_hash
+ );
+ }
+
+ #[test]
+ fn test_union() {
+ let patch1 = PatchDictSchema {
+ start_version: Some(0),
+ end_version: Some(1),
+ rel_patch_path: "a".into(),
+ metadata: None,
+ platforms: BTreeSet::from(["x".into()]),
+ };
+ let patch2 = PatchDictSchema {
+ rel_patch_path: "b".into(),
+ platforms: BTreeSet::from(["x".into(), "y".into()]),
+ ..patch1.clone()
+ };
+ let patch3 = PatchDictSchema {
+ platforms: BTreeSet::from(["z".into(), "x".into()]),
+ ..patch1.clone()
+ };
+ let collection1 = PatchCollection {
+ workdir: PathBuf::new(),
+ patches: vec![patch1, patch2],
+ };
+ let collection2 = PatchCollection {
+ workdir: PathBuf::new(),
+ patches: vec![patch3],
+ };
+ let union = collection1
+ .union_helper(
+ &collection2,
+ |p| Ok(p.rel_patch_path.to_string()),
+ |p| Ok(p.rel_patch_path.to_string()),
+ )
+ .expect("could not create union");
+ assert_eq!(union.patches.len(), 2);
+ assert_eq!(
+ union.patches[0].platforms.iter().collect::<Vec<&String>>(),
+ vec!["x", "z"]
+ );
+ assert_eq!(
+ union.patches[1].platforms.iter().collect::<Vec<&String>>(),
+ vec!["x", "y"]
+ );
+ }
+}
diff --git a/llvm_tools/patch_sync/src/version_control.rs b/llvm_tools/patch_sync/src/version_control.rs
new file mode 100644
index 00000000..3dc5aae9
--- /dev/null
+++ b/llvm_tools/patch_sync/src/version_control.rs
@@ -0,0 +1,238 @@
+use anyhow::{anyhow, bail, ensure, Context, Result};
+use regex::Regex;
+use std::ffi::OsStr;
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Output};
+
+const CHROMIUMOS_OVERLAY_REL_PATH: &str = "src/third_party/chromiumos-overlay";
+const ANDROID_LLVM_REL_PATH: &str = "toolchain/llvm_android";
+
+/// Context struct to keep track of both Chromium OS and Android checkouts.
+#[derive(Debug)]
+pub struct RepoSetupContext {
+ pub cros_checkout: PathBuf,
+ pub android_checkout: PathBuf,
+ /// Run `repo sync` before doing any comparisons.
+ pub sync_before: bool,
+}
+
+impl RepoSetupContext {
+ pub fn setup(&self) -> Result<()> {
+ if self.sync_before {
+ repo_cd_cmd(&self.cros_checkout, &["sync", CHROMIUMOS_OVERLAY_REL_PATH])?;
+ repo_cd_cmd(&self.android_checkout, &["sync", ANDROID_LLVM_REL_PATH])?;
+ }
+ Ok(())
+ }
+
+ pub fn cros_repo_upload(&self) -> Result<()> {
+ let llvm_dir = self
+ .cros_checkout
+ .join(&CHROMIUMOS_OVERLAY_REL_PATH)
+ .join("sys-devel/llvm");
+ ensure!(
+ llvm_dir.is_dir(),
+ "CrOS LLVM dir {} is not a directory",
+ llvm_dir.display()
+ );
+ Self::rev_bump_llvm(&llvm_dir)?;
+ Self::repo_upload(
+ &self.cros_checkout,
+ CHROMIUMOS_OVERLAY_REL_PATH,
+ &Self::build_commit_msg("android", "chromiumos", "BUG=None\nTEST=CQ"),
+ )
+ }
+
+ pub fn android_repo_upload(&self) -> Result<()> {
+ Self::repo_upload(
+ &self.android_checkout,
+ ANDROID_LLVM_REL_PATH,
+ &Self::build_commit_msg("chromiumos", "android", "Test: N/A"),
+ )
+ }
+
+ fn repo_upload(path: &Path, git_wd: &str, commit_msg: &str) -> Result<()> {
+ // TODO(ajordanr): Need to clean up if there's any failures during upload.
+ let git_path = &path.join(&git_wd);
+ ensure!(
+ git_path.is_dir(),
+ "git_path {} is not a directory",
+ git_path.display()
+ );
+ repo_cd_cmd(path, &["start", "patch_sync_branch", git_wd])?;
+ git_cd_cmd(git_path, &["add", "."])?;
+ git_cd_cmd(git_path, &["commit", "-m", commit_msg])?;
+ repo_cd_cmd(path, &["upload", "-y", "--verify", git_wd])?;
+ Ok(())
+ }
+
+ pub fn android_patches_path(&self) -> PathBuf {
+ self.android_checkout
+ .join(&ANDROID_LLVM_REL_PATH)
+ .join("patches/PATCHES.json")
+ }
+
+ pub fn cros_patches_path(&self) -> PathBuf {
+ self.cros_checkout
+ .join(&CHROMIUMOS_OVERLAY_REL_PATH)
+ .join("sys-devel/llvm/files/PATCHES.json")
+ }
+
+ /// Increment LLVM's revision number
+ fn rev_bump_llvm(llvm_dir: &Path) -> Result<PathBuf> {
+ let ebuild = find_ebuild(llvm_dir)
+ .with_context(|| format!("finding ebuild in {} to rev bump", llvm_dir.display()))?;
+ let ebuild_dir = ebuild.parent().unwrap();
+ let suffix_matcher = Regex::new(r"-r([0-9]+)\.ebuild").unwrap();
+ let ebuild_name = ebuild
+ .file_name()
+ .unwrap()
+ .to_str()
+ .ok_or_else(|| anyhow!("converting ebuild filename to utf-8"))?;
+ let new_path = if let Some(captures) = suffix_matcher.captures(ebuild_name) {
+ let full_suffix = captures.get(0).unwrap().as_str();
+ let cur_version = captures.get(1).unwrap().as_str().parse::<u32>().unwrap();
+ let new_filename =
+ ebuild_name.replace(full_suffix, &format!("-r{}.ebuild", cur_version + 1_u32));
+ let new_path = ebuild_dir.join(new_filename);
+ fs::rename(&ebuild, &new_path)?;
+ new_path
+ } else {
+ // File did not end in a revision. We should append -r1 to the end.
+ let new_filename = ebuild.file_stem().unwrap().to_string_lossy() + "-r1.ebuild";
+ let new_path = ebuild_dir.join(new_filename.as_ref());
+ fs::rename(&ebuild, &new_path)?;
+ new_path
+ };
+ Ok(new_path)
+ }
+
+ /// Return the contents of the old PATCHES.json from Chromium OS
+ #[allow(dead_code)]
+ pub fn old_cros_patch_contents(&self, hash: &str) -> Result<String> {
+ Self::old_file_contents(
+ hash,
+ &self.cros_checkout.join(CHROMIUMOS_OVERLAY_REL_PATH),
+ Path::new("sys-devel/llvm/files/PATCHES.json"),
+ )
+ }
+
+ /// Return the contents of the old PATCHES.json from android
+ #[allow(dead_code)]
+ pub fn old_android_patch_contents(&self, hash: &str) -> Result<String> {
+ Self::old_file_contents(
+ hash,
+ &self.android_checkout.join(ANDROID_LLVM_REL_PATH),
+ Path::new("patches/PATCHES.json"),
+ )
+ }
+
+ /// Return the contents of an old file in git
+ #[allow(dead_code)]
+ fn old_file_contents(hash: &str, pwd: &Path, file: &Path) -> Result<String> {
+ let git_ref = format!(
+ "{}:{}",
+ hash,
+ file.to_str()
+ .ok_or_else(|| anyhow!("failed to convert filepath to str"))?
+ );
+ let output = git_cd_cmd(pwd, &["show", &git_ref])?;
+ if !output.status.success() {
+ bail!("could not get old file contents for {}", &git_ref)
+ }
+ String::from_utf8(output.stdout)
+ .with_context(|| format!("converting {} file contents to UTF-8", &git_ref))
+ }
+
+ /// Create the commit message
+ fn build_commit_msg(from: &str, to: &str, footer: &str) -> String {
+ format!(
+ "[patch_sync] Synchronize patches from {}\n\n\
+ Copies new PATCHES.json changes from {} to {}\n\n{}",
+ from, from, to, footer
+ )
+ }
+}
+
+/// Return the path of an ebuild located within the given directory.
+fn find_ebuild(dir: &Path) -> Result<PathBuf> {
+ // TODO(ajordanr): Maybe use OnceCell for this regex?
+ let ebuild_matcher = Regex::new(r"(-r[0-9]+)?\.ebuild").unwrap();
+ for entry in fs::read_dir(dir)? {
+ let path = entry?.path();
+ if let Some(name) = path.file_name() {
+ if ebuild_matcher.is_match(
+ name.to_str()
+ .ok_or_else(|| anyhow!("converting filepath to UTF-8"))?,
+ ) {
+ return Ok(path);
+ }
+ }
+ }
+ bail!("could not find ebuild")
+}
+
+/// Run a given git command from inside a specified git dir.
+pub fn git_cd_cmd<I, S>(pwd: &Path, args: I) -> Result<Output>
+where
+ I: IntoIterator<Item = S>,
+ S: AsRef<OsStr>,
+{
+ let output = Command::new("git").current_dir(&pwd).args(args).output()?;
+ if !output.status.success() {
+ bail!("git command failed")
+ }
+ Ok(output)
+}
+
+pub fn repo_cd_cmd<I, S>(pwd: &Path, args: I) -> Result<()>
+where
+ I: IntoIterator<Item = S>,
+ S: AsRef<OsStr>,
+{
+ let status = Command::new("repo").current_dir(&pwd).args(args).status()?;
+ if !status.success() {
+ bail!("repo command failed")
+ }
+ Ok(())
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+ use rand::prelude::Rng;
+ use std::env;
+ use std::fs::File;
+
+ #[test]
+ fn test_revbump_ebuild() {
+ // Random number to append at the end of the test folder to prevent conflicts.
+ let rng: u32 = rand::thread_rng().gen();
+ let llvm_dir = env::temp_dir().join(format!("patch_sync_test_{}", rng));
+ fs::create_dir(&llvm_dir).expect("creating llvm dir in temp directory");
+
+ {
+ // With revision
+ let ebuild_name = "llvm-13.0_pre433403_p20211019-r10.ebuild";
+ let ebuild_path = llvm_dir.join(ebuild_name);
+ File::create(&ebuild_path).expect("creating test ebuild file");
+ let new_ebuild_path =
+ RepoSetupContext::rev_bump_llvm(&llvm_dir).expect("rev bumping the ebuild");
+ assert!(new_ebuild_path.ends_with("llvm-13.0_pre433403_p20211019-r11.ebuild"));
+ fs::remove_file(new_ebuild_path).expect("removing renamed ebuild file");
+ }
+ {
+ // Without revision
+ let ebuild_name = "llvm-13.0_pre433403_p20211019.ebuild";
+ let ebuild_path = llvm_dir.join(ebuild_name);
+ File::create(&ebuild_path).expect("creating test ebuild file");
+ let new_ebuild_path =
+ RepoSetupContext::rev_bump_llvm(&llvm_dir).expect("rev bumping the ebuild");
+ assert!(new_ebuild_path.ends_with("llvm-13.0_pre433403_p20211019-r1.ebuild"));
+ fs::remove_file(new_ebuild_path).expect("removing renamed ebuild file");
+ }
+
+ fs::remove_dir(&llvm_dir).expect("removing temp test dir");
+ }
+}
diff --git a/llvm_tools/update_chromeos_llvm_hash.py b/llvm_tools/update_chromeos_llvm_hash.py
index ab83321a..4e9b9104 100755
--- a/llvm_tools/update_chromeos_llvm_hash.py
+++ b/llvm_tools/update_chromeos_llvm_hash.py
@@ -25,6 +25,15 @@ import get_llvm_hash
import git
import llvm_patch_management
+DEFAULT_PACKAGES = [
+ 'dev-util/lldb-server',
+ 'sys-devel/llvm',
+ 'sys-libs/compiler-rt',
+ 'sys-libs/libcxx',
+ 'sys-libs/libcxxabi',
+ 'sys-libs/llvm-libunwind',
+]
+
# Specify which LLVM hash to update
class LLVMVariant(enum.Enum):
@@ -69,26 +78,23 @@ def GetCommandLineArgs():
description="Updates the build's hash for llvm-next.")
# Add argument for a specific chroot path.
- parser.add_argument(
- '--chroot_path',
- default=defaultCrosRoot(),
- help='the path to the chroot (default: %(default)s)')
+ parser.add_argument('--chroot_path',
+ default=defaultCrosRoot(),
+ help='the path to the chroot (default: %(default)s)')
# Add argument for specific builds to uprev and update their llvm-next hash.
- parser.add_argument(
- '--update_packages',
- default=['sys-devel/llvm'],
- required=False,
- nargs='+',
- help='the ebuilds to update their hash for llvm-next '
- '(default: %(default)s)')
+ parser.add_argument('--update_packages',
+ default=DEFAULT_PACKAGES,
+ required=False,
+ nargs='+',
+ help='the ebuilds to update their hash for llvm-next '
+ '(default: %(default)s)')
# Add argument for whether to display command contents to `stdout`.
- parser.add_argument(
- '--verbose',
- action='store_true',
- help='display contents of a command to the terminal '
- '(default: %(default)s)')
+ parser.add_argument('--verbose',
+ action='store_true',
+ help='display contents of a command to the terminal '
+ '(default: %(default)s)')
# Add argument for the LLVM hash to update
parser.add_argument(
@@ -314,9 +320,11 @@ def UprevEbuildToVersion(symlink, svn_version, git_hash):
count=1)
# any other package
else:
- new_ebuild, is_changed = re.subn(
- r'(\d+)\.(\d+)_pre([0-9]+)',
- '%s.\\2_pre%s' % (llvm_major_version, svn_version), ebuild, count=1)
+ new_ebuild, is_changed = re.subn(r'(\d+)\.(\d+)_pre([0-9]+)',
+ '%s.\\2_pre%s' %
+ (llvm_major_version, svn_version),
+ ebuild,
+ count=1)
if not is_changed: # failed to increment the revision number
raise ValueError('Failed to uprev the ebuild.')
@@ -397,7 +405,8 @@ def StagePatchMetadataFileForCommit(patch_metadata_file_path):
# Cmd to stage the patch metadata file for commit.
subprocess.check_output([
'git', '-C',
- os.path.dirname(patch_metadata_file_path), 'add', patch_metadata_file_path
+ os.path.dirname(patch_metadata_file_path), 'add',
+ patch_metadata_file_path
])
@@ -419,9 +428,9 @@ def StagePackagesPatchResultsForCommit(package_info_dict, commit_messages):
# changed, if so, add which patches have changed to the commit
# message.
for package_name, patch_info_dict in package_info_dict.items():
- if (patch_info_dict['disabled_patches'] or
- patch_info_dict['removed_patches'] or
- patch_info_dict['modified_metadata']):
+ if (patch_info_dict['disabled_patches']
+ or patch_info_dict['removed_patches']
+ or patch_info_dict['modified_metadata']):
cur_package_header = '\nFor the package %s:' % package_name
commit_messages.append(cur_package_header)
@@ -600,16 +609,16 @@ def main():
git_hash, svn_version = get_llvm_hash.GetLLVMHashAndVersionFromSVNOption(
git_hash_source)
- change_list = UpdatePackages(
- args_output.update_packages,
- llvm_variant,
- git_hash,
- svn_version,
- args_output.chroot_path,
- args_output.patch_metadata_file,
- failure_modes.FailureModes(args_output.failure_mode),
- git_hash_source,
- extra_commit_msg=None)
+ change_list = UpdatePackages(args_output.update_packages,
+ llvm_variant,
+ git_hash,
+ svn_version,
+ args_output.chroot_path,
+ args_output.patch_metadata_file,
+ failure_modes.FailureModes(
+ args_output.failure_mode),
+ git_hash_source,
+ extra_commit_msg=None)
print('Successfully updated packages to %s (%d)' % (git_hash, svn_version))
print('Gerrit URL: %s' % change_list.url)
diff --git a/llvm_tools/update_packages_and_run_tests.py b/llvm_tools/update_packages_and_run_tests.py
index dd01253e..2e4a9058 100755
--- a/llvm_tools/update_packages_and_run_tests.py
+++ b/llvm_tools/update_packages_and_run_tests.py
@@ -51,10 +51,9 @@ def GetCommandLineArgs():
'of updating the packages')
# Add argument for a specific chroot path.
- parser.add_argument(
- '--chroot_path',
- default=cros_root,
- help='the path to the chroot (default: %(default)s)')
+ parser.add_argument('--chroot_path',
+ default=cros_root,
+ help='the path to the chroot (default: %(default)s)')
# Add argument to choose between llvm and llvm-next.
parser.add_argument(
@@ -71,65 +70,58 @@ def GetCommandLineArgs():
'arguments.')
# Add argument for the LLVM version to use.
- parser.add_argument(
- '--llvm_version',
- type=get_llvm_hash.IsSvnOption,
- required=True,
- help='which git hash of LLVM to find '
- '{google3, ToT, <svn_version>} '
- '(default: finds the git hash of the google3 LLVM '
- 'version)')
+ parser.add_argument('--llvm_version',
+ type=get_llvm_hash.IsSvnOption,
+ required=True,
+ help='which git hash of LLVM to find '
+ '{google3, ToT, <svn_version>} '
+ '(default: finds the git hash of the google3 LLVM '
+ 'version)')
# Add argument to add reviewers for the created CL.
- parser.add_argument(
- '--reviewers',
- nargs='+',
- default=[],
- help='The reviewers for the package update changelist')
+ parser.add_argument('--reviewers',
+ nargs='+',
+ default=[],
+ help='The reviewers for the package update changelist')
# Add argument for whether to display command contents to `stdout`.
- parser.add_argument(
- '--verbose',
- action='store_true',
- help='display contents of a command to the terminal '
- '(default: %(default)s)')
+ parser.add_argument('--verbose',
+ action='store_true',
+ help='display contents of a command to the terminal '
+ '(default: %(default)s)')
subparsers = parser.add_subparsers(dest='subparser_name')
subparser_names = []
# Testing with the tryjobs.
tryjob_subparser = subparsers.add_parser('tryjobs')
subparser_names.append('tryjobs')
- tryjob_subparser.add_argument(
- '--builders',
- required=True,
- nargs='+',
- default=[],
- help='builders to use for the tryjob testing')
+ tryjob_subparser.add_argument('--builders',
+ required=True,
+ nargs='+',
+ default=[],
+ help='builders to use for the tryjob testing')
# Add argument for custom options for the tryjob.
- tryjob_subparser.add_argument(
- '--options',
- required=False,
- nargs='+',
- default=[],
- help='options to use for the tryjob testing')
+ tryjob_subparser.add_argument('--options',
+ required=False,
+ nargs='+',
+ default=[],
+ help='options to use for the tryjob testing')
# Testing with the recipe builders
recipe_subparser = subparsers.add_parser('recipe')
subparser_names.append('recipe')
- recipe_subparser.add_argument(
- '--options',
- required=False,
- nargs='+',
- default=[],
- help='options passed to the recipe builders')
-
- recipe_subparser.add_argument(
- '--builders',
- required=True,
- nargs='+',
- default=[],
- help='recipe builders to launch')
+ recipe_subparser.add_argument('--options',
+ required=False,
+ nargs='+',
+ default=[],
+ help='options passed to the recipe builders')
+
+ recipe_subparser.add_argument('--builders',
+ required=True,
+ nargs='+',
+ default=[],
+ help='recipe builders to launch')
# Testing with CQ.
cq_subparser = subparsers.add_parser('cq')
@@ -360,7 +352,8 @@ def GetCQDependString(dependent_cls):
return None
# Cq-Depend must start a new paragraph prefixed with "Cq-Depend".
- return '\nCq-Depend: ' + ', '.join(('chromium:%s' % i) for i in dependent_cls)
+ return '\nCq-Depend: ' + ', '.join(
+ ('chromium:%s' % i) for i in dependent_cls)
def GetCQIncludeTrybotsString(trybot):
@@ -402,11 +395,6 @@ def main():
args_output = GetCommandLineArgs()
- update_packages = [
- 'sys-devel/llvm', 'sys-libs/compiler-rt', 'sys-libs/libcxx',
- 'sys-libs/libcxxabi', 'sys-libs/llvm-libunwind'
- ]
-
patch_metadata_file = 'PATCHES.json'
svn_option = args_output.llvm_version
@@ -420,8 +408,8 @@ def main():
# If --last_tested is specified, check if the current run has the same
# arguments last time --last_tested is used.
if args_output.last_tested:
- chroot_file_paths = chroot.GetChrootEbuildPaths(args_output.chroot_path,
- update_packages)
+ chroot_file_paths = chroot.GetChrootEbuildPaths(
+ args_output.chroot_path, update_chromeos_llvm_hash.DEFAULT_PACKAGES)
arg_dict = {
'svn_version': svn_version,
'ebuilds': chroot_file_paths,
@@ -449,7 +437,7 @@ def main():
extra_commit_msg += cq_trybot_msg
change_list = update_chromeos_llvm_hash.UpdatePackages(
- update_packages,
+ update_chromeos_llvm_hash.DEFAULT_PACKAGES,
llvm_variant,
git_hash,
svn_version,
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-20 15:33:49 +0000