Add a flag to ignore patch application failures

Tested by corrupting one patch file, and it continued the build when
--continue-on-errors is supplied.

Part of b/328250257

Change-Id: Ic68393e877a1529342961d7dc2686d7b7af43406

3 files changed, 121 insertions, 25 deletions

diff --git a/do_build.py b/do_build.py
index c1c2a86..8408d47 100755
--- a/do_build.py
+++ b/do_build.py
@@ -35,6 +35,7 @@ import configs
 import hosts
 import paths
 import source_manager
+import toolchain_errors
 import timer
 import toolchains
 import utils
@@ -823,6 +824,12 @@ def parse_args():
         help='Skip applying local patches. This allows building a vanilla upstream version.')
 
     parser.add_argument(
+        '--continue-on-errors',
+        action='store_true',
+        default=False,
+        help='Continue build on error. This allows catching all errors at once.')
+
+    parser.add_argument(
         '--create-tar',
         action='store_true',
         default=False,
@@ -1006,10 +1013,15 @@ def main():
     else:
         logger().info('Tensorflow found: ' + paths.get_tensorflow_path())
 
+    build_errors : List[toolchain_errors.ToolchainError] = []
     # Clone sources to be built and apply patches.
     if not args.skip_source_setup:
-        source_manager.setup_sources(git_am=args.git_am, llvm_rev=args.llvm_rev,
-                                     skip_apply_patches=args.skip_apply_patches)
+        setup_source_result = source_manager.setup_sources(git_am=args.git_am,
+                                         llvm_rev=args.llvm_rev,
+                                         skip_apply_patches=args.skip_apply_patches,
+                                         continue_on_patch_errors=args.continue_on_errors)
+        if setup_source_result:
+            build_errors.append(setup_source_result)
 
     # Build the stage1 Clang for the build host
     instrumented = hosts.build_host().is_linux and args.build_instrumented
@@ -1181,6 +1193,9 @@ def main():
             with_runtimes=do_runtimes,
             create_tar=args.create_tar)
 
+    if build_errors:
+        logger().info(toolchain_errors.combine_toolchain_errors(build_errors))
+        return len(build_errors)
     return 0
 
 
diff --git a/source_manager.py b/source_manager.py
index db616e7..15b282f 100644
--- a/source_manager.py
+++ b/source_manager.py
@@ -19,6 +19,7 @@ Package to manage LLVM sources when building a toolchain.
 
 import logging
 from pathlib import Path
+from typing import List, Optional
 import os
 import re
 import shutil
@@ -27,6 +28,7 @@ import subprocess
 import sys
 
 import android_version
+from toolchain_errors import ToolchainErrorCode, ToolchainError
 import hosts
 import paths
 import utils
@@ -38,13 +40,13 @@ def logger():
 
 
 def apply_patches(source_dir, svn_version, patch_json, patch_dir, git_am,
-                  failure_mode='fail'):
+                  failure_mode):
     """Apply patches in $patch_dir/$patch_json to $source_dir.
 
     Invokes external/toolchain-utils/llvm_tools/patch_manager.py to apply the
     patches.
     """
-
+    assert failure_mode, "Invalid failure_mode"
     patch_manager_cmd = [
         sys.executable,
         str(paths.TOOLCHAIN_UTILS_DIR / 'llvm_tools' / 'patch_manager.py'),
@@ -62,12 +64,15 @@ def apply_patches(source_dir, svn_version, patch_json, patch_dir, git_am,
 
     return utils.check_output(patch_manager_cmd, cwd=patch_dir)
 
+class PatchInfo:
+    """Holds info for a round of patch applications."""
+    def __init__(self) -> None:
+        self.applied_patches = []
+        self.failed_patches = []
+        self.inapplicable_patches = []
 
-def write_source_info(source_dir: str, patch_output: str) -> None:
-    url_prefix = 'https://android.googlesource.com/toolchain/llvm_android/+/' +\
-        '{{scripts_sha}}'
-
-    def _get_subject(patch_file: Path) -> str:
+    @staticmethod
+    def get_subject(patch_file: Path) -> str:
         contents = patch_file.read_text()
         # Parse patch generated by `git format-patch`.
         matches = re.search('Subject: (.*)\n', contents)
@@ -83,47 +88,77 @@ def write_source_info(source_dir: str, patch_output: str) -> None:
         subject = matches.group(1)
         return subject
 
-
-    def _format_patch_line(patch_file: Path) -> str:
+    @staticmethod
+    def format_patch_line(patch_file: Path) -> str:
+        url_prefix = 'https://android.googlesource.com/toolchain/llvm_android/+/' +\
+            '{{scripts_sha}}'
         assert patch_file.is_file(), f"patch file doesn't exist: {patch_file}"
         patch_name = patch_file.name
         if re.match('([0-9a-f]+)(_v[0-9]+)?\.patch$', patch_name):
             url_suffix = '/patches/cherry/' + patch_name
-            link_text = _get_subject(patch_file)
+            link_text = PatchInfo.get_subject(patch_file)
         else:
             url_suffix = '/patches/' + patch_name
             link_text = patch_name
         return f'- [{link_text}]({url_prefix}{url_suffix})'
 
+def write_source_info(source_dir: str, pi: PatchInfo) -> None:
     output = []
     base_revision = android_version.get_git_sha()
     github_url = 'https://github.com/llvm/llvm-project/commits/' + base_revision
     output.append(f'Base revision: [{base_revision}]({github_url})')
     output.append('')
 
-    patch_lines = []
+    applied_patches = []
+    for patch in pi.applied_patches:
+        applied_patches.append(pi.format_patch_line(Path(patch)))
+
+    output.extend(sorted(applied_patches))
+
+    with open(paths.OUT_DIR / 'clang_source_info.md', 'w') as outfile:
+        outfile.write('\n'.join(output))
+
+
+def get_source_info(source_dir: str, patch_output: str) -> PatchInfo:
+    pi = PatchInfo()
     patches = patch_output.strip().splitlines()
     patches_iter = iter(patches)
     assert next(patches_iter) == 'The following patches applied successfully:'
+
+    applied = True # applied/successful patches
+    failed = False # failed patches
+    inapplicable = False # inapplicable patches
+
     while True:
         patch = next(patches_iter, None)
         # We may optionally have an empty line followed by patches that were not
         # applicable.
         if patch == '':
-            assert next(
-                patches_iter) == 'The following patches were not applicable:'
-            break
+            ni = next(patches_iter)
+            if ni == 'The following patches were not applicable:':
+                applied = False
+                failed = False
+                inapplicable = True
+                continue
+            if ni == 'The following patches failed to apply:':
+                applied = False
+                failed = True
+                inapplicable = False
+                continue
         elif patch is None:
             break
+        print("patch={},{},{}$$".format(patch, type(patch), len(patch)))
         assert patch.endswith('.patch')
-        patch_lines.append(_format_patch_line(Path(patch)))
+        if applied:
+            pi.applied_patches.append(patch)
+        if failed:
+            pi.failed_patches.append(patch)
+        if inapplicable:
+            pi.inapplicable_patches.append(patch)
 
-    output.extend(sorted(patch_lines))
-    with open(paths.OUT_DIR / 'clang_source_info.md', 'w') as outfile:
-        outfile.write('\n'.join(output))
+    return pi
 
-
-def setup_sources(git_am=False, llvm_rev=None, skip_apply_patches=False):
+def setup_sources(git_am=False, llvm_rev=None, skip_apply_patches=False, continue_on_patch_errors=False) -> Optional[ToolchainError]:
     """Setup toolchain sources into paths.LLVM_PATH.
 
     Copy toolchain/llvm-project into paths.LLVM_PATH or clone from upstream.
@@ -131,7 +166,8 @@ def setup_sources(git_am=False, llvm_rev=None, skip_apply_patches=False):
     toolchain/llvm_android/patches/PATCHES.json.  The function overwrites
     paths.LLVM_PATH only if necessary to avoid recompiles during incremental builds.
     """
-
+    # Return the error messages upon failure.
+    ret: Optional[ToolchainError] = []
     source_dir = paths.LLVM_PATH
     tmp_source_dir = source_dir.parent / (source_dir.name + '.tmp')
     if os.path.exists(tmp_source_dir):
@@ -191,10 +227,13 @@ def setup_sources(git_am=False, llvm_rev=None, skip_apply_patches=False):
     svn_version = android_version.get_svn_revision_number()
 
     if not skip_apply_patches:
+      failure_mode = 'continue' if continue_on_patch_errors else 'fail'
       patch_output = apply_patches(tmp_source_dir, svn_version, patch_json,
-                                   patch_dir, git_am)
+                                   patch_dir, git_am, failure_mode)
       logger().info(patch_output)
-      write_source_info(tmp_source_dir, patch_output)
+      pi = get_source_info(tmp_source_dir, patch_output)
+      write_source_info(tmp_source_dir, pi)
+      ret = ToolchainError(ToolchainErrorCode.PATCH_ERROR, str(pi.failed_patches))
 
     # Copy tmp_source_dir to source_dir if they are different.  This avoids
     # invalidating prior build outputs.
@@ -215,6 +254,7 @@ def setup_sources(git_am=False, llvm_rev=None, skip_apply_patches=False):
         shutil.rmtree(tmp_source_dir)
     remote, url = try_set_git_remote(source_dir)
     logger().info(f'git remote url: remote: {remote} url: {url}')
+    return ret
 
 
 def try_set_git_remote(source_dir):
diff --git a/toolchain_errors.py b/toolchain_errors.py
new file mode 100644
index 0000000..2700a0f
--- /dev/null
+++ b/toolchain_errors.py
@@ -0,0 +1,41 @@
+#
+# Copyright (C) 2024 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Report toolchain build errors"""
+
+from enum import Enum, auto
+from typing import List
+
+class ToolchainErrorCode(Enum):
+    UNKNOWN_ERROR = auto()
+    PATCH_ERROR = auto()
+    STAGE1_BUILD_ERROR = auto()
+    STAGE1_TEST_ERROR = auto()
+    STAGE2_BUILD_ERROR = auto()
+    STAGE2_TEST_ERROR = auto()
+    RUNTIME_BUILD_ERROR = auto()
+
+class ToolchainError:
+    code: ToolchainErrorCode
+    msg: str
+    def __init__(self, code: ToolchainErrorCode, msg: str) -> None:
+        self.code = code
+        self.msg = msg
+
+    def __repr__(self):
+        return repr(self.code.name + ':' + self.msg)
+
+def combine_toolchain_errors(errs: List[ToolchainError]) -> str:
+    return ',\n'.join(map(str, errs))