diff options
| -rw-r--r-- | cbuildbot/commands.py | 17 | ||||
| -rw-r--r-- | cbuildbot/failures_lib.py | 33 | ||||
| -rw-r--r-- | cbuildbot/prebuilts.py | 2 | ||||
| -rw-r--r-- | cbuildbot/validation_pool.py | 15 | ||||
| -rw-r--r-- | lib/parallel.py | 35 |
5 files changed, 22 insertions, 80 deletions
diff --git a/cbuildbot/commands.py b/cbuildbot/commands.py index a1f5540c7..50384e273 100644 --- a/cbuildbot/commands.py +++ b/cbuildbot/commands.py @@ -59,8 +59,7 @@ class SuiteTimedOut(failures_lib.TestLabFailure): # =========================== Command Helpers ================================= -def RunBuildScript(buildroot, cmd, chromite_cmd=False, possibly_flaky=False, - **kwargs): +def RunBuildScript(buildroot, cmd, chromite_cmd=False, **kwargs): """Run a build script, wrapping exceptions as needed. This wraps RunCommand(cmd, cwd=buildroot, **kwargs), adding extra logic to @@ -79,8 +78,6 @@ def RunBuildScript(buildroot, cmd, chromite_cmd=False, possibly_flaky=False, cmd: The command to run. chromite_cmd: Whether the command should be evaluated relative to the chromite/bin subdir of the |buildroot|. - possibly_flaky: Whether this failure is likely to fail occasionally due - to flakiness (e.g. network flakiness). kwargs: Optional args passed to RunCommand; see RunCommand for specifics. """ assert not kwargs.get('shell', False), 'Cannot execute shell commands' @@ -121,8 +118,7 @@ def RunBuildScript(buildroot, cmd, chromite_cmd=False, possibly_flaky=False, raise failures_lib.PackageBuildFailure(ex, cmd[0], failed_packages) # Looks like a generic failure. Raise a BuildScriptFailure. - raise failures_lib.BuildScriptFailure(ex, cmd[0], - possibly_flaky=possibly_flaky) + raise failures_lib.BuildScriptFailure(ex, cmd[0]) def GetInput(prompt): @@ -525,10 +521,8 @@ def RunTestSuite(buildroot, board, image_dir, results_dir, test_type, error = '%s exited with code %d' % (' '.join(cmd), result.returncode) with open(results_dir_in_chroot + '/failed_test_command', 'w') as failed: failed.write(error) - # We already retry VMTest inline, so we can assume that failures in VMTest - # are not flaky. - raise TestFailure('** VMTests failed with code %d **' - % result.returncode, possibly_flaky=False) + + raise TestFailure('** VMTests failed with code %d **' % result.returncode) def RunDevModeTest(buildroot, board, image_dir): @@ -789,8 +783,7 @@ def RunHWTestSuite(build, suite, board, pool=None, num=None, file_bugs=None, elif result.returncode in timeout_codes: raise SuiteTimedOut('** Suite timed out before completion **') elif result.returncode != 0: - raise TestFailure('** HWTest failed (code %d) **' % result.returncode, - possibly_flaky=True) + raise TestFailure('** HWTest failed (code %d) **' % result.returncode) def _GetAbortCQHWTestsURL(version, suite): diff --git a/cbuildbot/failures_lib.py b/cbuildbot/failures_lib.py index e7d2b7918..0e5b93668 100644 --- a/cbuildbot/failures_lib.py +++ b/cbuildbot/failures_lib.py @@ -10,7 +10,6 @@ import traceback from chromite.cbuildbot import portage_utilities from chromite.lib import cros_build_lib -from chromite.lib import timeout_util class StepFailure(Exception): @@ -24,16 +23,14 @@ class StepFailure(Exception): 3) __str__() should be brief enough to include in a Commit Queue failure message. """ - def __init__(self, message='', possibly_flaky=False): + def __init__(self, message=''): """Constructor. Args: message: An error message. - possibly_flaky: Whether this failure might be flaky. """ Exception.__init__(self, message) - self.possibly_flaky = possibly_flaky - self.args = (message, possibly_flaky) + self.args = (message,) def __str__(self): """Stringify the message.""" @@ -69,21 +66,19 @@ def CreateExceptInfo(exception, tb): class CompoundFailure(StepFailure): """An exception that contains a list of ExceptInfo objects.""" - def __init__(self, message='', exc_infos=None, possibly_flaky=False): + def __init__(self, message='', exc_infos=None): """Initializes an CompoundFailure instance. Args: message: A string describing the failure. exc_infos: A list of ExceptInfo objects. - possibly_flaky: Whether this failure might be flaky. """ self.exc_infos = exc_infos if exc_infos else [] if not message: # By default, print the type and string of each ExceptInfo object. message = '\n'.join(['%s: %s' % (e.type, e.str) for e in self.exc_infos]) - super(CompoundFailure, self).__init__(message=message, - possibly_flaky=possibly_flaky) + super(CompoundFailure, self).__init__(message=message) def ToFullMessage(self): """Returns a string with all information in self.exc_infos.""" @@ -178,19 +173,18 @@ class BuildScriptFailure(StepFailure): commands (e.g. build_packages) fail. """ - def __init__(self, exception, shortname, possibly_flaky=False): + def __init__(self, exception, shortname): """Construct a BuildScriptFailure object. Args: exception: A RunCommandError object. shortname: Short name for the command we're running. - possibly_flaky: Whether this failure might be flaky. """ - StepFailure.__init__(self, possibly_flaky=possibly_flaky) + StepFailure.__init__(self) assert isinstance(exception, cros_build_lib.RunCommandError) self.exception = exception self.shortname = shortname - self.args = (exception, shortname, possibly_flaky) + self.args = (exception, shortname) def __str__(self): """Summarize a build command failure briefly.""" @@ -292,19 +286,6 @@ class BuildFailureMessage(object): def __str__(self): return self.message - def MightBeFlakyFailure(self): - """Check if there is a good chance this is a flaky failure.""" - # We only consider a failed build to be flaky if there is only one failure, - # and that failure is a flaky failure. - flaky = False - if len(self.tracebacks) == 1: - # TimeoutErrors are often flaky. - exc = self.tracebacks[0].exception - if (isinstance(exc, StepFailure) and exc.possibly_flaky or - isinstance(exc, timeout_util.TimeoutError)): - flaky = True - return flaky - def MatchesFailureType(self, cls): """Check if all of the tracebacks match the specified failure type.""" for tb in self.tracebacks: diff --git a/cbuildbot/prebuilts.py b/cbuildbot/prebuilts.py index 093481b99..d97735e0d 100644 --- a/cbuildbot/prebuilts.py +++ b/cbuildbot/prebuilts.py @@ -194,4 +194,4 @@ def _UploadPrebuilts(buildroot, board, extra_args): cmd.extend(['--board', board]) cmd.extend(extra_args) - commands.RunBuildScript(buildroot, cmd, cwd=cwd, possibly_flaky=True) + commands.RunBuildScript(buildroot, cmd, cwd=cwd) diff --git a/cbuildbot/validation_pool.py b/cbuildbot/validation_pool.py index 50aa66413..41e94eb09 100644 --- a/cbuildbot/validation_pool.py +++ b/cbuildbot/validation_pool.py @@ -1083,19 +1083,6 @@ class CalculateSuspects(object): return suspects @classmethod - def _MightBeFlakyFailure(cls, messages): - """Check if there is a good chance this is a flaky failure. - - Args: - messages: A list of build failure messages, of type - BuildFailureMessage or of type NoneType. - """ - # We consider a failed commit queue run to be flaky if only one builder - # failed, and that failure is flaky. - return (len(messages) == 1 and messages[0] and - messages[0].MightBeFlakyFailure()) - - @classmethod def _FindPreviouslyFailedChanges(cls, candidates): """Find what changes that have previously failed the CQ. @@ -1213,8 +1200,6 @@ class CalculateSuspects(object): for message in messages): # If we are here, there are no None messages. suspects = cls._FindPackageBuildFailureSuspects(candidates, messages) - elif cls._MightBeFlakyFailure(messages): - suspects = cls._FindPreviouslyFailedChanges(changes) else: suspects.update(candidates) diff --git a/lib/parallel.py b/lib/parallel.py index 2e7b8dc05..f337e8c7f 100644 --- a/lib/parallel.py +++ b/lib/parallel.py @@ -30,7 +30,6 @@ from chromite.cbuildbot import failures_lib from chromite.cbuildbot import results_lib from chromite.lib import cros_build_lib from chromite.lib import osutils -from chromite.lib import timeout_util _BUFSIZE = 1024 @@ -227,13 +226,12 @@ class _BackgroundTask(multiprocessing.Process): with open(self._output.name, 'r') as output: pos = 0 running, exited_cleanly, task_errors, all_errors = (True, False, [], []) - possibly_flaky = False while running: # Check whether the process is still alive. running = self.is_alive() try: - errors, results, possibly_flaky = \ + errors, results = \ self._queue.get(True, self.PRINT_INTERVAL) if errors: task_errors.extend(errors) @@ -254,11 +252,8 @@ class _BackgroundTask(multiprocessing.Process): failures_lib.CreateExceptInfo(ProcessExitTimeout(msg), '')) self._KillChildren([self]) elif not exited_cleanly: - # Treat SIGKILL signals as potentially flaky. - if self.exitcode == -signal.SIGKILL: - possibly_flaky = True - msg = ('%r exited unexpectedly with code %s' % - (self, self.exitcode)) + msg = ('%r exited unexpectedly with code %s' + % (self, self.exitcode)) all_errors.extend( failures_lib.CreateExceptInfo(ProcessUnexpectedExit(msg), '')) @@ -275,9 +270,6 @@ class _BackgroundTask(multiprocessing.Process): failures_lib.CreateExceptInfo(ProcessSilentTimeout(msg), '')) self._KillChildren([self]) - # Timeouts are possibly flaky. - possibly_flaky = True - # Read remaining output from the process. output.seek(pos) buf = output.read(_BUFSIZE) @@ -309,7 +301,7 @@ class _BackgroundTask(multiprocessing.Process): self.Cleanup(silent=True) # If an error occurred, return it. - return all_errors, possibly_flaky + return all_errors def start(self): """Invoke multiprocessing.Process.start after flushing output/err.""" @@ -330,14 +322,13 @@ class _BackgroundTask(multiprocessing.Process): errors = failures_lib.CreateExceptInfo( UnexpectedException('Unexpected exception in %r' % self), '') - possibly_flaky = False pid = os.getpid() try: - errors, possibly_flaky = self._Run() + errors = self._Run() finally: if not self._killing.is_set() and os.getpid() == pid: results = results_lib.Results.Get() - self._queue.put((errors, results, possibly_flaky)) + self._queue.put((errors, results)) if self._semaphore is not None: self._semaphore.release() @@ -354,7 +345,6 @@ class _BackgroundTask(multiprocessing.Process): sys.stdout.flush() sys.stderr.flush() errors = [] - possibly_flaky = False # Send all output to a named temporary file. with open(self._output.name, 'w', 0) as output: # Back up sys.std{err,out}. These aren't used, but we keep a copy so @@ -382,18 +372,16 @@ class _BackgroundTask(multiprocessing.Process): except failures_lib.StepFailure as ex: errors.extend(failures_lib.CreateExceptInfo( ex, traceback.format_exc())) - possibly_flaky = ex.possibly_flaky except BaseException as ex: errors.extend(failures_lib.CreateExceptInfo( ex, traceback.format_exc())) - possibly_flaky = isinstance(ex, timeout_util.TimeoutError) if self._killing.is_set(): traceback.print_exc() finally: sys.stdout.flush() sys.stderr.flush() - return errors, possibly_flaky + return errors @classmethod def _KillChildren(cls, bg_tasks, log_level=logging.WARNING): @@ -459,7 +447,6 @@ class _BackgroundTask(multiprocessing.Process): """ semaphore = None - possibly_flaky = False if max_parallel is not None: semaphore = multiprocessing.Semaphore(max_parallel) @@ -476,12 +463,10 @@ class _BackgroundTask(multiprocessing.Process): finally: # Wait for each step to complete. errors = [] - flaky_tasks = [] while bg_tasks: task = bg_tasks.popleft() - task_errors, possibly_flaky = task.Wait() + task_errors = task.Wait() if task_errors: - flaky_tasks.append(possibly_flaky) errors.extend(task_errors) if halt_on_error: break @@ -492,9 +477,7 @@ class _BackgroundTask(multiprocessing.Process): # Propagate any exceptions. if errors: - possibly_flaky = flaky_tasks and all(flaky_tasks) - raise BackgroundFailure(exc_infos=errors, - possibly_flaky=possibly_flaky) + raise BackgroundFailure(exc_infos=errors) @staticmethod def TaskRunner(queue, task, onexit=None, task_args=None, task_kwargs=None): |