From 88b8d78c5da4a385ee42ca699ba400a5306ef111 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Tue, 13 May 2014 17:30:55 -0700 Subject: Rename chromite.buildbot -> chromite.cbuildbot. The cbuildbot script lives in a directory named buildbot. Which is a bit confusing. BUG=chromium:373277 TEST=Unitests + cros lint. CQ-DEPEND=CL:200157 CQ-DEPEND=CL:200165 CQ-DEPEND=CL:200149 CQ-DEPEND=CL:*163598 CQ-DEPEND=CL:*163760 CQ-DEPEND=CL:*163786 Change-Id: Ia5953a4506e8b47d27e1a6908ecb938a439da8c2 Reviewed-on: https://chromium-review.googlesource.com/199664 Reviewed-by: Don Garrett Commit-Queue: Don Garrett Tested-by: Don Garrett --- cbuildbot/stages/sync_stages.py | 981 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 981 insertions(+) create mode 100644 cbuildbot/stages/sync_stages.py (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py new file mode 100644 index 000000000..282284643 --- /dev/null +++ b/cbuildbot/stages/sync_stages.py @@ -0,0 +1,981 @@ +# Copyright (c) 2013 The Chromium OS Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Module containing the sync stages.""" + +import contextlib +import datetime +import logging +import os +import sys +from xml.etree import ElementTree + +from chromite.cbuildbot import cbuildbot_config +from chromite.cbuildbot import cbuildbot_failures as failures_lib +from chromite.cbuildbot import constants +from chromite.cbuildbot import lkgm_manager +from chromite.cbuildbot import manifest_version +from chromite.cbuildbot import repository +from chromite.cbuildbot import trybot_patch_pool +from chromite.cbuildbot import validation_pool +from chromite.cbuildbot.stages import generic_stages +from chromite.cbuildbot.stages import build_stages +from chromite.lib import commandline +from chromite.lib import cros_build_lib +from chromite.lib import git +from chromite.lib import osutils +from chromite.lib import patch as cros_patch +from chromite.lib import timeout_util + + +PRE_CQ = validation_pool.PRE_CQ + + +class PatchChangesStage(generic_stages.BuilderStage): + """Stage that patches a set of Gerrit changes to the buildroot source tree.""" + + def __init__(self, builder_run, patch_pool, **kwargs): + """Construct a PatchChangesStage. + + Args: + builder_run: BuilderRun object. + patch_pool: A TrybotPatchPool object containing the different types of + patches to apply. + """ + super(PatchChangesStage, self).__init__(builder_run, **kwargs) + self.patch_pool = patch_pool + + @staticmethod + def _CheckForDuplicatePatches(_series, changes): + conflicts = {} + duplicates = [] + for change in changes: + if change.id is None: + cros_build_lib.Warning( + "Change %s lacks a usable ChangeId; duplicate checking cannot " + "be done for this change. If cherry-picking fails, this is a " + "potential cause.", change) + continue + conflicts.setdefault(change.id, []).append(change) + + duplicates = [x for x in conflicts.itervalues() if len(x) > 1] + if not duplicates: + return changes + + for conflict in duplicates: + cros_build_lib.Error( + "Changes %s conflict with each other- they have same id %s.", + ', '.join(map(str, conflict)), conflict[0].id) + + cros_build_lib.Die("Duplicate patches were encountered: %s", duplicates) + + def _PatchSeriesFilter(self, series, changes): + return self._CheckForDuplicatePatches(series, changes) + + def _ApplyPatchSeries(self, series, patch_pool, **kwargs): + """Applies a patch pool using a patch series.""" + kwargs.setdefault('frozen', False) + # Honor the given ordering, so that if a gerrit/remote patch + # conflicts w/ a local patch, the gerrit/remote patch are + # blamed rather than local (patch ordering is typically + # local, gerrit, then remote). + kwargs.setdefault('honor_ordering', True) + kwargs['changes_filter'] = self._PatchSeriesFilter + + _applied, failed_tot, failed_inflight = series.Apply( + list(patch_pool), **kwargs) + + failures = failed_tot + failed_inflight + if failures: + self.HandleApplyFailures(failures) + + def HandleApplyFailures(self, failures): + cros_build_lib.Die("Failed applying patches: %s", + "\n".join(map(str, failures))) + + def PerformStage(self): + class NoisyPatchSeries(validation_pool.PatchSeries): + """Custom PatchSeries that adds links to buildbot logs for remote trys.""" + + def ApplyChange(self, change, dryrun=False): + if isinstance(change, cros_patch.GerritPatch): + cros_build_lib.PrintBuildbotLink(str(change), change.url) + elif isinstance(change, cros_patch.UploadedLocalPatch): + cros_build_lib.PrintBuildbotStepText(str(change)) + + return validation_pool.PatchSeries.ApplyChange(self, change, + dryrun=dryrun) + + # If we're an external builder, ignore internal patches. + helper_pool = validation_pool.HelperPool.SimpleCreate( + cros_internal=self._run.config.internal, cros=True) + + # Limit our resolution to non-manifest patches. + patch_series = NoisyPatchSeries( + self._build_root, + force_content_merging=True, + helper_pool=helper_pool, + deps_filter_fn=lambda p: not trybot_patch_pool.ManifestFilter(p)) + + self._ApplyPatchSeries(patch_series, self.patch_pool) + + +class BootstrapStage(PatchChangesStage): + """Stage that patches a chromite repo and re-executes inside it. + + Attributes: + returncode - the returncode of the cbuildbot re-execution. Valid after + calling stage.Run(). + """ + option_name = 'bootstrap' + + def __init__(self, builder_run, chromite_patch_pool, + manifest_patch_pool=None, **kwargs): + super(BootstrapStage, self).__init__( + builder_run, trybot_patch_pool.TrybotPatchPool(), **kwargs) + self.chromite_patch_pool = chromite_patch_pool + self.manifest_patch_pool = manifest_patch_pool + self.returncode = None + + def _ApplyManifestPatches(self, patch_pool): + """Apply a pool of manifest patches to a temp manifest checkout. + + Args: + patch_pool: The pool to apply. + + Returns: + The path to the patched manifest checkout. + + Raises: + Exception, if the new patched manifest cannot be parsed. + """ + checkout_dir = os.path.join(self.tempdir, 'manfest-checkout') + repository.CloneGitRepo(checkout_dir, + self._run.config.manifest_repo_url) + + patch_series = validation_pool.PatchSeries.WorkOnSingleRepo( + checkout_dir, tracking_branch=self._run.manifest_branch) + + self._ApplyPatchSeries(patch_series, patch_pool) + # Create the branch that 'repo init -b -u ' + # will look for. + cmd = ['branch', '-f', self._run.manifest_branch, + constants.PATCH_BRANCH] + git.RunGit(checkout_dir, cmd) + + # Verify that the patched manifest loads properly. Propagate any errors as + # exceptions. + manifest = os.path.join(checkout_dir, self._run.config.manifest) + git.Manifest.Cached(manifest, manifest_include_dir=checkout_dir) + return checkout_dir + + @staticmethod + def _FilterArgsForApi(parsed_args, api_minor): + """Remove arguments that are introduced after an api version.""" + def filter_fn(passed_arg): + return passed_arg.opt_inst.api_version <= api_minor + + accepted, removed = commandline.FilteringParser.FilterArgs( + parsed_args, filter_fn) + + if removed: + cros_build_lib.Warning('The following arguments were removed due to api: ' + "'%s'" % ' '.join(removed)) + return accepted + + @classmethod + def FilterArgsForTargetCbuildbot(cls, buildroot, cbuildbot_path, options): + _, minor = cros_build_lib.GetTargetChromiteApiVersion(buildroot) + args = [cbuildbot_path] + args.extend(options.build_targets) + args.extend(cls._FilterArgsForApi(options.parsed_args, minor)) + + # Only pass down --cache-dir if it was specified. By default, we want + # the cache dir to live in the root of each checkout, so this means that + # each instance of cbuildbot needs to calculate the default separately. + if minor >= 2 and options.cache_dir_specified: + args += ['--cache-dir', options.cache_dir] + + return args + + def HandleApplyFailures(self, failures): + """Handle the case where patches fail to apply.""" + if self._run.options.pre_cq or self._run.config.pre_cq: + # Let the PreCQSync stage handle this failure. The PreCQSync stage will + # comment on CLs with the appropriate message when they fail to apply. + # + # WARNING: For manifest patches, the Pre-CQ attempts to apply external + # patches to the internal manifest, and this means we may flag a conflict + # here even if the patch applies cleanly. TODO(davidjames): Fix this. + cros_build_lib.PrintBuildbotStepWarnings() + cros_build_lib.Error('Failed applying patches: %s', + '\n'.join(map(str, failures))) + else: + PatchChangesStage.HandleApplyFailures(self, failures) + + #pylint: disable=E1101 + @osutils.TempDirDecorator + def PerformStage(self): + # The plan for the builders is to use master branch to bootstrap other + # branches. Now, if we wanted to test patches for both the bootstrap code + # (on master) and the branched chromite (say, R20), we need to filter the + # patches by branch. + filter_branch = self._run.manifest_branch + if self._run.options.test_bootstrap: + filter_branch = 'master' + + chromite_dir = os.path.join(self.tempdir, 'chromite') + reference_repo = os.path.join(constants.SOURCE_ROOT, 'chromite', '.git') + repository.CloneGitRepo(chromite_dir, constants.CHROMITE_URL, + reference=reference_repo) + git.RunGit(chromite_dir, ['checkout', filter_branch]) + + def BranchAndChromiteFilter(patch): + return (trybot_patch_pool.BranchFilter(filter_branch, patch) and + trybot_patch_pool.ChromiteFilter(patch)) + + patch_series = validation_pool.PatchSeries.WorkOnSingleRepo( + chromite_dir, filter_branch, + deps_filter_fn=BranchAndChromiteFilter) + + filtered_pool = self.chromite_patch_pool.FilterBranch(filter_branch) + if filtered_pool: + self._ApplyPatchSeries(patch_series, filtered_pool) + + cbuildbot_path = constants.PATH_TO_CBUILDBOT + if not os.path.exists(os.path.join(self.tempdir, cbuildbot_path)): + cbuildbot_path = 'chromite/cbuildbot/cbuildbot' + # pylint: disable=W0212 + cmd = self.FilterArgsForTargetCbuildbot(self.tempdir, cbuildbot_path, + self._run.options) + + extra_params = ['--sourceroot=%s' % self._run.options.sourceroot] + extra_params.extend(self._run.options.bootstrap_args) + if self._run.options.test_bootstrap: + # We don't want re-executed instance to see this. + cmd = [a for a in cmd if a != '--test-bootstrap'] + else: + # If we've already done the desired number of bootstraps, disable + # bootstrapping for the next execution. Also pass in the patched manifest + # repository. + extra_params.append('--nobootstrap') + if self.manifest_patch_pool: + manifest_dir = self._ApplyManifestPatches(self.manifest_patch_pool) + extra_params.extend(['--manifest-repo-url', manifest_dir]) + + cmd += extra_params + result_obj = cros_build_lib.RunCommand( + cmd, cwd=self.tempdir, kill_timeout=30, error_code_ok=True) + self.returncode = result_obj.returncode + + +class SyncStage(generic_stages.BuilderStage): + """Stage that performs syncing for the builder.""" + + option_name = 'sync' + output_manifest_sha1 = True + + def __init__(self, builder_run, **kwargs): + super(SyncStage, self).__init__(builder_run, **kwargs) + self.repo = None + self.skip_sync = False + + # TODO(mtennant): Why keep a duplicate copy of this config value + # at self.internal when it can always be retrieved from config? + self.internal = self._run.config.internal + + def _GetManifestVersionsRepoUrl(self, read_only=False): + return cbuildbot_config.GetManifestVersionsRepoUrl( + self.internal, + read_only=read_only) + + def Initialize(self): + self._InitializeRepo() + + def _InitializeRepo(self): + """Set up the RepoRepository object.""" + self.repo = self.GetRepoRepository() + + def GetNextManifest(self): + """Returns the manifest to use.""" + return self._run.config.manifest + + def ManifestCheckout(self, next_manifest): + """Checks out the repository to the given manifest.""" + self._Print('\n'.join(['BUILDROOT: %s' % self.repo.directory, + 'TRACKING BRANCH: %s' % self.repo.branch, + 'NEXT MANIFEST: %s' % next_manifest])) + + if not self.skip_sync: + self.repo.Sync(next_manifest) + + print >> sys.stderr, self.repo.ExportManifest( + mark_revision=self.output_manifest_sha1) + + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) + def PerformStage(self): + self.Initialize() + with osutils.TempDir() as tempdir: + # Save off the last manifest. + fresh_sync = True + if os.path.exists(self.repo.directory) and not self._run.options.clobber: + old_filename = os.path.join(tempdir, 'old.xml') + try: + old_contents = self.repo.ExportManifest() + except cros_build_lib.RunCommandError as e: + cros_build_lib.Warning(str(e)) + else: + osutils.WriteFile(old_filename, old_contents) + fresh_sync = False + + # Sync. + self.ManifestCheckout(self.GetNextManifest()) + + # Print the blamelist. + if fresh_sync: + cros_build_lib.PrintBuildbotStepText('(From scratch)') + elif self._run.options.buildbot: + lkgm_manager.GenerateBlameList(self.repo, old_filename) + + +class LKGMSyncStage(SyncStage): + """Stage that syncs to the last known good manifest blessed by builders.""" + + output_manifest_sha1 = False + + def GetNextManifest(self): + """Override: Gets the LKGM.""" + # TODO(sosa): Should really use an initialized manager here. + if self.internal: + mv_dir = 'manifest-versions-internal' + else: + mv_dir = 'manifest-versions' + + manifest_path = os.path.join(self._build_root, mv_dir) + manifest_repo = self._GetManifestVersionsRepoUrl(read_only=True) + manifest_version.RefreshManifestCheckout(manifest_path, manifest_repo) + return os.path.join(manifest_path, lkgm_manager.LKGMManager.LKGM_PATH) + + +class ManifestVersionedSyncStage(SyncStage): + """Stage that generates a unique manifest file, and sync's to it.""" + + # TODO(mtennant): Make this into a builder run value. + output_manifest_sha1 = False + + def __init__(self, builder_run, **kwargs): + # Perform the sync at the end of the stage to the given manifest. + super(ManifestVersionedSyncStage, self).__init__(builder_run, **kwargs) + self.repo = None + self.manifest_manager = None + + # If a builder pushes changes (even with dryrun mode), we need a writable + # repository. Otherwise, the push will be rejected by the server. + self.manifest_repo = self._GetManifestVersionsRepoUrl(read_only=False) + + # 1. If we're uprevving Chrome, Chrome might have changed even if the + # manifest has not, so we should force a build to double check. This + # means that we'll create a new manifest, even if there are no changes. + # 2. If we're running with --debug, we should always run through to + # completion, so as to ensure a complete test. + self._force = self._chrome_rev or self._run.options.debug + + def HandleSkip(self): + """Initializes a manifest manager to the specified version if skipped.""" + super(ManifestVersionedSyncStage, self).HandleSkip() + if self._run.options.force_version: + self.Initialize() + self.ForceVersion(self._run.options.force_version) + + def ForceVersion(self, version): + """Creates a manifest manager from given version and returns manifest.""" + cros_build_lib.PrintBuildbotStepText(version) + return self.manifest_manager.BootstrapFromVersion(version) + + def VersionIncrementType(self): + """Return which part of the version number should be incremented.""" + if self._run.manifest_branch == 'master': + return 'build' + + return 'branch' + + def RegisterManifestManager(self, manifest_manager): + """Save the given manifest manager for later use in this run. + + Args: + manifest_manager: Expected to be a BuildSpecsManager. + """ + self._run.attrs.manifest_manager = self.manifest_manager = manifest_manager + + def Initialize(self): + """Initializes a manager that manages manifests for associated stages.""" + + dry_run = self._run.options.debug + + self._InitializeRepo() + + # If chrome_rev is somehow set, fail. + assert not self._chrome_rev, \ + 'chrome_rev is unsupported on release builders.' + + self.RegisterManifestManager(manifest_version.BuildSpecsManager( + source_repo=self.repo, + manifest_repo=self.manifest_repo, + manifest=self._run.config.manifest, + build_names=self._run.GetBuilderIds(), + incr_type=self.VersionIncrementType(), + force=self._force, + branch=self._run.manifest_branch, + dry_run=dry_run, + master=self._run.config.master)) + + def GetNextManifest(self): + """Uses the initialized manifest manager to get the next manifest.""" + assert self.manifest_manager, \ + 'Must run GetStageManager before checkout out build.' + + to_return = self.manifest_manager.GetNextBuildSpec( + dashboard_url=self.ConstructDashboardURL()) + previous_version = self.manifest_manager.GetLatestPassingSpec() + target_version = self.manifest_manager.current_version + + # Print the Blamelist here. + url_prefix = 'http://chromeos-images.corp.google.com/diff/report?' + url = url_prefix + 'from=%s&to=%s' % (previous_version, target_version) + cros_build_lib.PrintBuildbotLink('Blamelist', url) + # The testManifestVersionedSyncOnePartBranch interacts badly with this + # function. It doesn't fully initialize self.manifest_manager which + # causes target_version to be None. Since there isn't a clean fix in + # either direction, just throw this through str(). In the normal case, + # it's already a string anyways. + cros_build_lib.PrintBuildbotStepText(str(target_version)) + + return to_return + + @contextlib.contextmanager + def LocalizeManifest(self, manifest, filter_cros=False): + """Remove restricted checkouts from the manifest if needed. + + Args: + manifest: The manifest to localize. + filter_cros: If set, then only checkouts with a remote of 'cros' or + 'cros-internal' are kept, and the rest are filtered out. + """ + if filter_cros: + with osutils.TempDir() as tempdir: + filtered_manifest = os.path.join(tempdir, 'filtered.xml') + doc = ElementTree.parse(manifest) + root = doc.getroot() + for node in root.findall('project'): + remote = node.attrib.get('remote') + if remote and remote not in constants.GIT_REMOTES: + root.remove(node) + doc.write(filtered_manifest) + yield filtered_manifest + else: + yield manifest + + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) + def PerformStage(self): + self.Initialize() + if self._run.options.force_version: + next_manifest = self.ForceVersion(self._run.options.force_version) + else: + next_manifest = self.GetNextManifest() + + if not next_manifest: + cros_build_lib.Info('Found no work to do.') + if self._run.attrs.manifest_manager.DidLastBuildFail(): + raise failures_lib.StepFailure('The previous build failed.') + else: + sys.exit(0) + + # Log this early on for the release team to grep out before we finish. + if self.manifest_manager: + self._Print('\nRELEASETAG: %s\n' % ( + self.manifest_manager.current_version)) + + # To keep local trybots working, remove restricted checkouts from the + # official manifest we get from manifest-versions. + with self.LocalizeManifest( + next_manifest, filter_cros=self._run.options.local) as new_manifest: + self.ManifestCheckout(new_manifest) + + +class MasterSlaveSyncStage(ManifestVersionedSyncStage): + """Stage that generates a unique manifest file candidate, and sync's to it.""" + + # TODO(mtennant): Turn this into self._run.attrs.sub_manager or similar. + # An instance of lkgm_manager.LKGMManager for slave builds. + sub_manager = None + + def __init__(self, builder_run, **kwargs): + super(MasterSlaveSyncStage, self).__init__(builder_run, **kwargs) + # lkgm_manager deals with making sure we're synced to whatever manifest + # we get back in GetNextManifest so syncing again is redundant. + self.skip_sync = True + + def _GetInitializedManager(self, internal): + """Returns an initialized lkgm manager. + + Args: + internal: Boolean. True if this is using an internal manifest. + + Returns: + lkgm_manager.LKGMManager. + """ + increment = self.VersionIncrementType() + return lkgm_manager.LKGMManager( + source_repo=self.repo, + manifest_repo=cbuildbot_config.GetManifestVersionsRepoUrl( + internal, read_only=False), + manifest=self._run.config.manifest, + build_names=self._run.GetBuilderIds(), + build_type=self._run.config.build_type, + incr_type=increment, + force=self._force, + branch=self._run.manifest_branch, + dry_run=self._run.options.debug, + master=self._run.config.master) + + def Initialize(self): + """Override: Creates an LKGMManager rather than a ManifestManager.""" + self._InitializeRepo() + self.RegisterManifestManager(self._GetInitializedManager(self.internal)) + if (self._run.config.master and self._GetSlaveConfigs()): + assert self.internal, 'Unified masters must use an internal checkout.' + MasterSlaveSyncStage.sub_manager = self._GetInitializedManager(False) + + def ForceVersion(self, version): + manifest = super(MasterSlaveSyncStage, self).ForceVersion(version) + if MasterSlaveSyncStage.sub_manager: + MasterSlaveSyncStage.sub_manager.BootstrapFromVersion(version) + + return manifest + + def GetNextManifest(self): + """Gets the next manifest using LKGM logic.""" + assert self.manifest_manager, \ + 'Must run Initialize before we can get a manifest.' + assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \ + 'Manifest manager instantiated with wrong class.' + + if self._run.config.master: + manifest = self.manifest_manager.CreateNewCandidate() + if MasterSlaveSyncStage.sub_manager: + MasterSlaveSyncStage.sub_manager.CreateFromManifest( + manifest, dashboard_url=self.ConstructDashboardURL()) + return manifest + else: + return self.manifest_manager.GetLatestCandidate( + dashboard_url=self.ConstructDashboardURL()) + + +class CommitQueueSyncStage(MasterSlaveSyncStage): + """Commit Queue Sync stage that handles syncing and applying patches. + + This stage handles syncing to a manifest, passing around that manifest to + other builders and finding the Gerrit Reviews ready to be committed and + applying them into its own checkout. + """ + + def __init__(self, builder_run, **kwargs): + super(CommitQueueSyncStage, self).__init__(builder_run, **kwargs) + # Figure out the builder's name from the buildbot waterfall. + builder_name = self._run.config.paladin_builder_name + self.builder_name = builder_name if builder_name else self._run.config.name + + # The pool of patches to be picked up by the commit queue. + # - For the master commit queue, it's initialized in GetNextManifest. + # - For slave commit queues, it's initialized in _SetPoolFromManifest. + # + # In all cases, the pool is saved to disk, and refreshed after bootstrapping + # by HandleSkip. + self.pool = None + + def HandleSkip(self): + """Handles skip and initializes validation pool from manifest.""" + super(CommitQueueSyncStage, self).HandleSkip() + filename = self._run.options.validation_pool + if filename: + self.pool = validation_pool.ValidationPool.Load(filename, + metadata=self._run.attrs.metadata) + else: + self._SetPoolFromManifest(self.manifest_manager.GetLocalManifest()) + + def _ChangeFilter(self, pool, changes, non_manifest_changes): + # First, look for changes that were tested by the Pre-CQ. + changes_to_test = [] + for change in changes: + status = pool.GetCLStatus(PRE_CQ, change) + if status == manifest_version.BuilderStatus.STATUS_PASSED: + changes_to_test.append(change) + + # If we only see changes that weren't verified by Pre-CQ, try all of the + # changes. This ensures that the CQ continues to work even if the Pre-CQ is + # down. + if not changes_to_test: + changes_to_test = changes + + return changes_to_test, non_manifest_changes + + def _SetPoolFromManifest(self, manifest): + """Sets validation pool based on manifest path passed in.""" + # Note that GetNextManifest() calls GetLatestCandidate() in this case, + # so the repo will already be sync'd appropriately. This means that + # AcquirePoolFromManifest does not need to sync. + self.pool = validation_pool.ValidationPool.AcquirePoolFromManifest( + manifest, self._run.config.overlays, self.repo, + self._run.buildnumber, self.builder_name, + self._run.config.master, self._run.options.debug, + metadata=self._run.attrs.metadata) + + def GetNextManifest(self): + """Gets the next manifest using LKGM logic.""" + assert self.manifest_manager, \ + 'Must run Initialize before we can get a manifest.' + assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \ + 'Manifest manager instantiated with wrong class.' + + if self._run.config.master: + try: + # In order to acquire a pool, we need an initialized buildroot. + if not git.FindRepoDir(self.repo.directory): + self.repo.Initialize() + self.pool = pool = validation_pool.ValidationPool.AcquirePool( + self._run.config.overlays, self.repo, + self._run.buildnumber, self.builder_name, + self._run.options.debug, + check_tree_open=not self._run.options.debug or + self._run.options.mock_tree_status, + changes_query=self._run.options.cq_gerrit_override, + change_filter=self._ChangeFilter, throttled_ok=True, + metadata=self._run.attrs.metadata) + + except validation_pool.TreeIsClosedException as e: + cros_build_lib.Warning(str(e)) + return None + + manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool) + if MasterSlaveSyncStage.sub_manager: + MasterSlaveSyncStage.sub_manager.CreateFromManifest( + manifest, dashboard_url=self.ConstructDashboardURL()) + + return manifest + else: + manifest = self.manifest_manager.GetLatestCandidate( + dashboard_url=self.ConstructDashboardURL()) + if manifest: + if self._run.config.build_before_patching: + pre_build_passed = self._RunPrePatchBuild() + cros_build_lib.PrintBuildbotStepName( + 'CommitQueueSync : Apply Patches') + if not pre_build_passed: + cros_build_lib.PrintBuildbotStepText('Pre-patch build failed.') + + self._SetPoolFromManifest(manifest) + self.pool.ApplyPoolIntoRepo() + + return manifest + + def _RunPrePatchBuild(self): + """Run through a pre-patch build to prepare for incremental build. + + This function runs though the InitSDKStage, SetupBoardStage, and + BuildPackagesStage. It is intended to be called before applying + any patches under test, to prepare the chroot and sysroot in a state + corresponding to ToT prior to an incremental build. + + Returns: + True if all stages were successful, False if any of them failed. + """ + suffix = ' (pre-Patch)' + try: + build_stages.InitSDKStage( + self._run, chroot_replace=True, suffix=suffix).Run() + for builder_run in self._run.GetUngroupedBuilderRuns(): + for board in builder_run.config.boards: + build_stages.SetupBoardStage( + builder_run, board=board, suffix=suffix).Run() + build_stages.BuildPackagesStage( + builder_run, board=board, suffix=suffix).Run() + except failures_lib.StepFailure: + return False + + return True + + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) + def PerformStage(self): + """Performs normal stage and prints blamelist at end.""" + if self._run.options.force_version: + self.HandleSkip() + else: + ManifestVersionedSyncStage.PerformStage(self) + + +class PreCQSyncStage(SyncStage): + """Sync and apply patches to test if they compile.""" + + def __init__(self, builder_run, patches, **kwargs): + super(PreCQSyncStage, self).__init__(builder_run, **kwargs) + + # The list of patches to test. + self.patches = patches + + # The ValidationPool of patches to test. Initialized in PerformStage, and + # refreshed after bootstrapping by HandleSkip. + self.pool = None + + def HandleSkip(self): + """Handles skip and loads validation pool from disk.""" + super(PreCQSyncStage, self).HandleSkip() + filename = self._run.options.validation_pool + if filename: + self.pool = validation_pool.ValidationPool.Load(filename, + metadata=self._run.attrs.metadata) + + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) + def PerformStage(self): + super(PreCQSyncStage, self).PerformStage() + self.pool = validation_pool.ValidationPool.AcquirePreCQPool( + self._run.config.overlays, self._build_root, + self._run.buildnumber, self._run.config.name, + dryrun=self._run.options.debug_forced, changes=self.patches, + metadata=self._run.attrs.metadata) + self.pool.ApplyPoolIntoRepo() + + +class PreCQLauncherStage(SyncStage): + """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" + + STATUS_INFLIGHT = validation_pool.ValidationPool.STATUS_INFLIGHT + STATUS_PASSED = validation_pool.ValidationPool.STATUS_PASSED + STATUS_FAILED = validation_pool.ValidationPool.STATUS_FAILED + STATUS_LAUNCHING = validation_pool.ValidationPool.STATUS_LAUNCHING + STATUS_WAITING = validation_pool.ValidationPool.STATUS_WAITING + + # The number of minutes we allow before considering a launch attempt failed. + # If this window isn't hit in a given launcher run, the window will start + # again from scratch in the next run. + LAUNCH_DELAY = 30 + + # The number of minutes we allow before considering an in-flight + # job failed. If this window isn't hit in a given launcher run, the window + # will start again from scratch in the next run. + INFLIGHT_DELAY = 90 + + # The maximum number of patches we will allow in a given trybot run. This is + # needed because our trybot infrastructure can only handle so many patches at + # once. + MAX_PATCHES_PER_TRYBOT_RUN = 50 + + def __init__(self, builder_run, **kwargs): + super(PreCQLauncherStage, self).__init__(builder_run, **kwargs) + self.skip_sync = True + # Mapping from launching changes to the first known time when they + # were launching. + self.launching = {} + # Mapping from inflight changes to the first known time when they + # were inflight. + self.inflight = {} + self.retried = set() + + def _HasLaunchTimedOut(self, change): + """Check whether a given |change| has timed out on its trybot launch. + + Assumes that the change is in the middle of being launched. + + Returns: + True if the change has timed out. False otherwise. + """ + diff = datetime.timedelta(minutes=self.LAUNCH_DELAY) + return datetime.datetime.now() - self.launching[change] > diff + + def _HasInflightTimedOut(self, change): + """Check whether a given |change| has timed out while trybot inflight. + + Assumes that the change's trybot is inflight. + + Returns: + True if the change has timed out. False otherwise. + """ + diff = datetime.timedelta(minutes=self.INFLIGHT_DELAY) + return datetime.datetime.now() - self.inflight[change] > diff + + def GetPreCQStatus(self, pool, changes): + """Get the Pre-CQ status of a list of changes. + + Side effect: reject or retry changes that have timed out. + + Args: + pool: The validation pool. + changes: Changes to examine. + + Returns: + busy: The set of CLs that are currently being tested. + passed: The set of CLs that have been verified. + """ + busy, passed = set(), set() + + for change in changes: + status = pool.GetCLStatus(PRE_CQ, change) + + if status != self.STATUS_LAUNCHING: + # The trybot is not launching, so we should remove it from our + # launching timeout map. + self.launching.pop(change, None) + + if status != self.STATUS_INFLIGHT: + # The trybot is not inflight, so we should remove it from our + # inflight timeout map. + self.inflight.pop(change, None) + + if status == self.STATUS_LAUNCHING: + # The trybot is in the process of launching. + busy.add(change) + if change not in self.launching: + # Record the launch time of changes. + self.launching[change] = datetime.datetime.now() + elif self._HasLaunchTimedOut(change): + if change in self.retried: + msg = ('We were not able to launch a pre-cq trybot for your change.' + '\n\n' + 'This problem can happen if the trybot waterfall is very ' + 'busy, or if there is an infrastructure issue. Please ' + 'notify the sheriff and mark your change as ready again. If ' + 'this problem occurs multiple times in a row, please file a ' + 'bug.') + + pool.SendNotification(change, '%(details)s', details=msg) + pool.RemoveCommitReady(change) + pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, + self._run.options.debug) + self.retried.discard(change) + else: + # Try the change again. + self.retried.add(change) + pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, + self._run.options.debug) + elif status == self.STATUS_INFLIGHT: + # Once a Pre-CQ run actually starts, it'll set the status to + # STATUS_INFLIGHT. + busy.add(change) + if change not in self.inflight: + # Record the inflight start time. + self.inflight[change] = datetime.datetime.now() + elif self._HasInflightTimedOut(change): + msg = ('The pre-cq trybot for your change timed out after %s minutes.' + '\n\n' + 'This problem can happen if your change causes the builder ' + 'to hang, or if there is some infrastructure issue. If your ' + 'change is not at fault you may mark your change as ready ' + 'again. If this problem occurs multiple times please notify ' + 'the sheriff and file a bug.' % self.INFLIGHT_DELAY) + + pool.SendNotification(change, '%(details)s', details=msg) + pool.RemoveCommitReady(change) + pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, + self._run.options.debug) + elif status == self.STATUS_FAILED: + # The Pre-CQ run failed for this change. It's possible that we got + # unlucky and this change was just marked as 'Not Ready' by a bot. To + # test this, mark the CL as 'waiting' for now. If the CL is still marked + # as 'Ready' next time we check, we'll know the CL is truly still ready. + busy.add(change) + pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, + self._run.options.debug) + elif status == self.STATUS_PASSED: + passed.add(change) + + return busy, passed + + def LaunchTrybot(self, pool, plan): + """Launch a Pre-CQ run with the provided list of CLs. + + Args: + pool: ValidationPool corresponding to |plan|. + plan: The list of patches to test in the Pre-CQ run. + """ + cmd = ['cbuildbot', '--remote', constants.PRE_CQ_BUILDER_NAME] + if self._run.options.debug: + cmd.append('--debug') + for patch in plan: + cmd += ['-g', cros_patch.AddPrefix(patch, patch.gerrit_number)] + cros_build_lib.RunCommand(cmd, cwd=self._build_root) + for patch in plan: + if pool.GetCLStatus(PRE_CQ, patch) != self.STATUS_PASSED: + pool.UpdateCLStatus(PRE_CQ, patch, self.STATUS_LAUNCHING, + self._run.options.debug) + + def GetDisjointTransactionsToTest(self, pool, changes): + """Get the list of disjoint transactions to test. + + Side effect: reject or retry changes that have timed out. + + Returns: + A list of disjoint transactions to test. Each transaction should be sent + to a different Pre-CQ trybot. + """ + busy, passed = self.GetPreCQStatus(pool, changes) + + # Create a list of disjoint transactions to test. + manifest = git.ManifestCheckout.Cached(self._build_root) + plans = pool.CreateDisjointTransactions( + manifest, max_txn_length=self.MAX_PATCHES_PER_TRYBOT_RUN) + for plan in plans: + # If any of the CLs in the plan are currently "busy" being tested, + # wait until they're done before launching our trybot run. This helps + # avoid race conditions. + # + # Similarly, if all of the CLs in the plan have already been validated, + # there's no need to launch a trybot run. + plan = set(plan) + if plan.issubset(passed): + logging.info('CLs already verified: %r', ' '.join(map(str, plan))) + elif plan.intersection(busy): + logging.info('CLs currently being verified: %r', + ' '.join(map(str, plan.intersection(busy)))) + if plan.difference(busy): + logging.info('CLs waiting on verification of dependencies: %r', + ' '.join(map(str, plan.difference(busy)))) + else: + yield plan + + def ProcessChanges(self, pool, changes, _non_manifest_changes): + """Process a list of changes that were marked as Ready. + + From our list of changes that were marked as Ready, we create a + list of disjoint transactions and send each one to a separate Pre-CQ + trybot. + + Non-manifest changes are just submitted here because they don't need to be + verified by either the Pre-CQ or CQ. + """ + # Submit non-manifest changes if we can. + if timeout_util.IsTreeOpen( + validation_pool.ValidationPool.STATUS_URL): + pool.SubmitNonManifestChanges(check_tree_open=False) + + # Launch trybots for manifest changes. + for plan in self.GetDisjointTransactionsToTest(pool, changes): + self.LaunchTrybot(pool, plan) + + # Tell ValidationPool to keep waiting for more changes until we hit + # its internal timeout. + return [], [] + + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) + def PerformStage(self): + # Setup and initialize the repo. + super(PreCQLauncherStage, self).PerformStage() + + # Loop through all of the changes until we hit a timeout. + validation_pool.ValidationPool.AcquirePool( + self._run.config.overlays, self.repo, + self._run.buildnumber, + constants.PRE_CQ_LAUNCHER_NAME, + dryrun=self._run.options.debug, + changes_query=self._run.options.cq_gerrit_override, + check_tree_open=False, change_filter=self.ProcessChanges, + metadata=self._run.attrs.metadata) -- cgit v1.2.3 From b2620d25dd49149a55a2200664a88da25ead8f0c Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Fri, 30 May 2014 10:23:09 -0700 Subject: Increate PreCQ runtime timeout from 90 minutes to 2 hours. We've been timing out sometimes, just because the builder was slow. BUG=chromium:378773 TEST=Pretty much no testing. Change-Id: Ic756c2f0a25aac54c89281b327e57dc056b23717 Reviewed-on: https://chromium-review.googlesource.com/202147 Tested-by: Don Garrett Reviewed-by: Yu-Ju Hong Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 282284643..2fed1c67d 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -763,7 +763,7 @@ class PreCQLauncherStage(SyncStage): # The number of minutes we allow before considering an in-flight # job failed. If this window isn't hit in a given launcher run, the window # will start again from scratch in the next run. - INFLIGHT_DELAY = 90 + INFLIGHT_DELAY = 120 # The maximum number of patches we will allow in a given trybot run. This is # needed because our trybot infrastructure can only handle so many patches at -- cgit v1.2.3 From 14e977789978c15ca823587b18e9f9cccf3e9a12 Mon Sep 17 00:00:00 2001 From: David James Date: Wed, 4 Jun 2014 18:44:49 -0700 Subject: Start renaming cbuildbot.(cbuildbot_)* to cbuildbot.*. - Rename cbuildbot.cbuildbot_commands -> cbuildbot.commands. - Rename cbuildbot.cbuildbot_failures -> cbuildbot.failures_lib. - Rename cbuildbot.cbuildbot_results -> cbuildbot.results_lib. I have intentionally not touched the order of import statements yet to put them back in order, since this change can be landed separately. BUG=none CQ-DEPEND=CL:*165269 TEST=All unit tests. TEST=Full release trybot run. TEST=Paladin trybot run. Change-Id: Ib40a0f03ddc97da300a7889f324c59bbe8212dc3 Reviewed-on: https://chromium-review.googlesource.com/202650 Commit-Queue: David James Reviewed-by: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 2fed1c67d..5aa2d6a5d 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -12,7 +12,7 @@ import sys from xml.etree import ElementTree from chromite.cbuildbot import cbuildbot_config -from chromite.cbuildbot import cbuildbot_failures as failures_lib +from chromite.cbuildbot import failures_lib from chromite.cbuildbot import constants from chromite.cbuildbot import lkgm_manager from chromite.cbuildbot import manifest_version -- cgit v1.2.3 From 97851d3b247ec6896a08e215e77f8b8523d320d8 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Tue, 10 Jun 2014 16:00:44 -0700 Subject: tree_status: move tree status code to its own module BUG=None TEST=`cbuildbot/run_tests` Change-Id: I1a29177c6a0d94d291fe68bf11893d771a9f9a8b Reviewed-on: https://chromium-review.googlesource.com/203306 Reviewed-by: Yu-Ju Hong Commit-Queue: Yu-Ju Hong Tested-by: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 5aa2d6a5d..4a8ce8e6e 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -17,6 +17,7 @@ from chromite.cbuildbot import constants from chromite.cbuildbot import lkgm_manager from chromite.cbuildbot import manifest_version from chromite.cbuildbot import repository +from chromite.cbuildbot import tree_status from chromite.cbuildbot import trybot_patch_pool from chromite.cbuildbot import validation_pool from chromite.cbuildbot.stages import generic_stages @@ -26,7 +27,6 @@ from chromite.lib import cros_build_lib from chromite.lib import git from chromite.lib import osutils from chromite.lib import patch as cros_patch -from chromite.lib import timeout_util PRE_CQ = validation_pool.PRE_CQ @@ -953,8 +953,7 @@ class PreCQLauncherStage(SyncStage): verified by either the Pre-CQ or CQ. """ # Submit non-manifest changes if we can. - if timeout_util.IsTreeOpen( - validation_pool.ValidationPool.STATUS_URL): + if tree_status.IsTreeOpen(): pool.SubmitNonManifestChanges(check_tree_open=False) # Launch trybots for manifest changes. -- cgit v1.2.3 From 9902542fa77552a9f3e9e9b30fffef80f682991e Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 17 Jun 2014 13:32:01 -0700 Subject: Fix "file content merging is disabled" errors. Currently, we sometimes see patches fail to apply in a repository with the following error: CL:*166236 conflicted with ToT because file content merging is disabled for this project. These errors can occur even if file content merging is really enabled -- when file content merging is set to the default setting (inherited), cbuildbot just assumes that it is disabled, when in fact we enable it by default. Since we've standardized on enabling content merging for all of our projects now, there is no need to check for content merging anymore, so eliminate the check and reduce the amount of code we have. This in turn eliminates the concept of applying a patch in dry-run mode. Previously, a patch would always have three-way-merging enabled in dry-run mode, but might have it disabled when we apply it for real. BUG=chromium:385764 TEST=All unit tests. TEST=cbuildbot --buildbot --debug x86-alex-paladin Change-Id: Ieb5d30f9002cb93277a7b7963cbf5d4c4ba44e93 Reviewed-on: https://chromium-review.googlesource.com/204370 Reviewed-by: Yu-Ju Hong Commit-Queue: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 4a8ce8e6e..96fedcd9c 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -98,14 +98,13 @@ class PatchChangesStage(generic_stages.BuilderStage): class NoisyPatchSeries(validation_pool.PatchSeries): """Custom PatchSeries that adds links to buildbot logs for remote trys.""" - def ApplyChange(self, change, dryrun=False): + def ApplyChange(self, change): if isinstance(change, cros_patch.GerritPatch): cros_build_lib.PrintBuildbotLink(str(change), change.url) elif isinstance(change, cros_patch.UploadedLocalPatch): cros_build_lib.PrintBuildbotStepText(str(change)) - return validation_pool.PatchSeries.ApplyChange(self, change, - dryrun=dryrun) + return validation_pool.PatchSeries.ApplyChange(self, change) # If we're an external builder, ignore internal patches. helper_pool = validation_pool.HelperPool.SimpleCreate( @@ -114,7 +113,6 @@ class PatchChangesStage(generic_stages.BuilderStage): # Limit our resolution to non-manifest patches. patch_series = NoisyPatchSeries( self._build_root, - force_content_merging=True, helper_pool=helper_pool, deps_filter_fn=lambda p: not trybot_patch_pool.ManifestFilter(p)) -- cgit v1.2.3 From ff6389dc2361455d2072625a91e18672bdf03547 Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Fri, 20 Jun 2014 12:49:06 -0400 Subject: PreCQLauncherStage: log links to CLs passed/failed/tested This makes browsing the buildbot summary page a bit nicer to see what any particular CQ run is doing. BUG=None TEST=`./cbuildbot/run_tests` passes Change-Id: I982067053e62a651c4ca16b41362f4df726f5068 Reviewed-on: https://chromium-review.googlesource.com/204882 Tested-by: Mike Frysinger Reviewed-by: David James Reviewed-by: Yu-Ju Hong Commit-Queue: Mike Frysinger --- cbuildbot/stages/sync_stages.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 96fedcd9c..15e4322ed 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -801,6 +801,16 @@ class PreCQLauncherStage(SyncStage): diff = datetime.timedelta(minutes=self.INFLIGHT_DELAY) return datetime.datetime.now() - self.inflight[change] > diff + @staticmethod + def _PrintPatchStatus(patch, status): + """Print a link to |patch| with |status| info.""" + items = ( + status, + os.path.basename(patch.project), + str(patch), + ) + cros_build_lib.PrintBuildbotLink(' | '.join(items), patch.url) + def GetPreCQStatus(self, pool, changes): """Get the Pre-CQ status of a list of changes. @@ -883,8 +893,10 @@ class PreCQLauncherStage(SyncStage): busy.add(change) pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, self._run.options.debug) + self._PrintPatchStatus(change, 'failed') elif status == self.STATUS_PASSED: passed.add(change) + self._PrintPatchStatus(change, 'passed') return busy, passed @@ -900,6 +912,7 @@ class PreCQLauncherStage(SyncStage): cmd.append('--debug') for patch in plan: cmd += ['-g', cros_patch.AddPrefix(patch, patch.gerrit_number)] + self._PrintPatchStatus(patch, 'testing') cros_build_lib.RunCommand(cmd, cwd=self._build_root) for patch in plan: if pool.GetCLStatus(PRE_CQ, patch) != self.STATUS_PASSED: -- cgit v1.2.3 From 669eb5ed489ba2e37516b0403a56ec0b663a6819 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Mon, 23 Jun 2014 08:53:01 -0700 Subject: pass metadata via dumped json file into cbuildbot re-executions BUG=chromium:356930 TEST=`cbuildbot --local --test-boot-strap --buildbot --debug master-paladin` -> patches picked up in CommitQueueSync appear in metadata.json file, and only appear once. Change-Id: Ib0a33ca0042e9c4ab3b0c75f9fb08b5df6fc263d Reviewed-on: https://chromium-review.googlesource.com/205222 Tested-by: Aviv Keshet Reviewed-by: Yu-Ju Hong Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 15e4322ed..ce7b2c13f 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -597,8 +597,14 @@ class CommitQueueSyncStage(MasterSlaveSyncStage): super(CommitQueueSyncStage, self).HandleSkip() filename = self._run.options.validation_pool if filename: + # Temporary code to check if we are using a metadata object that was + # dumped by an outer cbuildbot run. If so, do not re-write CL actions. + # This code will be removed by a future change, once this change has + # landed. + record_patches = not self._run.attrs.metadata.GetDict().has_key( + 'dumped_dict') self.pool = validation_pool.ValidationPool.Load(filename, - metadata=self._run.attrs.metadata) + metadata=self._run.attrs.metadata, record_patches=record_patches) else: self._SetPoolFromManifest(self.manifest_manager.GetLocalManifest()) -- cgit v1.2.3 From 8c35e3d41a89843cfbe5ee5bf405a77588138b9f Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Mon, 23 Jun 2014 12:08:23 -0700 Subject: sync_stages.py: never record actions during CommitQueueSync HandleSkip This is a follow up to CL:205222 and requires that change to have already landed. BUG=chromium:356930 TEST=None Change-Id: If40554e8c89ac9eaf102c637d2c1d6f843ac7b43 Reviewed-on: https://chromium-review.googlesource.com/205234 Tested-by: Aviv Keshet Reviewed-by: Yu-Ju Hong Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index ce7b2c13f..aedeaa186 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -588,8 +588,7 @@ class CommitQueueSyncStage(MasterSlaveSyncStage): # - For the master commit queue, it's initialized in GetNextManifest. # - For slave commit queues, it's initialized in _SetPoolFromManifest. # - # In all cases, the pool is saved to disk, and refreshed after bootstrapping - # by HandleSkip. + # In all cases, the pool is saved to disk. self.pool = None def HandleSkip(self): @@ -597,14 +596,8 @@ class CommitQueueSyncStage(MasterSlaveSyncStage): super(CommitQueueSyncStage, self).HandleSkip() filename = self._run.options.validation_pool if filename: - # Temporary code to check if we are using a metadata object that was - # dumped by an outer cbuildbot run. If so, do not re-write CL actions. - # This code will be removed by a future change, once this change has - # landed. - record_patches = not self._run.attrs.metadata.GetDict().has_key( - 'dumped_dict') self.pool = validation_pool.ValidationPool.Load(filename, - metadata=self._run.attrs.metadata, record_patches=record_patches) + metadata=self._run.attrs.metadata, record_patches=False) else: self._SetPoolFromManifest(self.manifest_manager.GetLocalManifest()) -- cgit v1.2.3 From 173aa1c6246761b167637e77796112bd8b08fa46 Mon Sep 17 00:00:00 2001 From: Prathmesh Prabhu Date: Mon, 23 Jun 2014 11:28:49 -0700 Subject: cbuildbot: Launch a pre-patch build for incremental paladin trybots. CL:184432 added a new config parameter |build_before_patching| to request that we launch a build before patching changes. This is needed by incremental builders to obtain a state corresponding to the current ToT. Although this worked for paladin buildbots, it did not work as expected for trybots. This CL adds the same logic to |SyncStage| so that trybots are also covered. BUG=chromium:383060 TEST=- $ ./cbuildbot/run_tests passed - $ cbuildbot --remote --buildbot --debug lumpy-incremental-paladin -p chromiumos/chromite --test-bootstrap passed, and included pre-patch build stages. This validates existing behaviour. - $ cbuildbot --remote --debug lumpy-incremental-paladin -p chromium/chromite --test-bootstrap passed, and included pre-patch build stages. This tests new behaviour. (--debug flag is not strictly required in this case as --debug is implied with --remote when --buildbot is not supplied.) Change-Id: I35e16028d022f92ef44683a98e63f75ff7dce55b Reviewed-on: https://chromium-review.googlesource.com/205280 Reviewed-by: Aviv Keshet Reviewed-by: Yu-Ju Hong Commit-Queue: Prathmesh Prabhu Tested-by: Prathmesh Prabhu --- cbuildbot/stages/sync_stages.py | 60 ++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 27 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index aedeaa186..9e3e00ee2 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -311,6 +311,32 @@ class SyncStage(generic_stages.BuilderStage): print >> sys.stderr, self.repo.ExportManifest( mark_revision=self.output_manifest_sha1) + def RunPrePatchBuild(self): + """Run through a pre-patch build to prepare for incremental build. + + This function runs though the InitSDKStage, SetupBoardStage, and + BuildPackagesStage. It is intended to be called before applying + any patches under test, to prepare the chroot and sysroot in a state + corresponding to ToT prior to an incremental build. + + Returns: + True if all stages were successful, False if any of them failed. + """ + suffix = ' (pre-Patch)' + try: + build_stages.InitSDKStage( + self._run, chroot_replace=True, suffix=suffix).Run() + for builder_run in self._run.GetUngroupedBuilderRuns(): + for board in builder_run.config.boards: + build_stages.SetupBoardStage( + builder_run, board=board, suffix=suffix).Run() + build_stages.BuildPackagesStage( + builder_run, board=board, suffix=suffix).Run() + except failures_lib.StepFailure: + return False + + return True + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): self.Initialize() @@ -336,6 +362,12 @@ class SyncStage(generic_stages.BuilderStage): elif self._run.options.buildbot: lkgm_manager.GenerateBlameList(self.repo, old_filename) + # Incremental builds request an additional build before patching changes. + if self._run.config.build_before_patching: + pre_build_passed = self.RunPrePatchBuild() + if not pre_build_passed: + cros_build_lib.PrintBuildbotStepText('Pre-patch build failed.') + class LKGMSyncStage(SyncStage): """Stage that syncs to the last known good manifest blessed by builders.""" @@ -665,7 +697,7 @@ class CommitQueueSyncStage(MasterSlaveSyncStage): dashboard_url=self.ConstructDashboardURL()) if manifest: if self._run.config.build_before_patching: - pre_build_passed = self._RunPrePatchBuild() + pre_build_passed = self.RunPrePatchBuild() cros_build_lib.PrintBuildbotStepName( 'CommitQueueSync : Apply Patches') if not pre_build_passed: @@ -676,32 +708,6 @@ class CommitQueueSyncStage(MasterSlaveSyncStage): return manifest - def _RunPrePatchBuild(self): - """Run through a pre-patch build to prepare for incremental build. - - This function runs though the InitSDKStage, SetupBoardStage, and - BuildPackagesStage. It is intended to be called before applying - any patches under test, to prepare the chroot and sysroot in a state - corresponding to ToT prior to an incremental build. - - Returns: - True if all stages were successful, False if any of them failed. - """ - suffix = ' (pre-Patch)' - try: - build_stages.InitSDKStage( - self._run, chroot_replace=True, suffix=suffix).Run() - for builder_run in self._run.GetUngroupedBuilderRuns(): - for board in builder_run.config.boards: - build_stages.SetupBoardStage( - builder_run, board=board, suffix=suffix).Run() - build_stages.BuildPackagesStage( - builder_run, board=board, suffix=suffix).Run() - except failures_lib.StepFailure: - return False - - return True - @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): """Performs normal stage and prints blamelist at end.""" -- cgit v1.2.3 From 3980632c376c6c289b7a2a5e1fb829e26e9ad186 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Tue, 24 Jun 2014 16:46:32 -0700 Subject: Write the Chrome version to manifest PFQ master should write the version of the Chrome in the manifest it creates. This ensures the the master and all slaves build the same version. Note that this CL only writes the version in the manifest, and the next CL will force the slaves to use the version written the manifest. BUG=chromium:387252 TEST=`cbuildbot/run_tests` TEST=`cbuildbot --remote --debug --buildbot x86-generic-chromium-pfq` Change-Id: I49c7834493410bd023ffa85a6ef94d7dc80a069e Reviewed-on: https://chromium-review.googlesource.com/205523 Reviewed-by: Yu-Ju Hong Commit-Queue: Yu-Ju Hong Tested-by: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 9e3e00ee2..3146689de 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -24,9 +24,11 @@ from chromite.cbuildbot.stages import generic_stages from chromite.cbuildbot.stages import build_stages from chromite.lib import commandline from chromite.lib import cros_build_lib +from chromite.lib import gclient from chromite.lib import git from chromite.lib import osutils from chromite.lib import patch as cros_patch +from chromite.scripts import cros_mark_chrome_as_stable PRE_CQ = validation_pool.PRE_CQ @@ -545,6 +547,7 @@ class MasterSlaveSyncStage(ManifestVersionedSyncStage): # lkgm_manager deals with making sure we're synced to whatever manifest # we get back in GetNextManifest so syncing again is redundant. self.skip_sync = True + self._chrome_version = None def _GetInitializedManager(self, internal): """Returns an initialized lkgm manager. @@ -592,7 +595,8 @@ class MasterSlaveSyncStage(ManifestVersionedSyncStage): 'Manifest manager instantiated with wrong class.' if self._run.config.master: - manifest = self.manifest_manager.CreateNewCandidate() + manifest = self.manifest_manager.CreateNewCandidate( + chrome_version=self._chrome_version) if MasterSlaveSyncStage.sub_manager: MasterSlaveSyncStage.sub_manager.CreateFromManifest( manifest, dashboard_url=self.ConstructDashboardURL()) @@ -601,6 +605,21 @@ class MasterSlaveSyncStage(ManifestVersionedSyncStage): return self.manifest_manager.GetLatestCandidate( dashboard_url=self.ConstructDashboardURL()) + def GetLatestChromeVersion(self): + """Returns the version of Chrome to uprev.""" + return cros_mark_chrome_as_stable.GetLatestRelease(gclient.GetBaseURLs()[0]) + + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) + def PerformStage(self): + """Performs the stage.""" + if (self._chrome_rev == constants.CHROME_REV_LATEST and + self._run.config.master): + # PFQ master needs to determine what version of Chrome to build + # for all slaves. + self._chrome_version = self.GetLatestChromeVersion() + + ManifestVersionedSyncStage.PerformStage(self) + class CommitQueueSyncStage(MasterSlaveSyncStage): """Commit Queue Sync stage that handles syncing and applying patches. -- cgit v1.2.3 From 7d3d1ad304de455d6e0b0f705678e22ca9804907 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Fri, 20 Jun 2014 14:29:56 -0700 Subject: pre-cq-group should stop if no changes can be applied If the changes conflict with ToT, PreCQSync stage may not apply any changes. In this case, it will comment on the changes (on gerrit) about the conflicts, while continuing running the rest of the stages. This is a waste of resource. pre-cq-group should exit when there are no applicable changes. BUG=chromium:348652 TEST=`cbuildbot/run_tests` Change-Id: I15e2e4657a8e99aa855dcb63c0982ae66a4ea018 Reviewed-on: https://chromium-review.googlesource.com/204980 Reviewed-by: Yu-Ju Hong Commit-Queue: Yu-Ju Hong Tested-by: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 3146689de..86715d6ea 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -757,7 +757,6 @@ class PreCQSyncStage(SyncStage): self.pool = validation_pool.ValidationPool.Load(filename, metadata=self._run.attrs.metadata) - @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): super(PreCQSyncStage, self).PerformStage() self.pool = validation_pool.ValidationPool.AcquirePreCQPool( @@ -767,6 +766,9 @@ class PreCQSyncStage(SyncStage): metadata=self._run.attrs.metadata) self.pool.ApplyPoolIntoRepo() + if len(self.pool.changes) == 0: + cros_build_lib.Die('No changes have been applied.') + class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" -- cgit v1.2.3 From 7397b351cdbed60dcce2668a29333af00bf9eb0f Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Wed, 25 Jun 2014 10:57:53 -0700 Subject: PFQ slaves should use the chrome version specified in the manifest Recently, CL:205523 modifies the code to let PFQ master write the chrome version to the manifest. This CL modifies the PFQ builders to read this value from the manifest and uprev the Chrome to the specified version. This ensures that all master and slaves build the same version of Chrome, which is key to the correctness of PFQ runs. Note that to preserve the version read from the manifest through re-executions, we update the metadata dictionary as soon as the version is read from manifest. BUG=chromium:387252 TEST=`cbuildbot/run_tests` TEST=`cbuildbot --remote --debug --buildbot lumpy-chrome-pfq` TEST=`cbuildbot --remote lumpy-chrome-pfq` (sanity check) Change-Id: I1e0b5ecc65e053ebfda80c88ff4094b26bf89264 Reviewed-on: https://chromium-review.googlesource.com/205721 Reviewed-by: Yu-Ju Hong Commit-Queue: Yu-Ju Hong Tested-by: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 86715d6ea..b6b053580 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -10,6 +10,7 @@ import logging import os import sys from xml.etree import ElementTree +from xml.dom import minidom from chromite.cbuildbot import cbuildbot_config from chromite.cbuildbot import failures_lib @@ -462,6 +463,28 @@ class ManifestVersionedSyncStage(SyncStage): dry_run=dry_run, master=self._run.config.master)) + def _SetChromeVersionIfApplicable(self, manifest): + """If 'chrome' is in |manifest|, write the version to the BuilderRun object. + + Args: + manifest: Path to the manifest. + """ + manifest_dom = minidom.parse(manifest) + elements = manifest_dom.getElementsByTagName(lkgm_manager.CHROME_ELEMENT) + + if elements: + chrome_version = elements[0].getAttribute( + lkgm_manager.CHROME_VERSION_ATTR) + logging.info( + 'Chrome version was found in the manifest: %s', chrome_version) + # Update the metadata dictionary. This is necessary because the + # metadata dictionary is preserved through re-executions, so + # SyncChromeStage can read the version from the dictionary + # later. This is easier than parsing the manifest again after + # the re-execution. + self._run.attrs.metadata.UpdateKeyDictWithDict( + 'version', {'chrome': chrome_version}) + def GetNextManifest(self): """Uses the initialized manifest manager to get the next manifest.""" assert self.manifest_manager, \ @@ -528,6 +551,7 @@ class ManifestVersionedSyncStage(SyncStage): self._Print('\nRELEASETAG: %s\n' % ( self.manifest_manager.current_version)) + self._SetChromeVersionIfApplicable(next_manifest) # To keep local trybots working, remove restricted checkouts from the # official manifest we get from manifest-versions. with self.LocalizeManifest( -- cgit v1.2.3 From 33edd8e27bac3f022808abfd3653d8079d5939be Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Tue, 22 Jul 2014 17:21:23 -0700 Subject: Move GetBuildersStatus from LKGMManager to BuildSpecsManager Currently ManifestVersionedSyncStage (used by canaries) uses a BuildSpecsManager, while MasterSlaveSyncStage (used by PFQ, CQ) uses a LKGMManager, which derives from BuildSpecsManager. There is no reason why a master-slave model needs to be tied with a LKGMManager. In fact, to make canaries adop the master-slave model, we need to allow BuildSpecsManager to be used in this case. This CL moves one method, GetBuildersStatus, which is required for master-slave model, from LKGMManager to BuildSpecsManager. It also move the timeout values from the manager itself to the Sync/Completion stage so that they are more visible and easier to maintain. BUG=chromium:385267 TEST=`cbuildbot/run_tests` TEST=`cbuildbot --buildbot --debug --remote master-paladin` Change-Id: Ibf0098fff31bad0c84958b78a7b55a5dc9d2afc9 Reviewed-on: https://chromium-review.googlesource.com/209742 Tested-by: Yu-Ju Hong Reviewed-by: Aviv Keshet Commit-Queue: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index b6b053580..b56d2c5f4 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -562,6 +562,9 @@ class ManifestVersionedSyncStage(SyncStage): class MasterSlaveSyncStage(ManifestVersionedSyncStage): """Stage that generates a unique manifest file candidate, and sync's to it.""" + # Timeout for waiting on the latest candidate manifest. + LATEST_CANDIDATE_TIMEOUT_SECONDS = 20 * 60 + # TODO(mtennant): Turn this into self._run.attrs.sub_manager or similar. # An instance of lkgm_manager.LKGMManager for slave builds. sub_manager = None @@ -627,7 +630,8 @@ class MasterSlaveSyncStage(ManifestVersionedSyncStage): return manifest else: return self.manifest_manager.GetLatestCandidate( - dashboard_url=self.ConstructDashboardURL()) + dashboard_url=self.ConstructDashboardURL(), + timeout=self.LATEST_CANDIDATE_TIMEOUT_SECONDS) def GetLatestChromeVersion(self): """Returns the version of Chrome to uprev.""" -- cgit v1.2.3 From 02d4ff6624f78ec2d4fd6a1854a14dca704bc1d0 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Mon, 28 Jul 2014 14:56:19 -0700 Subject: Include build_id in published manifest commit messages This change will cause commit messages for published manifests to include the build id, in the form 'build_id: 1234'. The build id is taken from the builder run's metadata, if the build id exists, otherwise none is written. 1) Updated manifest_version.py:GetNextBuildSpec and all its callers. 2) Updated lkgm_manager.py:CreateNewCandidate and all its callers. 3) Updated lkgm_manager.py:CreateFromManifest and all its callers. 4) Updated manifest_version.py:PublishManifest and all its callers. Associated unit test fixes in lkgm_manager_unittest and sync_stage_unittest. New unit tests of PublishManifest's commit message behavior. BUG=chromium:398267 TEST=New unit tests and existing tests pass. Change-Id: I47e88b234301bccbece29f5bd94837ce30c4b2b9 Reviewed-on: https://chromium-review.googlesource.com/210094 Reviewed-by: Aviv Keshet Commit-Queue: Aviv Keshet Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index b56d2c5f4..f43f30a5a 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -490,8 +490,11 @@ class ManifestVersionedSyncStage(SyncStage): assert self.manifest_manager, \ 'Must run GetStageManager before checkout out build.' + build_id = self._run.attrs.metadata.GetDict().get('build_id') + to_return = self.manifest_manager.GetNextBuildSpec( - dashboard_url=self.ConstructDashboardURL()) + dashboard_url=self.ConstructDashboardURL(), + build_id=build_id) previous_version = self.manifest_manager.GetLatestPassingSpec() target_version = self.manifest_manager.current_version @@ -622,11 +625,14 @@ class MasterSlaveSyncStage(ManifestVersionedSyncStage): 'Manifest manager instantiated with wrong class.' if self._run.config.master: + build_id = self._run.attrs.metadata.GetDict().get('build_id') manifest = self.manifest_manager.CreateNewCandidate( - chrome_version=self._chrome_version) + chrome_version=self._chrome_version, + build_id=build_id) if MasterSlaveSyncStage.sub_manager: MasterSlaveSyncStage.sub_manager.CreateFromManifest( - manifest, dashboard_url=self.ConstructDashboardURL()) + manifest, dashboard_url=self.ConstructDashboardURL(), + build_id=build_id) return manifest else: return self.manifest_manager.GetLatestCandidate( @@ -714,6 +720,8 @@ class CommitQueueSyncStage(MasterSlaveSyncStage): assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \ 'Manifest manager instantiated with wrong class.' + build_id = self._run.attrs.metadata.GetDict().get('build_id') + if self._run.config.master: try: # In order to acquire a pool, we need an initialized buildroot. @@ -733,10 +741,12 @@ class CommitQueueSyncStage(MasterSlaveSyncStage): cros_build_lib.Warning(str(e)) return None - manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool) + manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool, + build_id=build_id) if MasterSlaveSyncStage.sub_manager: MasterSlaveSyncStage.sub_manager.CreateFromManifest( - manifest, dashboard_url=self.ConstructDashboardURL()) + manifest, dashboard_url=self.ConstructDashboardURL(), + build_id=build_id) return manifest else: -- cgit v1.2.3 From 78a4107fa8b747d45c7fbab69152c1e07cb3320a Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Wed, 30 Jul 2014 16:55:36 -0700 Subject: Make canaries use the true master-slave model This CL changes the sync/completion stages of the canary master to ManifestVersionedSyncStage and MasterSlaveCompletionStage. This enables the master to wait and collect slave statuses. It also turns on "important=True" for canary group builders and non-experimental, individual canary builders (e.g. link-release) so that the master would wait for them. Some other code cleanup included: - Rename MasterSlaveSyncStage to MasterSlaveLKGMSyncStage because it uses the lkgm manager. - Move AbortCQHWTest from MasterSlaveSyncCompletionStage to CommmitQueueCompletionStage because no other compeltion stage should be able to abort HWTest on CQ. BUG=chromium:385267 CQ-DEPEND=CL:209742 TEST=`cbuildbot --remote --debug --buildbot master-release` Change-Id: I6736cd40854c6ce9064bba3ce87e88e20c0ab7ac Reviewed-on: https://chromium-review.googlesource.com/209630 Reviewed-by: Yu-Ju Hong Commit-Queue: Yu-Ju Hong Tested-by: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index f43f30a5a..c64c984de 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -562,8 +562,12 @@ class ManifestVersionedSyncStage(SyncStage): self.ManifestCheckout(new_manifest) -class MasterSlaveSyncStage(ManifestVersionedSyncStage): - """Stage that generates a unique manifest file candidate, and sync's to it.""" +class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): + """Stage that generates a unique manifest file candidate, and sync's to it. + + This stage uses an LKGM manifest manager that handles LKGM + candidates and their states. + """ # Timeout for waiting on the latest candidate manifest. LATEST_CANDIDATE_TIMEOUT_SECONDS = 20 * 60 @@ -573,7 +577,7 @@ class MasterSlaveSyncStage(ManifestVersionedSyncStage): sub_manager = None def __init__(self, builder_run, **kwargs): - super(MasterSlaveSyncStage, self).__init__(builder_run, **kwargs) + super(MasterSlaveLKGMSyncStage, self).__init__(builder_run, **kwargs) # lkgm_manager deals with making sure we're synced to whatever manifest # we get back in GetNextManifest so syncing again is redundant. self.skip_sync = True @@ -608,12 +612,12 @@ class MasterSlaveSyncStage(ManifestVersionedSyncStage): self.RegisterManifestManager(self._GetInitializedManager(self.internal)) if (self._run.config.master and self._GetSlaveConfigs()): assert self.internal, 'Unified masters must use an internal checkout.' - MasterSlaveSyncStage.sub_manager = self._GetInitializedManager(False) + MasterSlaveLKGMSyncStage.sub_manager = self._GetInitializedManager(False) def ForceVersion(self, version): - manifest = super(MasterSlaveSyncStage, self).ForceVersion(version) - if MasterSlaveSyncStage.sub_manager: - MasterSlaveSyncStage.sub_manager.BootstrapFromVersion(version) + manifest = super(MasterSlaveLKGMSyncStage, self).ForceVersion(version) + if MasterSlaveLKGMSyncStage.sub_manager: + MasterSlaveLKGMSyncStage.sub_manager.BootstrapFromVersion(version) return manifest @@ -629,10 +633,9 @@ class MasterSlaveSyncStage(ManifestVersionedSyncStage): manifest = self.manifest_manager.CreateNewCandidate( chrome_version=self._chrome_version, build_id=build_id) - if MasterSlaveSyncStage.sub_manager: - MasterSlaveSyncStage.sub_manager.CreateFromManifest( - manifest, dashboard_url=self.ConstructDashboardURL(), - build_id=build_id) + if MasterSlaveLKGMSyncStage.sub_manager: + MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest( + manifest, dashboard_url=self.ConstructDashboardURL()) return manifest else: return self.manifest_manager.GetLatestCandidate( @@ -655,12 +658,16 @@ class MasterSlaveSyncStage(ManifestVersionedSyncStage): ManifestVersionedSyncStage.PerformStage(self) -class CommitQueueSyncStage(MasterSlaveSyncStage): +class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): """Commit Queue Sync stage that handles syncing and applying patches. - This stage handles syncing to a manifest, passing around that manifest to - other builders and finding the Gerrit Reviews ready to be committed and - applying them into its own checkout. + Similar to the MasterSlaveLKGMsync Stage, this stage handles syncing + to a manifest, passing around that manifest to other builders. + + What makes this stage different is that the CQ master finds the + patches on Gerrit which are ready to be committed, apply them, and + includes the pathces in the new manifest. The slaves sync to the + manifest, and apply the paches written in the manifest. """ def __init__(self, builder_run, **kwargs): @@ -743,8 +750,8 @@ class CommitQueueSyncStage(MasterSlaveSyncStage): manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool, build_id=build_id) - if MasterSlaveSyncStage.sub_manager: - MasterSlaveSyncStage.sub_manager.CreateFromManifest( + if MasterSlaveLKGMSyncStage.sub_manager: + MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest( manifest, dashboard_url=self.ConstructDashboardURL(), build_id=build_id) -- cgit v1.2.3 From 4e8d86d67c385fc643169d69077d229743b014c2 Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 7 Aug 2014 17:00:36 -0700 Subject: Submit changes in the Pre-CQ if requested. BUG=chromium:397312 TEST=pre-cq-launcher run with --debug, edited so that it submits changes TEST=unit tests Change-Id: Ib3e164dc571d6018b10ce534cdf25e6c79a5d3e5 Reviewed-on: https://chromium-review.googlesource.com/212306 Tested-by: David James Reviewed-by: Aviv Keshet Reviewed-by: Yu-Ju Hong Commit-Queue: David James --- cbuildbot/stages/sync_stages.py | 52 ++++++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 11 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index c64c984de..9168109f0 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -698,7 +698,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): changes_to_test = [] for change in changes: status = pool.GetCLStatus(PRE_CQ, change) - if status == manifest_version.BuilderStatus.STATUS_PASSED: + if status == validation_pool.ValidationPool.STATUS_PASSED: changes_to_test.append(change) # If we only see changes that weren't verified by Pre-CQ, try all of the @@ -818,12 +818,24 @@ class PreCQSyncStage(SyncStage): class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" + # The CL is currently being tested by a Pre-CQ builder. STATUS_INFLIGHT = validation_pool.ValidationPool.STATUS_INFLIGHT + + # The CL has passed the Pre-CQ. STATUS_PASSED = validation_pool.ValidationPool.STATUS_PASSED + + # The CL has failed the Pre-CQ. STATUS_FAILED = validation_pool.ValidationPool.STATUS_FAILED + + # We have requested a Pre-CQ trybot but it has not started yet. STATUS_LAUNCHING = validation_pool.ValidationPool.STATUS_LAUNCHING + + # The CL is ready to be retried. STATUS_WAITING = validation_pool.ValidationPool.STATUS_WAITING + # The CL has passed the Pre-CQ and is ready to be submitted. + STATUS_READY_TO_SUBMIT = validation_pool.ValidationPool.STATUS_READY_TO_SUBMIT + # The number of minutes we allow before considering a launch attempt failed. # If this window isn't hit in a given launcher run, the window will start # again from scratch in the next run. @@ -882,7 +894,7 @@ class PreCQLauncherStage(SyncStage): ) cros_build_lib.PrintBuildbotLink(' | '.join(items), patch.url) - def GetPreCQStatus(self, pool, changes): + def GetPreCQStatus(self, pool, changes, status_map): """Get the Pre-CQ status of a list of changes. Side effect: reject or retry changes that have timed out. @@ -890,6 +902,7 @@ class PreCQLauncherStage(SyncStage): Args: pool: The validation pool. changes: Changes to examine. + status_map: Dict mapping changes to their CL status. Returns: busy: The set of CLs that are currently being tested. @@ -898,7 +911,7 @@ class PreCQLauncherStage(SyncStage): busy, passed = set(), set() for change in changes: - status = pool.GetCLStatus(PRE_CQ, change) + status = status_map[change] if status != self.STATUS_LAUNCHING: # The trybot is not launching, so we should remove it from our @@ -964,10 +977,13 @@ class PreCQLauncherStage(SyncStage): busy.add(change) pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, self._run.options.debug) - self._PrintPatchStatus(change, 'failed') + self._PrintPatchStatus(change, status) elif status == self.STATUS_PASSED: passed.add(change) - self._PrintPatchStatus(change, 'passed') + self._PrintPatchStatus(change, status) + elif status == self.STATUS_READY_TO_SUBMIT: + passed.add(change) + self._PrintPatchStatus(change, 'submitting') return busy, passed @@ -990,16 +1006,21 @@ class PreCQLauncherStage(SyncStage): pool.UpdateCLStatus(PRE_CQ, patch, self.STATUS_LAUNCHING, self._run.options.debug) - def GetDisjointTransactionsToTest(self, pool, changes): + def GetDisjointTransactionsToTest(self, pool, changes, status_map): """Get the list of disjoint transactions to test. Side effect: reject or retry changes that have timed out. + Args: + pool: The validation pool. + changes: Changes to examine. + status_map: Dict mapping changes to their CL status. + Returns: A list of disjoint transactions to test. Each transaction should be sent to a different Pre-CQ trybot. """ - busy, passed = self.GetPreCQStatus(pool, changes) + busy, passed = self.GetPreCQStatus(pool, changes, status_map) # Create a list of disjoint transactions to test. manifest = git.ManifestCheckout.Cached(self._build_root) @@ -1034,14 +1055,23 @@ class PreCQLauncherStage(SyncStage): Non-manifest changes are just submitted here because they don't need to be verified by either the Pre-CQ or CQ. """ - # Submit non-manifest changes if we can. - if tree_status.IsTreeOpen(): - pool.SubmitNonManifestChanges(check_tree_open=False) + # Get change status. + status_map = {} + for change in changes: + status = pool.GetCLStatus(PRE_CQ, change) + status_map[change] = status # Launch trybots for manifest changes. - for plan in self.GetDisjointTransactionsToTest(pool, changes): + for plan in self.GetDisjointTransactionsToTest(pool, changes, status_map): self.LaunchTrybot(pool, plan) + # Submit changes that don't need a CQ run if we can. + if tree_status.IsTreeOpen(): + pool.SubmitNonManifestChanges(check_tree_open=False) + submitting = [change for (change, status) in status_map.items() + if status == self.STATUS_READY_TO_SUBMIT] + pool.SubmitChanges(submitting, check_tree_open=False) + # Tell ValidationPool to keep waiting for more changes until we hit # its internal timeout. return [], [] -- cgit v1.2.3 From 76e44d207c6f23d4021212fc1e676ca7c7ae2335 Mon Sep 17 00:00:00 2001 From: Stefan Zager Date: Fri, 15 Aug 2014 21:33:37 -0700 Subject: Convert ebuild to pull chrome sources from git. BUG=chromium:376027 TEST=scripts/cros_mark_chrome_as_stable_unittest.py Change-Id: I6359b233d64423c38162b3223a64db33717cca23 Reviewed-on: https://chromium-review.googlesource.com/212790 Reviewed-by: David James Tested-by: Stefan Zager --- cbuildbot/stages/sync_stages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 9168109f0..f56e5999f 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -25,7 +25,6 @@ from chromite.cbuildbot.stages import generic_stages from chromite.cbuildbot.stages import build_stages from chromite.lib import commandline from chromite.lib import cros_build_lib -from chromite.lib import gclient from chromite.lib import git from chromite.lib import osutils from chromite.lib import patch as cros_patch @@ -644,7 +643,8 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): def GetLatestChromeVersion(self): """Returns the version of Chrome to uprev.""" - return cros_mark_chrome_as_stable.GetLatestRelease(gclient.GetBaseURLs()[0]) + return cros_mark_chrome_as_stable.GetLatestRelease( + constants.CHROMIUM_GOB_URL) @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): -- cgit v1.2.3 From 826b00e93f5c8bb354f693c0f77f2329f6296582 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Thu, 21 Aug 2014 01:29:31 +0000 Subject: Revert "Convert ebuild to pull chrome sources from git." This reverts commit 76e44d207c6f23d4021212fc1e676ca7c7ae2335. BUG=chromium:405708 TEST=None Change-Id: Ide1a5247fe8d26ba8f475079a1d6c2c3f60bc860 Reviewed-on: https://chromium-review.googlesource.com/213372 Reviewed-by: David James Tested-by: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index f56e5999f..9168109f0 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -25,6 +25,7 @@ from chromite.cbuildbot.stages import generic_stages from chromite.cbuildbot.stages import build_stages from chromite.lib import commandline from chromite.lib import cros_build_lib +from chromite.lib import gclient from chromite.lib import git from chromite.lib import osutils from chromite.lib import patch as cros_patch @@ -643,8 +644,7 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): def GetLatestChromeVersion(self): """Returns the version of Chrome to uprev.""" - return cros_mark_chrome_as_stable.GetLatestRelease( - constants.CHROMIUM_GOB_URL) + return cros_mark_chrome_as_stable.GetLatestRelease(gclient.GetBaseURLs()[0]) @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): -- cgit v1.2.3 From d49d9ff866f4c333f48128f852aea8de37cc3715 Mon Sep 17 00:00:00 2001 From: Stefan Zager Date: Fri, 15 Aug 2014 21:33:37 -0700 Subject: Convert ebuild to pull chrome sources from git. Re-landing this: https://chromium-review.googlesource.com/212790 It should work now, after this change to gclient: https://codereview.chromium.org/490233003 ... and after rolling that change into the manifest: https://chromium-review.googlesource.com/#/c/213524/ https://chrome-internal-review.googlesource.com/#/c/173395/ BUG=chromium:376027 TEST=scripts/cros_mark_chrome_as_stable_unittest.py Change-Id: I20c2a6d3a221fae0a3849628b688cded996834b4 Reviewed-on: https://chromium-review.googlesource.com/213460 Reviewed-by: David James Tested-by: Stefan Zager --- cbuildbot/stages/sync_stages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 9168109f0..f56e5999f 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -25,7 +25,6 @@ from chromite.cbuildbot.stages import generic_stages from chromite.cbuildbot.stages import build_stages from chromite.lib import commandline from chromite.lib import cros_build_lib -from chromite.lib import gclient from chromite.lib import git from chromite.lib import osutils from chromite.lib import patch as cros_patch @@ -644,7 +643,8 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): def GetLatestChromeVersion(self): """Returns the version of Chrome to uprev.""" - return cros_mark_chrome_as_stable.GetLatestRelease(gclient.GetBaseURLs()[0]) + return cros_mark_chrome_as_stable.GetLatestRelease( + constants.CHROMIUM_GOB_URL) @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): -- cgit v1.2.3 From 629e7fff90deafe8334c8310a3120e7d2cc6859d Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Tue, 19 Aug 2014 21:27:06 -0700 Subject: metadata: eliminate post-unpickle recording of validation_pool clactions This CL eliminates legacy code left over from before metadata was being preserved across cbuildbot reexecutions. It also fixes a minor bug where pre-cq builds were duplicately recording 'picked_up' cl actions. BUG=None TEST=Unit tests pass, trybot manually verified Change-Id: I3ce3c4a698f33a5ee0c59556eee82c90812d4019 Reviewed-on: https://chromium-review.googlesource.com/213233 Tested-by: Aviv Keshet Reviewed-by: Yu-Ju Hong Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index f56e5999f..22633ad0b 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -689,7 +689,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): filename = self._run.options.validation_pool if filename: self.pool = validation_pool.ValidationPool.Load(filename, - metadata=self._run.attrs.metadata, record_patches=False) + metadata=self._run.attrs.metadata) else: self._SetPoolFromManifest(self.manifest_manager.GetLocalManifest()) -- cgit v1.2.3 From e13ef65d4e05b61ad09013ff453188fea30ade22 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Fri, 5 Sep 2014 09:55:31 -0700 Subject: cidb: record pre-cq status changes as cl actions, to database BUG=chromium:410546 TEST=Unit tests pass; `./cbuildbot --remote --debug --buildbot pre-cq-launcher` records the expected launching action to database Change-Id: I21d84f3c355e6d7ef218bb3ecb2706d6868efb65 Reviewed-on: https://chromium-review.googlesource.com/216711 Tested-by: Aviv Keshet Reviewed-by: David James Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 22633ad0b..b208c24c2 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -862,6 +862,8 @@ class PreCQLauncherStage(SyncStage): self.inflight = {} self.retried = set() + self._build_id = self._run.attrs.metadata.GetValue('build_id') + def _HasLaunchTimedOut(self, change): """Check whether a given |change| has timed out on its trybot launch. @@ -942,13 +944,15 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, - self._run.options.debug) + self._run.options.debug, + build_id=self._build_id) self.retried.discard(change) else: # Try the change again. self.retried.add(change) pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, - self._run.options.debug) + self._run.options.debug, + build_id=self._build_id) elif status == self.STATUS_INFLIGHT: # Once a Pre-CQ run actually starts, it'll set the status to # STATUS_INFLIGHT. @@ -968,7 +972,8 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, - self._run.options.debug) + self._run.options.debug, + build_id=self._build_id) elif status == self.STATUS_FAILED: # The Pre-CQ run failed for this change. It's possible that we got # unlucky and this change was just marked as 'Not Ready' by a bot. To @@ -976,7 +981,8 @@ class PreCQLauncherStage(SyncStage): # as 'Ready' next time we check, we'll know the CL is truly still ready. busy.add(change) pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, - self._run.options.debug) + self._run.options.debug, + build_id=self._build_id) self._PrintPatchStatus(change, status) elif status == self.STATUS_PASSED: passed.add(change) @@ -1004,7 +1010,8 @@ class PreCQLauncherStage(SyncStage): for patch in plan: if pool.GetCLStatus(PRE_CQ, patch) != self.STATUS_PASSED: pool.UpdateCLStatus(PRE_CQ, patch, self.STATUS_LAUNCHING, - self._run.options.debug) + self._run.options.debug, + build_id=self._build_id) def GetDisjointTransactionsToTest(self, pool, changes, status_map): """Get the list of disjoint transactions to test. -- cgit v1.2.3 From 840d27121f6bef7ea024e87086d0eb9d3567a4df Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 9 Sep 2014 15:28:47 -0700 Subject: Increase timeout for Pre-CQ Launcher. BUG=chromium:412564 TEST=unit tests Change-Id: I78c08222a4fa5d54875641ecc4f27a351d87f987 Reviewed-on: https://chromium-review.googlesource.com/217370 Reviewed-by: Aviv Keshet Tested-by: David James --- cbuildbot/stages/sync_stages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index b208c24c2..242c687c1 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -839,12 +839,12 @@ class PreCQLauncherStage(SyncStage): # The number of minutes we allow before considering a launch attempt failed. # If this window isn't hit in a given launcher run, the window will start # again from scratch in the next run. - LAUNCH_DELAY = 30 + LAUNCH_DELAY = 90 # The number of minutes we allow before considering an in-flight # job failed. If this window isn't hit in a given launcher run, the window # will start again from scratch in the next run. - INFLIGHT_DELAY = 120 + INFLIGHT_DELAY = 180 # The maximum number of patches we will allow in a given trybot run. This is # needed because our trybot infrastructure can only handle so many patches at -- cgit v1.2.3 From e2b61abf09e515b77341d9c1cb1484226b030653 Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 9 Sep 2014 15:34:53 -0700 Subject: Set a timeout for Pre-CQ runs. If Pre-CQ runs take too long, make sure they time out. BUG=chromium:412564 TEST=Unit tests. Change-Id: I145dbe58bf51d023e70bdcd881ccb68a2c4ace17 Reviewed-on: https://chromium-review.googlesource.com/217371 Reviewed-by: Aviv Keshet Tested-by: David James --- cbuildbot/stages/sync_stages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 242c687c1..7485eb981 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1004,7 +1004,8 @@ class PreCQLauncherStage(SyncStage): if self._run.options.debug: cmd.append('--debug') for patch in plan: - cmd += ['-g', cros_patch.AddPrefix(patch, patch.gerrit_number)] + cmd += ['-g', cros_patch.AddPrefix(patch, patch.gerrit_number), + '--timeout', self.INFLIGHT_DELAY] self._PrintPatchStatus(patch, 'testing') cros_build_lib.RunCommand(cmd, cwd=self._build_root) for patch in plan: -- cgit v1.2.3 From 4ba031df44532e998f7f1f834f11af9e7f8ee3e0 Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 9 Sep 2014 16:13:21 -0700 Subject: Typo fix: Use string for timeout. BUG=chromium:412564 TEST=Unit test. Change-Id: I8a0c0255696a3c54aa2fcd3c89c2670da9d8d07c Reviewed-on: https://chromium-review.googlesource.com/217374 Reviewed-by: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 7485eb981..2b2d1acc1 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1005,7 +1005,7 @@ class PreCQLauncherStage(SyncStage): cmd.append('--debug') for patch in plan: cmd += ['-g', cros_patch.AddPrefix(patch, patch.gerrit_number), - '--timeout', self.INFLIGHT_DELAY] + '--timeout', str(self.INFLIGHT_DELAY)] self._PrintPatchStatus(patch, 'testing') cros_build_lib.RunCommand(cmd, cwd=self._build_root) for patch in plan: -- cgit v1.2.3 From d69d43800b044cb3a9563429ec49cc6ee043766b Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 9 Sep 2014 20:52:18 -0700 Subject: Typo fix: Measure timeouts in minutes. BUG=chromium:412564 TEST=Unit test. Change-Id: I38f129f3707550870d6675631b9259da70147e24 Reviewed-on: https://chromium-review.googlesource.com/217401 Reviewed-by: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 2b2d1acc1..1cfab39e5 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1000,12 +1000,12 @@ class PreCQLauncherStage(SyncStage): pool: ValidationPool corresponding to |plan|. plan: The list of patches to test in the Pre-CQ run. """ - cmd = ['cbuildbot', '--remote', constants.PRE_CQ_BUILDER_NAME] + cmd = ['cbuildbot', '--remote', constants.PRE_CQ_BUILDER_NAME, + '--timeout', str(self.INFLIGHT_DELAY * 60)] if self._run.options.debug: cmd.append('--debug') for patch in plan: - cmd += ['-g', cros_patch.AddPrefix(patch, patch.gerrit_number), - '--timeout', str(self.INFLIGHT_DELAY)] + cmd += ['-g', cros_patch.AddPrefix(patch, patch.gerrit_number)] self._PrintPatchStatus(patch, 'testing') cros_build_lib.RunCommand(cmd, cwd=self._build_root) for patch in plan: -- cgit v1.2.3 From 383367e89fead21c09e22a2be3bffeeb59c99db9 Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Tue, 16 Sep 2014 15:06:17 -0400 Subject: convert to print_function BUG=chromium:414895 TEST=`./cbuildbot/run_tests` passes TEST=`cros lint` passes (it complains if print is used incorrectly) TEST=`cbuildbot chromiumos-sdk` passes TEST=`cbuildbot lumpy-release` passes Change-Id: I67e7e1290b03a16525e0c2b1afa74ebbd8d3508f Reviewed-on: https://chromium-review.googlesource.com/218572 Reviewed-by: David James Reviewed-by: Yu-Ju Hong Commit-Queue: Mike Frysinger Tested-by: Mike Frysinger --- cbuildbot/stages/sync_stages.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 1cfab39e5..101daf5db 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -4,6 +4,8 @@ """Module containing the sync stages.""" +from __future__ import print_function + import contextlib import datetime import logging @@ -310,8 +312,8 @@ class SyncStage(generic_stages.BuilderStage): if not self.skip_sync: self.repo.Sync(next_manifest) - print >> sys.stderr, self.repo.ExportManifest( - mark_revision=self.output_manifest_sha1) + print(self.repo.ExportManifest(mark_revision=self.output_manifest_sha1), + file=sys.stderr) def RunPrePatchBuild(self): """Run through a pre-patch build to prepare for incremental build. -- cgit v1.2.3 From 91f687c1b6254335bbe8fa37fa39e43994fe496f Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Tue, 30 Sep 2014 10:54:48 -0700 Subject: cidb: eliminate CL status files This CL eliminates use of CL status files and status counters. For the CQ, the status counters were used in generating summary text that included the number of prior failures (and in the past also in identifying suspects, but this is no longer the case). The counters have been replaced by counting the number of occurences of KICKED_OUT actions for the change. For the pre-CQ, status files were used to coordinate and track state, by the pre-cq-launcher, the pre-cq-group, and master-paladin. These are replaced by status transition actions that are recorded to cidb. The pre-cq status of a given change is the status corresponding to the most recent pre-cq status action. - Rename PRE_CQ_BUILDER_NAME for consistency. - Rename GetCLStatus to GetCLPreCQStatus, update all its callers, and rewrite it to use the last known pre-CQ action for a change as its pre-cq status. - Rename UpdateCLStatus to UpdateCLPreCQStatus, update all its callers (deleting callers that were CQ-specific), and rewrite it to only record a status transition action to cidb. - Delete _FindPreviouslyFailedChanges, which was not used anywhere. - Do not update a change's pass_count, this was only used when printing CL summary links, and not clear why this should ever be nonzero. - Delete CLStatusMock, instead use FakeCIDBConnection in unit tests and do not mock out the CL status methods. BUG=chromium:410546 TEST=`git grep PRE_CQ_BUILDER_NAME` -> no results `git grep GetCLStatus` -> no results `git grep UpdateCLStatus` -> no results `git grep _FindPreviouslyFailedChanges` -> no results New unit tests of GetCLActionCount, UpdateCLPreCQStatus, and GetCLPreCQStatus. Existing unit tests fixed and passing. Remote trybots. Local trybots. Change-Id: I50966ec233f2caea620a25f36d0b12e28eb34565 Reviewed-on: https://chromium-review.googlesource.com/220599 Reviewed-by: Aviv Keshet Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 101daf5db..f19f754f7 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -699,7 +699,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): # First, look for changes that were tested by the Pre-CQ. changes_to_test = [] for change in changes: - status = pool.GetCLStatus(PRE_CQ, change) + status = pool.GetCLPreCQStatus(change) if status == validation_pool.ValidationPool.STATUS_PASSED: changes_to_test.append(change) @@ -820,7 +820,7 @@ class PreCQSyncStage(SyncStage): class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" - # The CL is currently being tested by a Pre-CQ builder. + # The CL is currently being tested by a Pre-CQ trybot. STATUS_INFLIGHT = validation_pool.ValidationPool.STATUS_INFLIGHT # The CL has passed the Pre-CQ. @@ -945,16 +945,14 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) - pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, - self._run.options.debug, - build_id=self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED, + self._build_id) self.retried.discard(change) else: # Try the change again. self.retried.add(change) - pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, - self._run.options.debug, - build_id=self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING, + self._build_id) elif status == self.STATUS_INFLIGHT: # Once a Pre-CQ run actually starts, it'll set the status to # STATUS_INFLIGHT. @@ -973,18 +971,16 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) - pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, - self._run.options.debug, - build_id=self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED, + self._build_id) elif status == self.STATUS_FAILED: # The Pre-CQ run failed for this change. It's possible that we got # unlucky and this change was just marked as 'Not Ready' by a bot. To # test this, mark the CL as 'waiting' for now. If the CL is still marked # as 'Ready' next time we check, we'll know the CL is truly still ready. busy.add(change) - pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, - self._run.options.debug, - build_id=self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING, + self._build_id) self._PrintPatchStatus(change, status) elif status == self.STATUS_PASSED: passed.add(change) @@ -1002,7 +998,7 @@ class PreCQLauncherStage(SyncStage): pool: ValidationPool corresponding to |plan|. plan: The list of patches to test in the Pre-CQ run. """ - cmd = ['cbuildbot', '--remote', constants.PRE_CQ_BUILDER_NAME, + cmd = ['cbuildbot', '--remote', constants.PRE_CQ_GROUP_CONFIG, '--timeout', str(self.INFLIGHT_DELAY * 60)] if self._run.options.debug: cmd.append('--debug') @@ -1011,10 +1007,9 @@ class PreCQLauncherStage(SyncStage): self._PrintPatchStatus(patch, 'testing') cros_build_lib.RunCommand(cmd, cwd=self._build_root) for patch in plan: - if pool.GetCLStatus(PRE_CQ, patch) != self.STATUS_PASSED: - pool.UpdateCLStatus(PRE_CQ, patch, self.STATUS_LAUNCHING, - self._run.options.debug, - build_id=self._build_id) + if pool.GetCLPreCQStatus(patch) != self.STATUS_PASSED: + pool.UpdateCLPreCQStatus(patch, self.STATUS_LAUNCHING, + self._build_id) def GetDisjointTransactionsToTest(self, pool, changes, status_map): """Get the list of disjoint transactions to test. @@ -1068,7 +1063,7 @@ class PreCQLauncherStage(SyncStage): # Get change status. status_map = {} for change in changes: - status = pool.GetCLStatus(PRE_CQ, change) + status = pool.GetCLPreCQStatus(change) status_map[change] = status # Launch trybots for manifest changes. -- cgit v1.2.3 From 5456858c3e51154359706586c4d79dfa8763323b Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Fri, 3 Oct 2014 06:54:44 +0000 Subject: Revert "cidb: eliminate CL status files" This reverts commit 91f687c1b6254335bbe8fa37fa39e43994fe496f. Change-Id: I26c0c9bf70b7f21e27fb4d03776fb5a855a05b58 Reviewed-on: https://chromium-review.googlesource.com/221382 Reviewed-by: Aviv Keshet Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index f19f754f7..101daf5db 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -699,7 +699,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): # First, look for changes that were tested by the Pre-CQ. changes_to_test = [] for change in changes: - status = pool.GetCLPreCQStatus(change) + status = pool.GetCLStatus(PRE_CQ, change) if status == validation_pool.ValidationPool.STATUS_PASSED: changes_to_test.append(change) @@ -820,7 +820,7 @@ class PreCQSyncStage(SyncStage): class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" - # The CL is currently being tested by a Pre-CQ trybot. + # The CL is currently being tested by a Pre-CQ builder. STATUS_INFLIGHT = validation_pool.ValidationPool.STATUS_INFLIGHT # The CL has passed the Pre-CQ. @@ -945,14 +945,16 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED, - self._build_id) + pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, + self._run.options.debug, + build_id=self._build_id) self.retried.discard(change) else: # Try the change again. self.retried.add(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING, - self._build_id) + pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, + self._run.options.debug, + build_id=self._build_id) elif status == self.STATUS_INFLIGHT: # Once a Pre-CQ run actually starts, it'll set the status to # STATUS_INFLIGHT. @@ -971,16 +973,18 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED, - self._build_id) + pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, + self._run.options.debug, + build_id=self._build_id) elif status == self.STATUS_FAILED: # The Pre-CQ run failed for this change. It's possible that we got # unlucky and this change was just marked as 'Not Ready' by a bot. To # test this, mark the CL as 'waiting' for now. If the CL is still marked # as 'Ready' next time we check, we'll know the CL is truly still ready. busy.add(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING, - self._build_id) + pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, + self._run.options.debug, + build_id=self._build_id) self._PrintPatchStatus(change, status) elif status == self.STATUS_PASSED: passed.add(change) @@ -998,7 +1002,7 @@ class PreCQLauncherStage(SyncStage): pool: ValidationPool corresponding to |plan|. plan: The list of patches to test in the Pre-CQ run. """ - cmd = ['cbuildbot', '--remote', constants.PRE_CQ_GROUP_CONFIG, + cmd = ['cbuildbot', '--remote', constants.PRE_CQ_BUILDER_NAME, '--timeout', str(self.INFLIGHT_DELAY * 60)] if self._run.options.debug: cmd.append('--debug') @@ -1007,9 +1011,10 @@ class PreCQLauncherStage(SyncStage): self._PrintPatchStatus(patch, 'testing') cros_build_lib.RunCommand(cmd, cwd=self._build_root) for patch in plan: - if pool.GetCLPreCQStatus(patch) != self.STATUS_PASSED: - pool.UpdateCLPreCQStatus(patch, self.STATUS_LAUNCHING, - self._build_id) + if pool.GetCLStatus(PRE_CQ, patch) != self.STATUS_PASSED: + pool.UpdateCLStatus(PRE_CQ, patch, self.STATUS_LAUNCHING, + self._run.options.debug, + build_id=self._build_id) def GetDisjointTransactionsToTest(self, pool, changes, status_map): """Get the list of disjoint transactions to test. @@ -1063,7 +1068,7 @@ class PreCQLauncherStage(SyncStage): # Get change status. status_map = {} for change in changes: - status = pool.GetCLPreCQStatus(change) + status = pool.GetCLStatus(PRE_CQ, change) status_map[change] = status # Launch trybots for manifest changes. -- cgit v1.2.3 From d31fd83ef85741efef943c76bb07ab304fdeca50 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Tue, 30 Sep 2014 10:54:48 -0700 Subject: cidb: eliminate CL status files This CL eliminates use of CL status files and status counters. For the CQ, the status counters were used in generating summary text that included the number of prior failures (and in the past also in identifying suspects, but this is no longer the case). The counters have been replaced by counting the number of occurences of KICKED_OUT actions for the change. For the pre-CQ, status files were used to coordinate and track state, by the pre-cq-launcher, the pre-cq-group, and master-paladin. These are replaced by status transition actions that are recorded to cidb. The pre-cq status of a given change is the status corresponding to the most recent pre-cq status action. - Rename PRE_CQ_BUILDER_NAME for consistency. - Rename GetCLStatus to GetCLPreCQStatus, update all its callers, and rewrite it to use the last known pre-CQ action for a change as its pre-cq status. - Rename UpdateCLStatus to UpdateCLPreCQStatus, update all its callers (deleting callers that were CQ-specific), and rewrite it to only record a status transition action to cidb. - Delete _FindPreviouslyFailedChanges, which was not used anywhere. - Do not update a change's pass_count, this was only used when printing CL summary links, and not clear why this should ever be nonzero. - Delete CLStatusMock, instead use FakeCIDBConnection in unit tests and do not mock out the CL status methods. BUG=chromium:410546 TEST=`git grep PRE_CQ_BUILDER_NAME` -> no results `git grep GetCLStatus` -> no results `git grep UpdateCLStatus` -> no results `git grep _FindPreviouslyFailedChanges` -> no results New unit tests of GetCLActionCount, UpdateCLPreCQStatus, and GetCLPreCQStatus. Existing unit tests fixed and passing. Remote trybots. Local trybots. Change-Id: Ida3dffddad24ffc658bae0700ff7a394c356f838 Reviewed-on: https://chromium-review.googlesource.com/221359 Tested-by: Aviv Keshet Reviewed-by: David James --- cbuildbot/stages/sync_stages.py | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 101daf5db..f19f754f7 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -699,7 +699,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): # First, look for changes that were tested by the Pre-CQ. changes_to_test = [] for change in changes: - status = pool.GetCLStatus(PRE_CQ, change) + status = pool.GetCLPreCQStatus(change) if status == validation_pool.ValidationPool.STATUS_PASSED: changes_to_test.append(change) @@ -820,7 +820,7 @@ class PreCQSyncStage(SyncStage): class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" - # The CL is currently being tested by a Pre-CQ builder. + # The CL is currently being tested by a Pre-CQ trybot. STATUS_INFLIGHT = validation_pool.ValidationPool.STATUS_INFLIGHT # The CL has passed the Pre-CQ. @@ -945,16 +945,14 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) - pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, - self._run.options.debug, - build_id=self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED, + self._build_id) self.retried.discard(change) else: # Try the change again. self.retried.add(change) - pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, - self._run.options.debug, - build_id=self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING, + self._build_id) elif status == self.STATUS_INFLIGHT: # Once a Pre-CQ run actually starts, it'll set the status to # STATUS_INFLIGHT. @@ -973,18 +971,16 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) - pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, - self._run.options.debug, - build_id=self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED, + self._build_id) elif status == self.STATUS_FAILED: # The Pre-CQ run failed for this change. It's possible that we got # unlucky and this change was just marked as 'Not Ready' by a bot. To # test this, mark the CL as 'waiting' for now. If the CL is still marked # as 'Ready' next time we check, we'll know the CL is truly still ready. busy.add(change) - pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, - self._run.options.debug, - build_id=self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING, + self._build_id) self._PrintPatchStatus(change, status) elif status == self.STATUS_PASSED: passed.add(change) @@ -1002,7 +998,7 @@ class PreCQLauncherStage(SyncStage): pool: ValidationPool corresponding to |plan|. plan: The list of patches to test in the Pre-CQ run. """ - cmd = ['cbuildbot', '--remote', constants.PRE_CQ_BUILDER_NAME, + cmd = ['cbuildbot', '--remote', constants.PRE_CQ_GROUP_CONFIG, '--timeout', str(self.INFLIGHT_DELAY * 60)] if self._run.options.debug: cmd.append('--debug') @@ -1011,10 +1007,9 @@ class PreCQLauncherStage(SyncStage): self._PrintPatchStatus(patch, 'testing') cros_build_lib.RunCommand(cmd, cwd=self._build_root) for patch in plan: - if pool.GetCLStatus(PRE_CQ, patch) != self.STATUS_PASSED: - pool.UpdateCLStatus(PRE_CQ, patch, self.STATUS_LAUNCHING, - self._run.options.debug, - build_id=self._build_id) + if pool.GetCLPreCQStatus(patch) != self.STATUS_PASSED: + pool.UpdateCLPreCQStatus(patch, self.STATUS_LAUNCHING, + self._build_id) def GetDisjointTransactionsToTest(self, pool, changes, status_map): """Get the list of disjoint transactions to test. @@ -1068,7 +1063,7 @@ class PreCQLauncherStage(SyncStage): # Get change status. status_map = {} for change in changes: - status = pool.GetCLStatus(PRE_CQ, change) + status = pool.GetCLPreCQStatus(change) status_map[change] = status # Launch trybots for manifest changes. -- cgit v1.2.3 From 1cae123ab1fca465a8c56c83a467c9e3a2797427 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Fri, 3 Oct 2014 22:41:44 +0000 Subject: Revert "cidb: eliminate CL status files" This reverts commit d31fd83ef85741efef943c76bb07ab304fdeca50. Change-Id: I1b690758e9d4aeef36ebd78c335f31d356fc55bb Reviewed-on: https://chromium-review.googlesource.com/221181 Reviewed-by: Aviv Keshet Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index f19f754f7..101daf5db 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -699,7 +699,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): # First, look for changes that were tested by the Pre-CQ. changes_to_test = [] for change in changes: - status = pool.GetCLPreCQStatus(change) + status = pool.GetCLStatus(PRE_CQ, change) if status == validation_pool.ValidationPool.STATUS_PASSED: changes_to_test.append(change) @@ -820,7 +820,7 @@ class PreCQSyncStage(SyncStage): class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" - # The CL is currently being tested by a Pre-CQ trybot. + # The CL is currently being tested by a Pre-CQ builder. STATUS_INFLIGHT = validation_pool.ValidationPool.STATUS_INFLIGHT # The CL has passed the Pre-CQ. @@ -945,14 +945,16 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED, - self._build_id) + pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, + self._run.options.debug, + build_id=self._build_id) self.retried.discard(change) else: # Try the change again. self.retried.add(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING, - self._build_id) + pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, + self._run.options.debug, + build_id=self._build_id) elif status == self.STATUS_INFLIGHT: # Once a Pre-CQ run actually starts, it'll set the status to # STATUS_INFLIGHT. @@ -971,16 +973,18 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED, - self._build_id) + pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, + self._run.options.debug, + build_id=self._build_id) elif status == self.STATUS_FAILED: # The Pre-CQ run failed for this change. It's possible that we got # unlucky and this change was just marked as 'Not Ready' by a bot. To # test this, mark the CL as 'waiting' for now. If the CL is still marked # as 'Ready' next time we check, we'll know the CL is truly still ready. busy.add(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING, - self._build_id) + pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, + self._run.options.debug, + build_id=self._build_id) self._PrintPatchStatus(change, status) elif status == self.STATUS_PASSED: passed.add(change) @@ -998,7 +1002,7 @@ class PreCQLauncherStage(SyncStage): pool: ValidationPool corresponding to |plan|. plan: The list of patches to test in the Pre-CQ run. """ - cmd = ['cbuildbot', '--remote', constants.PRE_CQ_GROUP_CONFIG, + cmd = ['cbuildbot', '--remote', constants.PRE_CQ_BUILDER_NAME, '--timeout', str(self.INFLIGHT_DELAY * 60)] if self._run.options.debug: cmd.append('--debug') @@ -1007,9 +1011,10 @@ class PreCQLauncherStage(SyncStage): self._PrintPatchStatus(patch, 'testing') cros_build_lib.RunCommand(cmd, cwd=self._build_root) for patch in plan: - if pool.GetCLPreCQStatus(patch) != self.STATUS_PASSED: - pool.UpdateCLPreCQStatus(patch, self.STATUS_LAUNCHING, - self._build_id) + if pool.GetCLStatus(PRE_CQ, patch) != self.STATUS_PASSED: + pool.UpdateCLStatus(PRE_CQ, patch, self.STATUS_LAUNCHING, + self._run.options.debug, + build_id=self._build_id) def GetDisjointTransactionsToTest(self, pool, changes, status_map): """Get the list of disjoint transactions to test. @@ -1063,7 +1068,7 @@ class PreCQLauncherStage(SyncStage): # Get change status. status_map = {} for change in changes: - status = pool.GetCLPreCQStatus(change) + status = pool.GetCLStatus(PRE_CQ, change) status_map[change] = status # Launch trybots for manifest changes. -- cgit v1.2.3 From e1b9fd3110f7ffa98162af6f2abb529a42dbeb22 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Tue, 7 Oct 2014 11:49:36 -0700 Subject: cidb: prepare pre-CQ for elimination of CL status files This CL prepares the pre-CQ for CL status files to be eliminated, by recording INFLIGHT and PASSED statuses in the database in the way they will be expected by a future CL. BUG=chromium:410546 TEST=remote pre-cq-group trybot with --buildbot --debug flags, build inserts expected actions into debug instance of cidb. Change-Id: I3234257a0b720779c576fa252c197cdd8a065682 Reviewed-on: https://chromium-review.googlesource.com/221955 Tested-by: Aviv Keshet Reviewed-by: Yu-Ju Hong Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 101daf5db..47d661120 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -19,6 +19,7 @@ from chromite.cbuildbot import failures_lib from chromite.cbuildbot import constants from chromite.cbuildbot import lkgm_manager from chromite.cbuildbot import manifest_version +from chromite.cbuildbot import metadata_lib from chromite.cbuildbot import repository from chromite.cbuildbot import tree_status from chromite.cbuildbot import trybot_patch_pool @@ -26,6 +27,7 @@ from chromite.cbuildbot import validation_pool from chromite.cbuildbot.stages import generic_stages from chromite.cbuildbot.stages import build_stages from chromite.lib import commandline +from chromite.lib import cidb from chromite.lib import cros_build_lib from chromite.lib import git from chromite.lib import osutils @@ -816,6 +818,18 @@ class PreCQSyncStage(SyncStage): if len(self.pool.changes) == 0: cros_build_lib.Die('No changes have been applied.') + # Mark changes with pre-cq status inflight in database. + # This will be replaced by a call to UpdateCLPreCQStatus in a future CL. + if (cidb.CIDBConnectionFactory.IsCIDBSetup() and + cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()): + build_id = self._run.attrs.metadata.GetValue('build_id') + db = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder() + for change in self.pool.changes: + db.InsertCLActions( + build_id, + [metadata_lib.GetCLActionTuple( + change, constants.CL_ACTION_PRE_CQ_INFLIGHT)]) + class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" -- cgit v1.2.3 From 89075d587a92ef9b06e2b6d283bf800a6887d029 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Fri, 10 Oct 2014 01:52:05 +0000 Subject: Revert "cidb: prepare pre-CQ for elimination of CL status files" This reverts commit e1b9fd3110f7ffa98162af6f2abb529a42dbeb22. Change-Id: I6618755e781907fa9cd8b13c26fceb84cf2f985e Reviewed-on: https://chromium-review.googlesource.com/222740 Reviewed-by: Aviv Keshet Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 47d661120..101daf5db 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -19,7 +19,6 @@ from chromite.cbuildbot import failures_lib from chromite.cbuildbot import constants from chromite.cbuildbot import lkgm_manager from chromite.cbuildbot import manifest_version -from chromite.cbuildbot import metadata_lib from chromite.cbuildbot import repository from chromite.cbuildbot import tree_status from chromite.cbuildbot import trybot_patch_pool @@ -27,7 +26,6 @@ from chromite.cbuildbot import validation_pool from chromite.cbuildbot.stages import generic_stages from chromite.cbuildbot.stages import build_stages from chromite.lib import commandline -from chromite.lib import cidb from chromite.lib import cros_build_lib from chromite.lib import git from chromite.lib import osutils @@ -818,18 +816,6 @@ class PreCQSyncStage(SyncStage): if len(self.pool.changes) == 0: cros_build_lib.Die('No changes have been applied.') - # Mark changes with pre-cq status inflight in database. - # This will be replaced by a call to UpdateCLPreCQStatus in a future CL. - if (cidb.CIDBConnectionFactory.IsCIDBSetup() and - cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()): - build_id = self._run.attrs.metadata.GetValue('build_id') - db = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder() - for change in self.pool.changes: - db.InsertCLActions( - build_id, - [metadata_lib.GetCLActionTuple( - change, constants.CL_ACTION_PRE_CQ_INFLIGHT)]) - class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" -- cgit v1.2.3 From 87864946f7e8de5b5955b623c4fa77ca246df30f Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Tue, 7 Oct 2014 11:49:36 -0700 Subject: cidb: prepare pre-CQ for elimination of CL status files This CL prepares the pre-CQ for CL status files to be eliminated, by recording INFLIGHT and PASSED statuses in the database in the way they will be expected by a future CL. BUG=chromium:410546 TEST=remote pre-cq-group trybot with --buildbot --debug flags, build inserts expected actions into debug instance of cidb. Change-Id: Ifa104b99c8dfdc7a321050defbb460dacb3c2d0d Reviewed-on: https://chromium-review.googlesource.com/222759 Reviewed-by: Aviv Keshet Tested-by: Aviv Keshet Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 101daf5db..47d661120 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -19,6 +19,7 @@ from chromite.cbuildbot import failures_lib from chromite.cbuildbot import constants from chromite.cbuildbot import lkgm_manager from chromite.cbuildbot import manifest_version +from chromite.cbuildbot import metadata_lib from chromite.cbuildbot import repository from chromite.cbuildbot import tree_status from chromite.cbuildbot import trybot_patch_pool @@ -26,6 +27,7 @@ from chromite.cbuildbot import validation_pool from chromite.cbuildbot.stages import generic_stages from chromite.cbuildbot.stages import build_stages from chromite.lib import commandline +from chromite.lib import cidb from chromite.lib import cros_build_lib from chromite.lib import git from chromite.lib import osutils @@ -816,6 +818,18 @@ class PreCQSyncStage(SyncStage): if len(self.pool.changes) == 0: cros_build_lib.Die('No changes have been applied.') + # Mark changes with pre-cq status inflight in database. + # This will be replaced by a call to UpdateCLPreCQStatus in a future CL. + if (cidb.CIDBConnectionFactory.IsCIDBSetup() and + cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()): + build_id = self._run.attrs.metadata.GetValue('build_id') + db = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder() + for change in self.pool.changes: + db.InsertCLActions( + build_id, + [metadata_lib.GetCLActionTuple( + change, constants.CL_ACTION_PRE_CQ_INFLIGHT)]) + class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" -- cgit v1.2.3 From 1b50d4fccc39d792a12ea50651096568c6e4727c Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Tue, 30 Sep 2014 10:54:48 -0700 Subject: cidb: eliminate CL status files This CL eliminates use of CL status files and status counters. For the CQ, the status counters were used in generating summary text that included the number of prior failures (and in the past also in identifying suspects, but this is no longer the case). The counters have been replaced by counting the number of occurences of KICKED_OUT actions for the change. For the pre-CQ, status files were used to coordinate and track state, by the pre-cq-launcher, the pre-cq-group, and master-paladin. These are replaced by status transition actions that are recorded to cidb. The pre-cq status of a given change is the status corresponding to the most recent pre-cq status action. - Rename PRE_CQ_BUILDER_NAME for consistency. - Rename GetCLStatus to GetCLPreCQStatus, update all its callers, and rewrite it to use the last known pre-CQ action for a change as its pre-cq status. - Rename UpdateCLStatus to UpdateCLPreCQStatus, update all its callers (deleting callers that were CQ-specific), and rewrite it to only record a status transition action to cidb. - Delete _FindPreviouslyFailedChanges, which was not used anywhere. - Do not update a change's pass_count, this was only used when printing CL summary links, and not clear why this should ever be nonzero. - Delete CLStatusMock, instead use FakeCIDBConnection in unit tests and do not mock out the CL status methods. BUG=chromium:410546 TEST=`git grep PRE_CQ_BUILDER_NAME` -> no results `git grep GetCLStatus` -> no results `git grep UpdateCLStatus` -> no results `git grep _FindPreviouslyFailedChanges` -> no results New unit tests of GetCLActionCount, UpdateCLPreCQStatus, and GetCLPreCQStatus. Existing unit tests fixed and passing. Remote trybots. Local trybots. Change-Id: I91752207782ff7278e0a4ada4388fcf3509b1860 Reviewed-on: https://chromium-review.googlesource.com/221956 Reviewed-by: Aviv Keshet Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 50 ++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 28 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 47d661120..5826bc718 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -19,7 +19,6 @@ from chromite.cbuildbot import failures_lib from chromite.cbuildbot import constants from chromite.cbuildbot import lkgm_manager from chromite.cbuildbot import manifest_version -from chromite.cbuildbot import metadata_lib from chromite.cbuildbot import repository from chromite.cbuildbot import tree_status from chromite.cbuildbot import trybot_patch_pool @@ -701,7 +700,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): # First, look for changes that were tested by the Pre-CQ. changes_to_test = [] for change in changes: - status = pool.GetCLStatus(PRE_CQ, change) + status = pool.GetCLPreCQStatus(change) if status == validation_pool.ValidationPool.STATUS_PASSED: changes_to_test.append(change) @@ -818,23 +817,23 @@ class PreCQSyncStage(SyncStage): if len(self.pool.changes) == 0: cros_build_lib.Die('No changes have been applied.') - # Mark changes with pre-cq status inflight in database. - # This will be replaced by a call to UpdateCLPreCQStatus in a future CL. + # Mark changes that are not passed with pre-cq status inflight. if (cidb.CIDBConnectionFactory.IsCIDBSetup() and cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()): build_id = self._run.attrs.metadata.GetValue('build_id') - db = cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder() for change in self.pool.changes: - db.InsertCLActions( - build_id, - [metadata_lib.GetCLActionTuple( - change, constants.CL_ACTION_PRE_CQ_INFLIGHT)]) + current_status = validation_pool.ValidationPool.GetCLPreCQStatus( + change) + if current_status != validation_pool.ValidationPool.STATUS_PASSED: + validation_pool.ValidationPool.UpdateCLPreCQStatus( + change, validation_pool.ValidationPool.STATUS_INFLIGHT, + build_id) class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" - # The CL is currently being tested by a Pre-CQ builder. + # The CL is currently being tested by a Pre-CQ trybot. STATUS_INFLIGHT = validation_pool.ValidationPool.STATUS_INFLIGHT # The CL has passed the Pre-CQ. @@ -959,16 +958,14 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) - pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, - self._run.options.debug, - build_id=self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED, + self._build_id) self.retried.discard(change) else: # Try the change again. self.retried.add(change) - pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, - self._run.options.debug, - build_id=self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING, + self._build_id) elif status == self.STATUS_INFLIGHT: # Once a Pre-CQ run actually starts, it'll set the status to # STATUS_INFLIGHT. @@ -987,18 +984,16 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) - pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_FAILED, - self._run.options.debug, - build_id=self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED, + self._build_id) elif status == self.STATUS_FAILED: # The Pre-CQ run failed for this change. It's possible that we got # unlucky and this change was just marked as 'Not Ready' by a bot. To # test this, mark the CL as 'waiting' for now. If the CL is still marked # as 'Ready' next time we check, we'll know the CL is truly still ready. busy.add(change) - pool.UpdateCLStatus(PRE_CQ, change, self.STATUS_WAITING, - self._run.options.debug, - build_id=self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING, + self._build_id) self._PrintPatchStatus(change, status) elif status == self.STATUS_PASSED: passed.add(change) @@ -1016,7 +1011,7 @@ class PreCQLauncherStage(SyncStage): pool: ValidationPool corresponding to |plan|. plan: The list of patches to test in the Pre-CQ run. """ - cmd = ['cbuildbot', '--remote', constants.PRE_CQ_BUILDER_NAME, + cmd = ['cbuildbot', '--remote', constants.PRE_CQ_GROUP_CONFIG, '--timeout', str(self.INFLIGHT_DELAY * 60)] if self._run.options.debug: cmd.append('--debug') @@ -1025,10 +1020,9 @@ class PreCQLauncherStage(SyncStage): self._PrintPatchStatus(patch, 'testing') cros_build_lib.RunCommand(cmd, cwd=self._build_root) for patch in plan: - if pool.GetCLStatus(PRE_CQ, patch) != self.STATUS_PASSED: - pool.UpdateCLStatus(PRE_CQ, patch, self.STATUS_LAUNCHING, - self._run.options.debug, - build_id=self._build_id) + if pool.GetCLPreCQStatus(patch) != self.STATUS_PASSED: + pool.UpdateCLPreCQStatus(patch, self.STATUS_LAUNCHING, + self._build_id) def GetDisjointTransactionsToTest(self, pool, changes, status_map): """Get the list of disjoint transactions to test. @@ -1082,7 +1076,7 @@ class PreCQLauncherStage(SyncStage): # Get change status. status_map = {} for change in changes: - status = pool.GetCLStatus(PRE_CQ, change) + status = pool.GetCLPreCQStatus(change) status_map[change] = status # Launch trybots for manifest changes. -- cgit v1.2.3 From 2234c4c9dc0f5108e2301e147ddd42d74a221826 Mon Sep 17 00:00:00 2001 From: David James Date: Sat, 18 Oct 2014 11:26:43 -0700 Subject: Add ability to build incrementally in the CQ. This CL adds the ability to build incrementally in the CQ, but doesn't turn it on yet. BUG=chromium:422636 TEST=Turn on the feature, and run it on a real builder twice and make sure it reuses the chroot. TEST=Turn on the feature, and make sure it doesn't reuse chroot when version number is off. Change-Id: I3a65b1990c886bc689ead7289f02e1a288c2071b Reviewed-on: https://chromium-review.googlesource.com/224852 Reviewed-by: David James Commit-Queue: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 5826bc718..dbabd4961 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -15,6 +15,7 @@ from xml.etree import ElementTree from xml.dom import minidom from chromite.cbuildbot import cbuildbot_config +from chromite.cbuildbot import chroot_lib from chromite.cbuildbot import failures_lib from chromite.cbuildbot import constants from chromite.cbuildbot import lkgm_manager @@ -723,6 +724,15 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): self._run.config.master, self._run.options.debug, metadata=self._run.attrs.metadata) + def _GetLGKMVersionFromManifest(self, manifest): + manifest_dom = minidom.parse(manifest) + elements = manifest_dom.getElementsByTagName(lkgm_manager.LKGM_ELEMENT) + if elements: + lkgm_version = elements[0].getAttribute(lkgm_manager.LKGM_VERSION_ATTR) + logging.info( + 'LKGM version was found in the manifest: %s', lkgm_version) + return lkgm_version + def GetNextManifest(self): """Gets the next manifest using LKGM logic.""" assert self.manifest_manager, \ @@ -758,7 +768,6 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): manifest, dashboard_url=self.ConstructDashboardURL(), build_id=build_id) - return manifest else: manifest = self.manifest_manager.GetLatestCandidate( dashboard_url=self.ConstructDashboardURL()) @@ -773,7 +782,15 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): self._SetPoolFromManifest(manifest) self.pool.ApplyPoolIntoRepo() - return manifest + # Make sure the chroot version is valid. + lkgm_version = self._GetLGKMVersionFromManifest(manifest) + chroot_manager = chroot_lib.ChrootManager(self._build_root) + chroot_manager.EnsureChrootAtVersion(lkgm_version) + + # Clear the chroot version as we are in the middle of building it. + chroot_manager.ClearChrootVersion() + + return manifest @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): -- cgit v1.2.3 From 370731692e77837af3bbbb7ad7378cf8e6710d13 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Mon, 20 Oct 2014 11:20:56 -0700 Subject: clactions: Move CL action history logic to dedicated module A number of upcoming CLs will make more elaborate use of CL action history. This CL refactors much of the CL action logic into a new standalone module. To allow it to be standalone, a number of constants are consolidated in constants.py. BUG=chromium:425209 TEST=existing unit tests pass; cidb_integration_test; new placeholder unit test module added, however, clactions.py is already tested via validation_pool_unittest, cidb_integration_test, sync_stages_unittest, and others; gather_builder_stats works --clactions Change-Id: I7720964f9b9dd0aab56161171734b304c98117c5 Reviewed-on: https://chromium-review.googlesource.com/224630 Reviewed-by: Aviv Keshet Tested-by: Aviv Keshet Reviewed-by: David James Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index dbabd4961..585c5cc7d 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -702,7 +702,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): changes_to_test = [] for change in changes: status = pool.GetCLPreCQStatus(change) - if status == validation_pool.ValidationPool.STATUS_PASSED: + if status == constants.CL_STATUS_PASSED: changes_to_test.append(change) # If we only see changes that weren't verified by Pre-CQ, try all of the @@ -841,9 +841,9 @@ class PreCQSyncStage(SyncStage): for change in self.pool.changes: current_status = validation_pool.ValidationPool.GetCLPreCQStatus( change) - if current_status != validation_pool.ValidationPool.STATUS_PASSED: + if current_status != constants.CL_STATUS_PASSED: validation_pool.ValidationPool.UpdateCLPreCQStatus( - change, validation_pool.ValidationPool.STATUS_INFLIGHT, + change, constants.CL_STATUS_INFLIGHT, build_id) @@ -851,22 +851,22 @@ class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" # The CL is currently being tested by a Pre-CQ trybot. - STATUS_INFLIGHT = validation_pool.ValidationPool.STATUS_INFLIGHT + STATUS_INFLIGHT = constants.CL_STATUS_INFLIGHT # The CL has passed the Pre-CQ. - STATUS_PASSED = validation_pool.ValidationPool.STATUS_PASSED + STATUS_PASSED = constants.CL_STATUS_PASSED # The CL has failed the Pre-CQ. - STATUS_FAILED = validation_pool.ValidationPool.STATUS_FAILED + STATUS_FAILED = constants.CL_STATUS_FAILED # We have requested a Pre-CQ trybot but it has not started yet. - STATUS_LAUNCHING = validation_pool.ValidationPool.STATUS_LAUNCHING + STATUS_LAUNCHING = constants.CL_STATUS_LAUNCHING # The CL is ready to be retried. - STATUS_WAITING = validation_pool.ValidationPool.STATUS_WAITING + STATUS_WAITING = constants.CL_STATUS_WAITING # The CL has passed the Pre-CQ and is ready to be submitted. - STATUS_READY_TO_SUBMIT = validation_pool.ValidationPool.STATUS_READY_TO_SUBMIT + STATUS_READY_TO_SUBMIT = constants.CL_STATUS_READY_TO_SUBMIT # The number of minutes we allow before considering a launch attempt failed. # If this window isn't hit in a given launcher run, the window will start -- cgit v1.2.3 From 6b32b47fb2d653f0ca62673723cf84f8c4b32c61 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 23 Oct 2014 14:14:39 -0700 Subject: cidb: eliminate action cache, consolidate cidb creation boilerplate This CL cleans up the way in which builds access a cidb handle. Boilerplate code for accessing the build id and handle for the given build is moved to BuilderRun. To do this cleanly, this CL also cleans up the way in which validation_pool interacts with cidb. The "action history" cache is eliminated, without increasing the number of database queries. Several class methods are turned into instance methods or eliminiated. The validation pool now receives a BuilderRun for the run it is participating in, rather than just a metadata object. BUG=chromium:425209 TEST=unit tests; validation_pool_unittest.py --network; pre-cq-launcher --debug trybot; pre-cq-group trybot; pre-cq-master --debug trybot Change-Id: I83814f8dfb94dff22d8d9cf1fbd373c8fc774dc4 Reviewed-on: https://chromium-review.googlesource.com/225580 Reviewed-by: Aviv Keshet Tested-by: Aviv Keshet Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 57 +++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 28 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 585c5cc7d..6b5f2832d 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -26,8 +26,8 @@ from chromite.cbuildbot import trybot_patch_pool from chromite.cbuildbot import validation_pool from chromite.cbuildbot.stages import generic_stages from chromite.cbuildbot.stages import build_stages +from chromite.lib import clactions from chromite.lib import commandline -from chromite.lib import cidb from chromite.lib import cros_build_lib from chromite.lib import git from chromite.lib import osutils @@ -693,15 +693,19 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): filename = self._run.options.validation_pool if filename: self.pool = validation_pool.ValidationPool.Load(filename, - metadata=self._run.attrs.metadata) + builder_run=self._run) else: self._SetPoolFromManifest(self.manifest_manager.GetLocalManifest()) + # pylint: disable-msg=W0613 def _ChangeFilter(self, pool, changes, non_manifest_changes): # First, look for changes that were tested by the Pre-CQ. changes_to_test = [] + + _, db = self._run.GetCIDBHandle() + actions_for_changes = db.GetActionsForChanges(changes) for change in changes: - status = pool.GetCLPreCQStatus(change) + status = clactions.GetCLPreCQStatus(change, actions_for_changes) if status == constants.CL_STATUS_PASSED: changes_to_test.append(change) @@ -722,7 +726,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): manifest, self._run.config.overlays, self.repo, self._run.buildnumber, self.builder_name, self._run.config.master, self._run.options.debug, - metadata=self._run.attrs.metadata) + builder_run=self._run) def _GetLGKMVersionFromManifest(self, manifest): manifest_dom = minidom.parse(manifest) @@ -755,7 +759,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): self._run.options.mock_tree_status, changes_query=self._run.options.cq_gerrit_override, change_filter=self._ChangeFilter, throttled_ok=True, - metadata=self._run.attrs.metadata) + builder_run=self._run) except validation_pool.TreeIsClosedException as e: cros_build_lib.Warning(str(e)) @@ -820,7 +824,7 @@ class PreCQSyncStage(SyncStage): filename = self._run.options.validation_pool if filename: self.pool = validation_pool.ValidationPool.Load(filename, - metadata=self._run.attrs.metadata) + builder_run=self._run) def PerformStage(self): super(PreCQSyncStage, self).PerformStage() @@ -828,23 +832,20 @@ class PreCQSyncStage(SyncStage): self._run.config.overlays, self._build_root, self._run.buildnumber, self._run.config.name, dryrun=self._run.options.debug_forced, changes=self.patches, - metadata=self._run.attrs.metadata) + builder_run=self._run) self.pool.ApplyPoolIntoRepo() if len(self.pool.changes) == 0: cros_build_lib.Die('No changes have been applied.') # Mark changes that are not passed with pre-cq status inflight. - if (cidb.CIDBConnectionFactory.IsCIDBSetup() and - cidb.CIDBConnectionFactory.GetCIDBConnectionForBuilder()): - build_id = self._run.attrs.metadata.GetValue('build_id') + _, db = self._run.GetCIDBHandle() + if db: + action_history = db.GetActionsForChanges(self.pool.changes) for change in self.pool.changes: - current_status = validation_pool.ValidationPool.GetCLPreCQStatus( - change) + current_status = clactions.GetCLPreCQStatus(change, action_history) if current_status != constants.CL_STATUS_PASSED: - validation_pool.ValidationPool.UpdateCLPreCQStatus( - change, constants.CL_STATUS_INFLIGHT, - build_id) + self.pool.UpdateCLPreCQStatus(change, constants.CL_STATUS_INFLIGHT) class PreCQLauncherStage(SyncStage): @@ -975,14 +976,12 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED, - self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED) self.retried.discard(change) else: # Try the change again. self.retried.add(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING, - self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING) elif status == self.STATUS_INFLIGHT: # Once a Pre-CQ run actually starts, it'll set the status to # STATUS_INFLIGHT. @@ -1001,16 +1000,14 @@ class PreCQLauncherStage(SyncStage): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED, - self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED) elif status == self.STATUS_FAILED: # The Pre-CQ run failed for this change. It's possible that we got # unlucky and this change was just marked as 'Not Ready' by a bot. To # test this, mark the CL as 'waiting' for now. If the CL is still marked # as 'Ready' next time we check, we'll know the CL is truly still ready. busy.add(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING, - self._build_id) + pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING) self._PrintPatchStatus(change, status) elif status == self.STATUS_PASSED: passed.add(change) @@ -1036,10 +1033,12 @@ class PreCQLauncherStage(SyncStage): cmd += ['-g', cros_patch.AddPrefix(patch, patch.gerrit_number)] self._PrintPatchStatus(patch, 'testing') cros_build_lib.RunCommand(cmd, cwd=self._build_root) + _, db = self._run.GetCIDBHandle() + action_history = db.GetActionsForChanges(plan) for patch in plan: - if pool.GetCLPreCQStatus(patch) != self.STATUS_PASSED: - pool.UpdateCLPreCQStatus(patch, self.STATUS_LAUNCHING, - self._build_id) + if clactions.GetCLPreCQStatus(patch, + action_history) != self.STATUS_PASSED: + pool.UpdateCLPreCQStatus(patch, self.STATUS_LAUNCHING) def GetDisjointTransactionsToTest(self, pool, changes, status_map): """Get the list of disjoint transactions to test. @@ -1092,8 +1091,10 @@ class PreCQLauncherStage(SyncStage): """ # Get change status. status_map = {} + _, db = self._run.GetCIDBHandle() + action_history = db.GetActionsForChanges(changes) for change in changes: - status = pool.GetCLPreCQStatus(change) + status = clactions.GetCLPreCQStatus(change, action_history) status_map[change] = status # Launch trybots for manifest changes. @@ -1124,4 +1125,4 @@ class PreCQLauncherStage(SyncStage): dryrun=self._run.options.debug, changes_query=self._run.options.cq_gerrit_override, check_tree_open=False, change_filter=self.ProcessChanges, - metadata=self._run.attrs.metadata) + builder_run=self._run) -- cgit v1.2.3 From 3f8e9632ecbebde6f989c94ba1c26c0c3b38337a Mon Sep 17 00:00:00 2001 From: David James Date: Sat, 25 Oct 2014 12:35:26 -0700 Subject: Give the Pre-CQ time to verify changes before picking them up in the CQ. Currently, if two users mark their changes as ready within the same hour, the CQ may pick up both changes at once without waiting for them to be verified by the Pre-CQ. This results in confusion (why did the CQ pick up changes without waiting for the Pre-CQ to finish?) and improperly rejected changes. Fix this. This adds a 2 hour window for waiting for the Pre-CQ to finish. This window may be bypassed for changes marked CQ=+2. BUG=chromium:290254 TEST=New unit tests. Change-Id: I587afce71d0362a4563fa26c6664f021e9ebbe18 Reviewed-on: https://chromium-review.googlesource.com/225553 Reviewed-by: David James Commit-Queue: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 6b5f2832d..197ab0b2b 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -11,6 +11,7 @@ import datetime import logging import os import sys +import time from xml.etree import ElementTree from xml.dom import minidom @@ -674,6 +675,10 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): manifest, and apply the paches written in the manifest. """ + # The amount of time we wait before assuming that the Pre-CQ is down and + # that we should start testing changes that haven't been tested by the Pre-CQ. + PRE_CQ_TIMEOUT = 2 * 60 * 60 + def __init__(self, builder_run, **kwargs): super(CommitQueueSyncStage, self).__init__(builder_run, **kwargs) # Figure out the builder's name from the buildbot waterfall. @@ -709,11 +714,20 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): if status == constants.CL_STATUS_PASSED: changes_to_test.append(change) - # If we only see changes that weren't verified by Pre-CQ, try all of the - # changes. This ensures that the CQ continues to work even if the Pre-CQ is - # down. + # Allow Commit-Ready=+2 changes to bypass the Pre-CQ, if there are no other + # changes. if not changes_to_test: - changes_to_test = changes + changes_to_test = [x for x in changes if x.HasApproval('COMR', '2')] + + # If we only see changes that weren't verified by Pre-CQ, and some of them + # are really old changes, try all of the changes. This ensures that the CQ + # continues to work (albeit slowly) even if the Pre-CQ is down. + if changes and not changes_to_test: + oldest = min(x.approval_timestamp for x in changes) + if time.time() > oldest + self.PRE_CQ_TIMEOUT: + # It's safest to try all changes here because some of the old changes + # might depend on newer changes (e.g. via CQ-DEPEND). + changes_to_test = changes return changes_to_test, non_manifest_changes -- cgit v1.2.3 From 5f58cbb23337bca3332cf79ebaf92f7d6272dc08 Mon Sep 17 00:00:00 2001 From: David James Date: Sat, 25 Oct 2014 15:24:45 -0700 Subject: Wait 5 minutes before launching trybots. When a user marks their CL as ready, don't launch trybots right away, in case they are in the middle of marking their CLs as ready. Wait 5 minutes after their last CL is marked ready before actually launching the trybots. BUG=chromium:422630 TEST=New unit test. Change-Id: Ia4b0395a28a8206ea817b0f729d2bd39fd2ea0f4 Reviewed-on: https://chromium-review.googlesource.com/225554 Reviewed-by: David James Commit-Queue: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 197ab0b2b..4f260171c 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -883,15 +883,18 @@ class PreCQLauncherStage(SyncStage): # The CL has passed the Pre-CQ and is ready to be submitted. STATUS_READY_TO_SUBMIT = constants.CL_STATUS_READY_TO_SUBMIT + # The number of minutes we wait before launching Pre-CQ jobs + LAUNCH_DELAY = 5 + # The number of minutes we allow before considering a launch attempt failed. # If this window isn't hit in a given launcher run, the window will start # again from scratch in the next run. - LAUNCH_DELAY = 90 + LAUNCH_TIMEOUT = 90 # The number of minutes we allow before considering an in-flight # job failed. If this window isn't hit in a given launcher run, the window # will start again from scratch in the next run. - INFLIGHT_DELAY = 180 + INFLIGHT_TIMEOUT = 180 # The maximum number of patches we will allow in a given trybot run. This is # needed because our trybot infrastructure can only handle so many patches at @@ -919,7 +922,7 @@ class PreCQLauncherStage(SyncStage): Returns: True if the change has timed out. False otherwise. """ - diff = datetime.timedelta(minutes=self.LAUNCH_DELAY) + diff = datetime.timedelta(minutes=self.LAUNCH_TIMEOUT) return datetime.datetime.now() - self.launching[change] > diff def _HasInflightTimedOut(self, change): @@ -930,7 +933,7 @@ class PreCQLauncherStage(SyncStage): Returns: True if the change has timed out. False otherwise. """ - diff = datetime.timedelta(minutes=self.INFLIGHT_DELAY) + diff = datetime.timedelta(minutes=self.INFLIGHT_TIMEOUT) return datetime.datetime.now() - self.inflight[change] > diff @staticmethod @@ -1010,7 +1013,7 @@ class PreCQLauncherStage(SyncStage): 'to hang, or if there is some infrastructure issue. If your ' 'change is not at fault you may mark your change as ready ' 'again. If this problem occurs multiple times please notify ' - 'the sheriff and file a bug.' % self.INFLIGHT_DELAY) + 'the sheriff and file a bug.' % self.INFLIGHT_TIMEOUT) pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveCommitReady(change) @@ -1040,7 +1043,7 @@ class PreCQLauncherStage(SyncStage): plan: The list of patches to test in the Pre-CQ run. """ cmd = ['cbuildbot', '--remote', constants.PRE_CQ_GROUP_CONFIG, - '--timeout', str(self.INFLIGHT_DELAY * 60)] + '--timeout', str(self.INFLIGHT_TIMEOUT * 60)] if self._run.options.debug: cmd.append('--debug') for patch in plan: @@ -1090,6 +1093,9 @@ class PreCQLauncherStage(SyncStage): if plan.difference(busy): logging.info('CLs waiting on verification of dependencies: %r', ' '.join(map(str, plan.difference(busy)))) + elif any(x.approval_timestamp + self.LAUNCH_DELAY * 60 > time.time() + for x in plan): + logging.info('CLs waiting on launch delay: %r', plan) else: yield plan -- cgit v1.2.3 From 24b5c8d4c05832adf8d603927af5a3e8b1221af5 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Mon, 20 Oct 2014 11:22:58 -0700 Subject: detect when changes have been re-queued by developer This CL teaches the pre-cq-launcher to detect when a CL that was previously rejected has been re-queued in the CQ, and to record a timestamped CL action when this takes place. The rationale for doing this in the pre-cq-launcher is that that builder is continuously running and examining the validation pool. BUG=chromium:424037 TEST=New unit test Change-Id: I393ec4374885669fa20d230da3c03e2387834d5f Reviewed-on: https://chromium-review.googlesource.com/225865 Reviewed-by: Aviv Keshet Commit-Queue: Aviv Keshet Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 4f260171c..05c9da5b9 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1111,11 +1111,18 @@ class PreCQLauncherStage(SyncStage): """ # Get change status. status_map = {} - _, db = self._run.GetCIDBHandle() + build_id, db = self._run.GetCIDBHandle() action_history = db.GetActionsForChanges(changes) for change in changes: status = clactions.GetCLPreCQStatus(change, action_history) status_map[change] = status + # Detect changes that have been re-queued by the developer, and mark + # them as such in cidb. + was_requeued = clactions.WasChangeRequeued(change, action_history) + if was_requeued: + action = clactions.CLAction.FromGerritPatchAndAction( + change, constants.CL_ACTION_REQUEUED) + db.InsertCLActions(build_id, [action]) # Launch trybots for manifest changes. for plan in self.GetDisjointTransactionsToTest(pool, changes, status_map): -- cgit v1.2.3 From d8e30dc2d2ed03a629792e42b4afed5d2e091955 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Mon, 3 Nov 2014 17:35:40 -0800 Subject: Record the manifest path in ManifestVersionedSyncStage CQ master creates a new manifest for each CQ run. In the manifest, the revision (the commit hash) and the upstream (e.g. refs/head/master) are recorded for each project. The "upstream" attribute is necessary for each slave to map a cros_workon ebuild to a project and a branch. However, this information exists only in the original manifest, and is lost when exporting the manifest from a local checkout on the slave. This CL records the local path of the original manifest in the builder run object, so that a later stage (e.g. the completion stage) has access to the file. BUG=chromium:428824 TEST=`cbuildbot --remote --debug --buildbot link-paladin x86-generic-paladin` Change-Id: I5a5be93c6a6611bd2c9f9b8866f71798a73ed2fc Reviewed-on: https://chromium-review.googlesource.com/227251 Reviewed-by: David James Reviewed-by: Don Garrett Tested-by: Yu-Ju Hong Commit-Queue: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 05c9da5b9..1b367c916 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -553,6 +553,9 @@ class ManifestVersionedSyncStage(SyncStage): else: sys.exit(0) + # Record the path to the local manifest for this build. + self._run.attrs.manifest_path = next_manifest + # Log this early on for the release team to grep out before we finish. if self.manifest_manager: self._Print('\nRELEASETAG: %s\n' % ( -- cgit v1.2.3 From 0d6228aeeaa0c9be2504cb75ba83409736ee198d Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Tue, 4 Nov 2014 15:47:11 -0800 Subject: Record the manifest path in ManifestVersionedSyncStage CL:227251 recorded the path in the builder run object, which does not survive through cbuidlbot re-execution. This CL corrects that mistake by recording the path in the metadata dictionary. CQ master creates a new manifest for each CQ run. In the manifest, the revision (the commit hash) and the upstream (e.g. refs/head/master) are recorded for each project. The "upstream" attribute is necessary for each slave to map a cros_workon ebuild to a project and a branch. However, this information exists only in the original manifest, and is lost when exporting the manifest from a local checkout on the slave. This CL records the local path of the original manifest in the metadata dictionary, so that a later stage (e.g. the completion stage) has access to the file. BUG=chromium:428824 TEST=`cbuildbot --remote --debug --buildbot link-paladin` with modification to consume the path in the completion stage Change-Id: I07afa22719b1448842c47d83f8e11e411404929d Reviewed-on: https://chromium-review.googlesource.com/227512 Tested-by: Yu-Ju Hong Reviewed-by: Don Garrett Commit-Queue: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 1b367c916..b3350375a 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -538,6 +538,11 @@ class ManifestVersionedSyncStage(SyncStage): else: yield manifest + def RecordLocalManifestPath(self, manifest_path): + """Records |manifest_path| in the metadata dictionary.""" + self._run.attrs.metadata.UpdateWithDict( + {'local_manifest_path': manifest_path}) + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): self.Initialize() @@ -553,14 +558,14 @@ class ManifestVersionedSyncStage(SyncStage): else: sys.exit(0) - # Record the path to the local manifest for this build. - self._run.attrs.manifest_path = next_manifest - # Log this early on for the release team to grep out before we finish. if self.manifest_manager: self._Print('\nRELEASETAG: %s\n' % ( self.manifest_manager.current_version)) + # Record the path to the local manifest for this build. + self.RecordLocalManifestPath(next_manifest) + self._SetChromeVersionIfApplicable(next_manifest) # To keep local trybots working, remove restricted checkouts from the # official manifest we get from manifest-versions. -- cgit v1.2.3 From 67d08fa7b6f0e23d0694ba99153c1f6b931ddb25 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Wed, 29 Oct 2014 11:32:12 -0700 Subject: parallel pre-cq This CL enables the pre-cq-launcher to launch multiple separate tryjobs in parallel for a given change, with different configs, and wait for all of them to verify a change before marking it as passed. For now, all changes will still be tested with pre-cq-group. - The pre-cq status LAUNCHING and INFLIGHT are no longer used. Instead, this status is tracked on a per-config basis. - The database time is used to determine when a given status was reached, rather than a timeout map held by PreCQLauncherStage. This means that timeouts are no longer reset when the pre-cq-launcher is reset. - PreCQLauncherStage's ProcessChanges and its helper methods are now responsible for: screening a change to determine which configs to test it for, then launching and waiting on results from multiple tryjobs. - Unit tests for PreCQLauncherStage have been overhauled. BUG=chromium:427976 TEST=cidb_integration_test; new unit tests; pre-cq-launcher and master-paladin trybots Change-Id: I7db0059203e20e9492f5c1d52976393ed75c281c Reviewed-on: https://chromium-review.googlesource.com/226417 Reviewed-by: Aviv Keshet Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 410 ++++++++++++++++++++++++++-------------- 1 file changed, 263 insertions(+), 147 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index b3350375a..6bac17a44 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -6,6 +6,7 @@ from __future__ import print_function +import ConfigParser import contextlib import datetime import logging @@ -38,6 +39,23 @@ from chromite.scripts import cros_mark_chrome_as_stable PRE_CQ = validation_pool.PRE_CQ +PRECQ_LAUNCH_TIMEOUT_MSG = ( + 'We were not able to launch a %s trybot for your change within ' + '%s minutes.\n\n' + 'This problem can happen if the trybot waterfall is very ' + 'busy, or if there is an infrastructure issue. Please ' + 'notify the sheriff and mark your change as ready again. If ' + 'this problem occurs multiple times in a row, please file a ' + 'bug.') +PRECQ_INFLIGHT_TIMEOUT_MSG = ( + 'The %s trybot for your change timed out after %s minutes.' + '\n\n' + 'This problem can happen if your change causes the builder ' + 'to hang, or if there is some infrastructure issue. If your ' + 'change is not at fault you may mark your change as ready ' + 'again. If this problem occurs multiple times please notify ' + 'the sheriff and file a bug.') + class PatchChangesStage(generic_stages.BuilderStage): """Stage that patches a set of Gerrit changes to the buildroot source tree.""" @@ -912,37 +930,22 @@ class PreCQLauncherStage(SyncStage): def __init__(self, builder_run, **kwargs): super(PreCQLauncherStage, self).__init__(builder_run, **kwargs) self.skip_sync = True - # Mapping from launching changes to the first known time when they - # were launching. - self.launching = {} - # Mapping from inflight changes to the first known time when they - # were inflight. - self.inflight = {} - self.retried = set() - self._build_id = self._run.attrs.metadata.GetValue('build_id') - def _HasLaunchTimedOut(self, change): - """Check whether a given |change| has timed out on its trybot launch. + def _HasTimedOut(self, start, now, timeout_minutes): + """Check whether |timeout_minutes| has elapsed between |start| and |now|. - Assumes that the change is in the middle of being launched. + Args: + start: datetime.datetime start time. + now: datetime.datetime current time. + timeout_minutes: integer number of minutes for timeout. Returns: - True if the change has timed out. False otherwise. + True if (now-start) > timeout_minutes. """ - diff = datetime.timedelta(minutes=self.LAUNCH_TIMEOUT) - return datetime.datetime.now() - self.launching[change] > diff - - def _HasInflightTimedOut(self, change): - """Check whether a given |change| has timed out while trybot inflight. + diff = datetime.timedelta(minutes=timeout_minutes) + return (now - start) > diff - Assumes that the change's trybot is inflight. - - Returns: - True if the change has timed out. False otherwise. - """ - diff = datetime.timedelta(minutes=self.INFLIGHT_TIMEOUT) - return datetime.datetime.now() - self.inflight[change] > diff @staticmethod def _PrintPatchStatus(patch, status): @@ -954,146 +957,129 @@ class PreCQLauncherStage(SyncStage): ) cros_build_lib.PrintBuildbotLink(' | '.join(items), patch.url) - def GetPreCQStatus(self, pool, changes, status_map): - """Get the Pre-CQ status of a list of changes. - Side effect: reject or retry changes that have timed out. + @staticmethod + def VerificationsForChange(_change): + """Determine which configs to test |change| with. Args: - pool: The validation pool. - changes: Changes to examine. - status_map: Dict mapping changes to their CL status. + _change: GerritPatch instance to get configs-to-test for. Returns: - busy: The set of CLs that are currently being tested. - passed: The set of CLs that have been verified. + A list of configs. """ - busy, passed = set(), set() + # TODO(akeshet): Screen CL's based on the contents of the CL rather than + # hard coding a single test config. crbug.com/384169 + return [constants.PRE_CQ_GROUP_CONFIG] - for change in changes: - status = status_map[change] - if status != self.STATUS_LAUNCHING: - # The trybot is not launching, so we should remove it from our - # launching timeout map. - self.launching.pop(change, None) - - if status != self.STATUS_INFLIGHT: - # The trybot is not inflight, so we should remove it from our - # inflight timeout map. - self.inflight.pop(change, None) - - if status == self.STATUS_LAUNCHING: - # The trybot is in the process of launching. - busy.add(change) - if change not in self.launching: - # Record the launch time of changes. - self.launching[change] = datetime.datetime.now() - elif self._HasLaunchTimedOut(change): - if change in self.retried: - msg = ('We were not able to launch a pre-cq trybot for your change.' - '\n\n' - 'This problem can happen if the trybot waterfall is very ' - 'busy, or if there is an infrastructure issue. Please ' - 'notify the sheriff and mark your change as ready again. If ' - 'this problem occurs multiple times in a row, please file a ' - 'bug.') - - pool.SendNotification(change, '%(details)s', details=msg) - pool.RemoveCommitReady(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED) - self.retried.discard(change) - else: - # Try the change again. - self.retried.add(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING) - elif status == self.STATUS_INFLIGHT: - # Once a Pre-CQ run actually starts, it'll set the status to - # STATUS_INFLIGHT. - busy.add(change) - if change not in self.inflight: - # Record the inflight start time. - self.inflight[change] = datetime.datetime.now() - elif self._HasInflightTimedOut(change): - msg = ('The pre-cq trybot for your change timed out after %s minutes.' - '\n\n' - 'This problem can happen if your change causes the builder ' - 'to hang, or if there is some infrastructure issue. If your ' - 'change is not at fault you may mark your change as ready ' - 'again. If this problem occurs multiple times please notify ' - 'the sheriff and file a bug.' % self.INFLIGHT_TIMEOUT) - - pool.SendNotification(change, '%(details)s', details=msg) - pool.RemoveCommitReady(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED) - elif status == self.STATUS_FAILED: - # The Pre-CQ run failed for this change. It's possible that we got - # unlucky and this change was just marked as 'Not Ready' by a bot. To - # test this, mark the CL as 'waiting' for now. If the CL is still marked - # as 'Ready' next time we check, we'll know the CL is truly still ready. - busy.add(change) - pool.UpdateCLPreCQStatus(change, self.STATUS_WAITING) - self._PrintPatchStatus(change, status) - elif status == self.STATUS_PASSED: - passed.add(change) - self._PrintPatchStatus(change, status) - elif status == self.STATUS_READY_TO_SUBMIT: - passed.add(change) - self._PrintPatchStatus(change, 'submitting') - - return busy, passed - - def LaunchTrybot(self, pool, plan): + def ScreenChangeForPreCQ(self, change): + """Record which pre-cq tryjobs to test |change| with. + + This method determines which configs to test a given |change| with, and + writes those as pending tryjobs to the cidb. + + Args: + change: GerritPatch instance to screen. This change should not yet have + been screened. + """ + actions = [] + configs_to_test = self.VerificationsForChange(change) + for c in configs_to_test: + actions.append(clactions.CLAction.FromGerritPatchAndAction( + change, constants.CL_ACTION_VALIDATION_PENDING_PRE_CQ, + reason=c)) + actions.append(clactions.CLAction.FromGerritPatchAndAction( + change, constants.CL_ACTION_SCREENED_FOR_PRE_CQ)) + + build_id, db = self._run.GetCIDBHandle() + db.InsertCLActions(build_id, actions) + + def CanSubmitChangeInPreCQ(self, change): + """Look up whether |change| is configured to be submitted in the pre-CQ. + + This looks up the "submit-in-pre-cq" setting inside the project in + COMMIT-QUEUE.ini and checks whether it is set to "yes". + + [GENERAL] + submit-in-pre-cq: yes + + Args: + change: Change to examine. + + Returns: + A list of stages to ignore for the given |change|. + """ + result = None + try: + result = validation_pool.GetOptionForChange( + self._build_root, change, 'GENERAL', 'submit-in-pre-cq') + except ConfigParser.Error: + cros_build_lib.Error('%s has malformed config file', change, + exc_info=True) + return result and result.lower() == 'yes' + + + def LaunchTrybot(self, plan, config): """Launch a Pre-CQ run with the provided list of CLs. Args: pool: ValidationPool corresponding to |plan|. - plan: The list of patches to test in the Pre-CQ run. + plan: The list of patches to test in the pre-cq tryjob. + config: The pre-cq config name to launch. """ - cmd = ['cbuildbot', '--remote', constants.PRE_CQ_GROUP_CONFIG, + cmd = ['cbuildbot', '--remote', config, '--timeout', str(self.INFLIGHT_TIMEOUT * 60)] - if self._run.options.debug: - cmd.append('--debug') for patch in plan: cmd += ['-g', cros_patch.AddPrefix(patch, patch.gerrit_number)] self._PrintPatchStatus(patch, 'testing') - cros_build_lib.RunCommand(cmd, cwd=self._build_root) - _, db = self._run.GetCIDBHandle() - action_history = db.GetActionsForChanges(plan) - for patch in plan: - if clactions.GetCLPreCQStatus(patch, - action_history) != self.STATUS_PASSED: - pool.UpdateCLPreCQStatus(patch, self.STATUS_LAUNCHING) + if self._run.options.debug: + logging.debug('Would have launched tryjob with %s', cmd) + else: + cros_build_lib.RunCommand(cmd, cwd=self._build_root) + build_id, db = self._run.GetCIDBHandle() + actions = [ + clactions.CLAction.FromGerritPatchAndAction( + patch, constants.CL_ACTION_TRYBOT_LAUNCHING, config) + for patch in plan] + db.InsertCLActions(build_id, actions) + - def GetDisjointTransactionsToTest(self, pool, changes, status_map): + def GetDisjointTransactionsToTest(self, pool, progress_map): """Get the list of disjoint transactions to test. Side effect: reject or retry changes that have timed out. Args: pool: The validation pool. - changes: Changes to examine. - status_map: Dict mapping changes to their CL status. + progress_map: See return type of clactions.GetPreCQProgressMap. Returns: - A list of disjoint transactions to test. Each transaction should be sent - to a different Pre-CQ trybot. + A list of (transaction, config) tuples corresponding to different trybots + that should be launched. """ - busy, passed = self.GetPreCQStatus(pool, changes, status_map) + # Get the set of busy and passed CLs. + busy, verified = clactions.GetPreCQCategories(progress_map) # Create a list of disjoint transactions to test. manifest = git.ManifestCheckout.Cached(self._build_root) plans = pool.CreateDisjointTransactions( manifest, max_txn_length=self.MAX_PATCHES_PER_TRYBOT_RUN) + screened_changes = set(progress_map) for plan in plans: + # If any of the CLs in the plan is not yet screened, wait for them to + # be screened. + # # If any of the CLs in the plan are currently "busy" being tested, - # wait until they're done before launching our trybot run. This helps - # avoid race conditions. + # wait until they're done before starting to test this plan. # # Similarly, if all of the CLs in the plan have already been validated, # there's no need to launch a trybot run. plan = set(plan) - if plan.issubset(passed): + if not plan.issubset(screened_changes): + logging.info('CLs waiting to be screened: %r', + ' '.join(map(str, plan.difference(screened_changes)))) + elif plan.issubset(verified): logging.info('CLs already verified: %r', ' '.join(map(str, plan))) elif plan.intersection(busy): logging.info('CLs currently being verified: %r', @@ -1101,11 +1087,95 @@ class PreCQLauncherStage(SyncStage): if plan.difference(busy): logging.info('CLs waiting on verification of dependencies: %r', ' '.join(map(str, plan.difference(busy)))) + # TODO(akeshet): Consider using a database time rather than gerrit + # approval time and local clock for launch delay. elif any(x.approval_timestamp + self.LAUNCH_DELAY * 60 > time.time() for x in plan): logging.info('CLs waiting on launch delay: %r', plan) else: - yield plan + pending_configs = clactions.GetPreCQConfigsToTest(plan, progress_map) + for config in pending_configs: + yield (plan, config) + + def _ProcessRequeued(self, change, action_history): + """Detect if |change| was requeued by developer, and mark in cidb. + + Args: + change: GerritPatch instance to check. + action_history: List of CLActions. + + Returns: + True if the change was marked as requeued. + """ + if clactions.WasChangeRequeued(change, action_history): + action = clactions.CLAction.FromGerritPatchAndAction( + change, constants.CL_ACTION_REQUEUED) + build_id, db = self._run.GetCIDBHandle() + db.InsertCLActions(build_id, [action]) + return True + + def _ProcessTimeouts(self, change, progress_map, pool, current_time): + """Enforce per-config launch and inflight timeouts. + + Args: + change: GerritPatch instance to process. + progress_map: As returned by clactions.GetCLPreCQProgress a dict mapping + each change in |changes| to a dict mapping config names + to (status, timestamp) tuples for the configs under test. + pool: The current validation pool. + current_time: datetime.datetime timestamp giving current database time. + """ + # TODO(akeshet) restore trybot launch retries here (there was + # no straightforward existing mechanism to include them in the + # transition to parallel pre-cq). + timeout_statuses = (constants.CL_PRECQ_CONFIG_STATUS_LAUNCHED, + constants.CL_PRECQ_CONFIG_STATUS_INFLIGHT) + config_progress = progress_map[change] + for config, (config_status, timestamp) in config_progress.iteritems(): + if not config_status in timeout_statuses: + continue + launched = config_status == constants.CL_PRECQ_CONFIG_STATUS_LAUNCHED + timeout = self.LAUNCH_TIMEOUT if launched else self.INFLIGHT_TIMEOUT + msg = (PRECQ_LAUNCH_TIMEOUT_MSG if launched + else PRECQ_INFLIGHT_TIMEOUT_MSG) % (config, timeout) + timeout = self.LAUNCH_TIMEOUT + + if self._HasTimedOut(timestamp, current_time, timeout): + pool.SendNotification(change, '%(details)s', details=msg) + pool.RemoveCommitReady(change, reason=config) + pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED) + + + def _ProcessVerified(self, change, can_submit, will_submit): + """Process a change that is fully pre-cq verified. + + Args: + change: GerritPatch instance to process. + can_submit: set of changes that can be submitted by the pre-cq. + will_submit: set of changes that will be submitted by the pre-cq. + + Returns: + A tuple of (set of changes that should be submitted by pre-cq, + set of changes that should be passed by pre-cq) + """ + # If this change and all its dependencies are pre-cq submittable, + # and none of them have yet been marked as pre-cq passed, then + # mark them for submission. Otherwise, mark this change as passed. + if change in will_submit: + return set(), set() + + if change in can_submit: + logging.info('Attempting to determine if %s can be submitted.', change) + patch_series = validation_pool.PatchSeries(self._build_root) + try: + plan = patch_series.CreateTransaction(change, limit_to=can_submit) + return plan, set() + except cros_patch.DependencyError: + pass + + # Changes that cannot be submitted are marked as passed. + return set(), set([change]) + def ProcessChanges(self, pool, changes, _non_manifest_changes): """Process a list of changes that were marked as Ready. @@ -1117,31 +1187,77 @@ class PreCQLauncherStage(SyncStage): Non-manifest changes are just submitted here because they don't need to be verified by either the Pre-CQ or CQ. """ - # Get change status. - status_map = {} build_id, db = self._run.GetCIDBHandle() action_history = db.GetActionsForChanges(changes) + status_map = {c: clactions.GetCLPreCQStatus(c, action_history) + for c in changes} + progress_map = clactions.GetPreCQProgressMap(changes, action_history) + _, verified = clactions.GetPreCQCategories(progress_map) + current_db_time = db.GetTime() + + # TODO(akeshet): Once this change lands, we will no longer mark changes + # with pre-cq status READY_TO_SUBMIT, so simplify the status check below. + passed_statuses = (constants.CL_STATUS_PASSED, + constants.CL_STATUS_READY_TO_SUBMIT) + already_passed = set(c for c in changes + if status_map[c] in passed_statuses) + to_process = set(changes) - already_passed + # Changes that can be submitted, if their dependencies can be too. Only + # include changes that have not already been marked as passed. + can_submit = set(c for c in (verified and to_process) if + self.CanSubmitChangeInPreCQ(c)) + # Changes that will be submitted. + will_submit = set() + # Changes that will be passed + will_pass = set() + for change in changes: - status = clactions.GetCLPreCQStatus(change, action_history) - status_map[change] = status - # Detect changes that have been re-queued by the developer, and mark - # them as such in cidb. - was_requeued = clactions.WasChangeRequeued(change, action_history) - if was_requeued: - action = clactions.CLAction.FromGerritPatchAndAction( - change, constants.CL_ACTION_REQUEUED) - db.InsertCLActions(build_id, [action]) - - # Launch trybots for manifest changes. - for plan in self.GetDisjointTransactionsToTest(pool, changes, status_map): - self.LaunchTrybot(pool, plan) - - # Submit changes that don't need a CQ run if we can. + self._ProcessRequeued(change, action_history) + + for change in to_process: + status = status_map[change] + + # Detect if change is ready to be marked as passed, or ready to submit. + if change in verified: + to_submit, to_pass = self._ProcessVerified(change, can_submit, + will_submit) + will_submit.update(to_submit) + will_pass.update(to_pass) + continue + + # TODO(akeshet): Eliminate this block after this CL has landed and all + # previously inflight or launching CLs have made it through the pre-CQ. + legacy_statuses = (constants.CL_STATUS_LAUNCHING, + constants.CL_STATUS_INFLIGHT) + if status in legacy_statuses: + continue + + # Screen unscreened changes to determine which trybots to test them with. + if not any(a.action == constants.CL_ACTION_SCREENED_FOR_PRE_CQ + for a in clactions.ActionsForPatch(change, action_history)): + self.ScreenChangeForPreCQ(change) + continue + + self._ProcessTimeouts(change, progress_map, pool, current_db_time) + + # Launch any necessary tryjobs + for plan, config in self.GetDisjointTransactionsToTest( + pool, progress_map): + self.LaunchTrybot(plan, config) + + # Submit changes that are ready to submit, if we can. if tree_status.IsTreeOpen(): pool.SubmitNonManifestChanges(check_tree_open=False) - submitting = [change for (change, status) in status_map.items() - if status == self.STATUS_READY_TO_SUBMIT] - pool.SubmitChanges(submitting, check_tree_open=False) + pool.SubmitChanges(will_submit, check_tree_open=False) + + # Mark passed changes as passed + if will_pass: + # TODO(akeshet) Refactor this into a general purpose method somewhere to + # atomically update CL statuses. + a = clactions.TranslatePreCQStatusToAction(constants.CL_STATUS_PASSED) + actions = [clactions.CLAction.FromGerritPatchAndAction(c, a) + for c in will_pass] + db.InsertCLActions(build_id, actions) # Tell ValidationPool to keep waiting for more changes until we hit # its internal timeout. -- cgit v1.2.3 From cd3bc1d04a4edfb506091ca52403796df223e7dc Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Wed, 5 Nov 2014 22:58:17 -0800 Subject: sync_stages: do not mark changes as INFLIGHT on pre-cq tryjobs Marking changes an inflight on the trybots is no longer necessary, and is interfering with some of the logging on the master. BUG=None TEST=None Change-Id: I751969b47bf9ab521fd216e1929efa6464c03093 Reviewed-on: https://chromium-review.googlesource.com/228080 Reviewed-by: Aviv Keshet Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 9 --------- 1 file changed, 9 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 6bac17a44..80f4bbc54 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -878,15 +878,6 @@ class PreCQSyncStage(SyncStage): if len(self.pool.changes) == 0: cros_build_lib.Die('No changes have been applied.') - # Mark changes that are not passed with pre-cq status inflight. - _, db = self._run.GetCIDBHandle() - if db: - action_history = db.GetActionsForChanges(self.pool.changes) - for change in self.pool.changes: - current_status = clactions.GetCLPreCQStatus(change, action_history) - if current_status != constants.CL_STATUS_PASSED: - self.pool.UpdateCLPreCQStatus(change, constants.CL_STATUS_INFLIGHT) - class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" -- cgit v1.2.3 From 015ce16a7000833e0e954b2b80d99889d75fe5dd Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Wed, 5 Nov 2014 20:13:48 -0800 Subject: sync_stages: fix logging bug in pre-cq-launcher BUG=None TEST=None Change-Id: I7f3a4e5666d836b1bcde6e31b21eebd30a6356fe Reviewed-on: https://chromium-review.googlesource.com/227789 Commit-Queue: Aviv Keshet Tested-by: Aviv Keshet Reviewed-by: Aviv Keshet Reviewed-by: David James --- cbuildbot/stages/sync_stages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 80f4bbc54..6050db604 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1082,7 +1082,8 @@ class PreCQLauncherStage(SyncStage): # approval time and local clock for launch delay. elif any(x.approval_timestamp + self.LAUNCH_DELAY * 60 > time.time() for x in plan): - logging.info('CLs waiting on launch delay: %r', plan) + logging.info('CLs waiting on launch delay: %r', + ' '.join(map(str, plan))) else: pending_configs = clactions.GetPreCQConfigsToTest(plan, progress_map) for config in pending_configs: -- cgit v1.2.3 From b7d47f70b4285475e0722de47d4a14cf6bc22a50 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Wed, 5 Nov 2014 11:47:10 -0800 Subject: Move setting builder status to inflight to the end of the stage This is desirable for the reasons below: 1. When a slave fails midway through the sync stage, we often have to delete the status file in GS in order to relaunch the slave. By moving the status change to the end of the stage, we avoid hitting this situation again. 2. If a builder did not start ("inflight" was never set), we consider it an infrastructure failure. If the builder set status to inflight then timed out, we think the CLs may be at fault. However, as mentioned in (1), we have sync failures often due to gerrit/git problems, which are valid infra failures. Moving the status change to the end of the stage helps CQ categorize the failure more accurately. 3. CQ slave records CLs it picks up in the sync stage. If the build fails before the recording completes, we can detect that the pickup information is inaccurate (and should not be used) if the builder status is not set. BUG=chromium:422639 TEST=`cbuildbot/run_tests` Change-Id: I23e9999b016baea7d40cf62ef77b85057a7d4f33 Reviewed-on: https://chromium-review.googlesource.com/227654 Tested-by: Yu-Ju Hong Reviewed-by: Don Garrett Commit-Queue: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 6050db604..503d47266 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -514,9 +514,7 @@ class ManifestVersionedSyncStage(SyncStage): build_id = self._run.attrs.metadata.GetDict().get('build_id') - to_return = self.manifest_manager.GetNextBuildSpec( - dashboard_url=self.ConstructDashboardURL(), - build_id=build_id) + to_return = self.manifest_manager.GetNextBuildSpec(build_id=build_id) previous_version = self.manifest_manager.GetLatestPassingSpec() target_version = self.manifest_manager.current_version @@ -591,6 +589,13 @@ class ManifestVersionedSyncStage(SyncStage): next_manifest, filter_cros=self._run.options.local) as new_manifest: self.ManifestCheckout(new_manifest) + # Set the status inflight at the end of the ManifestVersionedSync + # stage. This guarantees that all syncing has completed. + if self.manifest_manager: + self.manifest_manager.SetInFlight( + self.manifest_manager.current_version, + dashboard_url=self.ConstructDashboardURL()) + class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): """Stage that generates a unique manifest file candidate, and sync's to it. @@ -664,12 +669,10 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): chrome_version=self._chrome_version, build_id=build_id) if MasterSlaveLKGMSyncStage.sub_manager: - MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest( - manifest, dashboard_url=self.ConstructDashboardURL()) + MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(manifest) return manifest else: return self.manifest_manager.GetLatestCandidate( - dashboard_url=self.ConstructDashboardURL(), timeout=self.LATEST_CANDIDATE_TIMEOUT_SECONDS) def GetLatestChromeVersion(self): @@ -809,12 +812,10 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): build_id=build_id) if MasterSlaveLKGMSyncStage.sub_manager: MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest( - manifest, dashboard_url=self.ConstructDashboardURL(), - build_id=build_id) + manifest, build_id=build_id) else: - manifest = self.manifest_manager.GetLatestCandidate( - dashboard_url=self.ConstructDashboardURL()) + manifest = self.manifest_manager.GetLatestCandidate() if manifest: if self._run.config.build_before_patching: pre_build_passed = self.RunPrePatchBuild() -- cgit v1.2.3 From c05e32d93af4e272d374542e1a5443c39cb96765 Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 6 Nov 2014 10:33:44 -0800 Subject: Reduce launch delay from 5 minutes to 2 minutes. A 5 minute launch delay is overkill. Use a smaller launch delay as users are usually pretty fast at marking CLs as ready. Note that this delay represents idle time for the patch series, so if a user takes 20 minutes to mark a series of 20 patches as ready, we won't start picking up any of the patches until the last patch has been idle for 2 minutes. BUG=chromium:422630 TEST=Unit tests. Change-Id: Iba5ce1d979535b479259cb1286ba51c3f74e4cc2 Reviewed-on: https://chromium-review.googlesource.com/228193 Reviewed-by: David James Commit-Queue: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 503d47266..baba40687 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -901,8 +901,11 @@ class PreCQLauncherStage(SyncStage): # The CL has passed the Pre-CQ and is ready to be submitted. STATUS_READY_TO_SUBMIT = constants.CL_STATUS_READY_TO_SUBMIT - # The number of minutes we wait before launching Pre-CQ jobs - LAUNCH_DELAY = 5 + # The number of minutes we wait before launching Pre-CQ jobs. This measures + # the idle time of a given patch series, so, for example, if a user takes + # 20 minutes to mark a series of 20 patches as ready, we won't launch a + # tryjob on any of the patches until the user has been idle for 2 minutes. + LAUNCH_DELAY = 2 # The number of minutes we allow before considering a launch attempt failed. # If this window isn't hit in a given launcher run, the window will start -- cgit v1.2.3 From 0c2a525d7c4e31565dbbcba76590c39e07efc1d1 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Thu, 6 Nov 2014 17:48:21 -0800 Subject: Remove recording local manifest path in metadata Even though the manifest file created from the CQ slave source checkout does not contain the upstream refs, the original manifest (created by the CQ master) has already been copied to SOURCE_ROOT/.repo/manifest.xml after syncing. When creating a ManifestCheckout instance using the current build root, the default manifest file SOURCE_ROOT/.repo/manifest.xml will be used. There is no need to pass the original manifest file explicitly. Remove redundant code introduced previously. BUG=chromium:422639 TEST=remote tryjobs Change-Id: I6204288d42a1efbbb20dbf03b63b9d8a6c229a60 Reviewed-on: https://chromium-review.googlesource.com/228327 Reviewed-by: David James Tested-by: Yu-Ju Hong Commit-Queue: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 8 -------- 1 file changed, 8 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index baba40687..b9e9e1ff6 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -554,11 +554,6 @@ class ManifestVersionedSyncStage(SyncStage): else: yield manifest - def RecordLocalManifestPath(self, manifest_path): - """Records |manifest_path| in the metadata dictionary.""" - self._run.attrs.metadata.UpdateWithDict( - {'local_manifest_path': manifest_path}) - @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): self.Initialize() @@ -579,9 +574,6 @@ class ManifestVersionedSyncStage(SyncStage): self._Print('\nRELEASETAG: %s\n' % ( self.manifest_manager.current_version)) - # Record the path to the local manifest for this build. - self.RecordLocalManifestPath(next_manifest) - self._SetChromeVersionIfApplicable(next_manifest) # To keep local trybots working, remove restricted checkouts from the # official manifest we get from manifest-versions. -- cgit v1.2.3 From b6a7f3fd730fe9d887b20337a552c895f1d9856a Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Fri, 7 Nov 2014 11:51:25 -0800 Subject: sync_stages: fix logic error in computing can_submit BUG=None TEST=unit tests pass Change-Id: I79d3e5da085dfaf3800dc24722df4046eee72c2c Reviewed-on: https://chromium-review.googlesource.com/228513 Reviewed-by: David James Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index b9e9e1ff6..f7b5a5c46 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1192,7 +1192,7 @@ class PreCQLauncherStage(SyncStage): to_process = set(changes) - already_passed # Changes that can be submitted, if their dependencies can be too. Only # include changes that have not already been marked as passed. - can_submit = set(c for c in (verified and to_process) if + can_submit = set(c for c in (verified.intersection(to_process)) if self.CanSubmitChangeInPreCQ(c)) # Changes that will be submitted. will_submit = set() -- cgit v1.2.3 From 03faec23994991547fcb5103ab62db69b6a4dcad Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 6 Nov 2014 13:12:28 -0800 Subject: sync_stages: read pre-cq configs to test from config file BUG=chromium:384169 TEST=unit tests Change-Id: I6a4fa378d67617418693ecededae2a2f8b762e3d Reviewed-on: https://chromium-review.googlesource.com/228290 Tested-by: Aviv Keshet Reviewed-by: David James Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index f7b5a5c46..40606ecde 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -945,19 +945,27 @@ class PreCQLauncherStage(SyncStage): cros_build_lib.PrintBuildbotLink(' | '.join(items), patch.url) - @staticmethod - def VerificationsForChange(_change): + def VerificationsForChange(self, change): """Determine which configs to test |change| with. Args: - _change: GerritPatch instance to get configs-to-test for. + change: GerritPatch instance to get configs-to-test for. Returns: A list of configs. """ - # TODO(akeshet): Screen CL's based on the contents of the CL rather than - # hard coding a single test config. crbug.com/384169 - return [constants.PRE_CQ_GROUP_CONFIG] + configs_to_test = constants.PRE_CQ_DEFAULT_CONFIGS + try: + result = validation_pool.GetOptionForChange( + self._build_root, change, 'GENERAL', 'pre-cq-configs') + result = result.split() + if result and all(c in cbuildbot_config.config for c in result): + configs_to_test = result + except ConfigParser.Error: + cros_build_lib.Error('%s has malformed config file', change, + exc_info=True) + + return configs_to_test def ScreenChangeForPreCQ(self, change): -- cgit v1.2.3 From 82b8e1f1ba8b943741853e0d0f4f2736edaa2d94 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Fri, 7 Nov 2014 16:09:38 -0800 Subject: sync_stages: fix VerificationsForChange handling of None BUG=None TEST=New unit test added. Verified that test fails without fix. Change-Id: Ie2ada9004930fb7b51970a64498ed8c01039897f Reviewed-on: https://chromium-review.googlesource.com/228650 Tested-by: Aviv Keshet Reviewed-by: Prathmesh Prabhu --- cbuildbot/stages/sync_stages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 40606ecde..034cd3931 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -958,9 +958,9 @@ class PreCQLauncherStage(SyncStage): try: result = validation_pool.GetOptionForChange( self._build_root, change, 'GENERAL', 'pre-cq-configs') - result = result.split() - if result and all(c in cbuildbot_config.config for c in result): - configs_to_test = result + if (result and result.split() and + all(c in cbuildbot_config.config for c in result.split())): + configs_to_test = result.split() except ConfigParser.Error: cros_build_lib.Error('%s has malformed config file', change, exc_info=True) -- cgit v1.2.3 From 5a0a75b2558058e9588a8cdf524ef4d2adf4b926 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 23 Oct 2014 14:49:42 -0700 Subject: precq: Prepare for speculative PreCQ runs. Create the infrastructure needed for speculative PreCQ runs, but leave it turned off. A follow on CL will enable it. Speculative means PreCQ tryjobs for any recenty CLs, which have a verified flag, and a +2 code review. They do NOT need a CQ flag. BUG=chromium:422625 TEST=pylint + unitests. Change-Id: Ib1e6ab7a8e6048e39190d38422e32592ffdd022e Reviewed-on: https://chromium-review.googlesource.com/225233 Reviewed-by: Don Garrett Commit-Queue: Don Garrett Tested-by: Don Garrett --- cbuildbot/stages/sync_stages.py | 59 +++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 17 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 034cd3931..6ce80e90d 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -786,13 +786,18 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): # In order to acquire a pool, we need an initialized buildroot. if not git.FindRepoDir(self.repo.directory): self.repo.Initialize() + + query = constants.CQ_READY_QUERY + if self._run.options.cq_gerrit_override: + query = (self._run.options.cq_gerrit_override, None) + self.pool = pool = validation_pool.ValidationPool.AcquirePool( self._run.config.overlays, self.repo, self._run.buildnumber, self.builder_name, - self._run.options.debug, + query, + dryrun=self._run.options.debug, check_tree_open=not self._run.options.debug or self._run.options.mock_tree_status, - changes_query=self._run.options.cq_gerrit_override, change_filter=self._ChangeFilter, throttled_ok=True, builder_run=self._run) @@ -1056,11 +1061,13 @@ class PreCQLauncherStage(SyncStage): # Get the set of busy and passed CLs. busy, verified = clactions.GetPreCQCategories(progress_map) + screened_changes = set(progress_map) + # Create a list of disjoint transactions to test. manifest = git.ManifestCheckout.Cached(self._build_root) plans = pool.CreateDisjointTransactions( - manifest, max_txn_length=self.MAX_PATCHES_PER_TRYBOT_RUN) - screened_changes = set(progress_map) + manifest, screened_changes, + max_txn_length=self.MAX_PATCHES_PER_TRYBOT_RUN) for plan in plans: # If any of the CLs in the plan is not yet screened, wait for them to # be screened. @@ -1093,22 +1100,23 @@ class PreCQLauncherStage(SyncStage): for config in pending_configs: yield (plan, config) - def _ProcessRequeued(self, change, action_history): + def _ProcessRequeuedAndSpeculative(self, change, action_history, + is_speculative): """Detect if |change| was requeued by developer, and mark in cidb. Args: change: GerritPatch instance to check. action_history: List of CLActions. - - Returns: - True if the change was marked as requeued. + is_speculative: Boolean indicating if |change| is speculative, i.e. it + does not have CQ approval. """ - if clactions.WasChangeRequeued(change, action_history): - action = clactions.CLAction.FromGerritPatchAndAction( - change, constants.CL_ACTION_REQUEUED) + action_string = clactions.GetRequeuedOrSpeculative( + change, action_history, is_speculative) + if action_string: build_id, db = self._run.GetCIDBHandle() + action = clactions.CLAction.FromGerritPatchAndAction( + change, action_string) db.InsertCLActions(build_id, [action]) - return True def _ProcessTimeouts(self, change, progress_map, pool, current_time): """Enforce per-config launch and inflight timeouts. @@ -1198,23 +1206,32 @@ class PreCQLauncherStage(SyncStage): already_passed = set(c for c in changes if status_map[c] in passed_statuses) to_process = set(changes) - already_passed + + # We don't know for sure they were initially part of a speculative PreCQ + # run. It might just be someone turned off the flag, mid-run. + speculative = set(c for c in changes + if not c.HasApproval('COMR', ('1', '2'))) + # Changes that can be submitted, if their dependencies can be too. Only # include changes that have not already been marked as passed. can_submit = set(c for c in (verified.intersection(to_process)) if self.CanSubmitChangeInPreCQ(c)) + can_submit.difference_update(speculative) + # Changes that will be submitted. will_submit = set() # Changes that will be passed will_pass = set() for change in changes: - self._ProcessRequeued(change, action_history) + self._ProcessRequeuedAndSpeculative(change, action_history, + change in speculative) for change in to_process: status = status_map[change] # Detect if change is ready to be marked as passed, or ready to submit. - if change in verified: + if change in verified and change not in speculative: to_submit, to_pass = self._ProcessVerified(change, can_submit, will_submit) will_submit.update(to_submit) @@ -1236,9 +1253,13 @@ class PreCQLauncherStage(SyncStage): self._ProcessTimeouts(change, progress_map, pool, current_db_time) - # Launch any necessary tryjobs + # Filter out speculative changes that have already failed before launching. + launchable_progress_map = { + k: v for k, v in progress_map.iteritems() + if k not in speculative or status_map[k] != self.STATUS_FAILED} + for plan, config in self.GetDisjointTransactionsToTest( - pool, progress_map): + pool, launchable_progress_map): self.LaunchTrybot(plan, config) # Submit changes that are ready to submit, if we can. @@ -1264,12 +1285,16 @@ class PreCQLauncherStage(SyncStage): # Setup and initialize the repo. super(PreCQLauncherStage, self).PerformStage() + query = constants.PRECQ_READY_QUERY + if self._run.options.cq_gerrit_override: + query = (self._run.options.cq_gerrit_override, None) + # Loop through all of the changes until we hit a timeout. validation_pool.ValidationPool.AcquirePool( self._run.config.overlays, self.repo, self._run.buildnumber, constants.PRE_CQ_LAUNCHER_NAME, + query, dryrun=self._run.options.debug, - changes_query=self._run.options.cq_gerrit_override, check_tree_open=False, change_filter=self.ProcessChanges, builder_run=self._run) -- cgit v1.2.3 From 20808a86ba340766ddf409282a1bfcba5b541f46 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Fri, 7 Nov 2014 12:59:54 -0800 Subject: Determine fully verified changes based on per-config change information Now that each CQ slave records relevant/irrelevant CLs in CIDB, the next step is to utilize the board-specific information to submit and reject changes: 1. All fully-verified changes should be submitted, even upon slave failures. This should increase the submission rate. 2. Only CLs relevant to the failed build/slave should be considered suspects and potentially be rejected. This should improve the false rejection rate. This CL implements the submission logic in (1), while the rejection logic (2) will be implemented in a separate CL. The new board-specific submission logic will co-exist with the existing logic until we verify that there is no regression. This means that even though CQ master runs the new submission logic, it will still use the old logic to submit CLs. Side effects of this CL: - Migrate CommitQueueCompletionStage unittest from mox to mock. - Move GetOptionForChange and its related functions from validation_pool to triage_lib. - Fix patch.GetChangesAsString to print internal changes with a prefix (*). BUG=chromium:422639 TEST=`cbuildbot/run_tests` TEST=`cbuildbot --remote --debug master-paladin` Change-Id: Ic018175b808a3029727547261a471c6d35d2c1f3 Reviewed-on: https://chromium-review.googlesource.com/228880 Tested-by: Yu-Ju Hong Reviewed-by: Aviv Keshet Commit-Queue: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 6ce80e90d..172717917 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -24,6 +24,7 @@ from chromite.cbuildbot import lkgm_manager from chromite.cbuildbot import manifest_version from chromite.cbuildbot import repository from chromite.cbuildbot import tree_status +from chromite.cbuildbot import triage_lib from chromite.cbuildbot import trybot_patch_pool from chromite.cbuildbot import validation_pool from chromite.cbuildbot.stages import generic_stages @@ -961,7 +962,7 @@ class PreCQLauncherStage(SyncStage): """ configs_to_test = constants.PRE_CQ_DEFAULT_CONFIGS try: - result = validation_pool.GetOptionForChange( + result = triage_lib.GetOptionForChange( self._build_root, change, 'GENERAL', 'pre-cq-configs') if (result and result.split() and all(c in cbuildbot_config.config for c in result.split())): @@ -1012,7 +1013,7 @@ class PreCQLauncherStage(SyncStage): """ result = None try: - result = validation_pool.GetOptionForChange( + result = triage_lib.GetOptionForChange( self._build_root, change, 'GENERAL', 'submit-in-pre-cq') except ConfigParser.Error: cros_build_lib.Error('%s has malformed config file', change, @@ -1079,22 +1080,24 @@ class PreCQLauncherStage(SyncStage): # there's no need to launch a trybot run. plan = set(plan) if not plan.issubset(screened_changes): - logging.info('CLs waiting to be screened: %r', - ' '.join(map(str, plan.difference(screened_changes)))) + logging.info('CLs waiting to be screened: %s', + cros_patch.GetChangesAsString( + plan.difference(screened_changes))) elif plan.issubset(verified): - logging.info('CLs already verified: %r', ' '.join(map(str, plan))) + logging.info('CLs already verified: %s', + cros_patch.GetChangesAsString(plan)) elif plan.intersection(busy): - logging.info('CLs currently being verified: %r', - ' '.join(map(str, plan.intersection(busy)))) + logging.info('CLs currently being verified: %s', + cros_patch.GetChangesAsString(plan.intersection(busy))) if plan.difference(busy): logging.info('CLs waiting on verification of dependencies: %r', - ' '.join(map(str, plan.difference(busy)))) + cros_patch.GetChangesAsString(plan.difference(busy))) # TODO(akeshet): Consider using a database time rather than gerrit # approval time and local clock for launch delay. elif any(x.approval_timestamp + self.LAUNCH_DELAY * 60 > time.time() for x in plan): - logging.info('CLs waiting on launch delay: %r', - ' '.join(map(str, plan))) + logging.info('CLs waiting on launch delay: %s', + cros_patch.GetChangesAsString(plan)) else: pending_configs = clactions.GetPreCQConfigsToTest(plan, progress_map) for config in pending_configs: -- cgit v1.2.3 From 58410662b8b1f211b43a6f1bcefeaf4d97186894 Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 11 Nov 2014 16:22:43 -0800 Subject: Ignore dependency errors for patches that aren't marked commit ready. If a patch isn't marked commit ready yet, and it depends on another change that hasn't been reviewed yet, it doesn't make sense to email the user and complain about it. Filter out these errors until the user marks the patch ready. BUG=chromium:422625 TEST=New unit test. Change-Id: I02d9e533816984f6c1a4941d7c2ca5257a9c974a Reviewed-on: https://chromium-review.googlesource.com/229165 Reviewed-by: David James Tested-by: David James Commit-Queue: David James --- cbuildbot/stages/sync_stages.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 172717917..a66e8dc96 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1212,8 +1212,7 @@ class PreCQLauncherStage(SyncStage): # We don't know for sure they were initially part of a speculative PreCQ # run. It might just be someone turned off the flag, mid-run. - speculative = set(c for c in changes - if not c.HasApproval('COMR', ('1', '2'))) + speculative = set(c for c in changes if not c.IsCommitReady()) # Changes that can be submitted, if their dependencies can be too. Only # include changes that have not already been marked as passed. -- cgit v1.2.3 From 422faf4b4a5283cb82d64ea9d3070660db153135 Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Wed, 12 Nov 2014 23:16:48 -0500 Subject: lint: fix superfluous parenthesis Newer pylint catches warnings like: C:1950, 0: Unnecessary parens after 'if' keyword (superfluous-parens) BUG=chromium:431514 TEST=`./cbuildbot/run_tests` passes TEST=`cros lint` no longer complains about superfluous-parens Change-Id: I47f901f8ceb06402e3c2e83303b7be38390ae535 Reviewed-on: https://chromium-review.googlesource.com/229510 Reviewed-by: Gabe Black Commit-Queue: Mike Frysinger Tested-by: Mike Frysinger --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index a66e8dc96..57aff5e2b 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -638,7 +638,7 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): """Override: Creates an LKGMManager rather than a ManifestManager.""" self._InitializeRepo() self.RegisterManifestManager(self._GetInitializedManager(self.internal)) - if (self._run.config.master and self._GetSlaveConfigs()): + if self._run.config.master and self._GetSlaveConfigs(): assert self.internal, 'Unified masters must use an internal checkout.' MasterSlaveLKGMSyncStage.sub_manager = self._GetInitializedManager(False) -- cgit v1.2.3 From 1132a70c2a0cea2c42849a6a0448d953f36b1aca Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Mon, 10 Nov 2014 21:50:14 -0500 Subject: lint: stop using {en,dis}able-msg Newer pylint issues a warning here: I: 67, 0: Pragma "disable-msg" is deprecated, use "disable" instead (deprecated-pragma) I: 52, 0: Pragma "enable-msg" is deprecated, use "enable" instead (deprecated-pragma) BUG=chromium:431514 TEST=`cros lint` no longer complains about deprecated-pragma Change-Id: I7aedc125571991031271d03c4018c360f3f6a65d Reviewed-on: https://chromium-review.googlesource.com/229500 Reviewed-by: Mike Frysinger Commit-Queue: Mike Frysinger Tested-by: Mike Frysinger --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 57aff5e2b..ddc061f8c 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -724,7 +724,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): else: self._SetPoolFromManifest(self.manifest_manager.GetLocalManifest()) - # pylint: disable-msg=W0613 + # pylint: disable=W0613 def _ChangeFilter(self, pool, changes, non_manifest_changes): # First, look for changes that were tested by the Pre-CQ. changes_to_test = [] -- cgit v1.2.3 From 6dc7dbf0352c8903a2ad2d6dc9e405fccae9dc51 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Wed, 12 Nov 2014 15:05:15 -0800 Subject: PreCQ: Close race condition when CLs are being submitted. Sometimes, the PreCQ will pickup and try to test CLs as they go through the submission cycle (create a new revision, submit it, merge it). Various Gerrit queries appear to lie for long enough to get past the PreCQ Master. So... if we can't stop the PreCQ Master from requesting Tryjobs for these CLs we can at least ignore them if they've already been submitted when the slave starts up. BUG=chromium:432706 TEST=cros lint + unittests. Not much test coverage. Change-Id: Iaa69ccea2f1caa1ba39c8edebcb698cc4b0c2761 Reviewed-on: https://chromium-review.googlesource.com/229421 Tested-by: Don Garrett Reviewed-by: David James Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index ddc061f8c..4840510ef 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -850,8 +850,12 @@ class PreCQSyncStage(SyncStage): def __init__(self, builder_run, patches, **kwargs): super(PreCQSyncStage, self).__init__(builder_run, **kwargs) - # The list of patches to test. - self.patches = patches + # As a workaround for crbug.com/432706, we scan patches to see if they + # are already being merged. If they are, we don't test them in the PreCQ. + self.patches = [p for p in patches if not p.IsBeingMerged()] + + if not self.patches: + cros_build_lib.Die('No patches that still need testing.') # The ValidationPool of patches to test. Initialized in PerformStage, and # refreshed after bootstrapping by HandleSkip. -- cgit v1.2.3 From 1d4752b78eaad48cf7b7e33534a140576e669835 Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Sat, 8 Nov 2014 04:00:18 -0500 Subject: lint: update support for pylint-1.x This doesn't fix the new warnings, but makes `cros lint` work. BUG=chromium:431514 TEST=`cros lint ` no longer crashes TEST=`cros/commands/lint_unittest.py` passes Change-Id: I908abd03027cfc348ed3e74454d9cc0e556ca29d Reviewed-on: https://chromium-review.googlesource.com/228721 Reviewed-by: Mike Frysinger Tested-by: Mike Frysinger --- cbuildbot/stages/sync_stages.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 4840510ef..a172647a5 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -4,6 +4,8 @@ """Module containing the sync stages.""" +# pylint: disable=bad-continuation + from __future__ import print_function import ConfigParser -- cgit v1.2.3 From 2b2e2c52959d74500a4b13435d611198add8ae5f Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 2 Dec 2014 19:32:07 -0800 Subject: Launch tryjobs when users mark CLs as Trybot-Ready. Also launch speculative tryjobs when CLs are CR+2, regardless of whether they have the verified bit. BUG=chromium:211660 TEST=Run master-paladin and pre-cq-launcher trybot runs. TEST=Updated unit test. TEST=pre-cq-launcher run Change-Id: If4fe439061dc2eed161e644f4667f1eb681e90a9 Reviewed-on: https://chromium-review.googlesource.com/232880 Reviewed-by: David James Commit-Queue: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index a172647a5..ec78edef0 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1109,18 +1109,15 @@ class PreCQLauncherStage(SyncStage): for config in pending_configs: yield (plan, config) - def _ProcessRequeuedAndSpeculative(self, change, action_history, - is_speculative): + def _ProcessRequeuedAndSpeculative(self, change, action_history): """Detect if |change| was requeued by developer, and mark in cidb. Args: change: GerritPatch instance to check. action_history: List of CLActions. - is_speculative: Boolean indicating if |change| is speculative, i.e. it - does not have CQ approval. """ action_string = clactions.GetRequeuedOrSpeculative( - change, action_history, is_speculative) + change, action_history, not change.IsMergeable()) if action_string: build_id, db = self._run.GetCIDBHandle() action = clactions.CLAction.FromGerritPatchAndAction( @@ -1155,7 +1152,7 @@ class PreCQLauncherStage(SyncStage): if self._HasTimedOut(timestamp, current_time, timeout): pool.SendNotification(change, '%(details)s', details=msg) - pool.RemoveCommitReady(change, reason=config) + pool.RemoveReady(change, reason=config) pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED) @@ -1216,15 +1213,10 @@ class PreCQLauncherStage(SyncStage): if status_map[c] in passed_statuses) to_process = set(changes) - already_passed - # We don't know for sure they were initially part of a speculative PreCQ - # run. It might just be someone turned off the flag, mid-run. - speculative = set(c for c in changes if not c.IsCommitReady()) - # Changes that can be submitted, if their dependencies can be too. Only # include changes that have not already been marked as passed. can_submit = set(c for c in (verified.intersection(to_process)) if - self.CanSubmitChangeInPreCQ(c)) - can_submit.difference_update(speculative) + c.IsMergeable() and self.CanSubmitChangeInPreCQ(c)) # Changes that will be submitted. will_submit = set() @@ -1232,14 +1224,13 @@ class PreCQLauncherStage(SyncStage): will_pass = set() for change in changes: - self._ProcessRequeuedAndSpeculative(change, action_history, - change in speculative) + self._ProcessRequeuedAndSpeculative(change, action_history) for change in to_process: status = status_map[change] # Detect if change is ready to be marked as passed, or ready to submit. - if change in verified and change not in speculative: + if change in verified and change.IsMergeable(): to_submit, to_pass = self._ProcessVerified(change, can_submit, will_submit) will_submit.update(to_submit) @@ -1261,10 +1252,11 @@ class PreCQLauncherStage(SyncStage): self._ProcessTimeouts(change, progress_map, pool, current_db_time) - # Filter out speculative changes that have already failed before launching. + # Filter out changes that have already failed, and aren't marked trybot + # ready or commit ready, before launching. launchable_progress_map = { k: v for k, v in progress_map.iteritems() - if k not in speculative or status_map[k] != self.STATUS_FAILED} + if k.HasReadyFlag() or status_map[k] != self.STATUS_FAILED} for plan, config in self.GetDisjointTransactionsToTest( pool, launchable_progress_map): -- cgit v1.2.3 From ddf191a651e04950a758e317dce4f4e9f0231fc2 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 4 Dec 2014 00:01:35 -0800 Subject: sync_stages: remove unneeded legacy code BUG=None TEST=None Change-Id: I78c1fd728611c67a60be5c7b6867ac0ef5624dcf Reviewed-on: https://chromium-review.googlesource.com/233048 Tested-by: Aviv Keshet Reviewed-by: David James Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 9 --------- 1 file changed, 9 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index ec78edef0..61811f3ab 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1227,8 +1227,6 @@ class PreCQLauncherStage(SyncStage): self._ProcessRequeuedAndSpeculative(change, action_history) for change in to_process: - status = status_map[change] - # Detect if change is ready to be marked as passed, or ready to submit. if change in verified and change.IsMergeable(): to_submit, to_pass = self._ProcessVerified(change, can_submit, @@ -1237,13 +1235,6 @@ class PreCQLauncherStage(SyncStage): will_pass.update(to_pass) continue - # TODO(akeshet): Eliminate this block after this CL has landed and all - # previously inflight or launching CLs have made it through the pre-CQ. - legacy_statuses = (constants.CL_STATUS_LAUNCHING, - constants.CL_STATUS_INFLIGHT) - if status in legacy_statuses: - continue - # Screen unscreened changes to determine which trybots to test them with. if not any(a.action == constants.CL_ACTION_SCREENED_FOR_PRE_CQ for a in clactions.ActionsForPatch(change, action_history)): -- cgit v1.2.3 From cc5aa647d44bbd972537da6a1b19fb612e80920d Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Mon, 8 Dec 2014 01:35:05 -0500 Subject: lint: cbuildbot: clean up a bit Mostly a focus on bad whitespace, but also drop some obsolete warnings. BUG=chromium:431514 TEST=no more lint errors TEST=`./cbuildbot/run_tests` passes Change-Id: Ie94bd5d6b2abf3a37cfc51e0af1d1abe815fcdac Reviewed-on: https://chromium-review.googlesource.com/233840 Reviewed-by: Aviv Keshet Commit-Queue: Mike Frysinger Tested-by: Mike Frysinger --- cbuildbot/stages/sync_stages.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 61811f3ab..1dc1e2cb2 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -4,8 +4,6 @@ """Module containing the sync stages.""" -# pylint: disable=bad-continuation - from __future__ import print_function import ConfigParser @@ -240,7 +238,6 @@ class BootstrapStage(PatchChangesStage): else: PatchChangesStage.HandleApplyFailures(self, failures) - #pylint: disable=E1101 @osutils.TempDirDecorator def PerformStage(self): # The plan for the builders is to use master branch to bootstrap other @@ -272,7 +269,6 @@ class BootstrapStage(PatchChangesStage): cbuildbot_path = constants.PATH_TO_CBUILDBOT if not os.path.exists(os.path.join(self.tempdir, cbuildbot_path)): cbuildbot_path = 'chromite/cbuildbot/cbuildbot' - # pylint: disable=W0212 cmd = self.FilterArgsForTargetCbuildbot(self.tempdir, cbuildbot_path, self._run.options) @@ -721,13 +717,12 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): super(CommitQueueSyncStage, self).HandleSkip() filename = self._run.options.validation_pool if filename: - self.pool = validation_pool.ValidationPool.Load(filename, - builder_run=self._run) + self.pool = validation_pool.ValidationPool.Load( + filename, builder_run=self._run) else: self._SetPoolFromManifest(self.manifest_manager.GetLocalManifest()) - # pylint: disable=W0613 - def _ChangeFilter(self, pool, changes, non_manifest_changes): + def _ChangeFilter(self, _pool, changes, non_manifest_changes): # First, look for changes that were tested by the Pre-CQ. changes_to_test = [] @@ -799,8 +794,8 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): self._run.buildnumber, self.builder_name, query, dryrun=self._run.options.debug, - check_tree_open=not self._run.options.debug or - self._run.options.mock_tree_status, + check_tree_open=(not self._run.options.debug or + self._run.options.mock_tree_status), change_filter=self._ChangeFilter, throttled_ok=True, builder_run=self._run) @@ -868,8 +863,8 @@ class PreCQSyncStage(SyncStage): super(PreCQSyncStage, self).HandleSkip() filename = self._run.options.validation_pool if filename: - self.pool = validation_pool.ValidationPool.Load(filename, - builder_run=self._run) + self.pool = validation_pool.ValidationPool.Load( + filename, builder_run=self._run) def PerformStage(self): super(PreCQSyncStage, self).PerformStage() @@ -1220,7 +1215,7 @@ class PreCQLauncherStage(SyncStage): # Changes that will be submitted. will_submit = set() - # Changes that will be passed + # Changes that will be passed. will_pass = set() for change in changes: @@ -1247,7 +1242,7 @@ class PreCQLauncherStage(SyncStage): # ready or commit ready, before launching. launchable_progress_map = { k: v for k, v in progress_map.iteritems() - if k.HasReadyFlag() or status_map[k] != self.STATUS_FAILED} + if k.HasReadyFlag() or status_map[k] != self.STATUS_FAILED} for plan, config in self.GetDisjointTransactionsToTest( pool, launchable_progress_map): -- cgit v1.2.3 From fbb3b5286f5b76cd001cd3c0ca47b9108f02bee7 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Mon, 8 Dec 2014 16:10:06 -0800 Subject: Send CL notifications from pre-cq-launcher We will soon be launching multiple pre-cq configs for a CL to improve the test coverage and performance. To reduce the gerrit comment/email spam, This CL does the following: * Individual pre-cq config no loger comments on the CL after applying the changes. Instead, pre-cq-launcher will update the CL with URLs to the build statuses after all configs are inflight. * Individual pre-cq config no loger comments on the CL after successfully verifies the change. Instead, pre-cq-launcher will update the CL when marking the CL as passed. Note that when the pre-cq configs fail, they will still comment on the CL individually. This issue should be addressed later. This CL also moves ConstructDashboardURL from validation_pool to tree_status and uses the waterfall information recorded in the metadata to construct the URL, as opposed to using the overlays and the trybot flag. This provides a more accurate information because overlays are not sufficient to distinguish between ToT, branch, and release builds. BUG=chromium:439595 TEST=remote pre-cq-launcher and pre-cq configs Change-Id: I99da0f99c1f1da15d66039979ddd3cd600bcc9ae Reviewed-on: https://chromium-review.googlesource.com/234044 Tested-by: Yu-Ju Hong Trybot-Ready: Yu-Ju Hong Reviewed-by: Yu-Ju Hong Commit-Queue: Yu-Ju Hong --- cbuildbot/stages/sync_stages.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 1dc1e2cb2..2578f7eb6 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1061,7 +1061,7 @@ class PreCQLauncherStage(SyncStage): that should be launched. """ # Get the set of busy and passed CLs. - busy, verified = clactions.GetPreCQCategories(progress_map) + busy, _, verified = clactions.GetPreCQCategories(progress_map) screened_changes = set(progress_map) @@ -1136,7 +1136,7 @@ class PreCQLauncherStage(SyncStage): timeout_statuses = (constants.CL_PRECQ_CONFIG_STATUS_LAUNCHED, constants.CL_PRECQ_CONFIG_STATUS_INFLIGHT) config_progress = progress_map[change] - for config, (config_status, timestamp) in config_progress.iteritems(): + for config, (config_status, timestamp, _) in config_progress.iteritems(): if not config_status in timeout_statuses: continue launched = config_status == constants.CL_PRECQ_CONFIG_STATUS_LAUNCHED @@ -1197,7 +1197,7 @@ class PreCQLauncherStage(SyncStage): status_map = {c: clactions.GetCLPreCQStatus(c, action_history) for c in changes} progress_map = clactions.GetPreCQProgressMap(changes, action_history) - _, verified = clactions.GetPreCQCategories(progress_map) + _, inflight, verified = clactions.GetPreCQCategories(progress_map) current_db_time = db.GetTime() # TODO(akeshet): Once this change lands, we will no longer mark changes @@ -1221,6 +1221,20 @@ class PreCQLauncherStage(SyncStage): for change in changes: self._ProcessRequeuedAndSpeculative(change, action_history) + for change in inflight: + if status_map[change] != constants.CL_STATUS_INFLIGHT: + build_ids = [x for _, _, x in progress_map[change].values()] + # Change the status to inflight. + pool.UpdateCLPreCQStatus(change, constants.CL_STATUS_INFLIGHT) + build_dicts = db.GetBuildStatuses(build_ids) + urls = [] + for b in build_dicts: + urls.append(tree_status.ConstructDashboardURL( + b['waterfall'], b['builder_name'], b['build_number'])) + + # Send notifications. + pool.HandleApplySuccess(change, build_log='\n'.join(urls)) + for change in to_process: # Detect if change is ready to be marked as passed, or ready to submit. if change in verified and change.IsMergeable(): @@ -1248,10 +1262,10 @@ class PreCQLauncherStage(SyncStage): pool, launchable_progress_map): self.LaunchTrybot(plan, config) - # Submit changes that are ready to submit, if we can. - if tree_status.IsTreeOpen(): - pool.SubmitNonManifestChanges(check_tree_open=False) - pool.SubmitChanges(will_submit, check_tree_open=False) + # Send notifications to all successfully verified changes. + success = will_pass.union(will_submit) + if success: + pool.HandlePreCQSuccess(success) # Mark passed changes as passed if will_pass: @@ -1262,6 +1276,11 @@ class PreCQLauncherStage(SyncStage): for c in will_pass] db.InsertCLActions(build_id, actions) + # Submit changes that are ready to submit, if we can. + if tree_status.IsTreeOpen(): + pool.SubmitNonManifestChanges(check_tree_open=False) + pool.SubmitChanges(will_submit, check_tree_open=False) + # Tell ValidationPool to keep waiting for more changes until we hit # its internal timeout. return [], [] -- cgit v1.2.3 From 1cba7fdd447c6482f4c51e8c4ea613ce3c8b6d31 Mon Sep 17 00:00:00 2001 From: Yu-Ju Hong Date: Thu, 11 Dec 2014 18:25:03 -0800 Subject: Pre-CQ: mark CLs as fully verified Currently, We mark a change as "passed" and comment on the CL if and only if the CL has been fully verified (passed all relevant pre-cq configs) and is mergeable (CR+2, CQ+1, V+1). This prevents speculative Pre-CQ from spamming the developers. However, developer might be unnecessarily waiting for the CL to be verified before marking it ready. This CL uses the "CL_STATUS_FULLY_VERIFIED" state to solve this issue. Once verified, the CL is marked "fully verified" and Pre-CQ will comment on the CL of this status change. The CL will then transition to "passed" or be submitted when it becomes mergeable. BUG=chromium:439595 TEST=`cbuildbot/run_tests` Change-Id: Ida0ff4fea708fb7b63307cee0251c1277488d75c Reviewed-on: https://chromium-review.googlesource.com/234695 Reviewed-by: Yu-Ju Hong Commit-Queue: Yu-Ju Hong Tested-by: Yu-Ju Hong Reviewed-by: David James --- cbuildbot/stages/sync_stages.py | 48 +++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 21 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 2578f7eb6..d28ceb6db 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1181,6 +1181,19 @@ class PreCQLauncherStage(SyncStage): # Changes that cannot be submitted are marked as passed. return set(), set([change]) + def UpdateChangeStatuses(self, changes, status): + """Update |changes| to |status|. + + Args: + changes: A set of GerritPatch instances. + status: One of constants.CL_STATUS_* statuses. + """ + if changes: + build_id, db = self._run.GetCIDBHandle() + a = clactions.TranslatePreCQStatusToAction(status) + actions = [clactions.CLAction.FromGerritPatchAndAction(c, a) + for c in changes] + db.InsertCLActions(build_id, actions) def ProcessChanges(self, pool, changes, _non_manifest_changes): """Process a list of changes that were marked as Ready. @@ -1192,7 +1205,7 @@ class PreCQLauncherStage(SyncStage): Non-manifest changes are just submitted here because they don't need to be verified by either the Pre-CQ or CQ. """ - build_id, db = self._run.GetCIDBHandle() + _, db = self._run.GetCIDBHandle() action_history = db.GetActionsForChanges(changes) status_map = {c: clactions.GetCLPreCQStatus(c, action_history) for c in changes} @@ -1200,13 +1213,17 @@ class PreCQLauncherStage(SyncStage): _, inflight, verified = clactions.GetPreCQCategories(progress_map) current_db_time = db.GetTime() - # TODO(akeshet): Once this change lands, we will no longer mark changes - # with pre-cq status READY_TO_SUBMIT, so simplify the status check below. - passed_statuses = (constants.CL_STATUS_PASSED, - constants.CL_STATUS_READY_TO_SUBMIT) - already_passed = set(c for c in changes - if status_map[c] in passed_statuses) - to_process = set(changes) - already_passed + to_process = set(c for c in changes + if status_map[c] != constants.CL_STATUS_PASSED) + + # Mark verified changes verified. + to_mark_verified = [c for c in verified.intersection(to_process) if + status_map[c] != constants.CL_STATUS_FULLY_VERIFIED] + self.UpdateChangeStatuses(to_mark_verified, + constants.CL_STATUS_FULLY_VERIFIED) + # Send notifications to the fully verified changes. + if to_mark_verified: + pool.HandlePreCQSuccess(to_mark_verified) # Changes that can be submitted, if their dependencies can be too. Only # include changes that have not already been marked as passed. @@ -1225,7 +1242,7 @@ class PreCQLauncherStage(SyncStage): if status_map[change] != constants.CL_STATUS_INFLIGHT: build_ids = [x for _, _, x in progress_map[change].values()] # Change the status to inflight. - pool.UpdateCLPreCQStatus(change, constants.CL_STATUS_INFLIGHT) + self.UpdateChangeStatuses([change], constants.CL_STATUS_INFLIGHT) build_dicts = db.GetBuildStatuses(build_ids) urls = [] for b in build_dicts: @@ -1262,19 +1279,8 @@ class PreCQLauncherStage(SyncStage): pool, launchable_progress_map): self.LaunchTrybot(plan, config) - # Send notifications to all successfully verified changes. - success = will_pass.union(will_submit) - if success: - pool.HandlePreCQSuccess(success) - # Mark passed changes as passed - if will_pass: - # TODO(akeshet) Refactor this into a general purpose method somewhere to - # atomically update CL statuses. - a = clactions.TranslatePreCQStatusToAction(constants.CL_STATUS_PASSED) - actions = [clactions.CLAction.FromGerritPatchAndAction(c, a) - for c in will_pass] - db.InsertCLActions(build_id, actions) + self.UpdateChangeStatuses(will_pass, constants.CL_STATUS_PASSED) # Submit changes that are ready to submit, if we can. if tree_status.IsTreeOpen(): -- cgit v1.2.3 From 9dc8ca015c54a61712319605a85564567f9b689e Mon Sep 17 00:00:00 2001 From: David James Date: Wed, 17 Dec 2014 14:49:24 -0800 Subject: Don't exit immediately when there are no Gerrit patches. BUG=chromium:443373 TEST=Run trybot run with no Gerrit patches. TEST=Run trybot run with a Gerrit patch that's already merged. TEST=Run trybot run with a Gerrit patch that has a conflict. Change-Id: I8755462bff8f345cf95ed2e30343ace096202563 Reviewed-on: https://chromium-review.googlesource.com/236417 Reviewed-by: Aviv Keshet Commit-Queue: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index d28ceb6db..e4d311c1d 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -851,7 +851,7 @@ class PreCQSyncStage(SyncStage): # are already being merged. If they are, we don't test them in the PreCQ. self.patches = [p for p in patches if not p.IsBeingMerged()] - if not self.patches: + if patches and not self.patches: cros_build_lib.Die('No patches that still need testing.') # The ValidationPool of patches to test. Initialized in PerformStage, and @@ -875,7 +875,7 @@ class PreCQSyncStage(SyncStage): builder_run=self._run) self.pool.ApplyPoolIntoRepo() - if len(self.pool.changes) == 0: + if len(self.pool.changes) == 0 and self.patches: cros_build_lib.Die('No changes have been applied.') -- cgit v1.2.3 From 80e05dfc060c165631a3d2b6f48811e08d3b00da Mon Sep 17 00:00:00 2001 From: Prathmesh Prabhu Date: Thu, 11 Dec 2014 15:20:33 -0800 Subject: cbuildbot: Use build 'deadline' to abort slaves when master quits. When builds are launched in the master-slave model, we'd like to abort the slaves about the same time when the master times out on them. After this CL, - The master posts a reasonable deadline for the build to cidb, and waits for slaves to finish till that deadline. - The slaves abort themselves if the deadline set by their master is hit. - CQ Master: waits for a long time in the sync stage for CLs to be +2'ed, so it extends the deadline after the sync stage so that slaves have enough time to finish their build. BUG=chromium:431836 TEST=(1) ./cbuildbot/run_tests (2) Launch a master trybot and verify that a sensible deadline is posted to cidb. (3) Launch a slave trybot pointing to the earlier master trybot as the master, and verify that the slave posts the same deadline and aborts when the deadline is hit. Change-Id: Idebd09630eda1be10afae462d38f41ff913004de Reviewed-on: https://chromium-review.googlesource.com/234853 Tested-by: Prathmesh Prabhu Reviewed-by: David James Commit-Queue: Prathmesh Prabhu --- cbuildbot/stages/sync_stages.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index e4d311c1d..502329de5 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -803,6 +803,16 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): cros_build_lib.Warning(str(e)) return None + # We must extend the builder deadline before publishing a new manifest to + # ensure that slaves have enough time to complete the builds about to + # start. + build_id, db = self._run.GetCIDBHandle() + if db: + timeout = constants.MASTER_BUILD_TIMEOUT_SECONDS.get( + self._run.config.build_type, + constants.MASTER_BUILD_TIMEOUT_DEFAULT_SECONDS) + db.ExtendDeadline(build_id, timeout) + manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool, build_id=build_id) if MasterSlaveLKGMSyncStage.sub_manager: -- cgit v1.2.3 From 71b380da44d382ea598c18523b8ae39dc9e80430 Mon Sep 17 00:00:00 2001 From: Chris Sosa Date: Mon, 3 Nov 2014 17:14:01 -0800 Subject: validation_pool: Add logic to apply exponential fallback. This change introduces a helper method to apply exponential fallback by calculating the fail streak for the given config and reducing the size of the changes if the fail streak is higher than 0. In addition, I've modified AcquirePool to use ApplyExponentialFallback when it detects the tree is throttled and no CQ+2 changes are available. As part of this CL I've slightly refactored AcquirePool. Mostly I've moved the core of AcquirePool into a helper method AcquireChanges that takes in the gerrit query and gets/filters changes for it. This both makes AcquirePool much more readable as well as testable (easier to mock our the changes component and test the rest of the logic). BUG=chromium:422626 TEST=Unittests! Still doing more testing. Change-Id: I2d06b0da82cfddde02327d38afd04e156b1d8029 Reviewed-on: https://chromium-review.googlesource.com/226959 Reviewed-by: Chris Sosa Commit-Queue: Chris Sosa Tested-by: Chris Sosa Trybot-Ready: Chris Sosa --- cbuildbot/stages/sync_stages.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 502329de5..71d177e16 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -796,9 +796,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): dryrun=self._run.options.debug, check_tree_open=(not self._run.options.debug or self._run.options.mock_tree_status), - change_filter=self._ChangeFilter, throttled_ok=True, - builder_run=self._run) - + change_filter=self._ChangeFilter, builder_run=self._run) except validation_pool.TreeIsClosedException as e: cros_build_lib.Warning(str(e)) return None -- cgit v1.2.3 From ee6b9a60a10d6c578f72e0eb8f5027d364373f85 Mon Sep 17 00:00:00 2001 From: Simran Basi Date: Fri, 5 Dec 2014 11:19:33 -0800 Subject: cbuildbot: Don't bootstrap chromite in /tmp For the bootstrap stage, we pull down a copy of chromite into /tmp and patch in chromite changes there. For systems that do not allow executing code in /tmp, this will fail. BUG=chromium:435339 TEST=mobbuild & trybot runs. Change-Id: I22106eb61a28b129712ac02a362b684c9bc14e2f Reviewed-on: https://chromium-review.googlesource.com/233554 Reviewed-by: Mike Frysinger Tested-by: Simran Basi Commit-Queue: Simran Basi --- cbuildbot/stages/sync_stages.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 71d177e16..d07adcdcb 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -161,6 +161,12 @@ class BootstrapStage(PatchChangesStage): self.chromite_patch_pool = chromite_patch_pool self.manifest_patch_pool = manifest_patch_pool self.returncode = None + # Bootstrap chromite in a subdirectory of the buildroot. This directory + # requires exec permissions so that cbuildbot can be re-executed after + # chromite is patched. + self.tempdir = os.path.join(self._run.options.buildroot, + 'chromite-bootstrap') + osutils.RmDir(self.tempdir, ignore_missing=True) def _ApplyManifestPatches(self, patch_pool): """Apply a pool of manifest patches to a temp manifest checkout. @@ -238,7 +244,6 @@ class BootstrapStage(PatchChangesStage): else: PatchChangesStage.HandleApplyFailures(self, failures) - @osutils.TempDirDecorator def PerformStage(self): # The plan for the builders is to use master branch to bootstrap other # branches. Now, if we wanted to test patches for both the bootstrap code -- cgit v1.2.3 From 24c61e5d02f0d62769261fa98725aa32bd67f4c4 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Tue, 6 Jan 2015 13:15:25 -0800 Subject: sync_stages: failed speculative changes should be ignored by launcher BUG=chromium:446555 TEST=New unit test added. Verified that test fails prior to the fix. Change-Id: Iec7525d8a4ab3f4af437be08000802f89e51dcd8 Reviewed-on: https://chromium-review.googlesource.com/238773 Trybot-Ready: Aviv Keshet Tested-by: Aviv Keshet Reviewed-by: Yu-Ju Hong Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index d07adcdcb..0bd137218 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1220,8 +1220,17 @@ class PreCQLauncherStage(SyncStage): """ _, db = self._run.GetCIDBHandle() action_history = db.GetActionsForChanges(changes) + for change in changes: + self._ProcessRequeuedAndSpeculative(change, action_history) + + status_map = {c: clactions.GetCLPreCQStatus(c, action_history) for c in changes} + + # Filter out failed speculative changes. + changes = [c for c in changes if status_map[c] != constants.CL_STATUS_FAILED + or c.HasReadyFlag()] + progress_map = clactions.GetPreCQProgressMap(changes, action_history) _, inflight, verified = clactions.GetPreCQCategories(progress_map) current_db_time = db.GetTime() @@ -1248,8 +1257,6 @@ class PreCQLauncherStage(SyncStage): # Changes that will be passed. will_pass = set() - for change in changes: - self._ProcessRequeuedAndSpeculative(change, action_history) for change in inflight: if status_map[change] != constants.CL_STATUS_INFLIGHT: -- cgit v1.2.3 From cda64c3323f33604c2950a8933e2104d6d81555a Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 8 Jan 2015 14:12:26 -0800 Subject: Use BUILDBOT_BUILDERNAME instead of paladin_builder_name. Buildbot supplies the current builder name as an environment variable, so there is no need to specify it in the cbuildbot config. Remove it from the config and use BUILDBOT_BUILDERNAME instead. BUG=none TEST=unit tests Change-Id: Ia6922f704b83d1dd019a69c369c3f88f92815fa5 Reviewed-on: https://chromium-review.googlesource.com/239574 Trybot-Ready: David James Reviewed-by: Aviv Keshet Commit-Queue: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 0bd137218..58a9147b0 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -706,9 +706,6 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): def __init__(self, builder_run, **kwargs): super(CommitQueueSyncStage, self).__init__(builder_run, **kwargs) - # Figure out the builder's name from the buildbot waterfall. - builder_name = self._run.config.paladin_builder_name - self.builder_name = builder_name if builder_name else self._run.config.name # The pool of patches to be picked up by the commit queue. # - For the master commit queue, it's initialized in GetNextManifest. @@ -762,7 +759,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): # AcquirePoolFromManifest does not need to sync. self.pool = validation_pool.ValidationPool.AcquirePoolFromManifest( manifest, self._run.config.overlays, self.repo, - self._run.buildnumber, self.builder_name, + self._run.buildnumber, self._run.GetBuilderName(), self._run.config.master, self._run.options.debug, builder_run=self._run) @@ -796,7 +793,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): self.pool = pool = validation_pool.ValidationPool.AcquirePool( self._run.config.overlays, self.repo, - self._run.buildnumber, self.builder_name, + self._run.buildnumber, self._run.GetBuilderName(), query, dryrun=self._run.options.debug, check_tree_open=(not self._run.options.debug or -- cgit v1.2.3 From cf1c36edf0c3084913920888f413007c825d000a Mon Sep 17 00:00:00 2001 From: David James Date: Mon, 12 Jan 2015 08:55:07 -0800 Subject: Remove unused code found by vulture. Using vulture 0.6.1 from https://pypi.python.org/pypi/vulture TEST=vulture says this code isn't used in chromite. TEST=git grep for each function deleted. TEST=Unit tests. TEST=release and cq trybot runs Change-Id: Ifcae7791421b708a518db20da4b9ffe6bffaf62f Reviewed-on: https://chromium-review.googlesource.com/240260 Reviewed-by: Mike Frysinger Reviewed-by: Luis Lozano Reviewed-by: David James Commit-Queue: David James Trybot-Ready: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 58a9147b0..4eb3740b0 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -892,24 +892,6 @@ class PreCQSyncStage(SyncStage): class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" - # The CL is currently being tested by a Pre-CQ trybot. - STATUS_INFLIGHT = constants.CL_STATUS_INFLIGHT - - # The CL has passed the Pre-CQ. - STATUS_PASSED = constants.CL_STATUS_PASSED - - # The CL has failed the Pre-CQ. - STATUS_FAILED = constants.CL_STATUS_FAILED - - # We have requested a Pre-CQ trybot but it has not started yet. - STATUS_LAUNCHING = constants.CL_STATUS_LAUNCHING - - # The CL is ready to be retried. - STATUS_WAITING = constants.CL_STATUS_WAITING - - # The CL has passed the Pre-CQ and is ready to be submitted. - STATUS_READY_TO_SUBMIT = constants.CL_STATUS_READY_TO_SUBMIT - # The number of minutes we wait before launching Pre-CQ jobs. This measures # the idle time of a given patch series, so, for example, if a user takes # 20 minutes to mark a series of 20 patches as ready, we won't launch a @@ -1158,8 +1140,7 @@ class PreCQLauncherStage(SyncStage): if self._HasTimedOut(timestamp, current_time, timeout): pool.SendNotification(change, '%(details)s', details=msg) pool.RemoveReady(change, reason=config) - pool.UpdateCLPreCQStatus(change, self.STATUS_FAILED) - + pool.UpdateCLPreCQStatus(change, constants.CL_STATUS_FAILED) def _ProcessVerified(self, change, can_submit, will_submit): """Process a change that is fully pre-cq verified. @@ -1290,7 +1271,7 @@ class PreCQLauncherStage(SyncStage): # ready or commit ready, before launching. launchable_progress_map = { k: v for k, v in progress_map.iteritems() - if k.HasReadyFlag() or status_map[k] != self.STATUS_FAILED} + if k.HasReadyFlag() or status_map[k] != constants.CL_STATUS_FAILED} for plan, config in self.GetDisjointTransactionsToTest( pool, launchable_progress_map): -- cgit v1.2.3 From 056abefd3af819cb52670e8560d106acd2faf151 Mon Sep 17 00:00:00 2001 From: David James Date: Wed, 14 Jan 2015 14:11:30 -0800 Subject: Pre-CQ: Update inflight timeout to actually be 120 minutes. The inflight timeout was configured to 180 minutes, but was accidentally set to 90 minutes due to a bug in the code introduced in CL:226417. Fix this, and compromise on 120 minutes. This caused weird bugs where CLs were rejected while the trybot continued and actually successfully verified the change. See the comments by Commit Bot in CL:240588 for an example. BUG=none TEST=unit tests Change-Id: I0db52694e9b519b207ef8fdaa14d8d922b676a28 Reviewed-on: https://chromium-review.googlesource.com/240770 Reviewed-by: Aviv Keshet Commit-Queue: David James Trybot-Ready: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 4eb3740b0..c9dc47bb9 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -906,7 +906,7 @@ class PreCQLauncherStage(SyncStage): # The number of minutes we allow before considering an in-flight # job failed. If this window isn't hit in a given launcher run, the window # will start again from scratch in the next run. - INFLIGHT_TIMEOUT = 180 + INFLIGHT_TIMEOUT = 120 # The maximum number of patches we will allow in a given trybot run. This is # needed because our trybot infrastructure can only handle so many patches at @@ -1135,7 +1135,6 @@ class PreCQLauncherStage(SyncStage): timeout = self.LAUNCH_TIMEOUT if launched else self.INFLIGHT_TIMEOUT msg = (PRECQ_LAUNCH_TIMEOUT_MSG if launched else PRECQ_INFLIGHT_TIMEOUT_MSG) % (config, timeout) - timeout = self.LAUNCH_TIMEOUT if self._HasTimedOut(timestamp, current_time, timeout): pool.SendNotification(change, '%(details)s', details=msg) -- cgit v1.2.3 From 45b258b2bedf4680e6003c2da550e332fbd9ebad Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 22 Jan 2015 10:16:08 -0800 Subject: pre-cq: expire PASSED status after a generous timeout BUG=chromium:448955 TEST=New unit test coverage Change-Id: Ic932ab622f87b136bce2915cc3da7781b731e679 Reviewed-on: https://chromium-review.googlesource.com/242601 Trybot-Ready: Aviv Keshet Tested-by: Aviv Keshet Reviewed-by: David James Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 50 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 5 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index c9dc47bb9..b90b07e5b 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -56,7 +56,13 @@ PRECQ_INFLIGHT_TIMEOUT_MSG = ( 'change is not at fault you may mark your change as ready ' 'again. If this problem occurs multiple times please notify ' 'the sheriff and file a bug.') - +PRECQ_EXPIRY_MSG = ( + 'The pre-cq verification for this change expired after %s minutes. No ' + 'action is required on your part.' + '\n\n' + 'In order to protect the CQ from picking up stale changes, the pre-cq ' + 'status for changes are cleared after a generous timeout. This change ' + 'will be re-tested by the pre-cq before the CQ picks it up.') class PatchChangesStage(generic_stages.BuilderStage): """Stage that patches a set of Gerrit changes to the buildroot source tree.""" @@ -908,6 +914,11 @@ class PreCQLauncherStage(SyncStage): # will start again from scratch in the next run. INFLIGHT_TIMEOUT = 120 + # The number of minutes we allow before expiring a pre-cq PASSED or + # FULLY_VERIFIED status. After this timeout is hit, a CL's status will be + # reset to None. This prevents very stale CLs from entering the CQ. + STATUS_EXPIRY_TIMEOUT = 60 * 24 * 7 + # The maximum number of patches we will allow in a given trybot run. This is # needed because our trybot infrastructure can only handle so many patches at # once. @@ -1111,6 +1122,31 @@ class PreCQLauncherStage(SyncStage): change, action_string) db.InsertCLActions(build_id, [action]) + def _ProcessExpiry(self, change, status, timestamp, pool, current_time): + """Enforce expiry of a PASSED or FULLY_VERIFIED status. + + Args: + change: GerritPatch instance to process. + status: |change|'s pre-cq status. + timestamp: datetime.datetime for when |status| was achieved. + pool: The current validation pool. + current_time: datetime.datetime for current database time. + """ + if not timestamp: + return + timed_out = self._HasTimedOut(timestamp, current_time, + self.STATUS_EXPIRY_TIMEOUT) + verified = status in (constants.CL_STATUS_PASSED, + constants.CL_STATUS_FULLY_VERIFIED) + if timed_out and verified: + msg = PRECQ_EXPIRY_MSG % self.STATUS_EXPIRY_TIMEOUT + build_id, db = self._run.GetCIDBHandle() + if db: + pool.SendNotification(change, '%(details)s', details=msg) + action = clactions.CLAction.FromGerritPatchAndAction( + change, constants.CL_ACTION_PRE_CQ_RESET) + db.InsertCLActions(build_id, [action]) + def _ProcessTimeouts(self, change, progress_map, pool, current_time): """Enforce per-config launch and inflight timeouts. @@ -1200,9 +1236,10 @@ class PreCQLauncherStage(SyncStage): for change in changes: self._ProcessRequeuedAndSpeculative(change, action_history) - - status_map = {c: clactions.GetCLPreCQStatus(c, action_history) - for c in changes} + status_and_timestamp_map = { + c: clactions.GetCLPreCQStatusAndTime(c, action_history) + for c in changes} + status_map = {c: v[0] for c, v in status_and_timestamp_map.items()} # Filter out failed speculative changes. changes = [c for c in changes if status_map[c] != constants.CL_STATUS_FAILED @@ -1234,7 +1271,6 @@ class PreCQLauncherStage(SyncStage): # Changes that will be passed. will_pass = set() - for change in inflight: if status_map[change] != constants.CL_STATUS_INFLIGHT: build_ids = [x for _, _, x in progress_map[change].values()] @@ -1279,6 +1315,10 @@ class PreCQLauncherStage(SyncStage): # Mark passed changes as passed self.UpdateChangeStatuses(will_pass, constants.CL_STATUS_PASSED) + # Expire any very stale passed or fully verified changes. + for c, v in status_and_timestamp_map.items(): + self._ProcessExpiry(c, v[0], v[1], pool, current_db_time) + # Submit changes that are ready to submit, if we can. if tree_status.IsTreeOpen(): pool.SubmitNonManifestChanges(check_tree_open=False) -- cgit v1.2.3 From ea779ac44a5ef6974da1f11ec43fc581f73decf8 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Fri, 23 Jan 2015 01:13:41 -0800 Subject: clactions: add helper method to determine if a change is screened This CL adds a helper method to clactions.py to determine if a change is pre-cq screened. It also uses the helper method in sync_stages.py. BUG=None TEST=unittests Change-Id: I79c0fefcb7eaae8ef856028c0b8533b6e10c30f0 Reviewed-on: https://chromium-review.googlesource.com/242800 Reviewed-by: Aviv Keshet Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index b90b07e5b..7b6b81ba0 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1295,8 +1295,7 @@ class PreCQLauncherStage(SyncStage): continue # Screen unscreened changes to determine which trybots to test them with. - if not any(a.action == constants.CL_ACTION_SCREENED_FOR_PRE_CQ - for a in clactions.ActionsForPatch(change, action_history)): + if not clactions.IsChangeScreened(change, action_history): self.ScreenChangeForPreCQ(change) continue -- cgit v1.2.3 From c58fead4133963539f500c67bec2b41e96362d39 Mon Sep 17 00:00:00 2001 From: Prathmesh Prabhu Date: Fri, 23 Jan 2015 15:28:14 -0800 Subject: Increase pre-cq build timeout to 4 hours. Before this CL, pre-cq jobs would time out after 2 hours. This timeout was too small for board specific pre-cq's for boards that do not have prebuilts available for many packages. Increase this timeout to 4 hours. BUG=chromium:451578 TEST=pre-cq passes for a jecht specific CL. Change-Id: Ib5fcf76de1b11a7870079dc2407492a69e3176f0 Reviewed-on: https://chromium-review.googlesource.com/243070 Reviewed-by: Prathmesh Prabhu Tested-by: Prathmesh Prabhu --- cbuildbot/stages/sync_stages.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 7b6b81ba0..ffa1c055a 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -905,14 +905,10 @@ class PreCQLauncherStage(SyncStage): LAUNCH_DELAY = 2 # The number of minutes we allow before considering a launch attempt failed. - # If this window isn't hit in a given launcher run, the window will start - # again from scratch in the next run. LAUNCH_TIMEOUT = 90 - # The number of minutes we allow before considering an in-flight - # job failed. If this window isn't hit in a given launcher run, the window - # will start again from scratch in the next run. - INFLIGHT_TIMEOUT = 120 + # The number of minutes we allow before considering an in-flight job failed. + INFLIGHT_TIMEOUT = 240 # The number of minutes we allow before expiring a pre-cq PASSED or # FULLY_VERIFIED status. After this timeout is hit, a CL's status will be -- cgit v1.2.3 From 36a9bcfc0d577f0dee10c2cfdd427744d6c6f1b9 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 29 Jan 2015 18:25:38 -0800 Subject: sync_stages: Start creating Project Sdk manifest. Create a method to generate Project SDK manifests and to generate and publish it as part of the Canary master. BUG=brillo:48 TEST=Unittests Change-Id: Ie947ab8af5ff20f1a3da30d5366e2c2e221fa823 Reviewed-on: https://chromium-review.googlesource.com/244602 Trybot-Ready: Don Garrett Tested-by: Don Garrett Reviewed-by: Chris Sosa Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 76 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index ffa1c055a..d80c403e3 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -11,7 +11,9 @@ import contextlib import datetime import logging import os +import shutil import sys +import tempfile import time from xml.etree import ElementTree from xml.dom import minidom @@ -564,6 +566,58 @@ class ManifestVersionedSyncStage(SyncStage): else: yield manifest + @staticmethod + def CommitProjectSDKManifest(manifest, release, current_version, debug): + """Create and submit the Product SDK Manifest. + + Create the Project SDK manifest, and push it to the external manifest + repository. + + Args: + manifest: Path to new manifest to commit. + release: Current release of the build (ie: 42 if building R42-6513.0.0) + current_version: Version to use when commiting manifest. + debug: Is this a debug build? + """ + external_manifest_url = cbuildbot_config.GetManifestVersionsRepoUrl( + internal_build=False, + read_only=False, + test=debug) + + # TODO(dgarrett): Find a persistent directory for this. + with osutils.TempDir() as git_repo: + repository.UpdateGitRepo(git_repo, external_manifest_url) + + sdk_manifest_path = os.path.join( + git_repo, 'project-sdk', '%s' % release, '%s.xml' % current_version) + + if os.path.exists(sdk_manifest_path): + raise failures_lib.StepFailure( + 'Project SDK Manifest already exists: %s' % sdk_manifest_path) + + # Create branch for pushing new manifest file. + branch = 'temp_project_sdk_creation_branch' + git.CreatePushBranch(branch, git_repo, sync=False) + + # Create new manifest file. + logging.info('Creating Project SDK Manifest as: %s', sdk_manifest_path) + osutils.SafeMakedirs(os.path.dirname(sdk_manifest_path)) + shutil.copyfile(manifest, sdk_manifest_path) + + # Commit it locally. + logging.info('Committing Project SDK Manifest.') + git.AddPath(sdk_manifest_path) + git.Commit(git_repo, 'Create project_sdk for %s-%s.' % + (release, current_version)) + + # Push it. + logging.info('Pushing Project SDK Manifest.') + git.PushWithRetry(branch, git_repo) + + logging.info('Project SDK Manifest \'%s\' published:', + os.path.basename(sdk_manifest_path)) + logging.info('%s', osutils.ReadFile(manifest)) + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): self.Initialize() @@ -591,6 +645,28 @@ class ManifestVersionedSyncStage(SyncStage): next_manifest, filter_cros=self._run.options.local) as new_manifest: self.ManifestCheckout(new_manifest) + # If we are a Canary Master, create an additional derivative Manifest for + # the Project SDK builders. + if (cbuildbot_config.IsCanaryType(self._run.config.build_type) and + self._run.config.master): + logging.info('Creating of Project SDK Manifest.') + sdk_manifest = None + try: + with tempfile.NamedTemporaryFile() as pinned_manifest_file: + pinned_manifest = self.repo.ExportManifest(mark_revision=True) + osutils.WriteFile(pinned_manifest_file.name, pinned_manifest) + sdk_manifest = manifest_version.ConvertToProjectSdkManifest( + pinned_manifest_file.name) + + self.CommitProjectSDKManifest( + sdk_manifest, + self.manifest_manager.GetCurrentVersionInfo().chrome_branch, + self.manifest_manager.current_version, + self._run.options.debug) + finally: + if sdk_manifest: + os.unlink(sdk_manifest) + # Set the status inflight at the end of the ManifestVersionedSync # stage. This guarantees that all syncing has completed. if self.manifest_manager: -- cgit v1.2.3 From 6b59c67f8da8bb9912383d886177bf21e6b159f3 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 12 Feb 2015 11:29:01 -0800 Subject: LKGMSyncStage: LKGM now based on config.manifest. The LKGMSyncStage was using a hard coded location for the manifest to sync to. Switch it to a config based path instead. This is prep work for the Project SDK builds using it, from a new path. BUG=brillo:186 TEST=Unitests. Change-Id: I6fe3a70b7f345ddaed4f6096875aa3944600b0d4 Reviewed-on: https://chromium-review.googlesource.com/249330 Trybot-Ready: Don Garrett Tested-by: Don Garrett Reviewed-by: Chris Sosa Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index d80c403e3..52a38d257 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -422,7 +422,7 @@ class LKGMSyncStage(SyncStage): manifest_path = os.path.join(self._build_root, mv_dir) manifest_repo = self._GetManifestVersionsRepoUrl(read_only=True) manifest_version.RefreshManifestCheckout(manifest_path, manifest_repo) - return os.path.join(manifest_path, lkgm_manager.LKGMManager.LKGM_PATH) + return os.path.join(manifest_path, self._run.config.manifest) class ManifestVersionedSyncStage(SyncStage): -- cgit v1.2.3 From 18545ca66a6bdae023f32cb7438e8c73fa0be5a5 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Fri, 13 Feb 2015 18:09:57 +0000 Subject: Revert "LKGMSyncStage: LKGM now based on config.manifest." This reverts commit 6b59c67f8da8bb9912383d886177bf21e6b159f3. BUG=chromium:458526 TEST=None Change-Id: I6659ab0f098568d8a8b3e3a21360b56a89b20ac0 Reviewed-on: https://chromium-review.googlesource.com/249751 Reviewed-by: Prathmesh Prabhu Tested-by: Don Garrett --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 52a38d257..d80c403e3 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -422,7 +422,7 @@ class LKGMSyncStage(SyncStage): manifest_path = os.path.join(self._build_root, mv_dir) manifest_repo = self._GetManifestVersionsRepoUrl(read_only=True) manifest_version.RefreshManifestCheckout(manifest_path, manifest_repo) - return os.path.join(manifest_path, self._run.config.manifest) + return os.path.join(manifest_path, lkgm_manager.LKGMManager.LKGM_PATH) class ManifestVersionedSyncStage(SyncStage): -- cgit v1.2.3 From d9cdda76fb05fefae9ee8b79e70295d502874ced Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Fri, 13 Feb 2015 10:18:30 -0800 Subject: LKGMSyncStage: LKGM now based on config.manifest. The LKGMSyncStage was using a hard coded location for the manifest to sync to. Switch it to a config based path instead. This is prep work for the Project SDK builds using it, from a new path. This is an improved version of the reverted CL:249330. BUG=brillo:186 TEST=Unitests. Change-Id: I58d58e719b30d8d79e2f96b58b39b8339d55333b Reviewed-on: https://chromium-review.googlesource.com/249770 Tested-by: Don Garrett Reviewed-by: David James Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index d80c403e3..da8fc1867 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -422,7 +422,7 @@ class LKGMSyncStage(SyncStage): manifest_path = os.path.join(self._build_root, mv_dir) manifest_repo = self._GetManifestVersionsRepoUrl(read_only=True) manifest_version.RefreshManifestCheckout(manifest_path, manifest_repo) - return os.path.join(manifest_path, lkgm_manager.LKGMManager.LKGM_PATH) + return os.path.join(manifest_path, self._run.config.lkgm_manifest) class ManifestVersionedSyncStage(SyncStage): -- cgit v1.2.3 From 46627dde181cc50795ecd4fec764dd577abdc570 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Wed, 11 Feb 2015 19:06:10 -0800 Subject: sync_stages: Remove Chrome release from Project SDK Manifest path. We were publishing Project SDK manifests using the paths of the form: project-sdk/42/6700.0.0.xml Instead use: project-sdk/6700.0.0.xml Also, create symlink to newest manifest for project-sdk builders to use: project-sdk/latest.xml While making this change, did a little cleanup to the publishing code. In particular, adjusted it so that we checkout manifest-versions to a fixed location to reduce the sync cost. BUG=brillo:165 TEST=Unittests + Local run. Change-Id: I0957d89c984c45f0429d4838b3e6f08ac4bfa840 Reviewed-on: https://chromium-review.googlesource.com/248936 Reviewed-by: Don Garrett Tested-by: Don Garrett Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 75 ++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 34 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index da8fc1867..6f5f988f1 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -415,9 +415,9 @@ class LKGMSyncStage(SyncStage): """Override: Gets the LKGM.""" # TODO(sosa): Should really use an initialized manager here. if self.internal: - mv_dir = 'manifest-versions-internal' + mv_dir = constants.INTERNAL_MANIFEST_VERSIONS_PATH else: - mv_dir = 'manifest-versions' + mv_dir = constants.EXTERNAL_MANIFEST_VERSIONS_PATH manifest_path = os.path.join(self._build_root, mv_dir) manifest_repo = self._GetManifestVersionsRepoUrl(read_only=True) @@ -566,8 +566,7 @@ class ManifestVersionedSyncStage(SyncStage): else: yield manifest - @staticmethod - def CommitProjectSDKManifest(manifest, release, current_version, debug): + def CommitProjectSDKManifest(self, manifest, current_version, debug): """Create and submit the Product SDK Manifest. Create the Project SDK manifest, and push it to the external manifest @@ -575,48 +574,57 @@ class ManifestVersionedSyncStage(SyncStage): Args: manifest: Path to new manifest to commit. - release: Current release of the build (ie: 42 if building R42-6513.0.0) - current_version: Version to use when commiting manifest. + current_version: Version to use when commiting manifest (e.g. 6770.0.0). debug: Is this a debug build? """ + # Use a static dir, but don't overlap with other users, we might conflict. + git_repo = os.path.join( + self._build_root, constants.PROJECT_SDK_MANIFEST_VERSIONS_PATH) external_manifest_url = cbuildbot_config.GetManifestVersionsRepoUrl( internal_build=False, read_only=False, test=debug) - # TODO(dgarrett): Find a persistent directory for this. - with osutils.TempDir() as git_repo: - repository.UpdateGitRepo(git_repo, external_manifest_url) + logging.info('Using manifest URL: %s', external_manifest_url) + manifest_version.RefreshManifestCheckout( + git_repo, external_manifest_url) + + sdk_manifest_name = '%s.xml' % current_version + latest_manifest_path = os.path.join( + git_repo, constants.LATEST_PROJECT_SDK_MANIFEST) + sdk_manifest_path = os.path.join( + git_repo, 'project-sdk', sdk_manifest_name) - sdk_manifest_path = os.path.join( - git_repo, 'project-sdk', '%s' % release, '%s.xml' % current_version) + if os.path.exists(sdk_manifest_path): + raise failures_lib.StepFailure( + 'Project SDK Manifest already exists: %s' % sdk_manifest_path) - if os.path.exists(sdk_manifest_path): - raise failures_lib.StepFailure( - 'Project SDK Manifest already exists: %s' % sdk_manifest_path) + # Create branch for pushing new manifest file. + branch = 'temp_project_sdk_creation_branch' + git.CreatePushBranch(branch, git_repo, sync=False) - # Create branch for pushing new manifest file. - branch = 'temp_project_sdk_creation_branch' - git.CreatePushBranch(branch, git_repo, sync=False) + # Create new manifest file. + logging.info('Creating Project SDK Manifest as: %s', sdk_manifest_path) + osutils.SafeMakedirs(os.path.dirname(sdk_manifest_path)) + shutil.copyfile(manifest, sdk_manifest_path) - # Create new manifest file. - logging.info('Creating Project SDK Manifest as: %s', sdk_manifest_path) - osutils.SafeMakedirs(os.path.dirname(sdk_manifest_path)) - shutil.copyfile(manifest, sdk_manifest_path) + # Create 'latest' link to new manifest. + osutils.SafeUnlink(latest_manifest_path) + os.symlink(sdk_manifest_name, latest_manifest_path) - # Commit it locally. - logging.info('Committing Project SDK Manifest.') - git.AddPath(sdk_manifest_path) - git.Commit(git_repo, 'Create project_sdk for %s-%s.' % - (release, current_version)) + # Commit it locally. + logging.info('Committing Project SDK Manifest.') + git.AddPath(sdk_manifest_path) + git.AddPath(latest_manifest_path) + git.Commit(git_repo, 'Create project_sdk for %s.' % current_version) - # Push it. - logging.info('Pushing Project SDK Manifest.') - git.PushWithRetry(branch, git_repo) + # Push it. + logging.info('Pushing Project SDK Manifest.') + git.PushWithRetry(branch, git_repo) - logging.info('Project SDK Manifest \'%s\' published:', - os.path.basename(sdk_manifest_path)) - logging.info('%s', osutils.ReadFile(manifest)) + logging.info('Project SDK Manifest \'%s\' published:', + os.path.basename(sdk_manifest_path)) + logging.info('%s', osutils.ReadFile(manifest)) @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): @@ -649,7 +657,7 @@ class ManifestVersionedSyncStage(SyncStage): # the Project SDK builders. if (cbuildbot_config.IsCanaryType(self._run.config.build_type) and self._run.config.master): - logging.info('Creating of Project SDK Manifest.') + logging.info('Creating Project SDK Manifest.') sdk_manifest = None try: with tempfile.NamedTemporaryFile() as pinned_manifest_file: @@ -660,7 +668,6 @@ class ManifestVersionedSyncStage(SyncStage): self.CommitProjectSDKManifest( sdk_manifest, - self.manifest_manager.GetCurrentVersionInfo().chrome_branch, self.manifest_manager.current_version, self._run.options.debug) finally: -- cgit v1.2.3 From f6b20ba26c87bcbf1fff5872e583470842cd1e7d Mon Sep 17 00:00:00 2001 From: David James Date: Wed, 18 Feb 2015 17:53:26 -0800 Subject: Use CIDB for synchronizing between CQ and slaves. If a master_build_id is provided, grab the platform version from CIDB rather than looking in Google Storage. When the build finally launches, fail if the master_build_id was too old. BUG=chromium:459811 TEST=Launch example runs and verify the following: (1) We use the requested version number. (2) We reject the master_build_id if it is old. TEST=Run all unit tests, including cidb_integration_test Change-Id: I0f80defc10a4027804204f5aead445ebd1644e74 Reviewed-on: https://chromium-review.googlesource.com/251250 Reviewed-by: Aviv Keshet Tested-by: David James Commit-Queue: David James --- cbuildbot/stages/sync_stages.py | 175 ++++++++++++++++++++++++---------------- 1 file changed, 105 insertions(+), 70 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 6f5f988f1..aaf748a01 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -626,13 +626,42 @@ class ManifestVersionedSyncStage(SyncStage): os.path.basename(sdk_manifest_path)) logging.info('%s', osutils.ReadFile(manifest)) + def _GetMasterVersion(self): + """Get the platform version associated with the master_build_id.""" + master_id = self._run.options.master_build_id + _, db = self._run.GetCIDBHandle() + if db and master_id: + master_build_status = db.GetBuildStatus(master_id) + return master_build_status['platform_version'] + + def _VerifyMasterId(self): + """Verify that our master id is current and valid.""" + master_id = self._run.options.master_build_id + _, db = self._run.GetCIDBHandle() + if db and master_id: + assert not self._run.options.force_version + master_build_status = db.GetBuildStatus(master_id) + latest = db.GetBuildHistory(master_build_status['build_config'], 1) + if latest and latest[0]['id'] != master_id: + raise failures_lib.MasterSlaveVersionMismatchFailure( + 'This slave\'s master (id=%s) has been supplanted by a newer ' + 'master (id=%s). Aborting.' % (master_id, latest[0]['id'])) + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): self.Initialize() - if self._run.options.force_version: - next_manifest = self.ForceVersion(self._run.options.force_version) + + self._VerifyMasterId() + version = self._run.options.force_version or self._GetMasterVersion() + next_manifest = None + if version: + next_manifest = self.ForceVersion(version) else: - next_manifest = self.GetNextManifest() + self.skip_sync = True + try: + next_manifest = self.GetNextManifest() + except validation_pool.TreeIsClosedException as e: + cros_build_lib.Warning(str(e)) if not next_manifest: cros_build_lib.Info('Found no work to do.') @@ -689,9 +718,6 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): candidates and their states. """ - # Timeout for waiting on the latest candidate manifest. - LATEST_CANDIDATE_TIMEOUT_SECONDS = 20 * 60 - # TODO(mtennant): Turn this into self._run.attrs.sub_manager or similar. # An instance of lkgm_manager.LKGMManager for slave builds. sub_manager = None @@ -700,7 +726,6 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): super(MasterSlaveLKGMSyncStage, self).__init__(builder_run, **kwargs) # lkgm_manager deals with making sure we're synced to whatever manifest # we get back in GetNextManifest so syncing again is redundant. - self.skip_sync = True self._chrome_version = None def _GetInitializedManager(self, internal): @@ -741,24 +766,31 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): return manifest + def _VerifyMasterId(self): + """Verify that our master id is current and valid.""" + super(MasterSlaveLKGMSyncStage, self)._VerifyMasterId() + master_id = self._run.options.master_build_id + if not self._run.config.master and not master_id: + raise failures_lib.StepFailure( + 'Cannot start build without a master_build_id. Did you hit force ' + 'build on a slave? Please hit force build on the master instead.') + def GetNextManifest(self): """Gets the next manifest using LKGM logic.""" assert self.manifest_manager, \ 'Must run Initialize before we can get a manifest.' assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \ 'Manifest manager instantiated with wrong class.' + assert self._run.config.master - if self._run.config.master: - build_id = self._run.attrs.metadata.GetDict().get('build_id') - manifest = self.manifest_manager.CreateNewCandidate( - chrome_version=self._chrome_version, - build_id=build_id) - if MasterSlaveLKGMSyncStage.sub_manager: - MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(manifest) - return manifest - else: - return self.manifest_manager.GetLatestCandidate( - timeout=self.LATEST_CANDIDATE_TIMEOUT_SECONDS) + build_id = self._run.attrs.metadata.GetDict().get('build_id') + manifest = self.manifest_manager.CreateNewCandidate( + chrome_version=self._chrome_version, + build_id=build_id) + if MasterSlaveLKGMSyncStage.sub_manager: + MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(manifest) + + return manifest def GetLatestChromeVersion(self): """Returns the version of Chrome to uprev.""" @@ -843,9 +875,8 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): def _SetPoolFromManifest(self, manifest): """Sets validation pool based on manifest path passed in.""" - # Note that GetNextManifest() calls GetLatestCandidate() in this case, - # so the repo will already be sync'd appropriately. This means that - # AcquirePoolFromManifest does not need to sync. + # Note that this function is only called after the repo is already + # sync'd, so AcquirePoolFromManifest does not need to sync. self.pool = validation_pool.ValidationPool.AcquirePoolFromManifest( manifest, self._run.config.overlays, self.repo, self._run.buildnumber, self._run.GetBuilderName(), @@ -867,69 +898,73 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): 'Must run Initialize before we can get a manifest.' assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \ 'Manifest manager instantiated with wrong class.' + assert self._run.config.master build_id = self._run.attrs.metadata.GetDict().get('build_id') - if self._run.config.master: - try: - # In order to acquire a pool, we need an initialized buildroot. - if not git.FindRepoDir(self.repo.directory): - self.repo.Initialize() - - query = constants.CQ_READY_QUERY - if self._run.options.cq_gerrit_override: - query = (self._run.options.cq_gerrit_override, None) - - self.pool = pool = validation_pool.ValidationPool.AcquirePool( - self._run.config.overlays, self.repo, - self._run.buildnumber, self._run.GetBuilderName(), - query, - dryrun=self._run.options.debug, - check_tree_open=(not self._run.options.debug or - self._run.options.mock_tree_status), - change_filter=self._ChangeFilter, builder_run=self._run) - except validation_pool.TreeIsClosedException as e: - cros_build_lib.Warning(str(e)) - return None + # In order to acquire a pool, we need an initialized buildroot. + if not git.FindRepoDir(self.repo.directory): + self.repo.Initialize() - # We must extend the builder deadline before publishing a new manifest to - # ensure that slaves have enough time to complete the builds about to - # start. - build_id, db = self._run.GetCIDBHandle() - if db: - timeout = constants.MASTER_BUILD_TIMEOUT_SECONDS.get( - self._run.config.build_type, - constants.MASTER_BUILD_TIMEOUT_DEFAULT_SECONDS) - db.ExtendDeadline(build_id, timeout) + query = constants.CQ_READY_QUERY + if self._run.options.cq_gerrit_override: + query = (self._run.options.cq_gerrit_override, None) - manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool, - build_id=build_id) - if MasterSlaveLKGMSyncStage.sub_manager: - MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest( - manifest, build_id=build_id) + self.pool = pool = validation_pool.ValidationPool.AcquirePool( + self._run.config.overlays, self.repo, + self._run.buildnumber, self._run.GetBuilderName(), + query, + dryrun=self._run.options.debug, + check_tree_open=(not self._run.options.debug or + self._run.options.mock_tree_status), + change_filter=self._ChangeFilter, builder_run=self._run) - else: - manifest = self.manifest_manager.GetLatestCandidate() - if manifest: - if self._run.config.build_before_patching: - pre_build_passed = self.RunPrePatchBuild() - cros_build_lib.PrintBuildbotStepName( - 'CommitQueueSync : Apply Patches') - if not pre_build_passed: - cros_build_lib.PrintBuildbotStepText('Pre-patch build failed.') - - self._SetPoolFromManifest(manifest) - self.pool.ApplyPoolIntoRepo() + # We must extend the builder deadline before publishing a new manifest to + # ensure that slaves have enough time to complete the builds about to + # start. + build_id, db = self._run.GetCIDBHandle() + if db: + timeout = constants.MASTER_BUILD_TIMEOUT_SECONDS.get( + self._run.config.build_type, + constants.MASTER_BUILD_TIMEOUT_DEFAULT_SECONDS) + db.ExtendDeadline(build_id, timeout) + + manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool, + build_id=build_id) + if MasterSlaveLKGMSyncStage.sub_manager: + MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest( + manifest, build_id=build_id) + + return manifest + + def ManifestCheckout(self, next_manifest): + """Checks out the repository to the given manifest.""" + if self._run.config.build_before_patching: + assert not self._run.config.master + pre_build_passed = self.RunPrePatchBuild() + cros_build_lib.PrintBuildbotStepName( + 'CommitQueueSync : Apply Patches') + if not pre_build_passed: + cros_build_lib.PrintBuildbotStepText('Pre-patch build failed.') # Make sure the chroot version is valid. - lkgm_version = self._GetLGKMVersionFromManifest(manifest) + lkgm_version = self._GetLGKMVersionFromManifest(next_manifest) chroot_manager = chroot_lib.ChrootManager(self._build_root) chroot_manager.EnsureChrootAtVersion(lkgm_version) # Clear the chroot version as we are in the middle of building it. chroot_manager.ClearChrootVersion() - return manifest + # Sync to the provided manifest on slaves. On the master, we're + # already synced to this manifest, so self.skip_sync is set and + # this is a no-op. + super(CommitQueueSyncStage, self).ManifestCheckout(next_manifest) + + # On slaves, initialize our pool and apply patches. On the master, + # we've already done that in GetNextManifest, so this is a no-op. + if not self._run.config.master: + self._SetPoolFromManifest(next_manifest) + self.pool.ApplyPoolIntoRepo() @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): -- cgit v1.2.3 From d4a67c2b8cef1da0882daa750407f66c1bd1124e Mon Sep 17 00:00:00 2001 From: Josafat Garcia Date: Thu, 26 Feb 2015 05:13:04 +0000 Subject: Revert "Use CIDB for synchronizing between CQ and slaves." This reverts commit f6b20ba26c87bcbf1fff5872e583470842cd1e7d. Change-Id: I4945781a40d940ed18d90d821dd34c2f14b35b09 Reviewed-on: https://chromium-review.googlesource.com/253931 Reviewed-by: David James Reviewed-by: Josafat Garcia Commit-Queue: Josafat Garcia Tested-by: Josafat Garcia --- cbuildbot/stages/sync_stages.py | 175 ++++++++++++++++------------------------ 1 file changed, 70 insertions(+), 105 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index aaf748a01..6f5f988f1 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -626,42 +626,13 @@ class ManifestVersionedSyncStage(SyncStage): os.path.basename(sdk_manifest_path)) logging.info('%s', osutils.ReadFile(manifest)) - def _GetMasterVersion(self): - """Get the platform version associated with the master_build_id.""" - master_id = self._run.options.master_build_id - _, db = self._run.GetCIDBHandle() - if db and master_id: - master_build_status = db.GetBuildStatus(master_id) - return master_build_status['platform_version'] - - def _VerifyMasterId(self): - """Verify that our master id is current and valid.""" - master_id = self._run.options.master_build_id - _, db = self._run.GetCIDBHandle() - if db and master_id: - assert not self._run.options.force_version - master_build_status = db.GetBuildStatus(master_id) - latest = db.GetBuildHistory(master_build_status['build_config'], 1) - if latest and latest[0]['id'] != master_id: - raise failures_lib.MasterSlaveVersionMismatchFailure( - 'This slave\'s master (id=%s) has been supplanted by a newer ' - 'master (id=%s). Aborting.' % (master_id, latest[0]['id'])) - @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): self.Initialize() - - self._VerifyMasterId() - version = self._run.options.force_version or self._GetMasterVersion() - next_manifest = None - if version: - next_manifest = self.ForceVersion(version) + if self._run.options.force_version: + next_manifest = self.ForceVersion(self._run.options.force_version) else: - self.skip_sync = True - try: - next_manifest = self.GetNextManifest() - except validation_pool.TreeIsClosedException as e: - cros_build_lib.Warning(str(e)) + next_manifest = self.GetNextManifest() if not next_manifest: cros_build_lib.Info('Found no work to do.') @@ -718,6 +689,9 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): candidates and their states. """ + # Timeout for waiting on the latest candidate manifest. + LATEST_CANDIDATE_TIMEOUT_SECONDS = 20 * 60 + # TODO(mtennant): Turn this into self._run.attrs.sub_manager or similar. # An instance of lkgm_manager.LKGMManager for slave builds. sub_manager = None @@ -726,6 +700,7 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): super(MasterSlaveLKGMSyncStage, self).__init__(builder_run, **kwargs) # lkgm_manager deals with making sure we're synced to whatever manifest # we get back in GetNextManifest so syncing again is redundant. + self.skip_sync = True self._chrome_version = None def _GetInitializedManager(self, internal): @@ -766,31 +741,24 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): return manifest - def _VerifyMasterId(self): - """Verify that our master id is current and valid.""" - super(MasterSlaveLKGMSyncStage, self)._VerifyMasterId() - master_id = self._run.options.master_build_id - if not self._run.config.master and not master_id: - raise failures_lib.StepFailure( - 'Cannot start build without a master_build_id. Did you hit force ' - 'build on a slave? Please hit force build on the master instead.') - def GetNextManifest(self): """Gets the next manifest using LKGM logic.""" assert self.manifest_manager, \ 'Must run Initialize before we can get a manifest.' assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \ 'Manifest manager instantiated with wrong class.' - assert self._run.config.master - - build_id = self._run.attrs.metadata.GetDict().get('build_id') - manifest = self.manifest_manager.CreateNewCandidate( - chrome_version=self._chrome_version, - build_id=build_id) - if MasterSlaveLKGMSyncStage.sub_manager: - MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(manifest) - return manifest + if self._run.config.master: + build_id = self._run.attrs.metadata.GetDict().get('build_id') + manifest = self.manifest_manager.CreateNewCandidate( + chrome_version=self._chrome_version, + build_id=build_id) + if MasterSlaveLKGMSyncStage.sub_manager: + MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(manifest) + return manifest + else: + return self.manifest_manager.GetLatestCandidate( + timeout=self.LATEST_CANDIDATE_TIMEOUT_SECONDS) def GetLatestChromeVersion(self): """Returns the version of Chrome to uprev.""" @@ -875,8 +843,9 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): def _SetPoolFromManifest(self, manifest): """Sets validation pool based on manifest path passed in.""" - # Note that this function is only called after the repo is already - # sync'd, so AcquirePoolFromManifest does not need to sync. + # Note that GetNextManifest() calls GetLatestCandidate() in this case, + # so the repo will already be sync'd appropriately. This means that + # AcquirePoolFromManifest does not need to sync. self.pool = validation_pool.ValidationPool.AcquirePoolFromManifest( manifest, self._run.config.overlays, self.repo, self._run.buildnumber, self._run.GetBuilderName(), @@ -898,73 +867,69 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): 'Must run Initialize before we can get a manifest.' assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \ 'Manifest manager instantiated with wrong class.' - assert self._run.config.master build_id = self._run.attrs.metadata.GetDict().get('build_id') - # In order to acquire a pool, we need an initialized buildroot. - if not git.FindRepoDir(self.repo.directory): - self.repo.Initialize() - - query = constants.CQ_READY_QUERY - if self._run.options.cq_gerrit_override: - query = (self._run.options.cq_gerrit_override, None) - - self.pool = pool = validation_pool.ValidationPool.AcquirePool( - self._run.config.overlays, self.repo, - self._run.buildnumber, self._run.GetBuilderName(), - query, - dryrun=self._run.options.debug, - check_tree_open=(not self._run.options.debug or - self._run.options.mock_tree_status), - change_filter=self._ChangeFilter, builder_run=self._run) + if self._run.config.master: + try: + # In order to acquire a pool, we need an initialized buildroot. + if not git.FindRepoDir(self.repo.directory): + self.repo.Initialize() + + query = constants.CQ_READY_QUERY + if self._run.options.cq_gerrit_override: + query = (self._run.options.cq_gerrit_override, None) + + self.pool = pool = validation_pool.ValidationPool.AcquirePool( + self._run.config.overlays, self.repo, + self._run.buildnumber, self._run.GetBuilderName(), + query, + dryrun=self._run.options.debug, + check_tree_open=(not self._run.options.debug or + self._run.options.mock_tree_status), + change_filter=self._ChangeFilter, builder_run=self._run) + except validation_pool.TreeIsClosedException as e: + cros_build_lib.Warning(str(e)) + return None - # We must extend the builder deadline before publishing a new manifest to - # ensure that slaves have enough time to complete the builds about to - # start. - build_id, db = self._run.GetCIDBHandle() - if db: - timeout = constants.MASTER_BUILD_TIMEOUT_SECONDS.get( - self._run.config.build_type, - constants.MASTER_BUILD_TIMEOUT_DEFAULT_SECONDS) - db.ExtendDeadline(build_id, timeout) - - manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool, - build_id=build_id) - if MasterSlaveLKGMSyncStage.sub_manager: - MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest( - manifest, build_id=build_id) + # We must extend the builder deadline before publishing a new manifest to + # ensure that slaves have enough time to complete the builds about to + # start. + build_id, db = self._run.GetCIDBHandle() + if db: + timeout = constants.MASTER_BUILD_TIMEOUT_SECONDS.get( + self._run.config.build_type, + constants.MASTER_BUILD_TIMEOUT_DEFAULT_SECONDS) + db.ExtendDeadline(build_id, timeout) - return manifest + manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool, + build_id=build_id) + if MasterSlaveLKGMSyncStage.sub_manager: + MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest( + manifest, build_id=build_id) - def ManifestCheckout(self, next_manifest): - """Checks out the repository to the given manifest.""" - if self._run.config.build_before_patching: - assert not self._run.config.master - pre_build_passed = self.RunPrePatchBuild() - cros_build_lib.PrintBuildbotStepName( - 'CommitQueueSync : Apply Patches') - if not pre_build_passed: - cros_build_lib.PrintBuildbotStepText('Pre-patch build failed.') + else: + manifest = self.manifest_manager.GetLatestCandidate() + if manifest: + if self._run.config.build_before_patching: + pre_build_passed = self.RunPrePatchBuild() + cros_build_lib.PrintBuildbotStepName( + 'CommitQueueSync : Apply Patches') + if not pre_build_passed: + cros_build_lib.PrintBuildbotStepText('Pre-patch build failed.') + + self._SetPoolFromManifest(manifest) + self.pool.ApplyPoolIntoRepo() # Make sure the chroot version is valid. - lkgm_version = self._GetLGKMVersionFromManifest(next_manifest) + lkgm_version = self._GetLGKMVersionFromManifest(manifest) chroot_manager = chroot_lib.ChrootManager(self._build_root) chroot_manager.EnsureChrootAtVersion(lkgm_version) # Clear the chroot version as we are in the middle of building it. chroot_manager.ClearChrootVersion() - # Sync to the provided manifest on slaves. On the master, we're - # already synced to this manifest, so self.skip_sync is set and - # this is a no-op. - super(CommitQueueSyncStage, self).ManifestCheckout(next_manifest) - - # On slaves, initialize our pool and apply patches. On the master, - # we've already done that in GetNextManifest, so this is a no-op. - if not self._run.config.master: - self._SetPoolFromManifest(next_manifest) - self.pool.ApplyPoolIntoRepo() + return manifest @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): -- cgit v1.2.3 From 87455812a8c1c2916064e794561c91ff0e03725b Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 26 Feb 2015 14:32:07 -0800 Subject: precq: do not launch tryjobs when tree is closed BUG=chromium:460130 TEST=new unit test Change-Id: Ieb6ae96d3fd205f834e5bd66f83c54565852b414 Reviewed-on: https://chromium-review.googlesource.com/254350 Trybot-Ready: Aviv Keshet Tested-by: Aviv Keshet Reviewed-by: David James Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 6f5f988f1..f66f0937a 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1386,9 +1386,14 @@ class PreCQLauncherStage(SyncStage): k: v for k, v in progress_map.iteritems() if k.HasReadyFlag() or status_map[k] != constants.CL_STATUS_FAILED} + is_tree_open = tree_status.IsTreeOpen(throttled_ok=True) for plan, config in self.GetDisjointTransactionsToTest( pool, launchable_progress_map): - self.LaunchTrybot(plan, config) + if is_tree_open: + self.LaunchTrybot(plan, config) + else: + logging.info('Tree is closed, not launching config %s for plan %s.', + config, cros_patch.GetChangesAsString(plan)) # Mark passed changes as passed self.UpdateChangeStatuses(will_pass, constants.CL_STATUS_PASSED) -- cgit v1.2.3 From 9d197e2d5dfda7a0df460afe2c908a49173176f5 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 5 Mar 2015 14:14:13 -0800 Subject: triage_lib, sync_stages: allow commit message to specify pre-cq-configs BUG=chromium:464503 TEST=unit tests Change-Id: I8d431aa147e2ff3800510da7b0123588d978e3d7 Reviewed-on: https://chromium-review.googlesource.com/256593 Tested-by: Aviv Keshet Reviewed-by: Prathmesh Prabhu Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index f66f0937a..a42b6cc62 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1043,19 +1043,33 @@ class PreCQLauncherStage(SyncStage): Returns: A list of configs. """ - configs_to_test = constants.PRE_CQ_DEFAULT_CONFIGS + configs_to_test = None try: - result = triage_lib.GetOptionForChange( - self._build_root, change, 'GENERAL', 'pre-cq-configs') - if (result and result.split() and - all(c in cbuildbot_config.config for c in result.split())): - configs_to_test = result.split() + # If a pre-cq config is specified in the commit message, use that. + # Otherwise, look in appropriate COMMIT-QUEUE.ini. Otherwise, default to + # constants.PRE_CQ_DEFAULT_CONFIGS + lines = cros_patch.GetOptionLinesFromCommitMessage( + change.commit_message, constants.PRE_CQ_CONFIGS_OPTION_REGEX) + if lines is not None: + configs_to_test = self._ParsePreCQOption(' '.join(lines)) + configs_to_test = configs_to_test or self._ParsePreCQOption( + triage_lib.GetOptionForChange( + self._build_root, change, 'GENERAL', + constants.PRE_CQ_CONFIGS_OPTION)) except ConfigParser.Error: cros_build_lib.Error('%s has malformed config file', change, exc_info=True) - return configs_to_test + return configs_to_test or constants.PRE_CQ_DEFAULT_CONFIGS + def _ParsePreCQOption(self, pre_cq_option): + """Gets a valid config list, or None, from |pre_cq_option|.""" + if (pre_cq_option and + pre_cq_option.split() and + all(c in cbuildbot_config.config for c in pre_cq_option.split())): + return pre_cq_option.split() + else: + return None def ScreenChangeForPreCQ(self, change): """Record which pre-cq tryjobs to test |change| with. -- cgit v1.2.3 From 9eaecc71a3768a182fe7e7e92c422283328c43dd Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Mon, 9 Mar 2015 23:33:38 -0700 Subject: completion_stages, sync_stages: record submission strategy to cidb BUG=chromium:458608 TEST=Unit tests. Change-Id: Ia90efc79aeedd6d5aa9b5c003deb52861f48e44a Reviewed-on: https://chromium-review.googlesource.com/258011 Trybot-Ready: Aviv Keshet Tested-by: Aviv Keshet Reviewed-by: David James Reviewed-by: Aviv Keshet Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index a42b6cc62..ef0df66ba 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1418,8 +1418,10 @@ class PreCQLauncherStage(SyncStage): # Submit changes that are ready to submit, if we can. if tree_status.IsTreeOpen(): - pool.SubmitNonManifestChanges(check_tree_open=False) - pool.SubmitChanges(will_submit, check_tree_open=False) + pool.SubmitNonManifestChanges(check_tree_open=False, + reason=constants.STRATEGY_NONMANIFEST) + pool.SubmitChanges(will_submit, check_tree_open=False, + reason=constants.STRATEGY_PRECQ_SUBMIT) # Tell ValidationPool to keep waiting for more changes until we hit # its internal timeout. -- cgit v1.2.3 From 91874ca836371a14fa7c19773aeae34f23580413 Mon Sep 17 00:00:00 2001 From: Ralph Nathan Date: Thu, 19 Mar 2015 13:29:41 -0700 Subject: logging: use cros_logging instead of logging Replace all instances of import logging in chromite with cros_logging. BUG=brillo:615 TEST=unittests + trybot CQ-DEPEND=CL:260455 Change-Id: I3cf7ad9301290e736de2eebea00e9085656a2d5b Reviewed-on: https://chromium-review.googlesource.com/261311 Reviewed-by: Don Garrett Commit-Queue: Ralph Nathan Tested-by: Ralph Nathan --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index ef0df66ba..4596638db 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -9,7 +9,6 @@ from __future__ import print_function import ConfigParser import contextlib import datetime -import logging import os import shutil import sys @@ -34,6 +33,7 @@ from chromite.cbuildbot.stages import build_stages from chromite.lib import clactions from chromite.lib import commandline from chromite.lib import cros_build_lib +from chromite.lib import cros_logging as logging from chromite.lib import git from chromite.lib import osutils from chromite.lib import patch as cros_patch -- cgit v1.2.3 From 934e6d86a1a1d6d5fe29e04f03d7dda0fec117ef Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 19 Mar 2015 17:39:36 -0700 Subject: Add a binhost-pre-cq and run it on overlay / chromite changes. BUG=chromium:443254 TEST=new unit tests, trybot job of the new builder Change-Id: Ia34740c033bbc04c55f28f32a778da065a6fc184 Reviewed-on: https://chromium-review.googlesource.com/261415 Reviewed-by: David James Commit-Queue: David James Trybot-Ready: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 47 ++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 10 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 4596638db..5f0dff6c0 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1033,15 +1033,18 @@ class PreCQLauncherStage(SyncStage): ) cros_build_lib.PrintBuildbotLink(' | '.join(items), patch.url) - - def VerificationsForChange(self, change): + def _ConfiguredVerificationsForChange(self, change): """Determine which configs to test |change| with. + This method returns only the configs that are asked for by the config + file. It does not include special-case logic for adding additional bots + based on the type of the repository (see VerificationsForChange for that). + Args: change: GerritPatch instance to get configs-to-test for. Returns: - A list of configs. + A set of configs to test. """ configs_to_test = None try: @@ -1060,16 +1063,40 @@ class PreCQLauncherStage(SyncStage): cros_build_lib.Error('%s has malformed config file', change, exc_info=True) - return configs_to_test or constants.PRE_CQ_DEFAULT_CONFIGS + return set(configs_to_test or constants.PRE_CQ_DEFAULT_CONFIGS) + + def VerificationsForChange(self, change): + """Determine which configs to test |change| with. + + Args: + change: GerritPatch instance to get configs-to-test for. + + Returns: + A set of configs to test. + """ + configs_to_test = set(self._ConfiguredVerificationsForChange(change)) + + # Add the BINHOST_PRE_CQ to any changes that affect an overlay. + if '/overlays/' in change.project: + configs_to_test.add(constants.BINHOST_PRE_CQ) + + return configs_to_test def _ParsePreCQOption(self, pre_cq_option): """Gets a valid config list, or None, from |pre_cq_option|.""" - if (pre_cq_option and - pre_cq_option.split() and - all(c in cbuildbot_config.config for c in pre_cq_option.split())): - return pre_cq_option.split() - else: - return None + if pre_cq_option and pre_cq_option.split(): + configs_to_test = set(pre_cq_option.split()) + + # Replace 'default' with the default configs. + if 'default' in configs_to_test: + configs_to_test.discard('default') + configs_to_test.update(constants.PRE_CQ_DEFAULT_CONFIGS) + + # Verify that all of the configs are valid. + if all(c in cbuildbot_config.config for c in configs_to_test): + return configs_to_test + + return None def ScreenChangeForPreCQ(self, change): """Record which pre-cq tryjobs to test |change| with. -- cgit v1.2.3 From 0304728da86806661096b2607c06984ab95fd5da Mon Sep 17 00:00:00 2001 From: Ralph Nathan Date: Mon, 23 Mar 2015 11:09:32 -0700 Subject: Use cros_logging instead of cros_build_lib.Info In chromite, replace all calls to cros_build_lib.info with cros_logging.info BUG=brillo:600 TEST=unittests Change-Id: Ic91490e8799741c01f17813a10615e838a730719 Reviewed-on: https://chromium-review.googlesource.com/261849 Reviewed-by: Ralph Nathan Commit-Queue: Ralph Nathan Tested-by: Ralph Nathan Trybot-Ready: Ralph Nathan --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 5f0dff6c0..0e8163e5f 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -635,7 +635,7 @@ class ManifestVersionedSyncStage(SyncStage): next_manifest = self.GetNextManifest() if not next_manifest: - cros_build_lib.Info('Found no work to do.') + logging.info('Found no work to do.') if self._run.attrs.manifest_manager.DidLastBuildFail(): raise failures_lib.StepFailure('The previous build failed.') else: -- cgit v1.2.3 From 854af65281bec4533c08fd63edaabdbf4a69a8e4 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Wed, 25 Mar 2015 10:24:56 -0700 Subject: sync_stages: fix a docstring and clarify a return statement BUG=None TEST=None Change-Id: I2ab2060cfc93b1fb63496d11855b9632c742bc5c Reviewed-on: https://chromium-review.googlesource.com/262378 Reviewed-by: Prathmesh Prabhu Commit-Queue: Aviv Keshet Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 0e8163e5f..0ee97cf15 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1133,7 +1133,8 @@ class PreCQLauncherStage(SyncStage): change: Change to examine. Returns: - A list of stages to ignore for the given |change|. + Boolean indicating if this change is configured to be submitted + in the pre-CQ. """ result = None try: @@ -1142,7 +1143,7 @@ class PreCQLauncherStage(SyncStage): except ConfigParser.Error: cros_build_lib.Error('%s has malformed config file', change, exc_info=True) - return result and result.lower() == 'yes' + return bool(result and result.lower() == 'yes') def LaunchTrybot(self, plan, config): -- cgit v1.2.3 From 446aee9554af42bf2ba4cff5750b30963b4c9048 Mon Sep 17 00:00:00 2001 From: Ralph Nathan Date: Mon, 23 Mar 2015 14:44:56 -0700 Subject: logging: Use cros_logging instead of cros_build_lib.Warning In chromite, replace all calls to cros_build_lib.Warning with calls to cros_logging.warning. BUG=brillo:600 TEST=unittests + trybot Change-Id: Ie57935e214449890671fff9ade50e672fc43b2c1 Reviewed-on: https://chromium-review.googlesource.com/262070 Reviewed-by: David Pursell Tested-by: Ralph Nathan Commit-Queue: Ralph Nathan Trybot-Ready: Ralph Nathan --- cbuildbot/stages/sync_stages.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 0ee97cf15..aec872c1d 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -86,7 +86,7 @@ class PatchChangesStage(generic_stages.BuilderStage): duplicates = [] for change in changes: if change.id is None: - cros_build_lib.Warning( + logging.warning( "Change %s lacks a usable ChangeId; duplicate checking cannot " "be done for this change. If cherry-picking fails, this is a " "potential cause.", change) @@ -218,8 +218,8 @@ class BootstrapStage(PatchChangesStage): parsed_args, filter_fn) if removed: - cros_build_lib.Warning('The following arguments were removed due to api: ' - "'%s'" % ' '.join(removed)) + logging.warning("The following arguments were removed due to api: '%s'" + % ' '.join(removed)) return accepted @classmethod @@ -385,7 +385,7 @@ class SyncStage(generic_stages.BuilderStage): try: old_contents = self.repo.ExportManifest() except cros_build_lib.RunCommandError as e: - cros_build_lib.Warning(str(e)) + logging.warning(str(e)) else: osutils.WriteFile(old_filename, old_contents) fresh_sync = False @@ -889,7 +889,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): self._run.options.mock_tree_status), change_filter=self._ChangeFilter, builder_run=self._run) except validation_pool.TreeIsClosedException as e: - cros_build_lib.Warning(str(e)) + logging.warning(str(e)) return None # We must extend the builder deadline before publishing a new manifest to -- cgit v1.2.3 From 599004268612f9d4f3f6eb63378321566e066600 Mon Sep 17 00:00:00 2001 From: Ralph Nathan Date: Tue, 24 Mar 2015 10:41:17 -0700 Subject: logging: Use cros_logging instead of cros_build_lib.Error In chromite, replace all calls to cros_build_lib.Error with calls to cros_logging.error. BUG=brillo:600 TEST=unittests + trybot Change-Id: Ib036983fced3d90036d6dbf4bcebc39e317479c8 Reviewed-on: https://chromium-review.googlesource.com/262240 Reviewed-by: David Pursell Commit-Queue: Ralph Nathan Trybot-Ready: Ralph Nathan Tested-by: Ralph Nathan --- cbuildbot/stages/sync_stages.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index aec872c1d..53a9aedaf 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -98,9 +98,9 @@ class PatchChangesStage(generic_stages.BuilderStage): return changes for conflict in duplicates: - cros_build_lib.Error( - "Changes %s conflict with each other- they have same id %s.", - ', '.join(map(str, conflict)), conflict[0].id) + logging.error( + "Changes %s conflict with each other- they have same id %s., " + .join(map(str, conflict)), conflict[0].id) cros_build_lib.Die("Duplicate patches were encountered: %s", duplicates) @@ -247,8 +247,7 @@ class BootstrapStage(PatchChangesStage): # patches to the internal manifest, and this means we may flag a conflict # here even if the patch applies cleanly. TODO(davidjames): Fix this. cros_build_lib.PrintBuildbotStepWarnings() - cros_build_lib.Error('Failed applying patches: %s', - '\n'.join(map(str, failures))) + logging.error('Failed applying patches: %s\n'.join(map(str, failures))) else: PatchChangesStage.HandleApplyFailures(self, failures) @@ -1060,8 +1059,7 @@ class PreCQLauncherStage(SyncStage): self._build_root, change, 'GENERAL', constants.PRE_CQ_CONFIGS_OPTION)) except ConfigParser.Error: - cros_build_lib.Error('%s has malformed config file', change, - exc_info=True) + logging.error('%s has malformed config file', change, exc_info=True) return set(configs_to_test or constants.PRE_CQ_DEFAULT_CONFIGS) @@ -1141,8 +1139,7 @@ class PreCQLauncherStage(SyncStage): result = triage_lib.GetOptionForChange( self._build_root, change, 'GENERAL', 'submit-in-pre-cq') except ConfigParser.Error: - cros_build_lib.Error('%s has malformed config file', change, - exc_info=True) + logging.error('%s has malformed config file', change, exc_info=True) return bool(result and result.lower() == 'yes') -- cgit v1.2.3 From f58da0733420add4dabe8326b8919819947874e2 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Mon, 30 Mar 2015 13:26:22 -0700 Subject: pre-cq: Allow pre-cq-submittable changes when throttled There is no need to hold non-manifest and pre-cq-submittable changes to a higher standard than CQ changes. Prior to this, they were only being committed if the tree was open. BUG=None TEST=Unit test Change-Id: I67aeb7d6bbb102f124ec31b01b5dfa9b1dfad07d Reviewed-on: https://chromium-review.googlesource.com/263110 Trybot-Ready: Aviv Keshet Tested-by: Aviv Keshet Reviewed-by: David James --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 53a9aedaf..e9fffa0ad 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1442,7 +1442,7 @@ class PreCQLauncherStage(SyncStage): self._ProcessExpiry(c, v[0], v[1], pool, current_db_time) # Submit changes that are ready to submit, if we can. - if tree_status.IsTreeOpen(): + if tree_status.IsTreeOpen(throttled_ok=True): pool.SubmitNonManifestChanges(check_tree_open=False, reason=constants.STRATEGY_NONMANIFEST) pool.SubmitChanges(will_submit, check_tree_open=False, -- cgit v1.2.3 From 999442f50f2b9803f1dcbd37fc0a28608afc405d Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 26 Mar 2015 18:16:33 -0700 Subject: sync_stage: Start publishing Project SDK manifests to GS. Publish each new Project SDK manifest to gs://brillo-releases/sdk-releases/ in addition to manifest-versions. We will eventually stop using manifest-versions for fetching SDKs, but will probably keep publishing manifests there, since that is what triggers the Project SDK builders. BUG=brillo:630 TEST=lint + unittests Change-Id: If9d1465c8b27361f062c2216504079192ef012d4 Reviewed-on: https://chromium-review.googlesource.com/262739 Reviewed-by: Don Garrett Tested-by: Don Garrett Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index e9fffa0ad..54564f4ee 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -35,6 +35,7 @@ from chromite.lib import commandline from chromite.lib import cros_build_lib from chromite.lib import cros_logging as logging from chromite.lib import git +from chromite.lib import gs from chromite.lib import osutils from chromite.lib import patch as cros_patch from chromite.scripts import cros_mark_chrome_as_stable @@ -617,10 +618,25 @@ class ManifestVersionedSyncStage(SyncStage): git.AddPath(latest_manifest_path) git.Commit(git_repo, 'Create project_sdk for %s.' % current_version) - # Push it. + # Push it to Gerrit. logging.info('Pushing Project SDK Manifest.') git.PushWithRetry(branch, git_repo) + # Push to GS. + gs_ctx = gs.GSContext(dry_run=self._run.debug) + publish_uri = os.path.join(constants.BRILLO_RELEASE_MANIFESTS_URL, + sdk_manifest_name) + + # We use the default ACL (public readable) for this file. + logging.info('Pushing Project SDK Manifest to: %s', publish_uri) + gs_ctx.Copy(manifest, publish_uri) + + # Populate a file on GS with the newest version published. + with tempfile.NamedTemporaryFile() as latest: + osutils.WriteFile(latest.name, current_version) + gs_ctx.Copy(latest.name, constants.BRILLO_LATEST_RELEASE_URL) + + # Log what we published. logging.info('Project SDK Manifest \'%s\' published:', os.path.basename(sdk_manifest_path)) logging.info('%s', osutils.ReadFile(manifest)) @@ -652,6 +668,7 @@ class ManifestVersionedSyncStage(SyncStage): next_manifest, filter_cros=self._run.options.local) as new_manifest: self.ManifestCheckout(new_manifest) + # TODO(dgarrett): Push this logic into it's own stage. # If we are a Canary Master, create an additional derivative Manifest for # the Project SDK builders. if (cbuildbot_config.IsCanaryType(self._run.config.build_type) and -- cgit v1.2.3 From 16f6005a66686458f38edca83f0e9400742db615 Mon Sep 17 00:00:00 2001 From: David James Date: Wed, 18 Feb 2015 17:53:26 -0800 Subject: Re-land "Use CIDB for synchronizing between CQ and slaves." If a master_build_id is provided, grab the platform version from CIDB rather than looking in Google Storage. When the build finally launches, fail if the master_build_id was too old. This version of the patch adds a timeout, waiting for the CQ master to set the platform_version. BUG=chromium:459811, chromium:461912 TEST=Launch example runs and verify the following: (1) We use the requested version number. (2) We reject the master_build_id if it is old. TEST=Run all unit tests, including cidb_integration_test Original-Change-Id: I0f80defc10a4027804204f5aead445ebd1644e74 Originally-Reviewed-on: https://chromium-review.googlesource.com/251250 Change-Id: Ide2b9e7fa3af792a2348583f3417d563d4945978 Reviewed-on: https://chromium-review.googlesource.com/264440 Reviewed-by: David James Tested-by: David James Trybot-Ready: David James Commit-Queue: David James --- cbuildbot/stages/sync_stages.py | 208 ++++++++++++++++++++++++++-------------- 1 file changed, 137 insertions(+), 71 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 54564f4ee..b8b2d69ee 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -38,6 +38,7 @@ from chromite.lib import git from chromite.lib import gs from chromite.lib import osutils from chromite.lib import patch as cros_patch +from chromite.lib import timeout_util from chromite.scripts import cros_mark_chrome_as_stable @@ -641,13 +642,69 @@ class ManifestVersionedSyncStage(SyncStage): os.path.basename(sdk_manifest_path)) logging.info('%s', osutils.ReadFile(manifest)) + def _GetMasterVersion(self, master_id, timeout=5 * 60): + """Get the platform version associated with the master_build_id. + + Args: + master_id: Our master build id. + timeout: How long to wait for the platform version to show up + in the database. This is needed because the slave builders are + triggered slightly before the platform version is written. Default + is 5 minutes. + """ + # TODO(davidjames): Remove the wait loop here once we've updated slave + # builders to only get triggered after the platform version is written. + def _PrintRemainingTime(remaining): + logging.info('%s until timeout...', remaining) + + def _GetPlatformVersion(): + return db.GetBuildStatus(master_id)['platform_version'] + + # Retry until non-None version is returned. + def _ShouldRetry(x): + return not x + + _, db = self._run.GetCIDBHandle() + return timeout_util.WaitForSuccess(_ShouldRetry, + _GetPlatformVersion, + timeout, + period=constants.SLEEP_TIMEOUT, + side_effect_func=_PrintRemainingTime) + + def _VerifyMasterId(self, master_id): + """Verify that our master id is current and valid. + + Args: + master_id: Our master build id. + """ + _, db = self._run.GetCIDBHandle() + if db and master_id: + assert not self._run.options.force_version + master_build_status = db.GetBuildStatus(master_id) + latest = db.GetBuildHistory(master_build_status['build_config'], 1) + if latest and latest[0]['id'] != master_id: + raise failures_lib.MasterSlaveVersionMismatchFailure( + 'This slave\'s master (id=%s) has been supplanted by a newer ' + 'master (id=%s). Aborting.' % (master_id, latest[0]['id'])) + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): self.Initialize() - if self._run.options.force_version: - next_manifest = self.ForceVersion(self._run.options.force_version) + + self._VerifyMasterId(self._run.options.master_build_id) + version = self._run.options.force_version + if self._run.options.master_build_id: + version = self._GetMasterVersion(self._run.options.master_build_id) + + next_manifest = None + if version: + next_manifest = self.ForceVersion(version) else: - next_manifest = self.GetNextManifest() + self.skip_sync = True + try: + next_manifest = self.GetNextManifest() + except validation_pool.TreeIsClosedException as e: + cros_build_lib.Warning(str(e)) if not next_manifest: logging.info('Found no work to do.') @@ -705,9 +762,6 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): candidates and their states. """ - # Timeout for waiting on the latest candidate manifest. - LATEST_CANDIDATE_TIMEOUT_SECONDS = 20 * 60 - # TODO(mtennant): Turn this into self._run.attrs.sub_manager or similar. # An instance of lkgm_manager.LKGMManager for slave builds. sub_manager = None @@ -716,7 +770,6 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): super(MasterSlaveLKGMSyncStage, self).__init__(builder_run, **kwargs) # lkgm_manager deals with making sure we're synced to whatever manifest # we get back in GetNextManifest so syncing again is redundant. - self.skip_sync = True self._chrome_version = None def _GetInitializedManager(self, internal): @@ -757,24 +810,30 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): return manifest + def _VerifyMasterId(self, master_id): + """Verify that our master id is current and valid.""" + super(MasterSlaveLKGMSyncStage, self)._VerifyMasterId(master_id) + if not self._run.config.master and not master_id: + raise failures_lib.StepFailure( + 'Cannot start build without a master_build_id. Did you hit force ' + 'build on a slave? Please hit force build on the master instead.') + def GetNextManifest(self): """Gets the next manifest using LKGM logic.""" assert self.manifest_manager, \ 'Must run Initialize before we can get a manifest.' assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \ 'Manifest manager instantiated with wrong class.' + assert self._run.config.master - if self._run.config.master: - build_id = self._run.attrs.metadata.GetDict().get('build_id') - manifest = self.manifest_manager.CreateNewCandidate( - chrome_version=self._chrome_version, - build_id=build_id) - if MasterSlaveLKGMSyncStage.sub_manager: - MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(manifest) - return manifest - else: - return self.manifest_manager.GetLatestCandidate( - timeout=self.LATEST_CANDIDATE_TIMEOUT_SECONDS) + build_id = self._run.attrs.metadata.GetDict().get('build_id') + manifest = self.manifest_manager.CreateNewCandidate( + chrome_version=self._chrome_version, + build_id=build_id) + if MasterSlaveLKGMSyncStage.sub_manager: + MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(manifest) + + return manifest def GetLatestChromeVersion(self): """Returns the version of Chrome to uprev.""" @@ -859,9 +918,8 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): def _SetPoolFromManifest(self, manifest): """Sets validation pool based on manifest path passed in.""" - # Note that GetNextManifest() calls GetLatestCandidate() in this case, - # so the repo will already be sync'd appropriately. This means that - # AcquirePoolFromManifest does not need to sync. + # Note that this function is only called after the repo is already + # sync'd, so AcquirePoolFromManifest does not need to sync. self.pool = validation_pool.ValidationPool.AcquirePoolFromManifest( manifest, self._run.config.overlays, self.repo, self._run.buildnumber, self._run.GetBuilderName(), @@ -883,69 +941,77 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): 'Must run Initialize before we can get a manifest.' assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \ 'Manifest manager instantiated with wrong class.' + assert self._run.config.master build_id = self._run.attrs.metadata.GetDict().get('build_id') - if self._run.config.master: - try: - # In order to acquire a pool, we need an initialized buildroot. - if not git.FindRepoDir(self.repo.directory): - self.repo.Initialize() - - query = constants.CQ_READY_QUERY - if self._run.options.cq_gerrit_override: - query = (self._run.options.cq_gerrit_override, None) - - self.pool = pool = validation_pool.ValidationPool.AcquirePool( - self._run.config.overlays, self.repo, - self._run.buildnumber, self._run.GetBuilderName(), - query, - dryrun=self._run.options.debug, - check_tree_open=(not self._run.options.debug or - self._run.options.mock_tree_status), - change_filter=self._ChangeFilter, builder_run=self._run) - except validation_pool.TreeIsClosedException as e: - logging.warning(str(e)) - return None - - # We must extend the builder deadline before publishing a new manifest to - # ensure that slaves have enough time to complete the builds about to - # start. - build_id, db = self._run.GetCIDBHandle() - if db: - timeout = constants.MASTER_BUILD_TIMEOUT_SECONDS.get( - self._run.config.build_type, - constants.MASTER_BUILD_TIMEOUT_DEFAULT_SECONDS) - db.ExtendDeadline(build_id, timeout) + try: + # In order to acquire a pool, we need an initialized buildroot. + if not git.FindRepoDir(self.repo.directory): + self.repo.Initialize() + + query = constants.CQ_READY_QUERY + if self._run.options.cq_gerrit_override: + query = (self._run.options.cq_gerrit_override, None) + + self.pool = pool = validation_pool.ValidationPool.AcquirePool( + self._run.config.overlays, self.repo, + self._run.buildnumber, self._run.GetBuilderName(), + query, + dryrun=self._run.options.debug, + check_tree_open=(not self._run.options.debug or + self._run.options.mock_tree_status), + change_filter=self._ChangeFilter, builder_run=self._run) + except validation_pool.TreeIsClosedException as e: + logging.warning(str(e)) + return None + + # We must extend the builder deadline before publishing a new manifest to + # ensure that slaves have enough time to complete the builds about to + # start. + build_id, db = self._run.GetCIDBHandle() + if db: + timeout = constants.MASTER_BUILD_TIMEOUT_SECONDS.get( + self._run.config.build_type, + constants.MASTER_BUILD_TIMEOUT_DEFAULT_SECONDS) + db.ExtendDeadline(build_id, timeout) + + manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool, + build_id=build_id) + if MasterSlaveLKGMSyncStage.sub_manager: + MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest( + manifest, build_id=build_id) - manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool, - build_id=build_id) - if MasterSlaveLKGMSyncStage.sub_manager: - MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest( - manifest, build_id=build_id) + return manifest - else: - manifest = self.manifest_manager.GetLatestCandidate() - if manifest: - if self._run.config.build_before_patching: - pre_build_passed = self.RunPrePatchBuild() - cros_build_lib.PrintBuildbotStepName( - 'CommitQueueSync : Apply Patches') - if not pre_build_passed: - cros_build_lib.PrintBuildbotStepText('Pre-patch build failed.') - - self._SetPoolFromManifest(manifest) - self.pool.ApplyPoolIntoRepo() + def ManifestCheckout(self, next_manifest): + """Checks out the repository to the given manifest.""" + if self._run.config.build_before_patching: + assert not self._run.config.master + pre_build_passed = self.RunPrePatchBuild() + cros_build_lib.PrintBuildbotStepName( + 'CommitQueueSync : Apply Patches') + if not pre_build_passed: + cros_build_lib.PrintBuildbotStepText('Pre-patch build failed.') # Make sure the chroot version is valid. - lkgm_version = self._GetLGKMVersionFromManifest(manifest) + lkgm_version = self._GetLGKMVersionFromManifest(next_manifest) chroot_manager = chroot_lib.ChrootManager(self._build_root) chroot_manager.EnsureChrootAtVersion(lkgm_version) # Clear the chroot version as we are in the middle of building it. chroot_manager.ClearChrootVersion() - return manifest + # Sync to the provided manifest on slaves. On the master, we're + # already synced to this manifest, so self.skip_sync is set and + # this is a no-op. + super(CommitQueueSyncStage, self).ManifestCheckout(next_manifest) + + # On slaves, initialize our pool and apply patches. On the master, + # we've already done that in GetNextManifest, so this is a no-op. + if not self._run.config.master: + self._SetPoolFromManifest(next_manifest) + self.pool.ApplyPoolIntoRepo() @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): -- cgit v1.2.3 From 8806366b8ec8d021bcee181e80251e082ee744ec Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 9 Apr 2015 11:34:36 -0700 Subject: sync_stages: shorten pre-cq launcher LAUNCH_TIMEOUT to 30 min 30 minute launch latency generally indicates that something went wrong. Shortening the timeout means the affected developer is informed sooner. BUG=chromium:475609 TEST=None Change-Id: I906a3677f5e620c9fa9aec608d4be4c8e09e42b4 Reviewed-on: https://chromium-review.googlesource.com/265003 Tested-by: Aviv Keshet Reviewed-by: Don Garrett --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index b8b2d69ee..b7ed04b5e 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1070,7 +1070,7 @@ class PreCQLauncherStage(SyncStage): LAUNCH_DELAY = 2 # The number of minutes we allow before considering a launch attempt failed. - LAUNCH_TIMEOUT = 90 + LAUNCH_TIMEOUT = 30 # The number of minutes we allow before considering an in-flight job failed. INFLIGHT_TIMEOUT = 240 -- cgit v1.2.3 From 0097963b8e20abeaf1b1e942bdfdf45ebdd29d7f Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 9 Apr 2015 13:59:44 -0700 Subject: sync_stages: add a rate-limit to launching of pre-cq tryjobs When the tree is reopened after downtime, there can be a large backlog of pre-cq tryjobs. Sometimes it is desireable to have only a few of these be launched, rather than have all of them be launched by the initial open-tree cycle of the launcher. Add a 20 launches per-cycle-per-cycle rate limit to the pre-cq launcher. This will cause the launch rate to ramp up at a limited pace, while still not imposing any upper limit to steady state maximum throughput. BUG=None TEST=New unit test added. Confirmed that test fails prior to feature. Change-Id: I78a1df66e094928e9cb9769c6c04b25d787164b4 Reviewed-on: https://chromium-review.googlesource.com/265063 Reviewed-by: David James Reviewed-by: Prathmesh Prabhu Commit-Queue: Aviv Keshet Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index b7ed04b5e..a63d605d2 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1085,9 +1085,15 @@ class PreCQLauncherStage(SyncStage): # once. MAX_PATCHES_PER_TRYBOT_RUN = 50 + # The maximum derivative of the number of tryjobs we will launch in a given + # cycle of ProcessChanges. Used to rate-limit the launcher when reopening the + # tree after building up a large backlog. + MAX_LAUNCHES_PER_CYCLE_DERIVATIVE = 20 + def __init__(self, builder_run, **kwargs): super(PreCQLauncherStage, self).__init__(builder_run, **kwargs) self.skip_sync = True + self.last_cycle_launch_count = 0 def _HasTimedOut(self, start, now, timeout_minutes): @@ -1509,14 +1515,26 @@ class PreCQLauncherStage(SyncStage): if k.HasReadyFlag() or status_map[k] != constants.CL_STATUS_FAILED} is_tree_open = tree_status.IsTreeOpen(throttled_ok=True) + launch_count = 0 + launch_count_limit = (self.last_cycle_launch_count + + self.MAX_LAUNCHES_PER_CYCLE_DERIVATIVE) for plan, config in self.GetDisjointTransactionsToTest( pool, launchable_progress_map): if is_tree_open: - self.LaunchTrybot(plan, config) + if launch_count < launch_count_limit: + self.LaunchTrybot(plan, config) + launch_count += 1 + else: + logging.info('Hit maximum launch count of %s this cycle, not ' + 'launching config %s for plan %s.', + launch_count_limit, config, + cros_patch.GetChangesAsString(plan)) else: logging.info('Tree is closed, not launching config %s for plan %s.', config, cros_patch.GetChangesAsString(plan)) + self.last_cycle_launch_count = launch_count + # Mark passed changes as passed self.UpdateChangeStatuses(will_pass, constants.CL_STATUS_PASSED) -- cgit v1.2.3 From 23e8d3f902daba9d651b818cfbadb7ab98eb4687 Mon Sep 17 00:00:00 2001 From: David James Date: Wed, 8 Apr 2015 15:23:27 -0700 Subject: sync_stages: Don't test internal changes on external configs. If you test an internal change on an external configuration, the internal CL may wait forever on a response from the external bot. This results in the change timing out. Fix this. BUG=chromium:475276 TEST=Unit tests. Change-Id: I30831941d7981bde2e934dc7259589d24c2aa970 Reviewed-on: https://chromium-review.googlesource.com/264924 Reviewed-by: Aviv Keshet Commit-Queue: David James Trybot-Ready: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index a63d605d2..cfd178f9c 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1167,6 +1167,12 @@ class PreCQLauncherStage(SyncStage): if '/overlays/' in change.project: configs_to_test.add(constants.BINHOST_PRE_CQ) + # Don't test internal configs on external bots. + if change.internal: + configs_to_test = [ + x for x in configs_to_test if not cbuildbot_config.config[x].internal + ] + return configs_to_test def _ParsePreCQOption(self, pre_cq_option): -- cgit v1.2.3 From 1e585bef006998da935c1d14e0c7e363779160de Mon Sep 17 00:00:00 2001 From: David James Date: Fri, 10 Apr 2015 17:06:58 +0000 Subject: Revert "sync_stages: Don't test internal changes on external configs." This reverts CL:264924. It may have broken the Pre-CQ launcher so we are reverting as a test. BUG=chromium:475982 TEST=None Change-Id: Ia657e11f25cdb88ef361e151975b71ed286001c8 Reviewed-on: https://chromium-review.googlesource.com/265146 Reviewed-by: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index cfd178f9c..a63d605d2 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1167,12 +1167,6 @@ class PreCQLauncherStage(SyncStage): if '/overlays/' in change.project: configs_to_test.add(constants.BINHOST_PRE_CQ) - # Don't test internal configs on external bots. - if change.internal: - configs_to_test = [ - x for x in configs_to_test if not cbuildbot_config.config[x].internal - ] - return configs_to_test def _ParsePreCQOption(self, pre_cq_option): -- cgit v1.2.3 From 65f6a57400182c01c391deeefd7525e4166dac6e Mon Sep 17 00:00:00 2001 From: David James Date: Fri, 10 Apr 2015 10:04:27 -0700 Subject: sync_stages: Tolerate the case where a key is missing. This is needed for debugging errors on the Pre-CQ. BUG=chromium:475982 TEST=None Change-Id: I65f5a3df74cb67195469aff7c8c2169bd01a4e81 Reviewed-on: https://chromium-review.googlesource.com/265188 Reviewed-by: Aviv Keshet Tested-by: David James --- cbuildbot/stages/sync_stages.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index a63d605d2..4cb29acde 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1506,6 +1506,12 @@ class PreCQLauncherStage(SyncStage): self.ScreenChangeForPreCQ(change) continue + # TODO(davidjames): Find out why changes are missing in progress_map. + # See http://crbug.com/475982 + if change not in progress_map: + logging.warn('Change %s is missing in progress_map', change) + continue + self._ProcessTimeouts(change, progress_map, pool, current_db_time) # Filter out changes that have already failed, and aren't marked trybot -- cgit v1.2.3 From 93cb3763793622df9aec062fe066ccf80e896d35 Mon Sep 17 00:00:00 2001 From: David James Date: Fri, 10 Apr 2015 17:22:53 +0000 Subject: Revert "sync_stages: Tolerate the case where a key is missing." This reverts CL:265188. We're going to do surgery on the DB instead. BUG=chromium:475982 TEST=None Change-Id: I293b967b66e393756d42ac0b23e691c85cc98f27 Reviewed-on: https://chromium-review.googlesource.com/265189 Reviewed-by: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 4cb29acde..a63d605d2 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1506,12 +1506,6 @@ class PreCQLauncherStage(SyncStage): self.ScreenChangeForPreCQ(change) continue - # TODO(davidjames): Find out why changes are missing in progress_map. - # See http://crbug.com/475982 - if change not in progress_map: - logging.warn('Change %s is missing in progress_map', change) - continue - self._ProcessTimeouts(change, progress_map, pool, current_db_time) # Filter out changes that have already failed, and aren't marked trybot -- cgit v1.2.3 From 57c1d2ca05fd9b29c029ba4704dbcab4cd6a255f Mon Sep 17 00:00:00 2001 From: David James Date: Wed, 29 Apr 2015 13:15:59 -0700 Subject: Print chumped changes on CQ slaves again. BUG=chromium:478176 TEST=Test that CQ slaves are now printing chumped changes. Change-Id: I9b6bddbeac940d0fe16f1acc985210be5a8637fd Reviewed-on: https://chromium-review.googlesource.com/268354 Reviewed-by: David James Commit-Queue: David James Trybot-Ready: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index a63d605d2..0fdb7e620 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1010,6 +1010,9 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): # On slaves, initialize our pool and apply patches. On the master, # we've already done that in GetNextManifest, so this is a no-op. if not self._run.config.master: + # Print the list of CHUMP changes since the LKGM, then apply changes and + # print the list of applied changes. + self.manifest_manager.GenerateBlameListSinceLKGM() self._SetPoolFromManifest(next_manifest) self.pool.ApplyPoolIntoRepo() -- cgit v1.2.3 From 7d034ce9230c36aaca5129c88eafeac24a291975 Mon Sep 17 00:00:00 2001 From: David James Date: Wed, 29 Apr 2015 11:45:06 -0700 Subject: Sync with unpinned manifest first prior to syncing with pinned manifest. Syncing to a pinned manifest ensures that we have the specified revisions, but, unfortunately, repo won't bother to update branches. Sync with an unpinned manifest first to ensure that branches are updated (e.g. in case somebody adds a new branch to a repo.) BUG=chromium:482077 TEST=Confirm that the new repo sync -n is run in a trybot run, and that all repos are updated with new branches. Confirm that repos are not updated with new branches without the fix. Change-Id: I9ed27975eb58c20015d8f8e5d0a6f43baafdd2da Reviewed-on: https://chromium-review.googlesource.com/268365 Reviewed-by: Gabe Black Commit-Queue: David James Trybot-Ready: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 0fdb7e620..b1fce3732 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1002,6 +1002,13 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): # Clear the chroot version as we are in the middle of building it. chroot_manager.ClearChrootVersion() + # Syncing to a pinned manifest ensures that we have the specified + # revisions, but, unfortunately, repo won't bother to update branches. + # Sync with an unpinned manifest first to ensure that branches are updated + # (e.g. in case somebody adds a new branch to a repo.) See crbug.com/482077 + if not self.skip_sync: + self.repo.Sync(self._run.config.manifest, network_only=True) + # Sync to the provided manifest on slaves. On the master, we're # already synced to this manifest, so self.skip_sync is set and # this is a no-op. -- cgit v1.2.3 From 2403af4ebe5c531ca74ae33960fe16c814a57224 Mon Sep 17 00:00:00 2001 From: Mike Frysinger Date: Thu, 30 Apr 2015 06:25:03 -0400 Subject: fix some old users of cros_build_lib output Looks like the fwgdb script was missed in the big migration, as was the sync_stages module (and there looks like a linting bug there). The others are more insidious -- they weren't logging their output, but instead using the builtin exceptions.Warning to construct an object. That's why the linter didn't complain -- the code was valid. BUG=brillo:600 TEST=`cros lint` is happy TEST=`./cbuildbot/run_tests` passes Change-Id: I45207422fbea059bdee70a2eedd120b5688b6350 Reviewed-on: https://chromium-review.googlesource.com/268383 Trybot-Ready: Mike Frysinger Reviewed-by: Ralph Nathan Commit-Queue: Mike Frysinger Tested-by: Mike Frysinger --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index b1fce3732..ebe9824aa 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -704,7 +704,7 @@ class ManifestVersionedSyncStage(SyncStage): try: next_manifest = self.GetNextManifest() except validation_pool.TreeIsClosedException as e: - cros_build_lib.Warning(str(e)) + logging.warning(str(e)) if not next_manifest: logging.info('Found no work to do.') -- cgit v1.2.3 From 65a07a5e2422c1604a66ed548d364b2c2971c9c5 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Mon, 4 May 2015 15:05:09 -0700 Subject: cbuildbot_config: Get rid of GetManifestVersionsRepoUrl. The method GetManifestVersionsRepoUrl never really made sense inside cbuildbot_config, since it's not really a config value. So... get rid of the method altogether. It was either being used in places where it had a static result (now replaced with constant values), or via a wrapper in sync_stages. Move the real implementation into sync_stages. TEST=cbuildbot/run_tests && cros lint *.py BUG=chromium:483566 Change-Id: Iddfee41340bfdbd63b6376469e3fbbad18361673 Reviewed-on: https://chromium-review.googlesource.com/269165 Reviewed-by: Aviv Keshet Reviewed-by: Don Garrett Commit-Queue: Don Garrett Tested-by: Don Garrett --- cbuildbot/stages/sync_stages.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index ebe9824aa..b3afccc11 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -321,10 +321,20 @@ class SyncStage(generic_stages.BuilderStage): # at self.internal when it can always be retrieved from config? self.internal = self._run.config.internal - def _GetManifestVersionsRepoUrl(self, read_only=False): - return cbuildbot_config.GetManifestVersionsRepoUrl( - self.internal, - read_only=read_only) + def _GetManifestVersionsRepoUrl(self, internal=None, test=False): + if internal is None: + internal = self._run.config.internal + + if internal: + if test: + return constants.MANIFEST_VERSIONS_INT_GOB_URL_TEST + else: + return constants.MANIFEST_VERSIONS_INT_GOB_URL + else: + if test: + return constants.MANIFEST_VERSIONS_GOB_URL_TEST + else: + return constants.MANIFEST_VERSIONS_GOB_URL def Initialize(self): self._InitializeRepo() @@ -421,7 +431,7 @@ class LKGMSyncStage(SyncStage): mv_dir = constants.EXTERNAL_MANIFEST_VERSIONS_PATH manifest_path = os.path.join(self._build_root, mv_dir) - manifest_repo = self._GetManifestVersionsRepoUrl(read_only=True) + manifest_repo = self._GetManifestVersionsRepoUrl() manifest_version.RefreshManifestCheckout(manifest_path, manifest_repo) return os.path.join(manifest_path, self._run.config.lkgm_manifest) @@ -440,7 +450,7 @@ class ManifestVersionedSyncStage(SyncStage): # If a builder pushes changes (even with dryrun mode), we need a writable # repository. Otherwise, the push will be rejected by the server. - self.manifest_repo = self._GetManifestVersionsRepoUrl(read_only=False) + self.manifest_repo = self._GetManifestVersionsRepoUrl() # 1. If we're uprevving Chrome, Chrome might have changed even if the # manifest has not, so we should force a build to double check. This @@ -581,10 +591,8 @@ class ManifestVersionedSyncStage(SyncStage): # Use a static dir, but don't overlap with other users, we might conflict. git_repo = os.path.join( self._build_root, constants.PROJECT_SDK_MANIFEST_VERSIONS_PATH) - external_manifest_url = cbuildbot_config.GetManifestVersionsRepoUrl( - internal_build=False, - read_only=False, - test=debug) + external_manifest_url = self._GetManifestVersionsRepoUrl( + internal=False, test=debug) logging.info('Using manifest URL: %s', external_manifest_url) manifest_version.RefreshManifestCheckout( @@ -784,8 +792,7 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): increment = self.VersionIncrementType() return lkgm_manager.LKGMManager( source_repo=self.repo, - manifest_repo=cbuildbot_config.GetManifestVersionsRepoUrl( - internal, read_only=False), + manifest_repo=self._GetManifestVersionsRepoUrl(internal=internal), manifest=self._run.config.manifest, build_names=self._run.GetBuilderIds(), build_type=self._run.config.build_type, -- cgit v1.2.3 From 541bd6941ffdd2b79943613c5ff33782745aff35 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Tue, 5 May 2015 17:40:57 -0700 Subject: cbuildbot_config: Stop exporting 'config'. Stop exporting the variable 'config' from cbuildbot_config and instead export 'GetConfig()'. Instead of giving every script access to the global variable 'config', export via a function. This is to simplify upcoming changes that may need to load the config dynamically, or handle not having one. TEST=cbuildbot/run_tests && cros lint *.py BUG=chromium:483566 CQ-DEPEND=CL:269775 Change-Id: Ifaa33a597907cfc18a7e5e99c218609e26eed683 Reviewed-on: https://chromium-review.googlesource.com/269770 Reviewed-by: Don Garrett Commit-Queue: Don Garrett Tested-by: Don Garrett --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index b3afccc11..04fd8f220 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1197,7 +1197,7 @@ class PreCQLauncherStage(SyncStage): configs_to_test.update(constants.PRE_CQ_DEFAULT_CONFIGS) # Verify that all of the configs are valid. - if all(c in cbuildbot_config.config for c in configs_to_test): + if all(c in cbuildbot_config.GetConfig() for c in configs_to_test): return configs_to_test return None -- cgit v1.2.3 From f99a89b7704e28bccb27288e550183f3d2d8bb3a Mon Sep 17 00:00:00 2001 From: Gabe Black Date: Tue, 12 May 2015 21:53:57 -0700 Subject: stats: Add stats for the precq in the sync stage. These stats measure how many CLs are in various states, and how many tryjobs have been launched. Since a CL being in a particular state is a persistent condition, I'm measuring those with guages. Launching a trybot is an event whose rate we want to track over time, and that sounds like what a counter is for. BUG=chromium:469860 TEST=Ran run_tests. Change-Id: Ib8b4f5ed712710051dbd4193a86769d61f2aa646 Reviewed-on: https://chromium-review.googlesource.com/270559 Reviewed-by: Dan Shi Reviewed-by: Aviv Keshet Commit-Queue: Gabe Black Tested-by: Gabe Black --- cbuildbot/stages/sync_stages.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 04fd8f220..646c10f07 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -35,6 +35,7 @@ from chromite.lib import commandline from chromite.lib import cros_build_lib from chromite.lib import cros_logging as logging from chromite.lib import git +from chromite.lib import graphite from chromite.lib import gs from chromite.lib import osutils from chromite.lib import patch as cros_patch @@ -1495,6 +1496,13 @@ class PreCQLauncherStage(SyncStage): # Changes that will be passed. will_pass = set() + status_counts = {} + for status in status_map.values(): + status_counts[status] = status_counts.get(status, 0) + 1 + status_guage = graphite.StatsFactory.GetInstance().Gauge('precq.status') + for status, count in status_map.items(): + status_guage.send(status, count) + for change in inflight: if status_map[change] != constants.CL_STATUS_INFLIGHT: build_ids = [x for _, _, x in progress_map[change].values()] @@ -1550,6 +1558,9 @@ class PreCQLauncherStage(SyncStage): logging.info('Tree is closed, not launching config %s for plan %s.', config, cros_patch.GetChangesAsString(plan)) + graphite.StatsFactory.GetInstance().Counter('precq').increment( + subname='launch_count', delta=launch_count) + self.last_cycle_launch_count = launch_count # Mark passed changes as passed -- cgit v1.2.3 From 974f846c23d451a26f40790f2ecf113f4664f089 Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 14 May 2015 05:29:34 +0000 Subject: Revert "stats: Add stats for the precq in the sync stage." This reverts CL:270559, which broke the Pre-CQ launcher. See chromegw/i/chromeos/builders/Pre-CQ%20Launcher/builds/4722 20:48:43: ERROR: : Traceback (most recent call last): File "/mnt/data/b/cbuild/internal_master/chromite/cbuildbot/failures_lib.py", line 160, in wrapped_functor return functor(*args, **kwargs) File "/mnt/data/b/cbuild/internal_master/chromite/cbuildbot/stages/sync_stages.py", line 1601, in PerformStage builder_run=self._run) File "/mnt/data/b/cbuild/internal_master/chromite/cbuildbot/validation_pool.py", line 1330, in AcquirePool if pool.AcquireChanges(gerrit_query, ready_fn, change_filter): File "/mnt/data/b/cbuild/internal_master/chromite/cbuildbot/validation_pool.py", line 1234, in AcquireChanges self, self.changes, self.non_manifest_changes) File "/mnt/data/b/cbuild/internal_master/chromite/cbuildbot/stages/sync_stages.py", line 1504, in ProcessChanges status_guage.send(status, count) File "/mnt/data/b/cbuild/internal_master/chromite/lib/graphite_lib/stats.py", line 139, in send statsd.Gauge.send(self, subname, value) File "/usr/local/lib/python2.7/dist-packages/statsd/gauge.py", line 31, in send assert isinstance(value, compat.NUM_TYPES) AssertionError BUG=chromium:469860 TEST=none Change-Id: I11977afa2feb3ba2c239d561bfd23f97c6dc7154 Reviewed-on: https://chromium-review.googlesource.com/271232 Reviewed-by: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 646c10f07..04fd8f220 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -35,7 +35,6 @@ from chromite.lib import commandline from chromite.lib import cros_build_lib from chromite.lib import cros_logging as logging from chromite.lib import git -from chromite.lib import graphite from chromite.lib import gs from chromite.lib import osutils from chromite.lib import patch as cros_patch @@ -1496,13 +1495,6 @@ class PreCQLauncherStage(SyncStage): # Changes that will be passed. will_pass = set() - status_counts = {} - for status in status_map.values(): - status_counts[status] = status_counts.get(status, 0) + 1 - status_guage = graphite.StatsFactory.GetInstance().Gauge('precq.status') - for status, count in status_map.items(): - status_guage.send(status, count) - for change in inflight: if status_map[change] != constants.CL_STATUS_INFLIGHT: build_ids = [x for _, _, x in progress_map[change].values()] @@ -1558,9 +1550,6 @@ class PreCQLauncherStage(SyncStage): logging.info('Tree is closed, not launching config %s for plan %s.', config, cros_patch.GetChangesAsString(plan)) - graphite.StatsFactory.GetInstance().Counter('precq').increment( - subname='launch_count', delta=launch_count) - self.last_cycle_launch_count = launch_count # Mark passed changes as passed -- cgit v1.2.3 From 8bc983793b45fa19b7a22da9a0cd444d7cac8e48 Mon Sep 17 00:00:00 2001 From: Gabe Black Date: Tue, 12 May 2015 21:53:57 -0700 Subject: stats: Add stats for the precq in the sync stage (attempt 2). These stats measure how many CLs are in various states, and how many tryjobs have been launched. Since a CL being in a particular state is a persistent condition, I'm measuring those with gauges. Launching a trybot is an event whose rate we want to track over time, and that sounds like what a counter is for. BUG=chromium:469860 TEST=Ran run_tests. Change-Id: Idfac735b1b41b403972ed0d890e312ff25ebbca2 Reviewed-on: https://chromium-review.googlesource.com/271324 Reviewed-by: Dan Shi Tested-by: Gabe Black Commit-Queue: Gabe Black --- cbuildbot/stages/sync_stages.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 04fd8f220..4cbcfcd35 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -35,6 +35,7 @@ from chromite.lib import commandline from chromite.lib import cros_build_lib from chromite.lib import cros_logging as logging from chromite.lib import git +from chromite.lib import graphite from chromite.lib import gs from chromite.lib import osutils from chromite.lib import patch as cros_patch @@ -1495,6 +1496,13 @@ class PreCQLauncherStage(SyncStage): # Changes that will be passed. will_pass = set() + status_counts = {} + for status in status_map.values(): + status_counts[status] = status_counts.get(status, 0) + 1 + status_gauge = graphite.StatsFactory.GetInstance().Gauge('precq.status') + for status, count in status_counts.items(): + status_gauge.send(status, count) + for change in inflight: if status_map[change] != constants.CL_STATUS_INFLIGHT: build_ids = [x for _, _, x in progress_map[change].values()] @@ -1550,6 +1558,9 @@ class PreCQLauncherStage(SyncStage): logging.info('Tree is closed, not launching config %s for plan %s.', config, cros_patch.GetChangesAsString(plan)) + graphite.StatsFactory.GetInstance().Counter('precq').increment( + subname='launch_count', delta=launch_count) + self.last_cycle_launch_count = launch_count # Mark passed changes as passed -- cgit v1.2.3 From 87b267e8dae01ac8827cccb8dd545e4626db9e46 Mon Sep 17 00:00:00 2001 From: David James Date: Fri, 1 May 2015 11:31:22 -0700 Subject: Launch multiple trybots together in Pre-CQ launcher. Pre-CQ launcher now submits each tryjob one by one (4 Pre-CQ configs -> 4 tryjobs). This is wasteful. This CL fixes that. BUG=chromium:446554 TEST=Unit tests. TEST=Pre-cq-launcher trybot run. Change-Id: I2ef3f3d31ed308b73b80374dda1c3351bbf63c3a Reviewed-on: https://chromium-review.googlesource.com/268851 Trybot-Ready: David James Reviewed-by: Aviv Keshet Tested-by: David James Commit-Queue: David James --- cbuildbot/stages/sync_stages.py | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 4cbcfcd35..d06d68cf7 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -9,6 +9,7 @@ from __future__ import print_function import ConfigParser import contextlib import datetime +import itertools import os import shutil import sys @@ -1249,17 +1250,16 @@ class PreCQLauncherStage(SyncStage): logging.error('%s has malformed config file', change, exc_info=True) return bool(result and result.lower() == 'yes') - - def LaunchTrybot(self, plan, config): + def LaunchTrybot(self, plan, configs): """Launch a Pre-CQ run with the provided list of CLs. Args: pool: ValidationPool corresponding to |plan|. plan: The list of patches to test in the pre-cq tryjob. - config: The pre-cq config name to launch. + configs: A list of pre-cq config names to launch. """ - cmd = ['cbuildbot', '--remote', config, - '--timeout', str(self.INFLIGHT_TIMEOUT * 60)] + cmd = ['cbuildbot', '--remote', + '--timeout', str(self.INFLIGHT_TIMEOUT * 60)] + configs for patch in plan: cmd += ['-g', cros_patch.AddPrefix(patch, patch.gerrit_number)] self._PrintPatchStatus(patch, 'testing') @@ -1267,14 +1267,14 @@ class PreCQLauncherStage(SyncStage): logging.debug('Would have launched tryjob with %s', cmd) else: cros_build_lib.RunCommand(cmd, cwd=self._build_root) + build_id, db = self._run.GetCIDBHandle() actions = [ clactions.CLAction.FromGerritPatchAndAction( patch, constants.CL_ACTION_TRYBOT_LAUNCHING, config) - for patch in plan] + for patch, config in itertools.product(plan, configs)] db.InsertCLActions(build_id, actions) - def GetDisjointTransactionsToTest(self, pool, progress_map): """Get the list of disjoint transactions to test. @@ -1543,20 +1543,23 @@ class PreCQLauncherStage(SyncStage): launch_count = 0 launch_count_limit = (self.last_cycle_launch_count + self.MAX_LAUNCHES_PER_CYCLE_DERIVATIVE) + launches = {} for plan, config in self.GetDisjointTransactionsToTest( pool, launchable_progress_map): - if is_tree_open: - if launch_count < launch_count_limit: - self.LaunchTrybot(plan, config) - launch_count += 1 - else: - logging.info('Hit maximum launch count of %s this cycle, not ' - 'launching config %s for plan %s.', - launch_count_limit, config, - cros_patch.GetChangesAsString(plan)) + launches.setdefault(frozenset(plan), []).append(config) + + for plan, configs in launches.iteritems(): + if not is_tree_open: + logging.info('Tree is closed, not launching configs %r for plan %s.', + configs, cros_patch.GetChangesAsString(plan)) + elif launch_count >= launch_count_limit: + logging.info('Hit or exceeded maximum launch count of %s this cycle, ' + 'not launching configs %r for plan %s.', + launch_count_limit, configs, + cros_patch.GetChangesAsString(plan)) else: - logging.info('Tree is closed, not launching config %s for plan %s.', - config, cros_patch.GetChangesAsString(plan)) + self.LaunchTrybot(plan, configs) + launch_count += len(configs) graphite.StatsFactory.GetInstance().Counter('precq').increment( subname='launch_count', delta=launch_count) -- cgit v1.2.3 From 41c5f67996aca09af71128e3e3ca75ae741c2910 Mon Sep 17 00:00:00 2001 From: Gabe Black Date: Fri, 15 May 2015 13:41:05 -0700 Subject: stats: Split out the gauges in the sync stage. These seem to have been squished together even though they had separate subnames. Instead of relying on subnames, lets give them explicitly distinct gauge stats. BUG=chromium:469860 TEST=Ran run_tests. Change-Id: I6df85f9785dfa6008f2d0c6a95345c2ce968dd80 Reviewed-on: https://chromium-review.googlesource.com/271551 Reviewed-by: Dan Shi Commit-Queue: Gabe Black Tested-by: Gabe Black --- cbuildbot/stages/sync_stages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index d06d68cf7..bb142806e 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1499,9 +1499,9 @@ class PreCQLauncherStage(SyncStage): status_counts = {} for status in status_map.values(): status_counts[status] = status_counts.get(status, 0) + 1 - status_gauge = graphite.StatsFactory.GetInstance().Gauge('precq.status') for status, count in status_counts.items(): - status_gauge.send(status, count) + name = '.'.join(['precq', 'status', status if status else 'None']) + graphite.StatsFactory.GetInstance().Gauge(name).send(status, count) for change in inflight: if status_map[change] != constants.CL_STATUS_INFLIGHT: -- cgit v1.2.3 From 84b132c7c8895e3b469ea84c2843a5de2a011ccc Mon Sep 17 00:00:00 2001 From: Dan Jacques Date: Tue, 14 Apr 2015 13:15:51 -0700 Subject: cbuildbot_config: Backport release updates to master. Port of non-release-specific parts of: https://chromium-review.googlesource.com/#/c/264929 https://chromium-review.googlesource.com/#/c/266093 - The 'internal_pfq_branch'-derived configs mark themselves as present on the release waterfall. - Allow explicitly-specified waterfall presense (i.e., don't override with default). - Fix bug with non-canary masters in 'sync_stages' (from release branch). - Fixed some formatting errors. BUG=chromium:456203 TEST=`./cbuildbot/run_tests` Change-Id: I1bbd849b7b3875601f86e08a714057112c8d002f Reviewed-on: https://chromium-review.googlesource.com/265727 Trybot-Ready: Daniel Jacques Reviewed-by: Daniel Jacques Commit-Queue: Daniel Jacques Tested-by: Daniel Jacques --- cbuildbot/stages/sync_stages.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index bb142806e..5ff0b6d1a 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -738,8 +738,7 @@ class ManifestVersionedSyncStage(SyncStage): # TODO(dgarrett): Push this logic into it's own stage. # If we are a Canary Master, create an additional derivative Manifest for # the Project SDK builders. - if (cbuildbot_config.IsCanaryType(self._run.config.build_type) and - self._run.config.master): + if self._run.config.name == constants.CANARY_MASTER: logging.info('Creating Project SDK Manifest.') sdk_manifest = None try: -- cgit v1.2.3 From 0dc51e06a5a61d1803dbfdfaa0eb59bea1727086 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Wed, 20 May 2015 00:33:08 +0000 Subject: Revert "cbuildbot: Don't bootstrap chromite in /tmp" This reverts commit ee6b9a60a10d6c578f72e0eb8f5027d364373f85. BUG=chromium:489870 TEST=None Change-Id: I15ec483865280a9c1ef7cda93a0170aa96afbbcb Reviewed-on: https://chromium-review.googlesource.com/272284 Reviewed-by: Prathmesh Prabhu Reviewed-by: David James Tested-by: David James --- cbuildbot/stages/sync_stages.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 5ff0b6d1a..590b7a97f 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -173,12 +173,6 @@ class BootstrapStage(PatchChangesStage): self.chromite_patch_pool = chromite_patch_pool self.manifest_patch_pool = manifest_patch_pool self.returncode = None - # Bootstrap chromite in a subdirectory of the buildroot. This directory - # requires exec permissions so that cbuildbot can be re-executed after - # chromite is patched. - self.tempdir = os.path.join(self._run.options.buildroot, - 'chromite-bootstrap') - osutils.RmDir(self.tempdir, ignore_missing=True) def _ApplyManifestPatches(self, patch_pool): """Apply a pool of manifest patches to a temp manifest checkout. @@ -255,6 +249,7 @@ class BootstrapStage(PatchChangesStage): else: PatchChangesStage.HandleApplyFailures(self, failures) + @osutils.TempDirDecorator def PerformStage(self): # The plan for the builders is to use master branch to bootstrap other # branches. Now, if we wanted to test patches for both the bootstrap code -- cgit v1.2.3 From f9f8963ac27a24bb34a259ae0acbfe8fb2553892 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 21 May 2015 14:57:56 -0700 Subject: pre-cq: add debug logging around stats being thrown Some of the CL statuses are showing up, but not the full expected set. Adding some logging to help diagnose. BUG=chromium:469860 TEST=None Change-Id: Ifc5bc370751e25128765eeb1b5909de7c84e074a Reviewed-on: https://chromium-review.googlesource.com/272697 Reviewed-by: Prathmesh Prabhu Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 590b7a97f..2a75e2caf 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1495,6 +1495,8 @@ class PreCQLauncherStage(SyncStage): status_counts[status] = status_counts.get(status, 0) + 1 for status, count in status_counts.items(): name = '.'.join(['precq', 'status', status if status else 'None']) + logging.debug('Sending stat (name, status, count): (%s, %s, %s)', + name, status, count) graphite.StatsFactory.GetInstance().Gauge(name).send(status, count) for change in inflight: -- cgit v1.2.3 From 5ead07ee7bb2d298f008aa757d0825a3dff42891 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 21 May 2015 15:28:15 -0700 Subject: pre-cq: upgrade a logging level to something visible BUG=None TEST=None Change-Id: If6e8ea9ecebb49b26d34afed992e535696dfb5eb Reviewed-on: https://chromium-review.googlesource.com/272685 Reviewed-by: Prathmesh Prabhu Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 2a75e2caf..bd823f369 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1495,8 +1495,8 @@ class PreCQLauncherStage(SyncStage): status_counts[status] = status_counts.get(status, 0) + 1 for status, count in status_counts.items(): name = '.'.join(['precq', 'status', status if status else 'None']) - logging.debug('Sending stat (name, status, count): (%s, %s, %s)', - name, status, count) + logging.info('Sending stat (name, status, count): (%s, %s, %s)', + name, status, count) graphite.StatsFactory.GetInstance().Gauge(name).send(status, count) for change in inflight: -- cgit v1.2.3 From 1c832d66980f4fae2c49ecb9e52e9c222a82c115 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 21 May 2015 16:40:26 -0700 Subject: pre-cq: separately log speculative and mergeable stats BUG=chromium:469860 TEST=unit tests Change-Id: I50b7eaedeecababbb60763cfd4c0348e706585db Reviewed-on: https://chromium-review.googlesource.com/272743 Trybot-Ready: Aviv Keshet Tested-by: Aviv Keshet Reviewed-by: Gabe Black Reviewed-by: Prathmesh Prabhu --- cbuildbot/stages/sync_stages.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index bd823f369..720178627 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1490,14 +1490,22 @@ class PreCQLauncherStage(SyncStage): # Changes that will be passed. will_pass = set() + # Separately count and log the number of mergable and speculative changes in + # each of the possible pre-cq statuses (or in status None). + POSSIBLE_STATUSES = clactions.PRE_CQ_CL_STATUSES | {None} status_counts = {} - for status in status_map.values(): - status_counts[status] = status_counts.get(status, 0) + 1 - for status, count in status_counts.items(): - name = '.'.join(['precq', 'status', status if status else 'None']) - logging.info('Sending stat (name, status, count): (%s, %s, %s)', - name, status, count) - graphite.StatsFactory.GetInstance().Gauge(name).send(status, count) + for count_bin in itertools.product((True, False), POSSIBLE_STATUSES): + status_counts[count_bin] = 0 + for c, status in status_map.iteritems(): + count_bin = (c.IsMergeable(), status) + status_counts[count_bin] = status_counts[count_bin] + 1 + for count_bin, count in sorted(status_counts.items()): + subtype = 'mergeable' if count_bin[0] else 'speculative' + status = count_bin[1] + name = '.'.join(['pre-cq-status', status if status else 'None']) + logging.info('Sending stat (name, subtype, count): (%s, %s, %s)', + name, subtype, count) + graphite.StatsFactory.GetInstance().Gauge(name).send(subtype, count) for change in inflight: if status_map[change] != constants.CL_STATUS_INFLIGHT: -- cgit v1.2.3 From 3354d7fc2d2c4152077269a8787b756352e82d56 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Tue, 26 May 2015 10:23:57 -0700 Subject: stats: change a few stat names and add a diagnostic one This CL makes a cosmetic change to the build_started stat, so that the chromite stats namespace is not filled with entries for every config ever launched. It renames the pre-cq launch_count stat slightly, and adds an additional len(plan) * len(configs) counter which should exactly correspond with the number of trybot_launching actions in cidb. This is to help track down an anomaly in the launch counts. (the delta= argument is also removed from counters; statsd documentation indicates this is not necessary, and I wonder if it is responsible for the anomaly). BUG=None TEST=pre-cq-launcher tryjob Change-Id: Ie6cff9708861f25269678fba1be8d4539a54d5ae Reviewed-on: https://chromium-review.googlesource.com/273274 Trybot-Ready: Aviv Keshet Tested-by: Aviv Keshet Reviewed-by: Gabe Black Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 720178627..a98303941 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1545,6 +1545,7 @@ class PreCQLauncherStage(SyncStage): is_tree_open = tree_status.IsTreeOpen(throttled_ok=True) launch_count = 0 + cl_launch_count = 0 launch_count_limit = (self.last_cycle_launch_count + self.MAX_LAUNCHES_PER_CYCLE_DERIVATIVE) launches = {} @@ -1564,9 +1565,12 @@ class PreCQLauncherStage(SyncStage): else: self.LaunchTrybot(plan, configs) launch_count += len(configs) + cl_launch_count += len(configs) * len(plan) - graphite.StatsFactory.GetInstance().Counter('precq').increment( - subname='launch_count', delta=launch_count) + graphite.StatsFactory.GetInstance().Counter('pre-cq').increment( + 'launch_count', launch_count) + graphite.StatsFactory.GetInstance().Counter('pre-cq').increment( + 'cl_launch_count', cl_launch_count) self.last_cycle_launch_count = launch_count -- cgit v1.2.3 From 45758fba5bd71a3f51ad080aa3e304f915d32b9c Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Wed, 27 May 2015 12:06:56 -0700 Subject: stats: add a tick_count to pre-cq-launcher This may help diagnosing issues with the pre-cq-launcher, as well as help tracing down why some counter stats seem to be getting lost. BUG=None TEST=None Change-Id: Id8c199fc824e7e10f3dc6eee2e220a864cb2471b Reviewed-on: https://chromium-review.googlesource.com/273546 Tested-by: Aviv Keshet Trybot-Ready: Aviv Keshet Reviewed-by: Gabe Black Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index a98303941..ebf431892 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1571,6 +1571,8 @@ class PreCQLauncherStage(SyncStage): 'launch_count', launch_count) graphite.StatsFactory.GetInstance().Counter('pre-cq').increment( 'cl_launch_count', cl_launch_count) + graphite.StatsFactory.GetInstance().Counter('pre-cq').increment( + 'tick_count') self.last_cycle_launch_count = launch_count -- cgit v1.2.3 From 867c117861f729b9c4841933b84524011a382571 Mon Sep 17 00:00:00 2001 From: Prathmesh Prabhu Date: Tue, 2 Jun 2015 17:57:59 -0700 Subject: cbuildbot: Add flag to specify bootsrap directory. cbuildbot by default bootstraps in a directory under /tmp. This CL adds a flag that lets the user specify the top level directory under which all bootstrapping directories should be created. This is especially useful on mobbuild, where /tmp is mounted noexec. This will also be used on our builders where we don't want to execute code from /tmp. The default behaviour remains unchanged, so that's what developers get. BUG=chromium:490935 TEST=`cbuildbot --test-bootstrap --bootstrap-dir path/to/custom/dir \ --local -p chromiumos/chromite binhost-pre-cq` and verify that all bootstrap re-executions use the custom directory. Change-Id: Iba4bcf29bc092815b30c9ccbb5a5cfa29ef1d583 Reviewed-on: https://chromium-review.googlesource.com/274913 Reviewed-by: Paul Hobbs Commit-Queue: Prathmesh Prabhu Tested-by: Prathmesh Prabhu --- cbuildbot/stages/sync_stages.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index ebf431892..84dd5f3cf 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -173,6 +173,7 @@ class BootstrapStage(PatchChangesStage): self.chromite_patch_pool = chromite_patch_pool self.manifest_patch_pool = manifest_patch_pool self.returncode = None + self.tempdir = None def _ApplyManifestPatches(self, patch_pool): """Apply a pool of manifest patches to a temp manifest checkout. @@ -249,8 +250,7 @@ class BootstrapStage(PatchChangesStage): else: PatchChangesStage.HandleApplyFailures(self, failures) - @osutils.TempDirDecorator - def PerformStage(self): + def _PerformStageInTempDir(self): # The plan for the builders is to use master branch to bootstrap other # branches. Now, if we wanted to test patches for both the bootstrap code # (on master) and the branched chromite (say, R20), we need to filter the @@ -302,6 +302,12 @@ class BootstrapStage(PatchChangesStage): cmd, cwd=self.tempdir, kill_timeout=30, error_code_ok=True) self.returncode = result_obj.returncode + def PerformStage(self): + with osutils.TempDir(base_dir=self._run.options.bootstrap_dir) as tempdir: + self.tempdir = tempdir + self._PerformStageInTempDir() + self.tempdir = None + class SyncStage(generic_stages.BuilderStage): """Stage that performs syncing for the builder.""" -- cgit v1.2.3 From e60455583b9600d8579d10d1a5f1195ae66b9577 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Fri, 5 Jun 2015 17:34:52 -0700 Subject: sync_stages: Switch to self._run.site_config. Stop loading cbuildbot_config, just use the site_config configured by cbuildbot at startup. BUG=chromium:483566 TEST=lint + run_tests Change-Id: I95f726c2e363fe04412d1e60368640dfbcfce282 Reviewed-on: https://chromium-review.googlesource.com/275740 Trybot-Ready: Don Garrett Tested-by: Don Garrett Reviewed-by: Paul Hobbs Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 84dd5f3cf..b6fc72d98 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -18,7 +18,6 @@ import time from xml.etree import ElementTree from xml.dom import minidom -from chromite.cbuildbot import cbuildbot_config from chromite.cbuildbot import chroot_lib from chromite.cbuildbot import failures_lib from chromite.cbuildbot import constants @@ -1199,7 +1198,7 @@ class PreCQLauncherStage(SyncStage): configs_to_test.update(constants.PRE_CQ_DEFAULT_CONFIGS) # Verify that all of the configs are valid. - if all(c in cbuildbot_config.GetConfig() for c in configs_to_test): + if all(c in self._run.site_config for c in configs_to_test): return configs_to_test return None -- cgit v1.2.3 From dbbdd60c7d2ee4dd5efa9b1d80eb1f04d8471b1e Mon Sep 17 00:00:00 2001 From: Fang Deng Date: Fri, 22 May 2015 12:05:48 -0700 Subject: [cbuildbot] MasterSlaveLKGMSyncStage generates blamelist MasterSlaveLKGMSyncStage (PFQ) looks for the last chromeos version built by the builder, and generates a blamelist. TEST=unittest; 1) cbuildbot -g xxx --remote --buildbot --debug master-chromium-pfq the stage went through successfully, except that it used the debug cidb which didn't return a valid hisory build number. 2) hard-code the milestone and platform version returned by GetLastChromeOSVersion() to a valid one, confirm blamelist can be generated successfully. BUG=chromium:483070 Change-Id: I11441c9caf1762891833382f61da3319be2708db Reviewed-on: https://chromium-review.googlesource.com/272931 Tested-by: Fang Deng Reviewed-by: David James Reviewed-by: Paul Hobbs Commit-Queue: Fang Deng --- cbuildbot/stages/sync_stages.py | 49 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index b6fc72d98..0a5c6a23f 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -6,11 +6,13 @@ from __future__ import print_function +import collections import ConfigParser import contextlib import datetime import itertools import os +import re import shutil import sys import tempfile @@ -774,6 +776,9 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): # TODO(mtennant): Turn this into self._run.attrs.sub_manager or similar. # An instance of lkgm_manager.LKGMManager for slave builds. sub_manager = None + MAX_BUILD_HISTORY_LENGTH = 10 + MilestoneVersion = collections.namedtuple( + 'MilestoneVersion', ['milestone', 'platform']) def __init__(self, builder_run, **kwargs): super(MasterSlaveLKGMSyncStage, self).__init__(builder_run, **kwargs) @@ -848,6 +853,39 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): return cros_mark_chrome_as_stable.GetLatestRelease( constants.CHROMIUM_GOB_URL) + def GetLastChromeOSVersion(self): + """Fetching ChromeOS version from the last run. + + Fetching the chromeos version from the last run that published a manifest + by querying CIDB. Master builds that failed before publishing a manifest + will be ignored. + + Returns: + A namedtuple MilestoneVersion, + e.g. MilestoneVersion(milestone='44', platform='7072.0.0-rc4') + or None if failed to retrieve milestone and platform versions. + """ + build_id, db = self._run.GetCIDBHandle() + + if db is None: + return None + + builds = db.GetBuildHistory( + build_config=self._run.config.name, + num_results=self.MAX_BUILD_HISTORY_LENGTH, + ignore_build_id=build_id) + full_versions = [b.get('full_version') for b in builds] + old_version = next(itertools.ifilter(bool, full_versions), None) + if old_version: + pattern = r'^R(\d+)-(\d+.\d+.\d+(-rc\d+)*)' + m = re.match(pattern, old_version) + if m: + milestone = m.group(1) + platform = m.group(2) + return self.MilestoneVersion( + milestone=milestone, platform=platform) + return None + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): """Performs the stage.""" @@ -859,6 +897,17 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): ManifestVersionedSyncStage.PerformStage(self) + # Generate blamelist + cros_version = self.GetLastChromeOSVersion() + if cros_version: + old_filename = self.manifest_manager.GetBuildSpecFilePath( + cros_version.milestone, cros_version.platform) + if not os.path.exists(old_filename): + logging.error('Could not generate blamelist, ' + 'manifest file does not exist: %s', old_filename) + else: + logging.debug('Generate blamelist against: %s', old_filename) + lkgm_manager.GenerateBlameList(self.repo, old_filename) class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): """Commit Queue Sync stage that handles syncing and applying patches. -- cgit v1.2.3 From d4ddeb294a27e27af45dc12cdaa91ed28c04b259 Mon Sep 17 00:00:00 2001 From: Shuqian Zhao Date: Mon, 8 Jun 2015 16:14:30 -0700 Subject: triage_lib: parse subsystem label from COMMIT-QUEUE.ini There will be a subsystem label seeded in COMMIT-QUEUE.ini for a given change. Now to teach triage_lib to parse a subsystem label from COMMIT-QUEUE.ini. Also, change the behavior of a function, correct the behaviors of all the founded callers of this function. BUG=chromium:495962 TEST=Test with unittest Change-Id: I51c4f3c014fa0eaf40275e6cbf61fc10f7fd2e51 Reviewed-on: https://chromium-review.googlesource.com/274916 Reviewed-by: Shuqian Zhao Commit-Queue: Shuqian Zhao Trybot-Ready: Shuqian Zhao Tested-by: Shuqian Zhao --- cbuildbot/stages/sync_stages.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 0a5c6a23f..23ecd7472 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1202,20 +1202,16 @@ class PreCQLauncherStage(SyncStage): A set of configs to test. """ configs_to_test = None - try: - # If a pre-cq config is specified in the commit message, use that. - # Otherwise, look in appropriate COMMIT-QUEUE.ini. Otherwise, default to - # constants.PRE_CQ_DEFAULT_CONFIGS - lines = cros_patch.GetOptionLinesFromCommitMessage( - change.commit_message, constants.PRE_CQ_CONFIGS_OPTION_REGEX) - if lines is not None: - configs_to_test = self._ParsePreCQOption(' '.join(lines)) - configs_to_test = configs_to_test or self._ParsePreCQOption( - triage_lib.GetOptionForChange( - self._build_root, change, 'GENERAL', - constants.PRE_CQ_CONFIGS_OPTION)) - except ConfigParser.Error: - logging.error('%s has malformed config file', change, exc_info=True) + # If a pre-cq config is specified in the commit message, use that. + # Otherwise, look in appropriate COMMIT-QUEUE.ini. Otherwise, default to + # constants.PRE_CQ_DEFAULT_CONFIGS + lines = cros_patch.GetOptionLinesFromCommitMessage( + change.commit_message, constants.PRE_CQ_CONFIGS_OPTION_REGEX) + if lines is not None: + configs_to_test = self._ParsePreCQOption(' '.join(lines)) + configs_to_test = configs_to_test or self._ParsePreCQOption( + triage_lib.GetOptionForChange(self._build_root, change, 'GENERAL', + constants.PRE_CQ_CONFIGS_OPTION)) return set(configs_to_test or constants.PRE_CQ_DEFAULT_CONFIGS) -- cgit v1.2.3 From dc8fc355dab30c48a4927734d8731b6353fcacc4 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Wed, 24 Jun 2015 13:02:26 -0700 Subject: sync_stages: add in missing build_id argument BUG=chromium:504077 TEST=None Change-Id: I2758f4443d736dc1ee43f3098fbef732858d70ab Reviewed-on: https://chromium-review.googlesource.com/281644 Trybot-Ready: Aviv Keshet Tested-by: Aviv Keshet Reviewed-by: Daniel Jacques Commit-Queue: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 23ecd7472..8cce75505 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -844,7 +844,8 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): chrome_version=self._chrome_version, build_id=build_id) if MasterSlaveLKGMSyncStage.sub_manager: - MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(manifest) + MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest( + manifest, build_id=build_id) return manifest -- cgit v1.2.3 From 1062bbefb4b4d7a24ff837c7f6712b6c12d794b2 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 25 Jun 2015 15:39:33 -0700 Subject: sync_stages: Stop creating project_sdk manifests. This stops the Canary master from creating new project_sdk manifests. This will stop triggering the project sdk builders from building. BUG=chromium:504582 TEST=run_tests Change-Id: If20e833d4b89c81374215e6af545c7ca0fe62b65 Reviewed-on: https://chromium-review.googlesource.com/282071 Trybot-Ready: Don Garrett Tested-by: Don Garrett Reviewed-by: Dan Shi Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 97 ----------------------------------------- 1 file changed, 97 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 8cce75505..276ebb279 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -13,9 +13,7 @@ import datetime import itertools import os import re -import shutil import sys -import tempfile import time from xml.etree import ElementTree from xml.dom import minidom @@ -38,7 +36,6 @@ from chromite.lib import cros_build_lib from chromite.lib import cros_logging as logging from chromite.lib import git from chromite.lib import graphite -from chromite.lib import gs from chromite.lib import osutils from chromite.lib import patch as cros_patch from chromite.lib import timeout_util @@ -581,79 +578,6 @@ class ManifestVersionedSyncStage(SyncStage): else: yield manifest - def CommitProjectSDKManifest(self, manifest, current_version, debug): - """Create and submit the Product SDK Manifest. - - Create the Project SDK manifest, and push it to the external manifest - repository. - - Args: - manifest: Path to new manifest to commit. - current_version: Version to use when commiting manifest (e.g. 6770.0.0). - debug: Is this a debug build? - """ - # Use a static dir, but don't overlap with other users, we might conflict. - git_repo = os.path.join( - self._build_root, constants.PROJECT_SDK_MANIFEST_VERSIONS_PATH) - external_manifest_url = self._GetManifestVersionsRepoUrl( - internal=False, test=debug) - - logging.info('Using manifest URL: %s', external_manifest_url) - manifest_version.RefreshManifestCheckout( - git_repo, external_manifest_url) - - sdk_manifest_name = '%s.xml' % current_version - latest_manifest_path = os.path.join( - git_repo, constants.LATEST_PROJECT_SDK_MANIFEST) - sdk_manifest_path = os.path.join( - git_repo, 'project-sdk', sdk_manifest_name) - - if os.path.exists(sdk_manifest_path): - raise failures_lib.StepFailure( - 'Project SDK Manifest already exists: %s' % sdk_manifest_path) - - # Create branch for pushing new manifest file. - branch = 'temp_project_sdk_creation_branch' - git.CreatePushBranch(branch, git_repo, sync=False) - - # Create new manifest file. - logging.info('Creating Project SDK Manifest as: %s', sdk_manifest_path) - osutils.SafeMakedirs(os.path.dirname(sdk_manifest_path)) - shutil.copyfile(manifest, sdk_manifest_path) - - # Create 'latest' link to new manifest. - osutils.SafeUnlink(latest_manifest_path) - os.symlink(sdk_manifest_name, latest_manifest_path) - - # Commit it locally. - logging.info('Committing Project SDK Manifest.') - git.AddPath(sdk_manifest_path) - git.AddPath(latest_manifest_path) - git.Commit(git_repo, 'Create project_sdk for %s.' % current_version) - - # Push it to Gerrit. - logging.info('Pushing Project SDK Manifest.') - git.PushWithRetry(branch, git_repo) - - # Push to GS. - gs_ctx = gs.GSContext(dry_run=self._run.debug) - publish_uri = os.path.join(constants.BRILLO_RELEASE_MANIFESTS_URL, - sdk_manifest_name) - - # We use the default ACL (public readable) for this file. - logging.info('Pushing Project SDK Manifest to: %s', publish_uri) - gs_ctx.Copy(manifest, publish_uri) - - # Populate a file on GS with the newest version published. - with tempfile.NamedTemporaryFile() as latest: - osutils.WriteFile(latest.name, current_version) - gs_ctx.Copy(latest.name, constants.BRILLO_LATEST_RELEASE_URL) - - # Log what we published. - logging.info('Project SDK Manifest \'%s\' published:', - os.path.basename(sdk_manifest_path)) - logging.info('%s', osutils.ReadFile(manifest)) - def _GetMasterVersion(self, master_id, timeout=5 * 60): """Get the platform version associated with the master_build_id. @@ -737,27 +661,6 @@ class ManifestVersionedSyncStage(SyncStage): next_manifest, filter_cros=self._run.options.local) as new_manifest: self.ManifestCheckout(new_manifest) - # TODO(dgarrett): Push this logic into it's own stage. - # If we are a Canary Master, create an additional derivative Manifest for - # the Project SDK builders. - if self._run.config.name == constants.CANARY_MASTER: - logging.info('Creating Project SDK Manifest.') - sdk_manifest = None - try: - with tempfile.NamedTemporaryFile() as pinned_manifest_file: - pinned_manifest = self.repo.ExportManifest(mark_revision=True) - osutils.WriteFile(pinned_manifest_file.name, pinned_manifest) - sdk_manifest = manifest_version.ConvertToProjectSdkManifest( - pinned_manifest_file.name) - - self.CommitProjectSDKManifest( - sdk_manifest, - self.manifest_manager.current_version, - self._run.options.debug) - finally: - if sdk_manifest: - os.unlink(sdk_manifest) - # Set the status inflight at the end of the ManifestVersionedSync # stage. This guarantees that all syncing has completed. if self.manifest_manager: -- cgit v1.2.3 From bfb3353ec5f07b2abaf44eb39e44b4f2b482c532 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Tue, 7 Jul 2015 16:56:43 -0700 Subject: cbuildbot: Fetch of site config git repo. If the current invocation of cbuildbot is a buildbot bootstrap checkout, fetch the site config repo to help start the bootstrap process. The Bootstrap stage will also checkout the site config repository, if it was specified on the command line. However, the Bootstrap stage will not yet apply patches to that repository during bootstrap stages. BUG=chromium:497284 TEST=Unittests git clone chromite into tmp dir. bin/cbuildbot --buildroot /tmp/test_buildroot --test-bootstrap \ --buildbot --debug \ --config_repo=https://chromium.googlesource.com/chromiumos/ chromite/test_site_config sync-test-cbuildbot Change-Id: Ifdb8b75c0127b3a93617fce3bbf7d4e0706980d4 Reviewed-on: https://chromium-review.googlesource.com/283978 Reviewed-by: Prathmesh Prabhu Tested-by: Don Garrett Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 276ebb279..4c0489135 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -165,11 +165,12 @@ class BootstrapStage(PatchChangesStage): option_name = 'bootstrap' def __init__(self, builder_run, chromite_patch_pool, - manifest_patch_pool=None, **kwargs): + manifest_patch_pool, **kwargs): super(BootstrapStage, self).__init__( builder_run, trybot_patch_pool.TrybotPatchPool(), **kwargs) self.chromite_patch_pool = chromite_patch_pool self.manifest_patch_pool = manifest_patch_pool + self.config_repo = self._run.options.config_repo self.returncode = None self.tempdir = None @@ -257,8 +258,9 @@ class BootstrapStage(PatchChangesStage): if self._run.options.test_bootstrap: filter_branch = 'master' + # Checkout the new version of chromite, and patch it. chromite_dir = os.path.join(self.tempdir, 'chromite') - reference_repo = os.path.join(constants.SOURCE_ROOT, 'chromite', '.git') + reference_repo = os.path.join(constants.CHROMITE_DIR, '.git') repository.CloneGitRepo(chromite_dir, constants.CHROMITE_URL, reference=reference_repo) git.RunGit(chromite_dir, ['checkout', filter_branch]) @@ -275,13 +277,23 @@ class BootstrapStage(PatchChangesStage): if filtered_pool: self._ApplyPatchSeries(patch_series, filtered_pool) + # Checkout the new version of site config (no patching logic, yet). + if self.config_repo: + site_config_dir = os.path.join(chromite_dir, 'config') + site_config_reference_repo = os.path.join(constants.SITE_CONFIG_DIR, + '.git') + repository.CloneGitRepo(site_config_dir, self.config_repo, + reference=site_config_reference_repo) + git.RunGit(site_config_dir, ['checkout', filter_branch]) + + # Re-exec into new instance of cbuildbot, with proper command line args. cbuildbot_path = constants.PATH_TO_CBUILDBOT if not os.path.exists(os.path.join(self.tempdir, cbuildbot_path)): cbuildbot_path = 'chromite/cbuildbot/cbuildbot' cmd = self.FilterArgsForTargetCbuildbot(self.tempdir, cbuildbot_path, self._run.options) - extra_params = ['--sourceroot=%s' % self._run.options.sourceroot] + extra_params = ['--sourceroot', self._run.options.sourceroot] extra_params.extend(self._run.options.bootstrap_args) if self._run.options.test_bootstrap: # We don't want re-executed instance to see this. -- cgit v1.2.3 From 17f0742270498c9ce88af2e61027caca0307ee78 Mon Sep 17 00:00:00 2001 From: Prathmesh Prabhu Date: Fri, 17 Jul 2015 11:40:40 -0700 Subject: logging: Add knob to suppress PrintBuildbot* logging functions. This CL adds a knob to modify the behavior of PrintBuildbot* functions. By default, these functions now dump logs in a way that buidbot does not recognize them as special log lines. When the knob is turned on, the buildbot recognizable special log lines are dumped. It also moves these functions to the cros_logging module, because they belong better to the logging module. Finally, by default, the cbuildbot entry point turns on this knob. BUG=chromium:510905 TEST=(1) unittests. (2) Trybot the hell out of this. Change-Id: I739dcbf13a60e20f68a1a692dba0c621ec297524 Reviewed-on: https://chromium-review.googlesource.com/286532 Reviewed-by: Prathmesh Prabhu Commit-Queue: Prathmesh Prabhu Tested-by: Prathmesh Prabhu --- cbuildbot/stages/sync_stages.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 4c0489135..3d93be851 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -136,9 +136,9 @@ class PatchChangesStage(generic_stages.BuilderStage): def ApplyChange(self, change): if isinstance(change, cros_patch.GerritPatch): - cros_build_lib.PrintBuildbotLink(str(change), change.url) + logging.PrintBuildbotLink(str(change), change.url) elif isinstance(change, cros_patch.UploadedLocalPatch): - cros_build_lib.PrintBuildbotStepText(str(change)) + logging.PrintBuildbotStepText(str(change)) return validation_pool.PatchSeries.ApplyChange(self, change) @@ -244,7 +244,7 @@ class BootstrapStage(PatchChangesStage): # WARNING: For manifest patches, the Pre-CQ attempts to apply external # patches to the internal manifest, and this means we may flag a conflict # here even if the patch applies cleanly. TODO(davidjames): Fix this. - cros_build_lib.PrintBuildbotStepWarnings() + logging.PrintBuildbotStepWarnings() logging.error('Failed applying patches: %s\n'.join(map(str, failures))) else: PatchChangesStage.HandleApplyFailures(self, failures) @@ -419,7 +419,7 @@ class SyncStage(generic_stages.BuilderStage): # Print the blamelist. if fresh_sync: - cros_build_lib.PrintBuildbotStepText('(From scratch)') + logging.PrintBuildbotStepText('(From scratch)') elif self._run.options.buildbot: lkgm_manager.GenerateBlameList(self.repo, old_filename) @@ -427,7 +427,7 @@ class SyncStage(generic_stages.BuilderStage): if self._run.config.build_before_patching: pre_build_passed = self.RunPrePatchBuild() if not pre_build_passed: - cros_build_lib.PrintBuildbotStepText('Pre-patch build failed.') + logging.PrintBuildbotStepText('Pre-patch build failed.') class LKGMSyncStage(SyncStage): @@ -481,7 +481,7 @@ class ManifestVersionedSyncStage(SyncStage): def ForceVersion(self, version): """Creates a manifest manager from given version and returns manifest.""" - cros_build_lib.PrintBuildbotStepText(version) + logging.PrintBuildbotStepText(version) return self.manifest_manager.BootstrapFromVersion(version) def VersionIncrementType(self): @@ -557,13 +557,13 @@ class ManifestVersionedSyncStage(SyncStage): # Print the Blamelist here. url_prefix = 'http://chromeos-images.corp.google.com/diff/report?' url = url_prefix + 'from=%s&to=%s' % (previous_version, target_version) - cros_build_lib.PrintBuildbotLink('Blamelist', url) + logging.PrintBuildbotLink('Blamelist', url) # The testManifestVersionedSyncOnePartBranch interacts badly with this # function. It doesn't fully initialize self.manifest_manager which # causes target_version to be None. Since there isn't a clean fix in # either direction, just throw this through str(). In the normal case, # it's already a string anyways. - cros_build_lib.PrintBuildbotStepText(str(target_version)) + logging.PrintBuildbotStepText(str(target_version)) return to_return @@ -962,10 +962,9 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): if self._run.config.build_before_patching: assert not self._run.config.master pre_build_passed = self.RunPrePatchBuild() - cros_build_lib.PrintBuildbotStepName( - 'CommitQueueSync : Apply Patches') + logging.PrintBuildbotStepName('CommitQueueSync : Apply Patches') if not pre_build_passed: - cros_build_lib.PrintBuildbotStepText('Pre-patch build failed.') + logging.PrintBuildbotStepText('Pre-patch build failed.') # Make sure the chroot version is valid. lkgm_version = self._GetLGKMVersionFromManifest(next_manifest) @@ -1102,7 +1101,7 @@ class PreCQLauncherStage(SyncStage): os.path.basename(patch.project), str(patch), ) - cros_build_lib.PrintBuildbotLink(' | '.join(items), patch.url) + logging.PrintBuildbotLink(' | '.join(items), patch.url) def _ConfiguredVerificationsForChange(self, change): """Determine which configs to test |change| with. -- cgit v1.2.3 From ace2b361a4b590c85a6bcf1c31650dffad055141 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Tue, 21 Jul 2015 15:57:42 -0700 Subject: cbuildbot: Remove --pre-cq. This option didn't really do what it said it did, and it's obsoleted by the precq build config, so just get rid of it. BUG=chromium:434413 TEST=Unittests (and, well, can it pass the PreCQ). Change-Id: I4b7338f31642a3c072a097ac04551cf791e16886 Reviewed-on: https://chromium-review.googlesource.com/287406 Tested-by: Don Garrett Reviewed-by: Aviv Keshet Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 3d93be851..487f8c437 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -237,7 +237,7 @@ class BootstrapStage(PatchChangesStage): def HandleApplyFailures(self, failures): """Handle the case where patches fail to apply.""" - if self._run.options.pre_cq or self._run.config.pre_cq: + if self._run.config.pre_cq: # Let the PreCQSync stage handle this failure. The PreCQSync stage will # comment on CLs with the appropriate message when they fail to apply. # -- cgit v1.2.3 From a3efc8b04929bf5aa8198189ef583773b68e231c Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Fri, 17 Jul 2015 17:04:34 -0700 Subject: cbuildbot_run: Use BUILDBOT_BUILDBOTURL to find waterfall URL. We have been finding the waterfall name using BUILDBOT_MASTERNAME, then using a hard coded map to find the URL for that waterfall. However, we have been receiving BUILDBOT_BUILDBOTURL as well, which contains the needed URL. Start using that value, so we don't break when running on a new site specific waterfall. However, there is code in sync_stages.PreCQLauncherStage. ProcessChanges which converts a waterfall name from CIDB into a URL. Leave that based on the constant for now, but we'll have to create a configuration value for this before we can support PreCQ for site specific builds. BUG=chromium:492908 TEST=Unittests Change-Id: Ie9a648f67d792a576d0bf62c53252de2d2a8fa28 Reviewed-on: https://chromium-review.googlesource.com/286574 Reviewed-by: Don Garrett Tested-by: Don Garrett Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 487f8c437..59adcd74e 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1480,8 +1480,9 @@ class PreCQLauncherStage(SyncStage): build_dicts = db.GetBuildStatuses(build_ids) urls = [] for b in build_dicts: + waterfall_url = constants.WATERFALL_TO_DASHBOARD[b['waterfall']] urls.append(tree_status.ConstructDashboardURL( - b['waterfall'], b['builder_name'], b['build_number'])) + waterfall_url, b['builder_name'], b['build_number'])) # Send notifications. pool.HandleApplySuccess(change, build_log='\n'.join(urls)) -- cgit v1.2.3 From 61591c61812bfabb91c05c5f49ba5f13cb626dbc Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 23 Jul 2015 04:10:06 +0000 Subject: Revert "cbuildbot_run: Use BUILDBOT_BUILDBOTURL to find waterfall URL." This reverts commit a3efc8b04929bf5aa8198189ef583773b68e231c. Change-Id: I4b1e42ec8212c7c6f0d62e03144d7cd0ae150221 Reviewed-on: https://chromium-review.googlesource.com/287761 Reviewed-by: Don Garrett Commit-Queue: Don Garrett Tested-by: Don Garrett --- cbuildbot/stages/sync_stages.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 59adcd74e..487f8c437 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1480,9 +1480,8 @@ class PreCQLauncherStage(SyncStage): build_dicts = db.GetBuildStatuses(build_ids) urls = [] for b in build_dicts: - waterfall_url = constants.WATERFALL_TO_DASHBOARD[b['waterfall']] urls.append(tree_status.ConstructDashboardURL( - waterfall_url, b['builder_name'], b['build_number'])) + b['waterfall'], b['builder_name'], b['build_number'])) # Send notifications. pool.HandleApplySuccess(change, build_log='\n'.join(urls)) -- cgit v1.2.3 From bda572e1a1ce8dd525baab6b8c1d1334e7cafaeb Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 23 Jul 2015 04:30:42 +0000 Subject: Revert "cbuildbot: Remove --pre-cq." This reverts commit ace2b361a4b590c85a6bcf1c31650dffad055141. Change-Id: Idbf9c73937e628d360a79468d80b8f77eecf4bb6 Reviewed-on: https://chromium-review.googlesource.com/287663 Reviewed-by: Don Garrett Commit-Queue: Don Garrett Tested-by: Don Garrett --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 487f8c437..3d93be851 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -237,7 +237,7 @@ class BootstrapStage(PatchChangesStage): def HandleApplyFailures(self, failures): """Handle the case where patches fail to apply.""" - if self._run.config.pre_cq: + if self._run.options.pre_cq or self._run.config.pre_cq: # Let the PreCQSync stage handle this failure. The PreCQSync stage will # comment on CLs with the appropriate message when they fail to apply. # -- cgit v1.2.3 From 8385ac3ef5c3b7fccade3a61ae890af4cd0259f8 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 23 Jul 2015 19:48:04 +0000 Subject: Revert "Revert "cbuildbot_run: Use BUILDBOT_BUILDBOTURL to find waterfall URL."" Reland this reverted CL, now that it's been cleared of the failure. https://chromium-review.googlesource.com/#/c/286574/ This reverts commit 61591c61812bfabb91c05c5f49ba5f13cb626dbc. Change-Id: Ifb3e347e5dbd209d37345377d9a2b1208dd2ccfa Reviewed-on: https://chromium-review.googlesource.com/287861 Reviewed-by: Don Garrett Commit-Queue: Don Garrett Tested-by: Don Garrett --- cbuildbot/stages/sync_stages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 3d93be851..e5c45e87b 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1480,8 +1480,9 @@ class PreCQLauncherStage(SyncStage): build_dicts = db.GetBuildStatuses(build_ids) urls = [] for b in build_dicts: + waterfall_url = constants.WATERFALL_TO_DASHBOARD[b['waterfall']] urls.append(tree_status.ConstructDashboardURL( - b['waterfall'], b['builder_name'], b['build_number'])) + waterfall_url, b['builder_name'], b['build_number'])) # Send notifications. pool.HandleApplySuccess(change, build_log='\n'.join(urls)) -- cgit v1.2.3 From 0bc8567add6f999f2f6482ebb1a4e81a466cd9e0 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 23 Jul 2015 19:46:00 +0000 Subject: Revert "Revert "cbuildbot: Remove --pre-cq."" Reland this CL, now that it's been shown to have not been the cause. This reverts commit bda572e1a1ce8dd525baab6b8c1d1334e7cafaeb. Change-Id: I6e969061e1a7eea5241ae76a0fd25566c19cccef Reviewed-on: https://chromium-review.googlesource.com/287870 Reviewed-by: Don Garrett Commit-Queue: Don Garrett Tested-by: Don Garrett --- cbuildbot/stages/sync_stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index e5c45e87b..59adcd74e 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -237,7 +237,7 @@ class BootstrapStage(PatchChangesStage): def HandleApplyFailures(self, failures): """Handle the case where patches fail to apply.""" - if self._run.options.pre_cq or self._run.config.pre_cq: + if self._run.config.pre_cq: # Let the PreCQSync stage handle this failure. The PreCQSync stage will # comment on CLs with the appropriate message when they fail to apply. # -- cgit v1.2.3 From f6b983d220d4e59309ae1bf41a7269da17dd346c Mon Sep 17 00:00:00 2001 From: Shuqian Zhao Date: Wed, 29 Jul 2015 22:42:01 -0700 Subject: cbuildbot: Add function to record changes to metadata in sync stage Prior to this, the changes are recorded in the report stage at the end of each build run. It is useful to record the changes right after the sync stage, then the following stages can use these data. BUG=None TEST=Test with unittest and tryjobs. Change-Id: I80d270c476c99730145b6022c68e787d4e17c52f Reviewed-on: https://chromium-review.googlesource.com/288195 Reviewed-by: Aviv Keshet Tested-by: Shuqian Zhao Trybot-Ready: Shuqian Zhao Commit-Queue: Shuqian Zhao --- cbuildbot/stages/sync_stages.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 59adcd74e..e16079dd6 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -398,6 +398,20 @@ class SyncStage(generic_stages.BuilderStage): return True + def WriteChangesToMetadata(self, changes): + """Write the changes under test into the metadata. + + Args: + changes: A list of GerritPatch instances. + """ + changes_list = self._run.attrs.metadata.GetDict().get('changes', []) + changes_list = changes_list + [c.GetAttributeDict() for c in set(changes)] + changes_list = sorted(changes_list, + key=lambda x: (x[cros_patch.ATTR_GERRIT_NUMBER], + x[cros_patch.ATTR_PATCH_NUMBER], + x[cros_patch.ATTR_REMOTE])) + self._run.attrs.metadata.UpdateWithDict({'changes': changes_list}) + @failures_lib.SetFailureType(failures_lib.InfrastructureFailure) def PerformStage(self): self.Initialize() @@ -1003,6 +1017,8 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): else: ManifestVersionedSyncStage.PerformStage(self) + self.WriteChangesToMetadata(self.pool.changes) + class PreCQSyncStage(SyncStage): """Sync and apply patches to test if they compile.""" @@ -1041,6 +1057,8 @@ class PreCQSyncStage(SyncStage): if len(self.pool.changes) == 0 and self.patches: cros_build_lib.Die('No changes have been applied.') + changes = self.pool.changes or self.patches + self.WriteChangesToMetadata(changes) class PreCQLauncherStage(SyncStage): """Scans for CLs and automatically launches Pre-CQ jobs to test them.""" -- cgit v1.2.3 From 19453d8220d3e686f934bfb3fff9753c86d43136 Mon Sep 17 00:00:00 2001 From: Matthew Sartori Date: Mon, 27 Jul 2015 15:16:31 -0700 Subject: cbuildbot: Add Manifest constants to SiteConfig SiteConfig objects now have the manifest and manifest manipulation constants defined in their SiteParameters. Some changes were also made to the default settings for a BuildConfig. Some parameters, such as MANIFEST_URL, that were site specific as well as build specific have been set to a default None value in config_lib and set appropriately in chromeos_config. Also deleted several that were previously separated but which the manifest constants depended on. BUG=chromium:492930,chromium:492931,chromium:513418 TEST=Unittests. Change-Id: I89c19e1cfc8ce6164bbe74972ae010608e6aa294 Reviewed-on: https://chromium-review.googlesource.com/288815 Tested-by: Matthew Sartori Reviewed-by: Matthew Sartori Commit-Queue: Matthew Sartori --- cbuildbot/stages/sync_stages.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index e16079dd6..d3f496036 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -19,8 +19,9 @@ from xml.etree import ElementTree from xml.dom import minidom from chromite.cbuildbot import chroot_lib -from chromite.cbuildbot import failures_lib +from chromite.cbuildbot import config_lib from chromite.cbuildbot import constants +from chromite.cbuildbot import failures_lib from chromite.cbuildbot import lkgm_manager from chromite.cbuildbot import manifest_version from chromite.cbuildbot import repository @@ -42,6 +43,9 @@ from chromite.lib import timeout_util from chromite.scripts import cros_mark_chrome_as_stable +site_config = config_lib.GetConfig() + + PRE_CQ = validation_pool.PRE_CQ PRECQ_LAUNCH_TIMEOUT_MSG = ( @@ -340,14 +344,14 @@ class SyncStage(generic_stages.BuilderStage): if internal: if test: - return constants.MANIFEST_VERSIONS_INT_GOB_URL_TEST + return site_config.params.MANIFEST_VERSIONS_INT_GOB_URL_TEST else: - return constants.MANIFEST_VERSIONS_INT_GOB_URL + return site_config.params.MANIFEST_VERSIONS_INT_GOB_URL else: if test: - return constants.MANIFEST_VERSIONS_GOB_URL_TEST + return site_config.params.MANIFEST_VERSIONS_GOB_URL_TEST else: - return constants.MANIFEST_VERSIONS_GOB_URL + return site_config.params.MANIFEST_VERSIONS_GOB_URL def Initialize(self): self._InitializeRepo() @@ -453,9 +457,9 @@ class LKGMSyncStage(SyncStage): """Override: Gets the LKGM.""" # TODO(sosa): Should really use an initialized manager here. if self.internal: - mv_dir = constants.INTERNAL_MANIFEST_VERSIONS_PATH + mv_dir = site_config.params.INTERNAL_MANIFEST_VERSIONS_PATH else: - mv_dir = constants.EXTERNAL_MANIFEST_VERSIONS_PATH + mv_dir = site_config.params.EXTERNAL_MANIFEST_VERSIONS_PATH manifest_path = os.path.join(self._build_root, mv_dir) manifest_repo = self._GetManifestVersionsRepoUrl() @@ -597,7 +601,7 @@ class ManifestVersionedSyncStage(SyncStage): root = doc.getroot() for node in root.findall('project'): remote = node.attrib.get('remote') - if remote and remote not in constants.GIT_REMOTES: + if remote and remote not in site_config.params.GIT_REMOTES: root.remove(node) doc.write(filtered_manifest) yield filtered_manifest -- cgit v1.2.3 From ea88dac901a31137e4e1afef7fcee62b772d781f Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Tue, 28 Jul 2015 18:49:41 -0700 Subject: BootstrapStage: Centralize knowledge of bootstrap patching. Right now, both the GenericBuilder, and the BootstrapStage have detailed knowledge of which patches need to be applied during the bootstrap process. In prep for patching site config repositories, move this knowledge into a single location. BUG=chromium:508601 TEST=lint + unittests Change-Id: Ieb747824107fc3a3ac0659848cf2705af55cfdc7 Reviewed-on: https://chromium-review.googlesource.com/289341 Trybot-Ready: Don Garrett Tested-by: Don Garrett Reviewed-by: Matthew Sartori Commit-Queue: Don Garrett --- cbuildbot/stages/sync_stages.py | 56 ++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 17 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index d3f496036..98ad07144 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -168,12 +168,11 @@ class BootstrapStage(PatchChangesStage): """ option_name = 'bootstrap' - def __init__(self, builder_run, chromite_patch_pool, - manifest_patch_pool, **kwargs): + def __init__(self, builder_run, patch_pool, **kwargs): super(BootstrapStage, self).__init__( builder_run, trybot_patch_pool.TrybotPatchPool(), **kwargs) - self.chromite_patch_pool = chromite_patch_pool - self.manifest_patch_pool = manifest_patch_pool + + self.patch_pool = patch_pool self.config_repo = self._run.options.config_repo self.returncode = None self.tempdir = None @@ -239,6 +238,27 @@ class BootstrapStage(PatchChangesStage): return args + @classmethod + def BootstrapPatchesNeeded(cls, builder_run, patch_pool): + """See if bootstrapping is needed for any of the given patches. + + Does NOT determine if they have already been applied. + + Args: + builder_run: BuilderRun object for this build. + patch_pool: All patches to be applied this run. + + Returns: + boolean True if bootstrapping is needed. + """ + chromite_pool = patch_pool.Filter(project=constants.CHROMITE_PROJECT) + if builder_run.config.internal: + manifest_pool = patch_pool.FilterIntManifest() + else: + manifest_pool = patch_pool.FilterExtManifest() + + return bool(chromite_pool or manifest_pool) + def HandleApplyFailures(self, failures): """Handle the case where patches fail to apply.""" if self._run.config.pre_cq: @@ -262,6 +282,9 @@ class BootstrapStage(PatchChangesStage): if self._run.options.test_bootstrap: filter_branch = 'master' + # Filter all requested patches for the branch. + branch_pool = self.patch_pool.FilterBranch(filter_branch) + # Checkout the new version of chromite, and patch it. chromite_dir = os.path.join(self.tempdir, 'chromite') reference_repo = os.path.join(constants.CHROMITE_DIR, '.git') @@ -269,17 +292,11 @@ class BootstrapStage(PatchChangesStage): reference=reference_repo) git.RunGit(chromite_dir, ['checkout', filter_branch]) - def BranchAndChromiteFilter(patch): - return (trybot_patch_pool.BranchFilter(filter_branch, patch) and - trybot_patch_pool.ChromiteFilter(patch)) - - patch_series = validation_pool.PatchSeries.WorkOnSingleRepo( - chromite_dir, filter_branch, - deps_filter_fn=BranchAndChromiteFilter) - - filtered_pool = self.chromite_patch_pool.FilterBranch(filter_branch) - if filtered_pool: - self._ApplyPatchSeries(patch_series, filtered_pool) + chromite_pool = branch_pool.Filter(project=constants.CHROMITE_PROJECT) + if chromite_pool: + patch_series = validation_pool.PatchSeries.WorkOnSingleRepo( + chromite_dir, filter_branch) + self._ApplyPatchSeries(patch_series, chromite_pool) # Checkout the new version of site config (no patching logic, yet). if self.config_repo: @@ -307,8 +324,13 @@ class BootstrapStage(PatchChangesStage): # bootstrapping for the next execution. Also pass in the patched manifest # repository. extra_params.append('--nobootstrap') - if self.manifest_patch_pool: - manifest_dir = self._ApplyManifestPatches(self.manifest_patch_pool) + if self._run.config.internal: + manifest_pool = branch_pool.FilterIntManifest() + else: + manifest_pool = branch_pool.FilterExtManifest() + + if manifest_pool: + manifest_dir = self._ApplyManifestPatches(manifest_pool) extra_params.extend(['--manifest-repo-url', manifest_dir]) cmd += extra_params -- cgit v1.2.3 From e480b1f3329eb4fe896d99011cb32737dbcc5c91 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Tue, 4 Aug 2015 11:03:01 -0700 Subject: sync_stages: include build config in pre-cq inflight message Our pre-cq jobs are now built by generic builders. This means it is no longer possible to tell at a glance by looking at your CL on Gerrit which configs have picked it up for testing. Add build config information to the in-flight comment to restore this information. BUG=None TEST=None Change-Id: I8df0541e5f8216a16af00a0dcfa7498c74ad10fd Reviewed-on: https://chromium-review.googlesource.com/290418 Trybot-Ready: Aviv Keshet Tested-by: Aviv Keshet Reviewed-by: David James --- cbuildbot/stages/sync_stages.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 98ad07144..0e2b9916a 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -1522,14 +1522,15 @@ class PreCQLauncherStage(SyncStage): # Change the status to inflight. self.UpdateChangeStatuses([change], constants.CL_STATUS_INFLIGHT) build_dicts = db.GetBuildStatuses(build_ids) - urls = [] + lines = [] for b in build_dicts: waterfall_url = constants.WATERFALL_TO_DASHBOARD[b['waterfall']] - urls.append(tree_status.ConstructDashboardURL( - waterfall_url, b['builder_name'], b['build_number'])) + url = tree_status.ConstructDashboardURL( + waterfall_url, b['builder_name'], b['build_number']) + lines.append('(%s) : %s' % (b['build_config'], url)) # Send notifications. - pool.HandleApplySuccess(change, build_log='\n'.join(urls)) + pool.HandleApplySuccess(change, build_log=('\n' + '\n'.join(lines))) for change in to_process: # Detect if change is ready to be marked as passed, or ready to submit. -- cgit v1.2.3 From fdd8aaacf2862d7417af1e9c99311aeb1e9314ba Mon Sep 17 00:00:00 2001 From: Matthew Sartori Date: Thu, 6 Aug 2015 18:32:22 -0700 Subject: cbuildbot: Site config -g patching This CL allows cbuildbot to patch a site config repo checked out to chromite/config during the bootstrap process. This CL supports -g patching. BUG=chromium:508601 TEST=Created a dummy branch in abuildbot_config with a trivial change and uploaded it to gerrit. Tested that the change was applied during bootstrap when using the -g option. Change-Id: I7e103eb7b12d20acb94fa1d0a328a0cd769287ea Reviewed-on: https://chromium-review.googlesource.com/291461 Reviewed-by: Matthew Sartori Tested-by: Matthew Sartori Commit-Queue: Matthew Sartori --- cbuildbot/stages/sync_stages.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index 0e2b9916a..fb9ab9f93 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -307,6 +307,12 @@ class BootstrapStage(PatchChangesStage): reference=site_config_reference_repo) git.RunGit(site_config_dir, ['checkout', filter_branch]) + site_config_pool = branch_pool.FilterGitRemoteUrl(self.config_repo) + if site_config_pool: + site_patch_series = validation_pool.PatchSeries.WorkOnSingleRepo( + site_config_dir, filter_branch) + self._ApplyPatchSeries(site_patch_series, site_config_pool) + # Re-exec into new instance of cbuildbot, with proper command line args. cbuildbot_path = constants.PATH_TO_CBUILDBOT if not os.path.exists(os.path.join(self.tempdir, cbuildbot_path)): -- cgit v1.2.3 From 15386a8075a4d6099a896ca7a0b718d29740b736 Mon Sep 17 00:00:00 2001 From: Aviv Keshet Date: Thu, 3 Sep 2015 17:06:45 -0700 Subject: sync_stages: better logging of where chrome_version comes from BUG=None TEST=None Change-Id: Ib06a74abcd05c6cba6d991e3b4dd3b4d26e986fc Reviewed-on: https://chromium-review.googlesource.com/297547 Reviewed-by: Don Garrett Tested-by: Aviv Keshet --- cbuildbot/stages/sync_stages.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'cbuildbot/stages/sync_stages.py') diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py index fb9ab9f93..dc2df54d3 100644 --- a/cbuildbot/stages/sync_stages.py +++ b/cbuildbot/stages/sync_stages.py @@ -801,6 +801,8 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): assert self._run.config.master build_id = self._run.attrs.metadata.GetDict().get('build_id') + logging.info('Creating new candidate manifest, including chrome version ' + '%s.', self._chrome_version) manifest = self.manifest_manager.CreateNewCandidate( chrome_version=self._chrome_version, build_id=build_id) @@ -855,7 +857,10 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage): self._run.config.master): # PFQ master needs to determine what version of Chrome to build # for all slaves. + logging.info('I am a master running with CHROME_REV_LATEST, ' + 'therefore getting latest chrome version.') self._chrome_version = self.GetLatestChromeVersion() + logging.info('Latest chrome version is: %s', self._chrome_version) ManifestVersionedSyncStage.PerformStage(self) @@ -995,6 +1000,7 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage): constants.MASTER_BUILD_TIMEOUT_DEFAULT_SECONDS) db.ExtendDeadline(build_id, timeout) + logging.info('Creating new candidate manifest.') manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool, build_id=build_id) if MasterSlaveLKGMSyncStage.sub_manager: -- cgit v1.2.3