Revert "Use CIDB for synchronizing between CQ and slaves."

This reverts commit f6b20ba26c87bcbf1fff5872e583470842cd1e7d.

Change-Id: I4945781a40d940ed18d90d821dd34c2f14b35b09
Reviewed-on: https://chromium-review.googlesource.com/253931
Reviewed-by: David James <davidjames@chromium.org>
Reviewed-by: Josafat Garcia <josafat@chromium.org>
Commit-Queue: Josafat Garcia <josafat@chromium.org>
Tested-by: Josafat Garcia <josafat@chromium.org>

4 files changed, 207 insertions, 120 deletions

diff --git a/cbuildbot/lkgm_manager.py b/cbuildbot/lkgm_manager.py
index c3e1a7487..0c581b8b2 100644
--- a/cbuildbot/lkgm_manager.py
+++ b/cbuildbot/lkgm_manager.py
@@ -16,6 +16,7 @@ from chromite.cbuildbot import constants
 from chromite.cbuildbot import manifest_version
 from chromite.lib import cros_build_lib
 from chromite.lib import git
+from chromite.lib import timeout_util
 
 
 # Paladin constants for manifest names.
@@ -372,6 +373,60 @@ class LKGMManager(manifest_version.BuildSpecsManager):
 
     raise manifest_version.GenerateBuildSpecException(last_error)
 
+  def GetLatestCandidate(self, timeout=10 * 60):
+    """Gets and syncs to the next candiate manifest.
+
+    Args:
+      timeout: The timeout in seconds.
+
+    Returns:
+      Local path to manifest to build or None in case of no need to build.
+
+    Raises:
+      GenerateBuildSpecException in case of failure to generate a buildspec
+    """
+    def _AttemptToGetLatestCandidate():
+      """Attempts to acquire latest candidate using manifest repo."""
+      self.RefreshManifestCheckout()
+      self.InitializeManifestVariables(self.GetCurrentVersionInfo())
+      if self.latest_unprocessed:
+        return self.latest_unprocessed
+      elif self.dry_run and self.latest:
+        return self.latest
+
+    def _PrintRemainingTime(remaining):
+      logging.info('Found nothing new to build, will keep trying for %s',
+                   remaining)
+      logging.info('If this is a PFQ, then you should have forced the master'
+                   ', which runs cbuildbot_master')
+
+    # TODO(sosa):  We only really need the overlay for the version info but we
+    # do a full checkout here because we have no way of refining it currently.
+    self.CheckoutSourceCode()
+    try:
+      version_to_build = timeout_util.WaitForSuccess(
+          lambda x: x is None,
+          _AttemptToGetLatestCandidate,
+          timeout,
+          period=self.SLEEP_TIMEOUT,
+          fallback_timeout=max(10, timeout),
+          side_effect_func=_PrintRemainingTime)
+    except timeout_util.TimeoutError:
+      _PrintRemainingTime(0)
+      version_to_build = _AttemptToGetLatestCandidate()
+
+    if version_to_build:
+      logging.info('Starting build spec: %s', version_to_build)
+      self.current_version = version_to_build
+
+      # Actually perform the sync.
+      manifest = self.GetLocalManifest(version_to_build)
+      self.cros_source.Sync(manifest)
+      self._GenerateBlameListSinceLKGM()
+      return manifest
+    else:
+      return None
+
   def PromoteCandidate(self, retries=manifest_version.NUM_RETRIES):
     """Promotes the current LKGM candidate to be a real versioned LKGM."""
     assert self.current_version, 'No current manifest exists.'
diff --git a/cbuildbot/lkgm_manager_unittest.py b/cbuildbot/lkgm_manager_unittest.py
index cf076be71..d591d1906 100644
--- a/cbuildbot/lkgm_manager_unittest.py
+++ b/cbuildbot/lkgm_manager_unittest.py
@@ -263,6 +263,87 @@ class LKGMManagerTest(cros_test_lib.MoxTempDirTestCase):
     self.assertEqual(candidate, None)
     self.mox.VerifyAll()
 
+  def testGetLatestCandidate(self):
+    """Makes sure we can get the latest created candidate manifest."""
+    self.mox.StubOutWithMock(repository.RepoRepository, 'Sync')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'GetCurrentVersionInfo')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
+                             'RefreshManifestCheckout')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
+                             'InitializeManifestVariables')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'CheckoutSourceCode')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'PushSpecChanges')
+
+    my_info = lkgm_manager._LKGMCandidateInfo('1.2.3')
+    most_recent_candidate = lkgm_manager._LKGMCandidateInfo('1.2.3-rc12')
+
+    # Do manifest refresh work.
+    lkgm_manager.LKGMManager.CheckoutSourceCode()
+    lkgm_manager.LKGMManager.RefreshManifestCheckout()
+    lkgm_manager.LKGMManager.GetCurrentVersionInfo().AndReturn(my_info)
+    lkgm_manager.LKGMManager.InitializeManifestVariables(my_info)
+
+    repository.RepoRepository.Sync(
+        self._GetPathToManifest(most_recent_candidate))
+
+    self.manager.latest_unprocessed = '1.2.3-rc12'
+    self.mox.ReplayAll()
+    candidate = self.manager.GetLatestCandidate()
+    self.assertEqual(candidate, self._GetPathToManifest(most_recent_candidate))
+    self.mox.VerifyAll()
+
+  def testGetLatestCandidateOneRetry(self):
+    """Makes sure we can get the latest candidate even on retry."""
+    self.mox.StubOutWithMock(repository.RepoRepository, 'Sync')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'GetCurrentVersionInfo')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
+                             'RefreshManifestCheckout')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
+                             'InitializeManifestVariables')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'CheckoutSourceCode')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'PushSpecChanges')
+
+    my_info = lkgm_manager._LKGMCandidateInfo('1.2.4')
+    most_recent_candidate = lkgm_manager._LKGMCandidateInfo('1.2.4-rc12',
+                                                            CHROME_BRANCH)
+
+    lkgm_manager.LKGMManager.CheckoutSourceCode()
+    lkgm_manager.LKGMManager.RefreshManifestCheckout()
+    lkgm_manager.LKGMManager.GetCurrentVersionInfo().AndReturn(my_info)
+    lkgm_manager.LKGMManager.InitializeManifestVariables(my_info)
+
+    repository.RepoRepository.Sync(
+        self._GetPathToManifest(most_recent_candidate))
+
+    self.manager.latest_unprocessed = '1.2.4-rc12'
+    self.mox.ReplayAll()
+    candidate = self.manager.GetLatestCandidate()
+    self.assertEqual(candidate, self._GetPathToManifest(most_recent_candidate))
+    self.mox.VerifyAll()
+
+  def testGetLatestCandidateNone(self):
+    """Makes sure we get nothing if there is no work to be done."""
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'GetCurrentVersionInfo')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
+                             'RefreshManifestCheckout')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
+                             'InitializeManifestVariables')
+    self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'CheckoutSourceCode')
+
+    my_info = lkgm_manager._LKGMCandidateInfo('1.2.4')
+    lkgm_manager.LKGMManager.CheckoutSourceCode()
+    for _ in range(2):
+      lkgm_manager.LKGMManager.RefreshManifestCheckout()
+      lkgm_manager.LKGMManager.GetCurrentVersionInfo().AndReturn(my_info)
+      lkgm_manager.LKGMManager.InitializeManifestVariables(my_info)
+
+    self.mox.ReplayAll()
+    self.manager.SLEEP_TIMEOUT = 0.2
+    # Only run once.
+    candidate = self.manager.GetLatestCandidate(timeout=0.1)
+    self.assertEqual(candidate, None)
+    self.mox.VerifyAll()
+
   def _CreateManifest(self):
     """Returns a created test manifest in tmpdir with its dir_pfx."""
     self.manager.current_version = '1.2.4-rc21'
diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py
index aaf748a01..6f5f988f1 100644
--- a/cbuildbot/stages/sync_stages.py
+++ b/cbuildbot/stages/sync_stages.py
@@ -626,42 +626,13 @@ class ManifestVersionedSyncStage(SyncStage):
                  os.path.basename(sdk_manifest_path))
     logging.info('%s', osutils.ReadFile(manifest))
 
-  def _GetMasterVersion(self):
-    """Get the platform version associated with the master_build_id."""
-    master_id = self._run.options.master_build_id
-    _, db = self._run.GetCIDBHandle()
-    if db and master_id:
-      master_build_status = db.GetBuildStatus(master_id)
-      return master_build_status['platform_version']
-
-  def _VerifyMasterId(self):
-    """Verify that our master id is current and valid."""
-    master_id = self._run.options.master_build_id
-    _, db = self._run.GetCIDBHandle()
-    if db and master_id:
-      assert not self._run.options.force_version
-      master_build_status = db.GetBuildStatus(master_id)
-      latest = db.GetBuildHistory(master_build_status['build_config'], 1)
-      if latest and latest[0]['id'] != master_id:
-        raise failures_lib.MasterSlaveVersionMismatchFailure(
-            'This slave\'s master (id=%s) has been supplanted by a newer '
-            'master (id=%s). Aborting.' % (master_id, latest[0]['id']))
-
   @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
   def PerformStage(self):
     self.Initialize()
-
-    self._VerifyMasterId()
-    version = self._run.options.force_version or self._GetMasterVersion()
-    next_manifest = None
-    if version:
-      next_manifest = self.ForceVersion(version)
+    if self._run.options.force_version:
+      next_manifest = self.ForceVersion(self._run.options.force_version)
     else:
-      self.skip_sync = True
-      try:
-        next_manifest = self.GetNextManifest()
-      except validation_pool.TreeIsClosedException as e:
-        cros_build_lib.Warning(str(e))
+      next_manifest = self.GetNextManifest()
 
     if not next_manifest:
       cros_build_lib.Info('Found no work to do.')
@@ -718,6 +689,9 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage):
   candidates and their states.
   """
 
+  # Timeout for waiting on the latest candidate manifest.
+  LATEST_CANDIDATE_TIMEOUT_SECONDS = 20 * 60
+
   # TODO(mtennant): Turn this into self._run.attrs.sub_manager or similar.
   # An instance of lkgm_manager.LKGMManager for slave builds.
   sub_manager = None
@@ -726,6 +700,7 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage):
     super(MasterSlaveLKGMSyncStage, self).__init__(builder_run, **kwargs)
     # lkgm_manager deals with making sure we're synced to whatever manifest
     # we get back in GetNextManifest so syncing again is redundant.
+    self.skip_sync = True
     self._chrome_version = None
 
   def _GetInitializedManager(self, internal):
@@ -766,31 +741,24 @@ class MasterSlaveLKGMSyncStage(ManifestVersionedSyncStage):
 
     return manifest
 
-  def _VerifyMasterId(self):
-    """Verify that our master id is current and valid."""
-    super(MasterSlaveLKGMSyncStage, self)._VerifyMasterId()
-    master_id = self._run.options.master_build_id
-    if not self._run.config.master and not master_id:
-      raise failures_lib.StepFailure(
-          'Cannot start build without a master_build_id. Did you hit force '
-          'build on a slave? Please hit force build on the master instead.')
-
   def GetNextManifest(self):
     """Gets the next manifest using LKGM logic."""
     assert self.manifest_manager, \
         'Must run Initialize before we can get a manifest.'
     assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \
         'Manifest manager instantiated with wrong class.'
-    assert self._run.config.master
-
-    build_id = self._run.attrs.metadata.GetDict().get('build_id')
-    manifest = self.manifest_manager.CreateNewCandidate(
-        chrome_version=self._chrome_version,
-        build_id=build_id)
-    if MasterSlaveLKGMSyncStage.sub_manager:
-      MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(manifest)
 
-    return manifest
+    if self._run.config.master:
+      build_id = self._run.attrs.metadata.GetDict().get('build_id')
+      manifest = self.manifest_manager.CreateNewCandidate(
+          chrome_version=self._chrome_version,
+          build_id=build_id)
+      if MasterSlaveLKGMSyncStage.sub_manager:
+        MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(manifest)
+      return manifest
+    else:
+      return self.manifest_manager.GetLatestCandidate(
+          timeout=self.LATEST_CANDIDATE_TIMEOUT_SECONDS)
 
   def GetLatestChromeVersion(self):
     """Returns the version of Chrome to uprev."""
@@ -875,8 +843,9 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage):
 
   def _SetPoolFromManifest(self, manifest):
     """Sets validation pool based on manifest path passed in."""
-    # Note that this function is only called after the repo is already
-    # sync'd, so AcquirePoolFromManifest does not need to sync.
+    # Note that GetNextManifest() calls GetLatestCandidate() in this case,
+    # so the repo will already be sync'd appropriately. This means that
+    # AcquirePoolFromManifest does not need to sync.
     self.pool = validation_pool.ValidationPool.AcquirePoolFromManifest(
         manifest, self._run.config.overlays, self.repo,
         self._run.buildnumber, self._run.GetBuilderName(),
@@ -898,73 +867,69 @@ class CommitQueueSyncStage(MasterSlaveLKGMSyncStage):
         'Must run Initialize before we can get a manifest.'
     assert isinstance(self.manifest_manager, lkgm_manager.LKGMManager), \
         'Manifest manager instantiated with wrong class.'
-    assert self._run.config.master
 
     build_id = self._run.attrs.metadata.GetDict().get('build_id')
 
-    # In order to acquire a pool, we need an initialized buildroot.
-    if not git.FindRepoDir(self.repo.directory):
-      self.repo.Initialize()
-
-    query = constants.CQ_READY_QUERY
-    if self._run.options.cq_gerrit_override:
-      query = (self._run.options.cq_gerrit_override, None)
-
-    self.pool = pool = validation_pool.ValidationPool.AcquirePool(
-        self._run.config.overlays, self.repo,
-        self._run.buildnumber, self._run.GetBuilderName(),
-        query,
-        dryrun=self._run.options.debug,
-        check_tree_open=(not self._run.options.debug or
-                         self._run.options.mock_tree_status),
-        change_filter=self._ChangeFilter, builder_run=self._run)
+    if self._run.config.master:
+      try:
+        # In order to acquire a pool, we need an initialized buildroot.
+        if not git.FindRepoDir(self.repo.directory):
+          self.repo.Initialize()
+
+        query = constants.CQ_READY_QUERY
+        if self._run.options.cq_gerrit_override:
+          query = (self._run.options.cq_gerrit_override, None)
+
+        self.pool = pool = validation_pool.ValidationPool.AcquirePool(
+            self._run.config.overlays, self.repo,
+            self._run.buildnumber, self._run.GetBuilderName(),
+            query,
+            dryrun=self._run.options.debug,
+            check_tree_open=(not self._run.options.debug or
+                             self._run.options.mock_tree_status),
+            change_filter=self._ChangeFilter, builder_run=self._run)
+      except validation_pool.TreeIsClosedException as e:
+        cros_build_lib.Warning(str(e))
+        return None
 
-    # We must extend the builder deadline before publishing a new manifest to
-    # ensure that slaves have enough time to complete the builds about to
-    # start.
-    build_id, db = self._run.GetCIDBHandle()
-    if db:
-      timeout = constants.MASTER_BUILD_TIMEOUT_SECONDS.get(
-          self._run.config.build_type,
-          constants.MASTER_BUILD_TIMEOUT_DEFAULT_SECONDS)
-      db.ExtendDeadline(build_id, timeout)
-
-    manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool,
-                                                        build_id=build_id)
-    if MasterSlaveLKGMSyncStage.sub_manager:
-      MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(
-          manifest, build_id=build_id)
+      # We must extend the builder deadline before publishing a new manifest to
+      # ensure that slaves have enough time to complete the builds about to
+      # start.
+      build_id, db = self._run.GetCIDBHandle()
+      if db:
+        timeout = constants.MASTER_BUILD_TIMEOUT_SECONDS.get(
+            self._run.config.build_type,
+            constants.MASTER_BUILD_TIMEOUT_DEFAULT_SECONDS)
+        db.ExtendDeadline(build_id, timeout)
 
-    return manifest
+      manifest = self.manifest_manager.CreateNewCandidate(validation_pool=pool,
+                                                          build_id=build_id)
+      if MasterSlaveLKGMSyncStage.sub_manager:
+        MasterSlaveLKGMSyncStage.sub_manager.CreateFromManifest(
+            manifest, build_id=build_id)
 
-  def ManifestCheckout(self, next_manifest):
-    """Checks out the repository to the given manifest."""
-    if self._run.config.build_before_patching:
-      assert not self._run.config.master
-      pre_build_passed = self.RunPrePatchBuild()
-      cros_build_lib.PrintBuildbotStepName(
-          'CommitQueueSync : Apply Patches')
-      if not pre_build_passed:
-        cros_build_lib.PrintBuildbotStepText('Pre-patch build failed.')
+    else:
+      manifest = self.manifest_manager.GetLatestCandidate()
+      if manifest:
+        if self._run.config.build_before_patching:
+          pre_build_passed = self.RunPrePatchBuild()
+          cros_build_lib.PrintBuildbotStepName(
+              'CommitQueueSync : Apply Patches')
+          if not pre_build_passed:
+            cros_build_lib.PrintBuildbotStepText('Pre-patch build failed.')
+
+        self._SetPoolFromManifest(manifest)
+        self.pool.ApplyPoolIntoRepo()
 
     # Make sure the chroot version is valid.
-    lkgm_version = self._GetLGKMVersionFromManifest(next_manifest)
+    lkgm_version = self._GetLGKMVersionFromManifest(manifest)
     chroot_manager = chroot_lib.ChrootManager(self._build_root)
     chroot_manager.EnsureChrootAtVersion(lkgm_version)
 
     # Clear the chroot version as we are in the middle of building it.
     chroot_manager.ClearChrootVersion()
 
-    # Sync to the provided manifest on slaves. On the master, we're
-    # already synced to this manifest, so self.skip_sync is set and
-    # this is a no-op.
-    super(CommitQueueSyncStage, self).ManifestCheckout(next_manifest)
-
-    # On slaves, initialize our pool and apply patches. On the master,
-    # we've already done that in GetNextManifest, so this is a no-op.
-    if not self._run.config.master:
-      self._SetPoolFromManifest(next_manifest)
-      self.pool.ApplyPoolIntoRepo()
+    return manifest
 
   @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
   def PerformStage(self):
diff --git a/cbuildbot/stages/sync_stages_unittest.py b/cbuildbot/stages/sync_stages_unittest.py
index d911bddba..f9249343c 100644
--- a/cbuildbot/stages/sync_stages_unittest.py
+++ b/cbuildbot/stages/sync_stages_unittest.py
@@ -82,14 +82,6 @@ class ManifestVersionedSyncStageTest(generic_stages_unittest.AbstractStageTest):
     self.PatchObject(manifest_version.BuildSpecsManager, 'GetLatestPassingSpec')
     self.PatchObject(sync_stages.SyncStage, 'ManifestCheckout',
                      return_value=self.next_version)
-    self.PatchObject(sync_stages.ManifestVersionedSyncStage,
-                     '_GetMasterVersion', return_value='foo',
-                     autospec=True)
-    self.PatchObject(sync_stages.ManifestVersionedSyncStage,
-                     '_VerifyMasterId', autospec=True)
-    self.PatchObject(manifest_version.BuildSpecsManager, 'BootstrapFromVersion',
-                     autospec=True)
-    self.PatchObject(repository.RepoRepository, 'Sync', autospec=True)
 
     self.sync_stage.Run()
 
@@ -322,14 +314,8 @@ class SlaveCQSyncTest(BaseCQTestCase):
 
   def testReload(self):
     """Test basic ability to sync and reload the patches from disk."""
-    self.PatchObject(sync_stages.ManifestVersionedSyncStage,
-                     '_GetMasterVersion', return_value='foo',
-                     autospec=True)
-    self.PatchObject(sync_stages.MasterSlaveLKGMSyncStage,
-                     '_VerifyMasterId', autospec=True)
-    self.PatchObject(lkgm_manager.LKGMManager, 'BootstrapFromVersion',
+    self.PatchObject(lkgm_manager.LKGMManager, 'GetLatestCandidate',
                      return_value=self.manifest_path, autospec=True)
-    self.PatchObject(repository.RepoRepository, 'Sync', autospec=True)
     self.sync_stage.PerformStage()
     self.ReloadPool()