sync_stages: Start creating Project Sdk manifest.

Create a method to generate Project SDK manifests and to
generate and publish it as part of the Canary master.

BUG=brillo:48
TEST=Unittests

Change-Id: Ie947ab8af5ff20f1a3da30d5366e2c2e221fa823
Reviewed-on: https://chromium-review.googlesource.com/244602
Trybot-Ready: Don Garrett <dgarrett@chromium.org>
Tested-by: Don Garrett <dgarrett@chromium.org>
Reviewed-by: Chris Sosa <sosa@chromium.org>
Commit-Queue: Don Garrett <dgarrett@chromium.org>

7 files changed, 462 insertions, 230 deletions

diff --git a/cbuildbot/constants.py b/cbuildbot/constants.py
index e7c1eb240..fe02c0375 100644
--- a/cbuildbot/constants.py
+++ b/cbuildbot/constants.py
@@ -292,6 +292,8 @@ CHANGE_PREFIX = {
 # List of remotes that are ok to include in the external manifest.
 EXTERNAL_REMOTES = (EXTERNAL_REMOTE, CHROMIUM_REMOTE)
 
+PROJECT_SDK_GROUPS = ('minilayout',)
+
 # Mapping 'remote name' -> regexp that matches names of repositories on that
 # remote that can be branched when creating CrOS branch. Branching script will
 # actually create a new git ref when branching these projects. It won't attempt
diff --git a/cbuildbot/lkgm_manager.py b/cbuildbot/lkgm_manager.py
index 05a6c9722..1e6996279 100644
--- a/cbuildbot/lkgm_manager.py
+++ b/cbuildbot/lkgm_manager.py
@@ -9,7 +9,6 @@ from __future__ import print_function
 import logging
 import os
 import re
-import tempfile
 from xml.dom import minidom
 
 from chromite.cbuildbot import cbuildbot_config
@@ -41,21 +40,11 @@ CHROME_VERSION_ATTR = 'version'
 LKGM_ELEMENT = 'lkgm'
 LKGM_VERSION_ATTR = 'version'
 
-MANIFEST_ELEMENT = 'manifest'
-DEFAULT_ELEMENT = 'default'
-PROJECT_ELEMENT = 'project'
-PROJECT_NAME_ATTR = 'name'
-PROJECT_REMOTE_ATTR = 'remote'
-
 
 class PromoteCandidateException(Exception):
   """Exception thrown for failure to promote manifest candidate."""
 
 
-class FilterManifestException(Exception):
-  """Exception thrown when failing to filter the internal manifest."""
-
-
 class _LKGMCandidateInfo(manifest_version.VersionInfo):
   """Class to encapsualte the chrome os lkgm candidate info
 
@@ -250,76 +239,6 @@ class LKGMManager(manifest_version.BuildSpecsManager):
     with open(manifest, 'w+') as manifest_file:
       manifest_dom.writexml(manifest_file)
 
-  @staticmethod
-  def _GetDefaultRemote(manifest_dom):
-    """Returns the default remote in a manifest (if any).
-
-    Args:
-      manifest_dom: DOM Document object representing the manifest.
-
-    Returns:
-      Default remote if one exists, None otherwise.
-    """
-    default_nodes = manifest_dom.getElementsByTagName(DEFAULT_ELEMENT)
-    if default_nodes:
-      if len(default_nodes) > 1:
-        raise FilterManifestException(
-            'More than one <default> element found in manifest')
-      return default_nodes[0].getAttribute(PROJECT_REMOTE_ATTR)
-    return None
-
-  @staticmethod
-  def _FilterCrosInternalProjectsFromManifest(
-      manifest, whitelisted_remotes=constants.EXTERNAL_REMOTES):
-    """Returns a path to a new manifest with internal repositories stripped.
-
-    Args:
-      manifest: Path to an existing manifest that may have internal
-        repositories.
-      whitelisted_remotes: Tuple of remotes to allow in the external manifest.
-        Only projects with those remotes will be included in the external
-        manifest.
-
-    Returns:
-      Path to a new manifest that is a copy of the original without internal
-        repositories or pending commits.
-    """
-    temp_fd, new_path = tempfile.mkstemp('external_manifest')
-    manifest_dom = minidom.parse(manifest)
-    manifest_node = manifest_dom.getElementsByTagName(MANIFEST_ELEMENT)[0]
-    projects = manifest_dom.getElementsByTagName(PROJECT_ELEMENT)
-    pending_commits = manifest_dom.getElementsByTagName(PALADIN_COMMIT_ELEMENT)
-
-    default_remote = LKGMManager._GetDefaultRemote(manifest_dom)
-    internal_projects = set()
-    for project_element in projects:
-      project_remote = project_element.getAttribute(PROJECT_REMOTE_ATTR)
-      project = project_element.getAttribute(PROJECT_NAME_ATTR)
-      if not project_remote:
-        if not default_remote:
-          # This should not happen for a valid manifest. Either each
-          # project must have a remote specified or there should
-          # be manifest default we could use.
-          raise FilterManifestException(
-              'Project %s has unspecified remote with no default' % project)
-        project_remote = default_remote
-      if project_remote not in whitelisted_remotes:
-        internal_projects.add(project)
-        manifest_node.removeChild(project_element)
-
-    for commit_element in pending_commits:
-      if commit_element.getAttribute(
-          PALADIN_PROJECT_ATTR) in internal_projects:
-        manifest_node.removeChild(commit_element)
-
-    with os.fdopen(temp_fd, 'w') as manifest_file:
-      # Filter out empty lines.
-      filtered_manifest_noempty = filter(
-          str.strip, manifest_dom.toxml('utf-8').splitlines())
-      manifest_file.write(os.linesep.join(filtered_manifest_noempty))
-
-    return new_path
-
   def CreateNewCandidate(self, validation_pool=None,
                          chrome_version=None,
                          retries=manifest_version.NUM_RETRIES,
@@ -430,7 +349,8 @@ class LKGMManager(manifest_version.BuildSpecsManager):
         manifest because of a git error or the manifest is already checked-in.
     """
     last_error = None
-    new_manifest = self._FilterCrosInternalProjectsFromManifest(manifest)
+    new_manifest = manifest_version.FilterManifest(
+        manifest, whitelisted_remotes=constants.EXTERNAL_REMOTES)
     version_info = self.GetCurrentVersionInfo()
     for _attempt in range(0, retries + 1):
       try:
diff --git a/cbuildbot/lkgm_manager_unittest.py b/cbuildbot/lkgm_manager_unittest.py
index 5b7cad473..9cc8d31b5 100644
--- a/cbuildbot/lkgm_manager_unittest.py
+++ b/cbuildbot/lkgm_manager_unittest.py
@@ -10,12 +10,12 @@ import contextlib
 import mock
 import mox
 import os
-import random
 import tempfile
 from xml.dom import minidom
 
 from chromite.cbuildbot import constants
 from chromite.cbuildbot import lkgm_manager
+from chromite.cbuildbot import manifest_version
 from chromite.cbuildbot import repository
 from chromite.lib import cros_build_lib
 from chromite.lib import cros_test_lib
@@ -34,9 +34,6 @@ CHROMEOS_PATCH=4
 CHROME_BRANCH=13
 """
 
-FAKE_WHITELISTED_REMOTES = ('cros', 'chromium')
-FAKE_NON_WHITELISTED_REMOTE = 'hottubtimemachine'
-
 
 # pylint: disable=protected-access
 # TODO: Re-enable when converted from mox to mock.
@@ -119,63 +116,6 @@ def TemporaryManifest():
 class LKGMManagerTest(cros_test_lib.MoxTempDirTestCase):
   """Tests for the BuildSpecs manager."""
 
-  def _CreateFakeManifest(self, num_internal, num_external, commits,
-                          has_default_remote=False):
-    """Creates a fake manifest with (optionally) some internal projects.
-
-    Args:
-      num_internal: Number of internal projects to add.
-      num_external: Number of external projects to add.
-      commits: Number of commits to add.
-      has_default_remote: If the manifest should have a default remote.
-
-    Returns:
-      A fake manifest for use in tests.
-    """
-    tmp_manifest = tempfile.mktemp('manifest')
-    # Create fake but empty manifest file.
-    new_doc = minidom.getDOMImplementation().createDocument(None, 'manifest',
-                                                            None)
-    m_element = new_doc.getElementsByTagName('manifest')[0]
-
-    default_remote = None
-    if has_default_remote:
-      default_remote = FAKE_WHITELISTED_REMOTES[0]
-      new_element = minidom.Element('default')
-      new_element.setAttribute('remote', default_remote)
-      m_element.appendChild(new_element)
-    remotes_to_use = list(FAKE_WHITELISTED_REMOTES) * (
-        num_external / len(FAKE_WHITELISTED_REMOTES))
-
-    internal_remotes = [FAKE_NON_WHITELISTED_REMOTE] * num_internal
-    remotes_to_use.extend(internal_remotes)
-    # Randomize the list of remotes to get wider test coverage for the
-    # filtering logic.
-    random.shuffle(remotes_to_use)
-
-    for idx in xrange(num_internal + num_external):
-      new_element = minidom.Element('project')
-      new_element.setAttribute('name', 'project_%d' % idx)
-      new_element.setAttribute('path', 'some_path/to/project_%d' % idx)
-      new_element.setAttribute('revision', 'revision_%d' % idx)
-      remote = remotes_to_use[idx % len(remotes_to_use)]
-      # Skip setting a remote attribute if this is a default remote.
-      if not has_default_remote or remote is not default_remote:
-        new_element.setAttribute('remote', remote)
-      m_element.appendChild(new_element)
-
-    for idx in xrange(commits):
-      new_element = minidom.Element('pending_commit')
-      new_element.setAttribute('project', 'project_%d' % idx)
-      new_element.setAttribute('change_id', 'changeid_%d' % idx)
-      new_element.setAttribute('commit', 'commit_%d' % idx)
-      m_element.appendChild(new_element)
-
-    with open(tmp_manifest, 'w+') as manifest_file:
-      new_doc.writexml(manifest_file, newl='\n')
-
-    return tmp_manifest
-
   def setUp(self):
     self.mox.StubOutWithMock(git, 'CreatePushBranch')
 
@@ -265,8 +205,7 @@ class LKGMManagerTest(cros_test_lib.MoxTempDirTestCase):
                              'RefreshManifestCheckout')
     self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
                              'InitializeManifestVariables')
-    self.mox.StubOutWithMock(lkgm_manager.LKGMManager,
-                             '_FilterCrosInternalProjectsFromManifest')
+    self.mox.StubOutWithMock(manifest_version, 'FilterManifest')
     self.mox.StubOutWithMock(lkgm_manager.LKGMManager, 'PublishManifest')
 
     version = '2010.0.0-rc7'
@@ -278,8 +217,9 @@ class LKGMManagerTest(cros_test_lib.MoxTempDirTestCase):
 
     build_id = 20162
 
-    lkgm_manager.LKGMManager._FilterCrosInternalProjectsFromManifest(
-        manifest).AndReturn(new_manifest)
+    manifest_version.FilterManifest(
+        manifest,
+        whitelisted_remotes=constants.EXTERNAL_REMOTES).AndReturn(new_manifest)
 
     # Do manifest refresh work.
     lkgm_manager.LKGMManager.GetCurrentVersionInfo().AndReturn(my_info)
@@ -584,86 +524,3 @@ class LKGMManagerTest(cros_test_lib.MoxTempDirTestCase):
           element.getAttribute(lkgm_manager.PALADIN_TOTAL_FAIL_COUNT_ATTR),
           str(gerrit_patch.total_fail_count))
 
-  def testFilterProjectsFromManifest(self):
-    """Tests whether we can remove internal projects from a manifest."""
-    fake_manifest = None
-    fake_new_manifest = None
-    try:
-      fake_manifest = self._CreateFakeManifest(num_internal=20,
-                                               num_external=80,
-                                               commits=100)
-      fake_new_manifest = \
-          lkgm_manager.LKGMManager._FilterCrosInternalProjectsFromManifest(
-              fake_manifest, whitelisted_remotes=FAKE_WHITELISTED_REMOTES)
-
-      new_dom = minidom.parse(fake_new_manifest)
-      projects = new_dom.getElementsByTagName('project')
-      # All external projects must be present in the new manifest.
-      self.assertEqual(len(projects), 80)
-      project_remote_dict = {}
-      # All projects should have whitelisted remotes.
-      for p in projects:
-        remote = p.getAttribute('remote')
-        self.assertIn(remote, FAKE_WHITELISTED_REMOTES)
-        project_remote_dict[p.getAttribute('name')] = remote
-
-      # Check commits. All commits should correspond to projects which
-      # have whitelisted remotes.
-      commits = new_dom.getElementsByTagName('pending_commit')
-      self.assertEqual(len(commits), 80)
-      for c in commits:
-        p = c.getAttribute('project')
-        self.assertIn(project_remote_dict[p], FAKE_WHITELISTED_REMOTES)
-
-    finally:
-      if fake_manifest:
-        os.remove(fake_manifest)
-      if fake_new_manifest:
-        os.remove(fake_new_manifest)
-
-  def testFilterProjectsFromExternalManifest(self):
-    """Tests filtering on a project where no filtering is needed."""
-    fake_manifest = None
-    fake_new_manifest = None
-    try:
-      fake_manifest = self._CreateFakeManifest(num_internal=0,
-                                               num_external=100,
-                                               commits=20)
-      fake_new_manifest = \
-          lkgm_manager.LKGMManager._FilterCrosInternalProjectsFromManifest(
-              fake_manifest, whitelisted_remotes=FAKE_WHITELISTED_REMOTES)
-
-      new_dom = minidom.parse(fake_new_manifest)
-      projects = new_dom.getElementsByTagName('project')
-      self.assertEqual(len(projects), 100)
-      commits = new_dom.getElementsByTagName('pending_commit')
-      self.assertEqual(len(commits), 20)
-
-    finally:
-      if fake_manifest:
-        os.remove(fake_manifest)
-      if fake_new_manifest:
-        os.remove(fake_new_manifest)
-
-  def testFilterDefaultProjectsFromManifest(self):
-    """Tests whether we correctly handle projects with default remotes."""
-    fake_manifest = None
-    fake_new_manifest = None
-    try:
-      fake_manifest = self._CreateFakeManifest(num_internal=20,
-                                               num_external=80,
-                                               commits=20,
-                                               has_default_remote=True)
-      fake_new_manifest = \
-          lkgm_manager.LKGMManager._FilterCrosInternalProjectsFromManifest(
-              fake_manifest, whitelisted_remotes=FAKE_WHITELISTED_REMOTES)
-
-      new_dom = minidom.parse(fake_new_manifest)
-      projects = new_dom.getElementsByTagName('project')
-      self.assertEqual(len(projects), 80)
-
-    finally:
-      if fake_manifest:
-        os.remove(fake_manifest)
-      if fake_new_manifest:
-        os.remove(fake_new_manifest)
diff --git a/cbuildbot/manifest_version.py b/cbuildbot/manifest_version.py
index c87fe5d38..1708a317f 100644
--- a/cbuildbot/manifest_version.py
+++ b/cbuildbot/manifest_version.py
@@ -14,6 +14,7 @@ import os
 import re
 import shutil
 import tempfile
+from xml.dom import minidom
 
 from chromite.cbuildbot import constants
 from chromite.cbuildbot import repository
@@ -29,6 +30,22 @@ BUILD_STATUS_URL = '%s/builder-status' % constants.MANIFEST_VERSIONS_GS_URL
 PUSH_BRANCH = 'temp_auto_checkin_branch'
 NUM_RETRIES = 20
 
+MANIFEST_ELEMENT = 'manifest'
+DEFAULT_ELEMENT = 'default'
+PROJECT_ELEMENT = 'project'
+REMOTE_ELEMENT = 'remote'
+PROJECT_NAME_ATTR = 'name'
+PROJECT_REMOTE_ATTR = 'remote'
+PROJECT_GROUP_ATTR = 'groups'
+REMOTE_NAME_ATTR = 'name'
+
+PALADIN_COMMIT_ELEMENT = 'pending_commit'
+PALADIN_PROJECT_ATTR = 'project'
+
+
+class FilterManifestException(Exception):
+  """Exception thrown when failing to filter the internal manifest."""
+
 
 class VersionUpdateException(Exception):
   """Exception gets thrown for failing to update the version file"""
@@ -937,3 +954,128 @@ class BuildSpecsManager(object):
     # Cleanse any failed local changes and throw an exception.
     self.RefreshManifestCheckout()
     raise StatusUpdateException(last_error)
+
+
+def _GetDefaultRemote(manifest_dom):
+  """Returns the default remote in a manifest (if any).
+
+  Args:
+    manifest_dom: DOM Document object representing the manifest.
+
+  Returns:
+    Default remote if one exists, None otherwise.
+  """
+  default_nodes = manifest_dom.getElementsByTagName(DEFAULT_ELEMENT)
+  if default_nodes:
+    if len(default_nodes) > 1:
+      raise FilterManifestException(
+          'More than one <default> element found in manifest')
+    return default_nodes[0].getAttribute(PROJECT_REMOTE_ATTR)
+  return None
+
+
+def _GetGroups(project_element):
+  """Returns the default remote in a manifest (if any).
+
+  Args:
+    project_element: DOM Document object representing a project.
+
+  Returns:
+    List of names of the groups the project belongs too.
+  """
+  group = project_element.getAttribute(PROJECT_GROUP_ATTR)
+  if not group:
+    return []
+
+  return [s.strip() for s in group.split(',')]
+
+
+def FilterManifest(manifest, whitelisted_remotes=None, whitelisted_groups=None):
+  """Returns a path to a new manifest with whitelists enforced.
+
+  Args:
+    manifest: Path to an existing manifest that should be filtered.
+    whitelisted_remotes: Tuple of remotes to allow in the generated manifest.
+      Only projects with those remotes will be included in the external
+      manifest. (None means all remotes are acceptable)
+    whitelisted_groups: Tuple of groups to allow in the generated manifest.
+      (None means all groups are acceptable)
+
+  Returns:
+    Path to a new manifest that is a filtered copy of the original.
+  """
+  temp_fd, new_path = tempfile.mkstemp('external_manifest')
+  manifest_dom = minidom.parse(manifest)
+  manifest_node = manifest_dom.getElementsByTagName(MANIFEST_ELEMENT)[0]
+  remotes = manifest_dom.getElementsByTagName(REMOTE_ELEMENT)
+  projects = manifest_dom.getElementsByTagName(PROJECT_ELEMENT)
+  pending_commits = manifest_dom.getElementsByTagName(PALADIN_COMMIT_ELEMENT)
+
+  default_remote = _GetDefaultRemote(manifest_dom)
+
+  # Remove remotes that don't match our whitelist.
+  for remote_element in remotes:
+    name = remote_element.getAttribute(REMOTE_NAME_ATTR)
+    if (name is not None and
+        whitelisted_remotes and
+        name not in whitelisted_remotes):
+      manifest_node.removeChild(remote_element)
+
+  filtered_projects = set()
+  for project_element in projects:
+    project_remote = project_element.getAttribute(PROJECT_REMOTE_ATTR)
+    project = project_element.getAttribute(PROJECT_NAME_ATTR)
+    if not project_remote:
+      if not default_remote:
+        # This should not happen for a valid manifest. Either each
+        # project must have a remote specified or there should
+        # be manifest default we could use.
+        raise FilterManifestException(
+            'Project %s has unspecified remote with no default' % project)
+      project_remote = default_remote
+
+    groups = _GetGroups(project_element)
+
+    filter_remote = (whitelisted_remotes and
+                     project_remote not in whitelisted_remotes)
+
+    filter_group = (whitelisted_groups and
+                    not any([g in groups for g in whitelisted_groups]))
+
+    if filter_remote or filter_group:
+      filtered_projects.add(project)
+      manifest_node.removeChild(project_element)
+
+  for commit_element in pending_commits:
+    if commit_element.getAttribute(
+        PALADIN_PROJECT_ATTR) in filtered_projects:
+      manifest_node.removeChild(commit_element)
+
+  with os.fdopen(temp_fd, 'w') as manifest_file:
+    # Filter out empty lines.
+    filtered_manifest_noempty = filter(
+        str.strip, manifest_dom.toxml('utf-8').splitlines())
+    manifest_file.write(os.linesep.join(filtered_manifest_noempty))
+
+  return new_path
+
+
+def ConvertToProjectSdkManifest(manifest):
+  """Converts a manifest to a ProjectSDK Manifest.
+
+  Project SDK manifests are based on the current manifest, but stripped
+  down in a variety of ways. If you want the project manifest to be pinned
+  to specific SHA1 values, then you should pass in a pinned manifest.
+
+  This is commonly done with: repo.ExportManifest(mark_revision=True)
+
+  Args:
+    manifest: Path to an existing manifest that should be converted.
+
+  Returns:
+    Path to a new manifest that is a converted copy of the original.
+  """
+  return FilterManifest(
+      manifest,
+      whitelisted_remotes=constants.EXTERNAL_REMOTES,
+      whitelisted_groups=constants.PROJECT_SDK_GROUPS)
diff --git a/cbuildbot/manifest_version_unittest.py b/cbuildbot/manifest_version_unittest.py
index 241a318e4..63e9a9b25 100644
--- a/cbuildbot/manifest_version_unittest.py
+++ b/cbuildbot/manifest_version_unittest.py
@@ -8,7 +8,9 @@ from __future__ import print_function
 
 import mox
 import os
+import random
 import tempfile
+from xml.dom import minidom
 
 from chromite.cbuildbot import constants
 from chromite.cbuildbot import failures_lib
@@ -27,6 +29,9 @@ CHROMEOS_PATCH=%(patch_number)s
 CHROME_BRANCH=%(chrome_branch)s
 """
 
+FAKE_WHITELISTED_REMOTES = ('cros', 'chromium')
+FAKE_NON_WHITELISTED_REMOTE = 'hottubtimemachine'
+
 FAKE_VERSION_STRING = '1.2.3'
 FAKE_VERSION_STRING_NEXT = '1.2.4'
 CHROME_BRANCH = '13'
@@ -402,3 +407,221 @@ class BuildSpecsManagerTest(cros_test_lib.MoxTempDirTestCase,
     statuses = self._GetBuildersStatus(['build1', 'build2'], status_runs)
     self.assertTrue(statuses['build1'].Failed())
     self.assertTrue(statuses['build2'].Passed())
+
+
+class ProjectSdkManifestTest(cros_test_lib.TestCase):
+  """Test cases for ManifestVersionedSyncStage.ConvertToProjectSdkManifest."""
+
+  def _CreateFakeManifest(self, num_internal, num_external, commits,
+                          has_default_remote=False):
+    """Creates a fake manifest with (optionally) some internal projects.
+
+    Args:
+      num_internal: Number of internal projects to add.
+      num_external: Number of external projects to add.
+      commits: Number of commits to add.
+      has_default_remote: If the manifest should have a default remote.
+
+    Returns:
+      A fake manifest for use in tests.
+    """
+    tmp_manifest = tempfile.mktemp('manifest')
+    # Create fake but empty manifest file.
+    new_doc = minidom.getDOMImplementation().createDocument(None, 'manifest',
+                                                            None)
+    m_element = new_doc.getElementsByTagName('manifest')[0]
+
+    default_remote = None
+    if has_default_remote:
+      default_remote = FAKE_WHITELISTED_REMOTES[0]
+      new_element = minidom.Element('default')
+      new_element.setAttribute('remote', default_remote)
+      m_element.appendChild(new_element)
+    remotes_to_use = list(FAKE_WHITELISTED_REMOTES) * (
+        num_external / len(FAKE_WHITELISTED_REMOTES))
+
+    internal_remotes = [FAKE_NON_WHITELISTED_REMOTE] * num_internal
+    remotes_to_use.extend(internal_remotes)
+    # Randomize the list of remotes to get wider test coverage for the
+    # filtering logic.
+    random.shuffle(remotes_to_use)
+
+    for idx in xrange(num_internal + num_external):
+      new_element = minidom.Element('project')
+      new_element.setAttribute('name', 'project_%d' % idx)
+      new_element.setAttribute('path', 'some_path/to/project_%d' % idx)
+      new_element.setAttribute('revision', 'revision_%d' % idx)
+      remote = remotes_to_use[idx % len(remotes_to_use)]
+      # Skip setting a remote attribute if this is a default remote.
+      if not has_default_remote or remote is not default_remote:
+        new_element.setAttribute('remote', remote)
+      m_element.appendChild(new_element)
+
+    for idx in xrange(commits):
+      new_element = minidom.Element('pending_commit')
+      new_element.setAttribute('project', 'project_%d' % idx)
+      new_element.setAttribute('change_id', 'changeid_%d' % idx)
+      new_element.setAttribute('commit', 'commit_%d' % idx)
+      m_element.appendChild(new_element)
+
+    with open(tmp_manifest, 'w+') as manifest_file:
+      new_doc.writexml(manifest_file, newl='\n')
+
+    return tmp_manifest
+
+  def testFilterProjectsFromManifest(self):
+    """Tests whether we can remove internal projects from a manifest."""
+    fake_manifest = None
+    fake_new_manifest = None
+    try:
+      fake_manifest = self._CreateFakeManifest(num_internal=20,
+                                               num_external=80,
+                                               commits=100)
+      fake_new_manifest = manifest_version.FilterManifest(
+          fake_manifest, whitelisted_remotes=FAKE_WHITELISTED_REMOTES)
+
+      new_dom = minidom.parse(fake_new_manifest)
+      projects = new_dom.getElementsByTagName('project')
+      # All external projects must be present in the new manifest.
+      self.assertEqual(len(projects), 80)
+      project_remote_dict = {}
+      # All projects should have whitelisted remotes.
+      for p in projects:
+        remote = p.getAttribute('remote')
+        self.assertIn(remote, FAKE_WHITELISTED_REMOTES)
+        project_remote_dict[p.getAttribute('name')] = remote
+
+      # Check commits. All commits should correspond to projects which
+      # have whitelisted remotes.
+      commits = new_dom.getElementsByTagName('pending_commit')
+      self.assertEqual(len(commits), 80)
+      for c in commits:
+        p = c.getAttribute('project')
+        self.assertIn(project_remote_dict[p], FAKE_WHITELISTED_REMOTES)
+
+    finally:
+      if fake_manifest:
+        os.remove(fake_manifest)
+      if fake_new_manifest:
+        os.remove(fake_new_manifest)
+
+  def testFilterProjectsFromExternalManifest(self):
+    """Tests filtering on a project where no filtering is needed."""
+    fake_manifest = None
+    fake_new_manifest = None
+    try:
+      fake_manifest = self._CreateFakeManifest(num_internal=0,
+                                               num_external=100,
+                                               commits=20)
+      fake_new_manifest = manifest_version.FilterManifest(
+          fake_manifest, whitelisted_remotes=FAKE_WHITELISTED_REMOTES)
+
+      new_dom = minidom.parse(fake_new_manifest)
+      projects = new_dom.getElementsByTagName('project')
+      self.assertEqual(len(projects), 100)
+      commits = new_dom.getElementsByTagName('pending_commit')
+      self.assertEqual(len(commits), 20)
+
+    finally:
+      if fake_manifest:
+        os.remove(fake_manifest)
+      if fake_new_manifest:
+        os.remove(fake_new_manifest)
+
+  def testFilterDefaultProjectsFromManifest(self):
+    """Tests whether we correctly handle projects with default remotes."""
+    fake_manifest = None
+    fake_new_manifest = None
+    try:
+      fake_manifest = self._CreateFakeManifest(num_internal=20,
+                                               num_external=80,
+                                               commits=20,
+                                               has_default_remote=True)
+      fake_new_manifest = manifest_version.FilterManifest(
+          fake_manifest, whitelisted_remotes=FAKE_WHITELISTED_REMOTES)
+
+      new_dom = minidom.parse(fake_new_manifest)
+      projects = new_dom.getElementsByTagName('project')
+      self.assertEqual(len(projects), 80)
+
+    finally:
+      if fake_manifest:
+        os.remove(fake_manifest)
+      if fake_new_manifest:
+        os.remove(fake_new_manifest)
+
+
+
+  def _HelperTestConvertToProjectSdkManifestEmpty(self, source, expected):
+    """Test that ConvertToProjectSdkManifest turns |source| into |expected|.
+
+    Args:
+      source: Starting manifest as a string.
+      expected: Expected result manifest, as a string.
+    """
+    manifest_out = None
+
+    try:      # Convert source into a file.
+      with tempfile.NamedTemporaryFile() as manifest_in:
+        osutils.WriteFile(manifest_in.name, source)
+        manifest_out = manifest_version.ConvertToProjectSdkManifest(
+            manifest_in.name)
+
+      # Convert output file into string.
+      result = osutils.ReadFile(manifest_out)
+
+    finally:
+      if manifest_out:
+        os.unlink(manifest_out)
+
+    # Verify the result.
+    self.assertEqual(expected, result)
+
+  def testConvertToProjectSdkManifestEmpty(self):
+    EMPTY = '<?xml version="1.0" encoding="UTF-8"?><manifest></manifest>'
+    EXPECTED = '<?xml version="1.0" encoding="utf-8"?><manifest/>'
+    self._HelperTestConvertToProjectSdkManifestEmpty(EMPTY, EXPECTED)
+
+  def testConvertToProjectSdkManifestMixed(self):
+    MIXED = '''<?xml version="1.0" encoding="utf-8"?>
+<manifest>
+  <remote alias="cros-int" fetch="https://int-url" name="chrome"/>
+  <remote alias="cros" fetch="https://url/" name="chromium"/>
+  <remote fetch="https://url" name="cros"/>
+  <remote fetch="https://int-url" name="cros-int"/>
+
+  <default remote="cros" revision="refs/heads/master" sync-j="8"/>
+
+  <project name="int" remote="cros-int" />
+  <project groups="foo,bar" name="int-other-groups" remote="cros-int" />
+  <project groups="minilayout" name="int-mini" remote="cros-int" />
+  <project groups="minilayout,foo" name="int-mini-groups" remote="cros-int" />
+
+  <project name="exp" remote="cros" />
+  <project groups="foo,bar" name="exp-other-groups" remote="cros" />
+  <project groups="minilayout" name="exp-mini" remote="cros" />
+  <project groups="minilayout,foo" name="exp-mini-groups" remote="cros" />
+
+  <project name="def-other-groups" />
+  <project groups="foo,bar" name="def-other-groups" />
+  <project groups="minilayout" name="def-mini" />
+  <project groups="minilayout,foo" name="def-mini-groups" />
+  <project groups="minilayout , foo" name="def-spacing" />
+
+  <repo-hooks enabled-list="pre-upload" in-project="chromiumos/repohooks"/>
+</manifest>
+'''
+
+    EXPECTED = '''<?xml version="1.0" encoding="utf-8"?><manifest>
+  <remote alias="cros" fetch="https://url/" name="chromium"/>
+  <remote fetch="https://url" name="cros"/>
+  <default remote="cros" revision="refs/heads/master" sync-j="8"/>
+  <project groups="minilayout" name="exp-mini" remote="cros"/>
+  <project groups="minilayout,foo" name="exp-mini-groups" remote="cros"/>
+  <project groups="minilayout" name="def-mini"/>
+  <project groups="minilayout,foo" name="def-mini-groups"/>
+  <project groups="minilayout , foo" name="def-spacing"/>
+  <repo-hooks enabled-list="pre-upload" in-project="chromiumos/repohooks"/>
+</manifest>'''
+
+    self._HelperTestConvertToProjectSdkManifestEmpty(MIXED, EXPECTED)
diff --git a/cbuildbot/stages/sync_stages.py b/cbuildbot/stages/sync_stages.py
index ffa1c055a..d80c403e3 100644
--- a/cbuildbot/stages/sync_stages.py
+++ b/cbuildbot/stages/sync_stages.py
@@ -11,7 +11,9 @@ import contextlib
 import datetime
 import logging
 import os
+import shutil
 import sys
+import tempfile
 import time
 from xml.etree import ElementTree
 from xml.dom import minidom
@@ -564,6 +566,58 @@ class ManifestVersionedSyncStage(SyncStage):
     else:
       yield manifest
 
+  @staticmethod
+  def CommitProjectSDKManifest(manifest, release, current_version, debug):
+    """Create and submit the Product SDK Manifest.
+
+    Create the Project SDK manifest, and push it to the external manifest
+    repository.
+
+    Args:
+      manifest: Path to new manifest to commit.
+      release: Current release of the build (ie: 42 if building R42-6513.0.0)
+      current_version: Version to use when commiting manifest.
+      debug: Is this a debug build?
+    """
+    external_manifest_url = cbuildbot_config.GetManifestVersionsRepoUrl(
+        internal_build=False,
+        read_only=False,
+        test=debug)
+
+    # TODO(dgarrett): Find a persistent directory for this.
+    with osutils.TempDir() as git_repo:
+      repository.UpdateGitRepo(git_repo, external_manifest_url)
+
+      sdk_manifest_path = os.path.join(
+          git_repo, 'project-sdk', '%s' % release, '%s.xml' % current_version)
+
+      if os.path.exists(sdk_manifest_path):
+        raise failures_lib.StepFailure(
+            'Project SDK Manifest already exists: %s' % sdk_manifest_path)
+
+      # Create branch for pushing new manifest file.
+      branch = 'temp_project_sdk_creation_branch'
+      git.CreatePushBranch(branch, git_repo, sync=False)
+
+      # Create new manifest file.
+      logging.info('Creating Project SDK Manifest as: %s', sdk_manifest_path)
+      osutils.SafeMakedirs(os.path.dirname(sdk_manifest_path))
+      shutil.copyfile(manifest, sdk_manifest_path)
+
+      # Commit it locally.
+      logging.info('Committing Project SDK Manifest.')
+      git.AddPath(sdk_manifest_path)
+      git.Commit(git_repo, 'Create project_sdk for %s-%s.' %
+                 (release, current_version))
+
+      # Push it.
+      logging.info('Pushing Project SDK Manifest.')
+      git.PushWithRetry(branch, git_repo)
+
+      logging.info('Project SDK Manifest \'%s\' published:',
+                   os.path.basename(sdk_manifest_path))
+      logging.info('%s', osutils.ReadFile(manifest))
+
   @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
   def PerformStage(self):
     self.Initialize()
@@ -591,6 +645,28 @@ class ManifestVersionedSyncStage(SyncStage):
         next_manifest, filter_cros=self._run.options.local) as new_manifest:
       self.ManifestCheckout(new_manifest)
 
+    # If we are a Canary Master, create an additional derivative Manifest for
+    # the Project SDK builders.
+    if (cbuildbot_config.IsCanaryType(self._run.config.build_type) and
+        self._run.config.master):
+      logging.info('Creating of Project SDK Manifest.')
+      sdk_manifest = None
+      try:
+        with tempfile.NamedTemporaryFile() as pinned_manifest_file:
+          pinned_manifest = self.repo.ExportManifest(mark_revision=True)
+          osutils.WriteFile(pinned_manifest_file.name, pinned_manifest)
+          sdk_manifest = manifest_version.ConvertToProjectSdkManifest(
+              pinned_manifest_file.name)
+
+        self.CommitProjectSDKManifest(
+            sdk_manifest,
+            self.manifest_manager.GetCurrentVersionInfo().chrome_branch,
+            self.manifest_manager.current_version,
+            self._run.options.debug)
+      finally:
+        if sdk_manifest:
+          os.unlink(sdk_manifest)
+
     # Set the status inflight at the end of the ManifestVersionedSync
     # stage. This guarantees that all syncing has completed.
     if self.manifest_manager:
diff --git a/cbuildbot/stages/sync_stages_unittest.py b/cbuildbot/stages/sync_stages_unittest.py
index 11f307145..f9dbff601 100644
--- a/cbuildbot/stages/sync_stages_unittest.py
+++ b/cbuildbot/stages/sync_stages_unittest.py
@@ -29,8 +29,10 @@ from chromite.cbuildbot.stages import generic_stages_unittest
 from chromite.lib import cros_build_lib_unittest
 from chromite.lib import cidb
 from chromite.lib import clactions
+from chromite.lib import cros_test_lib
 from chromite.lib import fake_cidb
 from chromite.lib import gerrit
+from chromite.lib import git
 from chromite.lib import git_unittest
 from chromite.lib import gob_util
 from chromite.lib import osutils
@@ -81,6 +83,16 @@ class ManifestVersionedSyncStageTest(generic_stages_unittest.AbstractStageTest):
 
     self.sync_stage.Run()
 
+  @cros_test_lib.NetworkTest()
+  @mock.patch.object(git, 'PushWithRetry')
+  def testCommitProjectSDKManifest(self, _mock_push):
+    """Tests that we can 'push' an SDK manifest."""
+    # Create test manifest
+    manifest = os.path.join(self.tempdir, 'sdk.xml')
+    osutils.WriteFile(manifest, 'bogus value')
+
+    sync_stages.ManifestVersionedSyncStage.CommitProjectSDKManifest(
+        manifest, 'test_release', 'test_version', True)
 
 class MockPatch(mock.MagicMock):
   """MagicMock for a GerritPatch-like object."""