Add new scheduler and change_source files to repo

Change-Id: I1a020323fb6a09e439c85c6058c36ff7d173e098

2 files changed, 222 insertions, 0 deletions

diff --git a/build/scripts/master/emu_gs_scheduler.py b/build/scripts/master/emu_gs_scheduler.py
new file mode 100644
index 00000000..3546e386
--- /dev/null
+++ b/build/scripts/master/emu_gs_scheduler.py
@@ -0,0 +1,66 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This SingleBranchScheduler download necessary emulator package and include it in build properties
+
+"""
+
+import os, sys
+import gcs_oauth2_boto_plugin
+import StringIO
+from boto import boto
+from twisted.python import log
+from twisted.internet import defer, utils
+from buildbot.schedulers.timed import Periodic
+from buildbot.schedulers.basic import SingleBranchScheduler
+
+google_storage = 'gs'
+
+class EmulatorSingleBranchScheduler(SingleBranchScheduler):
+  """Augmented 'SingleBranchScheduler' that adds emu_image properties"""
+
+  # Overrides 'SingleBranchScheduler.addBuildsetForChanges'
+  @defer.inlineCallbacks
+  def addBuildsetForChanges(self, *args, **kwargs):
+    for x in ['windows', 'linux', 'mac']:
+      if x in self.name:
+        emu_cache_file = 'emulator_%s_poller.cache' % x
+    try:
+        with open(emu_cache_file, 'r') as f:
+            content = f.readlines()
+            emu_revision = content[0]
+            emu_file = ','.join(content[1:]).rstrip('\n')
+    except:
+        log.msg("%s: Error - emulator cache file not available, cancle build" % self.name)
+        cancel_build = True
+    try:
+        with open('sys_image_lmp_poller.cache', 'r') as f:
+            content = f.readlines()
+            lmp_revision = content[0]
+            lmp_file = ','.join(content[1:]).rstrip('\n')
+    except:
+        lmp_revision = 'None'
+        lmp_file = ''
+    try:
+        with open('sys_image_mnc_poller.cache', 'r') as f:
+            content = f.readlines()
+            mnc_revision = content[0]
+            mnc_file = ','.join(content[1:]).rstrip('\n')
+    except:
+        mnc_revision = 'None'
+        mnc_file = ''
+
+    self.properties.setProperty('mnc_revision', mnc_revision, 'Scheduler')
+    self.properties.setProperty('mnc_system_image', mnc_file, 'Scheduler')
+    self.properties.setProperty('lmp_revision', lmp_revision, 'Scheduler')
+    self.properties.setProperty('lmp_system_image', lmp_file, 'Scheduler')
+    self.properties.setProperty('emu_revision', emu_revision, 'Scheduler')
+    self.properties.setProperty('emulator_image', emu_file, 'Scheduler')
+    self.properties.setProperty('got_revision', '%s-%s-%s' % (emu_revision, mnc_revision, lmp_revision), 'Scheduler')
+
+    rv = yield SingleBranchScheduler.addBuildsetForChanges(
+        self,
+        *args,
+        **kwargs)
+    defer.returnValue(rv)
diff --git a/build/scripts/master/gs_multi_poller.py b/build/scripts/master/gs_multi_poller.py
new file mode 100644
index 00000000..8082bc4d
--- /dev/null
+++ b/build/scripts/master/gs_multi_poller.py
@@ -0,0 +1,156 @@
+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This PollingChangeSource polls multiple Storage URL for change revisions.
+
+Each change is submitted to change master which triggers build steps.
+
+Notice that the gsutil configuration (.boto file) must be setup in either the
+default location (home dir) or by using the environment variables
+AWS_CREDENTIAL_FILE and BOTO_CONFIG.
+
+Example:
+To poll a change from buckets path1 and path2, use -
+from master import gs_multi_poller
+gs_bucket = "bucket_name"
+gs_path_list = ['path/to/path1', 'path/to/path2']
+poller = gs_multi_poller.GSMultiPoller("poller", gs_bucket, gs_path_list, pollInterval=10800)
+c['change_source'] = [poller]
+"""
+
+import os
+import sys
+
+from twisted.internet import defer
+from twisted.python import log
+
+from buildbot.changes import base
+import gcs_oauth2_boto_plugin
+import StringIO
+from boto import boto
+from twisted.python import log
+
+class GSMultiPoller(base.PollingChangeSource):
+  """Poll a Google Storage URL for change number and submit to change master."""
+
+  compare_attrs = ['changeurl', 'pollInterval']
+
+  # pylint runs this against the wrong buildbot version.
+  # In buildbot 8.4 base.PollingChangeSource has no __init__
+  # pylint: disable=W0231
+  def __init__(self, name, gs_bucket, gs_path_list, pollInterval=5*60,
+               project=None, branch=None, name_identifier='', category=None):
+    """Initialize GSMultiPoller.
+
+    Args:
+    gs_bucket: bucket name
+    gs_path_list: list of GS URLs to watch for
+    pollInterval: Time (in seconds) between queries for changes.
+    name_identifier: If given, used to identify if a file is important
+    category: Build category to trigger (optional).
+    """
+    #if not changeurl.startswith('gs://'):
+    #  raise Exception('GSMultiPoller changeurl must start with gs://')
+
+    self.name = name
+    self.cachepath = self.name + '.cache'
+    self.gs_bucket = gs_bucket
+    self.gs_path_list = gs_path_list
+    self.pollInterval = pollInterval
+    self.category = category
+    self.last_change = None
+    self.project = project
+    self.branch = branch
+    self.name_identifier = name_identifier
+
+    if os.path.exists(self.cachepath):
+      try:
+        with open(self.cachepath, "r") as f:
+          self.last_change = f.readline().strip()
+          log.msg("%s: Setting last_change to %s" % (self.name, self.last_change))
+      except:
+        self.cachepath = None
+        log.msg("%s: Cache file corrupt or unwriteable; skipping and not using" % self.name)
+        log.err()
+
+  def describe(self):
+    return '%s: watching %s' % (self.name, self.gs_path_list)
+
+  def poll(self):
+    log.msg('%s: polling %s' % (self.name, self.gs_path_list))
+    d = defer.succeed(None)
+    d.addCallback(self.find_latest_build)
+    d.addCallback(self._process_changes)
+    d.addErrback(self._finished_failure)
+    return d
+
+  def _finished_failure(self, res):
+    log.msg('%s: poll failed: %s. URL: %s' % (self.name, res, self.gs_path_list))
+
+  # return the latest complete build
+  def find_latest_build(self, _no_use):
+    bucket = boto.storage_uri(self.gs_bucket, 'gs').get_bucket()
+    maxtime = None
+    build_version = None
+    last_modified_file = None
+    for obj in bucket.list(self.gs_path_list[0]):
+      if (maxtime is None or maxtime < obj.last_modified) and (self.name_identifier in obj.name):
+        maxtime = obj.last_modified
+        last_modified_file = obj.name
+    if last_modified_file is not None:
+      # file path: "builds/[builder_name]/[build_version]/[random_hash]/[binary].zip"
+      build_version = last_modified_file.split('/')[2]
+    log.msg('%s: last_change %s, new_last_change %s' % (self.name, self.last_change, build_version))
+    if build_version is None or build_version == self.last_change:
+      return None
+    file_list = []
+    for path in self.gs_path_list:
+      objs = bucket.list(path + build_version + '/')
+      count = len(list(objs))
+      log.msg("%s: search %s, file count %d" % (self.name, path + build_version, count))
+      if count == 0:
+        log.msg("%s: Build incomplete, couldn't find %s" % (self.name, path + build_version))
+        return None
+      for obj in objs:
+        if self.name_identifier in obj.name:
+          file_list.append(obj.name)
+    return file_list
+
+  def _update_last_rev(self, new_revision):
+    log.msg("%s: last revision changed from %s to %s" % (self.name, self.last_change, new_revision))
+    self.last_change = new_revision
+    if self.cachepath:
+      with open(self.cachepath, "w") as f:
+          f.write("%s\n" % self.last_change)
+
+  def _download_image(self, src_path, dst_path):
+    log.msg("%s: downloadImage: from %s to %s" % (self.name, src_path, dst_path))
+    src_uri = boto.storage_uri(self.gs_bucket + '/' + src_path, 'gs')
+    object_contents = StringIO.StringIO()
+    src_uri.get_key().get_file(object_contents)
+    dst_uri = boto.storage_uri(dst_path, 'file')
+    object_contents.seek(0)
+    dst_uri.new_key().set_contents_from_file(object_contents)
+    object_contents.close()
+
+  def _process_changes(self, file_list):
+    if file_list is not None:
+      parsed_revision = file_list[0].split('/')[2]
+      self._update_last_rev(parsed_revision)
+      dst_file_list = []
+      for file in file_list:
+        ab_build_branch = file.split('/')[1]
+        dst_path = os.path.join(os.getcwd(), 'images', ab_build_branch, os.path.basename(file))
+        self._download_image(file, dst_path)
+        with open(self.cachepath, "a") as f:
+          f.write("%s\n" % dst_path)
+        dst_file_list.append(dst_path)
+
+      self.master.addChange(who=self.name,
+                            revision=parsed_revision,
+                            files=dst_file_list,
+                            project=self.project,
+                            branch=self.branch,
+                            comments='comment',
+                            category=self.category)