diff options
Diffstat (limited to 'cros_utils/buildbot_json.py')
| -rwxr-xr-x | cros_utils/buildbot_json.py | 1518 |
1 files changed, 1518 insertions, 0 deletions
diff --git a/cros_utils/buildbot_json.py b/cros_utils/buildbot_json.py new file mode 100755 index 00000000..8a9d9cb8 --- /dev/null +++ b/cros_utils/buildbot_json.py @@ -0,0 +1,1518 @@ +#!/usr/bin/env python2 +# Copyright (c) 2012 The Chromium Authors. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# NOTE: This file is NOT under GPL. See above. +"""Queries buildbot through the json interface. +""" + +from __future__ import print_function + +__author__ = 'maruel@chromium.org' +__version__ = '1.2' + +import code +import datetime +import functools +import json + +# Pylint recommends we use "from chromite.lib import cros_logging as logging". +# Chromite specific policy message, we want to keep using the standard logging. +# pylint: disable=cros-logging-import +import logging + +# pylint: disable=deprecated-module +import optparse + +import time +import urllib +import urllib2 +import sys + +try: + from natsort import natsorted +except ImportError: + # natsorted is a simple helper to sort "naturally", e.g. "vm40" is sorted + # after "vm7". Defaults to normal sorting. + natsorted = sorted + +# These values are buildbot constants used for Build and BuildStep. +# This line was copied from master/buildbot/status/builder.py. +SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = range(6) + +## Generic node caching code. + + +class Node(object): + """Root class for all nodes in the graph. + + Provides base functionality for any node in the graph, independent if it has + children or not or if its content can be addressed through an url or needs to + be fetched as part of another node. + + self.printable_attributes is only used for self documentation and for str() + implementation. + """ + printable_attributes = [] + + def __init__(self, parent, url): + self.printable_attributes = self.printable_attributes[:] + if url: + self.printable_attributes.append('url') + url = url.rstrip('/') + if parent is not None: + self.printable_attributes.append('parent') + self.url = url + self.parent = parent + + def __str__(self): + return self.to_string() + + def __repr__(self): + """Embeds key if present.""" + key = getattr(self, 'key', None) + if key is not None: + return '<%s key=%s>' % (self.__class__.__name__, key) + cached_keys = getattr(self, 'cached_keys', None) + if cached_keys is not None: + return '<%s keys=%s>' % (self.__class__.__name__, cached_keys) + return super(Node, self).__repr__() + + def to_string(self, maximum=100): + out = ['%s:' % self.__class__.__name__] + assert not 'printable_attributes' in self.printable_attributes + + def limit(txt): + txt = str(txt) + if maximum > 0: + if len(txt) > maximum + 2: + txt = txt[:maximum] + '...' + return txt + + for k in sorted(self.printable_attributes): + if k == 'parent': + # Avoid infinite recursion. + continue + out.append(limit(' %s: %r' % (k, getattr(self, k)))) + return '\n'.join(out) + + def refresh(self): + """Refreshes the data.""" + self.discard() + return self.cache() + + def cache(self): # pragma: no cover + """Caches the data.""" + raise NotImplementedError() + + def discard(self): # pragma: no cover + """Discards cached data. + + Pretty much everything is temporary except completed Build. + """ + raise NotImplementedError() + + +class AddressableBaseDataNode(Node): # pylint: disable=W0223 + """A node that contains a dictionary of data that can be fetched with an url. + + The node is directly addressable. It also often can be fetched by the parent. + """ + printable_attributes = Node.printable_attributes + ['data'] + + def __init__(self, parent, url, data): + super(AddressableBaseDataNode, self).__init__(parent, url) + self._data = data + + @property + def cached_data(self): + return self._data + + @property + def data(self): + self.cache() + return self._data + + def cache(self): + if self._data is None: + self._data = self._readall() + return True + return False + + def discard(self): + self._data = None + + def read(self, suburl): + assert self.url, self.__class__.__name__ + url = self.url + if suburl: + url = '%s/%s' % (self.url, suburl) + return self.parent.read(url) + + def _readall(self): + return self.read('') + + +class AddressableDataNode(AddressableBaseDataNode): # pylint: disable=W0223 + """Automatically encodes the url.""" + + def __init__(self, parent, url, data): + super(AddressableDataNode, self).__init__(parent, urllib.quote(url), data) + + +class NonAddressableDataNode(Node): # pylint: disable=W0223 + """A node that cannot be addressed by an unique url. + + The data comes directly from the parent. + """ + + def __init__(self, parent, subkey): + super(NonAddressableDataNode, self).__init__(parent, None) + self.subkey = subkey + + @property + def cached_data(self): + if self.parent.cached_data is None: + return None + return self.parent.cached_data[self.subkey] + + @property + def data(self): + return self.parent.data[self.subkey] + + def cache(self): + self.parent.cache() + + def discard(self): # pragma: no cover + """Avoid invalid state when parent recreate the object.""" + raise AttributeError('Call parent discard() instead') + + +class VirtualNodeList(Node): + """Base class for every node that has children. + + Adds partial supports for keys and iterator functionality. 'key' can be a + string or a int. Not to be used directly. + """ + printable_attributes = Node.printable_attributes + ['keys'] + + def __init__(self, parent, url): + super(VirtualNodeList, self).__init__(parent, url) + # Keeps the keys independently when ordering is needed. + self._is_cached = False + self._has_keys_cached = False + + def __contains__(self, key): + """Enables 'if i in obj:'.""" + return key in self.keys + + def __iter__(self): + """Enables 'for i in obj:'. It returns children.""" + self.cache_keys() + for key in self.keys: + yield self[key] + + def __len__(self): + """Enables 'len(obj)' to get the number of childs.""" + return len(self.keys) + + def discard(self): + """Discards data. + + The default behavior is to not invalidate cached keys. The only place where + keys need to be invalidated is with Builds. + """ + self._is_cached = False + self._has_keys_cached = False + + @property + def cached_children(self): # pragma: no cover + """Returns an iterator over the children that are cached.""" + raise NotImplementedError() + + @property + def cached_keys(self): # pragma: no cover + raise NotImplementedError() + + @property + def keys(self): # pragma: no cover + """Returns the keys for every children.""" + raise NotImplementedError() + + def __getitem__(self, key): # pragma: no cover + """Returns a child, without fetching its data. + + The children could be invalid since no verification is done. + """ + raise NotImplementedError() + + def cache(self): # pragma: no cover + """Cache all the children.""" + raise NotImplementedError() + + def cache_keys(self): # pragma: no cover + """Cache all children's keys.""" + raise NotImplementedError() + + +class NodeList(VirtualNodeList): # pylint: disable=W0223 + """Adds a cache of the keys.""" + + def __init__(self, parent, url): + super(NodeList, self).__init__(parent, url) + self._keys = [] + + @property + def cached_keys(self): + return self._keys + + @property + def keys(self): + self.cache_keys() + return self._keys + + +class NonAddressableNodeList(VirtualNodeList): # pylint: disable=W0223 + """A node that contains children but retrieves all its data from its parent. + + I.e. there's no url to get directly this data. + """ + # Child class object for children of this instance. For example, BuildSteps + # has BuildStep children. + _child_cls = None + + def __init__(self, parent, subkey): + super(NonAddressableNodeList, self).__init__(parent, None) + self.subkey = subkey + assert (not isinstance(self._child_cls, NonAddressableDataNode) and + issubclass(self._child_cls, NonAddressableDataNode)), ( + self._child_cls.__name__) + + @property + def cached_children(self): + if self.parent.cached_data is not None: + for i in xrange(len(self.parent.cached_data[self.subkey])): + yield self[i] + + @property + def cached_data(self): + if self.parent.cached_data is None: + return None + return self.parent.data.get(self.subkey, None) + + @property + def cached_keys(self): + if self.parent.cached_data is None: + return None + return range(len(self.parent.data.get(self.subkey, []))) + + @property + def data(self): + return self.parent.data[self.subkey] + + def cache(self): + self.parent.cache() + + def cache_keys(self): + self.parent.cache() + + def discard(self): # pragma: no cover + """Do not call. + + Avoid infinite recursion by having the caller calls the parent's + discard() explicitely. + """ + raise AttributeError('Call parent discard() instead') + + def __iter__(self): + """Enables 'for i in obj:'. It returns children.""" + if self.data: + for i in xrange(len(self.data)): + yield self[i] + + def __getitem__(self, key): + """Doesn't cache the value, it's not needed. + + TODO(maruel): Cache? + """ + if isinstance(key, int) and key < 0: + key = len(self.data) + key + # pylint: disable=E1102 + return self._child_cls(self, key) + + +class AddressableNodeList(NodeList): + """A node that has children that can be addressed with an url.""" + + # Child class object for children of this instance. For example, Builders has + # Builder children and Builds has Build children. + _child_cls = None + + def __init__(self, parent, url): + super(AddressableNodeList, self).__init__(parent, url) + self._cache = {} + assert (not isinstance(self._child_cls, AddressableDataNode) and + issubclass(self._child_cls, AddressableDataNode)), ( + self._child_cls.__name__) + + @property + def cached_children(self): + for item in self._cache.itervalues(): + if item.cached_data is not None: + yield item + + @property + def cached_keys(self): + return self._cache.keys() + + def __getitem__(self, key): + """Enables 'obj[i]'.""" + if self._has_keys_cached and not key in self._keys: + raise KeyError(key) + + if not key in self._cache: + # Create an empty object. + self._create_obj(key, None) + return self._cache[key] + + def cache(self): + if not self._is_cached: + data = self._readall() + for key in sorted(data): + self._create_obj(key, data[key]) + self._is_cached = True + self._has_keys_cached = True + + def cache_partial(self, children): + """Caches a partial number of children. + + This method is more efficient since it does a single request for all the + children instead of one request per children. + + It only grab objects not already cached. + """ + # pylint: disable=W0212 + if not self._is_cached: + to_fetch = [ + child + for child in children + if not (child in self._cache and self._cache[child].cached_data) + ] + if to_fetch: + # Similar to cache(). The only reason to sort is to simplify testing. + params = '&'.join('select=%s' % urllib.quote(str(v)) + for v in sorted(to_fetch)) + data = self.read('?' + params) + for key in sorted(data): + self._create_obj(key, data[key]) + + def cache_keys(self): + """Implement to speed up enumeration. Defaults to call cache().""" + if not self._has_keys_cached: + self.cache() + assert self._has_keys_cached + + def discard(self): + """Discards temporary children.""" + super(AddressableNodeList, self).discard() + for v in self._cache.itervalues(): + v.discard() + + def read(self, suburl): + assert self.url, self.__class__.__name__ + url = self.url + if suburl: + url = '%s/%s' % (self.url, suburl) + return self.parent.read(url) + + def _create_obj(self, key, data): + """Creates an object of type self._child_cls.""" + # pylint: disable=E1102 + obj = self._child_cls(self, key, data) + # obj.key and key may be different. + # No need to overide cached data with None. + if data is not None or obj.key not in self._cache: + self._cache[obj.key] = obj + if obj.key not in self._keys: + self._keys.append(obj.key) + + def _readall(self): + return self.read('') + + +class SubViewNodeList(VirtualNodeList): # pylint: disable=W0223 + """A node that shows a subset of children that comes from another structure. + + The node is not addressable. + + E.g. the keys are retrieved from parent but the actual data comes from + virtual_parent. + """ + + def __init__(self, parent, virtual_parent, subkey): + super(SubViewNodeList, self).__init__(parent, None) + self.subkey = subkey + self.virtual_parent = virtual_parent + assert isinstance(self.parent, AddressableDataNode) + assert isinstance(self.virtual_parent, NodeList) + + @property + def cached_children(self): + if self.parent.cached_data is not None: + for item in self.keys: + if item in self.virtual_parent.keys: + child = self[item] + if child.cached_data is not None: + yield child + + @property + def cached_keys(self): + return (self.parent.cached_data or {}).get(self.subkey, []) + + @property + def keys(self): + self.cache_keys() + return self.parent.data.get(self.subkey, []) + + def cache(self): + """Batch request for each child in a single read request.""" + if not self._is_cached: + self.virtual_parent.cache_partial(self.keys) + self._is_cached = True + + def cache_keys(self): + if not self._has_keys_cached: + self.parent.cache() + self._has_keys_cached = True + + def discard(self): + if self.parent.cached_data is not None: + for child in self.virtual_parent.cached_children: + if child.key in self.keys: + child.discard() + self.parent.discard() + super(SubViewNodeList, self).discard() + + def __getitem__(self, key): + """Makes sure the key is in our key but grab it from the virtual parent.""" + return self.virtual_parent[key] + + def __iter__(self): + self.cache() + return super(SubViewNodeList, self).__iter__() + +############################################################################### +## Buildbot-specific code + + +class Slave(AddressableDataNode): + """Buildbot slave class.""" + printable_attributes = AddressableDataNode.printable_attributes + [ + 'name', + 'key', + 'connected', + 'version', + ] + + def __init__(self, parent, name, data): + super(Slave, self).__init__(parent, name, data) + self.name = name + self.key = self.name + # TODO(maruel): Add SlaveBuilders and a 'builders' property. + # TODO(maruel): Add a 'running_builds' property. + + @property + def connected(self): + return self.data.get('connected', False) + + @property + def version(self): + return self.data.get('version') + + +class Slaves(AddressableNodeList): + """Buildbot slaves.""" + _child_cls = Slave + printable_attributes = AddressableNodeList.printable_attributes + ['names'] + + def __init__(self, parent): + super(Slaves, self).__init__(parent, 'slaves') + + @property + def names(self): + return self.keys + + +class BuilderSlaves(SubViewNodeList): + """Similar to Slaves but only list slaves connected to a specific builder.""" + printable_attributes = SubViewNodeList.printable_attributes + ['names'] + + def __init__(self, parent): + super(BuilderSlaves, self).__init__(parent, parent.parent.parent.slaves, + 'slaves') + + @property + def names(self): + return self.keys + + +class BuildStep(NonAddressableDataNode): + """Class for a buildbot build step.""" + printable_attributes = NonAddressableDataNode.printable_attributes + [ + 'name', + 'number', + 'start_time', + 'end_time', + 'duration', + 'is_started', + 'is_finished', + 'is_running', + 'result', + 'simplified_result', + ] + + def __init__(self, parent, number): + """Pre-loaded, since the data is retrieved via the Build object.""" + assert isinstance(number, int) + super(BuildStep, self).__init__(parent, number) + self.number = number + + @property + def start_time(self): + if self.data.get('times'): + return int(round(self.data['times'][0])) + + @property + def end_time(self): + times = self.data.get('times') + if times and len(times) == 2 and times[1]: + return int(round(times[1])) + + @property + def duration(self): + if self.start_time: + return (self.end_time or int(round(time.time()))) - self.start_time + + @property + def name(self): + return self.data['name'] + + @property + def is_started(self): + return self.data.get('isStarted', False) + + @property + def is_finished(self): + return self.data.get('isFinished', False) + + @property + def is_running(self): + return self.is_started and not self.is_finished + + @property + def result(self): + result = self.data.get('results') + if result is None: + # results may be 0, in that case with filter=1, the value won't be + # present. + if self.data.get('isFinished'): + result = self.data.get('results', 0) + while isinstance(result, list): + result = result[0] + return result + + @property + def simplified_result(self): + """Returns a simplified 3 state value, True, False or None.""" + result = self.result + if result in (SUCCESS, WARNINGS): + return True + elif result in (FAILURE, EXCEPTION, RETRY): + return False + assert result in (None, SKIPPED), (result, self.data) + return None + + +class BuildSteps(NonAddressableNodeList): + """Duplicates keys to support lookup by both step number and step name.""" + printable_attributes = NonAddressableNodeList.printable_attributes + [ + 'failed', + ] + _child_cls = BuildStep + + def __init__(self, parent): + """Pre-loaded, since the data is retrieved via the Build object.""" + super(BuildSteps, self).__init__(parent, 'steps') + + @property + def keys(self): + """Returns the steps name in order.""" + return [i['name'] for i in self.data or []] + + @property + def failed(self): + """Shortcuts that lists the step names of steps that failed.""" + return [step.name for step in self if step.simplified_result is False] + + def __getitem__(self, key): + """Accept step name in addition to index number.""" + if isinstance(key, basestring): + # It's a string, try to find the corresponding index. + for i, step in enumerate(self.data): + if step['name'] == key: + key = i + break + else: + raise KeyError(key) + return super(BuildSteps, self).__getitem__(key) + + +class Build(AddressableDataNode): + """Buildbot build info.""" + printable_attributes = AddressableDataNode.printable_attributes + [ + 'key', + 'number', + 'steps', + 'blame', + 'reason', + 'revision', + 'result', + 'simplified_result', + 'start_time', + 'end_time', + 'duration', + 'slave', + 'properties', + 'completed', + ] + + def __init__(self, parent, key, data): + super(Build, self).__init__(parent, str(key), data) + self.number = int(key) + self.key = self.number + self.steps = BuildSteps(self) + + @property + def blame(self): + return self.data.get('blame', []) + + @property + def builder(self): + """Returns the Builder object. + + Goes up the hierarchy to find the Buildbot.builders[builder] instance. + """ + return self.parent.parent.parent.parent.builders[self.data['builderName']] + + @property + def start_time(self): + if self.data.get('times'): + return int(round(self.data['times'][0])) + + @property + def end_time(self): + times = self.data.get('times') + if times and len(times) == 2 and times[1]: + return int(round(times[1])) + + @property + def duration(self): + if self.start_time: + return (self.end_time or int(round(time.time()))) - self.start_time + + @property + def eta(self): + return self.data.get('eta', 0) + + @property + def completed(self): + return self.data.get('currentStep') is None + + @property + def properties(self): + return self.data.get('properties', []) + + @property + def reason(self): + return self.data.get('reason') + + @property + def result(self): + result = self.data.get('results') + while isinstance(result, list): + result = result[0] + if result is None and self.steps: + # results may be 0, in that case with filter=1, the value won't be + # present. + result = self.steps[-1].result + return result + + @property + def revision(self): + return self.data.get('sourceStamp', {}).get('revision') + + @property + def simplified_result(self): + """Returns a simplified 3 state value, True, False or None.""" + result = self.result + if result in (SUCCESS, WARNINGS, SKIPPED): + return True + elif result in (FAILURE, EXCEPTION, RETRY): + return False + assert result is None, (result, self.data) + return None + + @property + def slave(self): + """Returns the Slave object. + + Goes up the hierarchy to find the Buildbot.slaves[slave] instance. + """ + return self.parent.parent.parent.parent.slaves[self.data['slave']] + + def discard(self): + """Completed Build isn't discarded.""" + if self._data and self.result is None: + assert not self.steps or not self.steps[-1].data.get('isFinished') + self._data = None + + +class CurrentBuilds(SubViewNodeList): + """Lists of the current builds.""" + + def __init__(self, parent): + super(CurrentBuilds, self).__init__(parent, parent.builds, 'currentBuilds') + + +class PendingBuilds(AddressableDataNode): + """List of the pending builds.""" + + def __init__(self, parent): + super(PendingBuilds, self).__init__(parent, 'pendingBuilds', None) + + +class Builds(AddressableNodeList): + """Supports iteration. + + Recommends using .cache() to speed up if a significant number of builds are + iterated over. + """ + _child_cls = Build + + def __init__(self, parent): + super(Builds, self).__init__(parent, 'builds') + + def __getitem__(self, key): + """Support for negative reference and enable retrieving non-cached builds. + + e.g. -1 is the last build, -2 is the previous build before the last one. + """ + key = int(key) + if key < 0: + # Convert negative to positive build number. + self.cache_keys() + # Since the negative value can be outside of the cache keys range, use the + # highest key value and calculate from it. + key = max(self._keys) + key + 1 + + if not key in self._cache: + # Create an empty object. + self._create_obj(key, None) + return self._cache[key] + + def __iter__(self): + """Returns cached Build objects in reversed order. + + The most recent build is returned first and then in reverse chronological + order, up to the oldest cached build by the server. Older builds can be + accessed but will trigger significantly more I/O so they are not included by + default in the iteration. + + To access the older builds, use self.iterall() instead. + """ + self.cache() + return reversed(self._cache.values()) + + def iterall(self): + """Returns Build objects in decreasing order unbounded up to build 0. + + The most recent build is returned first and then in reverse chronological + order. Older builds can be accessed and will trigger significantly more I/O + so use this carefully. + """ + # Only cache keys here. + self.cache_keys() + if self._keys: + for i in xrange(max(self._keys), -1, -1): + yield self[i] + + def cache_keys(self): + """Grabs the keys (build numbers) from the builder.""" + if not self._has_keys_cached: + for i in self.parent.data.get('cachedBuilds', []): + i = int(i) + self._cache.setdefault(i, Build(self, i, None)) + if i not in self._keys: + self._keys.append(i) + self._has_keys_cached = True + + def discard(self): + super(Builds, self).discard() + # Can't keep keys. + self._has_keys_cached = False + + def _readall(self): + return self.read('_all') + + +class Builder(AddressableDataNode): + """Builder status.""" + printable_attributes = AddressableDataNode.printable_attributes + [ + 'name', + 'key', + 'builds', + 'slaves', + 'pending_builds', + 'current_builds', + ] + + def __init__(self, parent, name, data): + super(Builder, self).__init__(parent, name, data) + self.name = name + self.key = name + self.builds = Builds(self) + self.slaves = BuilderSlaves(self) + self.current_builds = CurrentBuilds(self) + self.pending_builds = PendingBuilds(self) + + def discard(self): + super(Builder, self).discard() + self.builds.discard() + self.slaves.discard() + self.current_builds.discard() + + +class Builders(AddressableNodeList): + """Root list of builders.""" + _child_cls = Builder + + def __init__(self, parent): + super(Builders, self).__init__(parent, 'builders') + + +class Buildbot(AddressableBaseDataNode): + """This object should be recreated on a master restart as it caches data.""" + # Throttle fetches to not kill the server. + auto_throttle = None + printable_attributes = AddressableDataNode.printable_attributes + [ + 'slaves', + 'builders', + 'last_fetch', + ] + + def __init__(self, url): + super(Buildbot, self).__init__(None, url.rstrip('/') + '/json', None) + self._builders = Builders(self) + self._slaves = Slaves(self) + self.last_fetch = None + + @property + def builders(self): + return self._builders + + @property + def slaves(self): + return self._slaves + + def discard(self): + """Discards information about Builders and Slaves.""" + super(Buildbot, self).discard() + self._builders.discard() + self._slaves.discard() + + def read(self, suburl): + if self.auto_throttle: + if self.last_fetch: + delta = datetime.datetime.utcnow() - self.last_fetch + remaining = (datetime.timedelta(seconds=self.auto_throttle) - delta) + if remaining > datetime.timedelta(seconds=0): + logging.debug('Sleeping for %ss', remaining) + time.sleep(remaining.seconds) + self.last_fetch = datetime.datetime.utcnow() + url = '%s/%s' % (self.url, suburl) + if '?' in url: + url += '&filter=1' + else: + url += '?filter=1' + logging.info('read(%s)', suburl) + channel = urllib.urlopen(url) + data = channel.read() + try: + return json.loads(data) + except ValueError: + if channel.getcode() >= 400: + # Convert it into an HTTPError for easier processing. + raise urllib2.HTTPError(url, channel.getcode(), '%s:\n%s' % (url, data), + channel.headers, None) + raise + + def _readall(self): + return self.read('project') + +############################################################################### +## Controller code + + +def usage(more): + + def hook(fn): + fn.func_usage_more = more + return fn + + return hook + + +def need_buildbot(fn): + """Post-parse args to create a buildbot object.""" + + @functools.wraps(fn) + def hook(parser, args, *extra_args, **kwargs): + old_parse_args = parser.parse_args + + def new_parse_args(args): + options, args = old_parse_args(args) + if len(args) < 1: + parser.error('Need to pass the root url of the buildbot') + url = args.pop(0) + if not url.startswith('http'): + url = 'http://' + url + buildbot = Buildbot(url) + buildbot.auto_throttle = options.throttle + return options, args, buildbot + + parser.parse_args = new_parse_args + # Call the original function with the modified parser. + return fn(parser, args, *extra_args, **kwargs) + + hook.func_usage_more = '[options] <url>' + return hook + + +@need_buildbot +def CMDpending(parser, args): + """Lists pending jobs.""" + parser.add_option('-b', + '--builder', + dest='builders', + action='append', + default=[], + help='Builders to filter on') + options, args, buildbot = parser.parse_args(args) + if args: + parser.error('Unrecognized parameters: %s' % ' '.join(args)) + if not options.builders: + options.builders = buildbot.builders.keys + for builder in options.builders: + builder = buildbot.builders[builder] + pending_builds = builder.data.get('pendingBuilds', 0) + if not pending_builds: + continue + print('Builder %s: %d' % (builder.name, pending_builds)) + if not options.quiet: + for pending in builder.pending_builds.data: + if 'revision' in pending['source']: + print(' revision: %s' % pending['source']['revision']) + for change in pending['source']['changes']: + print(' change:') + print(' comment: %r' % unicode(change['comments'][:50])) + print(' who: %s' % change['who']) + return 0 + + +@usage('[options] <url> [commands] ...') +@need_buildbot +def CMDrun(parser, args): + """Runs commands passed as parameters. + + When passing commands on the command line, each command will be run as if it + was on its own line. + """ + parser.add_option('-f', '--file', help='Read script from file') + parser.add_option('-i', + dest='use_stdin', + action='store_true', + help='Read script on stdin') + # Variable 'buildbot' is not used directly. + # pylint: disable=W0612 + options, args, buildbot = parser.parse_args(args) + if (bool(args) + bool(options.use_stdin) + bool(options.file)) != 1: + parser.error('Need to pass only one of: <commands>, -f <file> or -i') + if options.use_stdin: + cmds = sys.stdin.read() + elif options.file: + cmds = open(options.file).read() + else: + cmds = '\n'.join(args) + compiled = compile(cmds, '<cmd line>', 'exec') + # pylint: disable=eval-used + eval(compiled, globals(), locals()) + return 0 + + +@need_buildbot +def CMDinteractive(parser, args): + """Runs an interactive shell to run queries.""" + _, args, buildbot = parser.parse_args(args) + if args: + parser.error('Unrecognized parameters: %s' % ' '.join(args)) + prompt = ( + 'Buildbot interactive console for "%s".\n' + 'Hint: Start with typing: \'buildbot.printable_attributes\' or ' + '\'print str(buildbot)\' to explore.') % buildbot.url[:-len('/json')] + local_vars = {'buildbot': buildbot, 'b': buildbot} + code.interact(prompt, None, local_vars) + + +@need_buildbot +def CMDidle(parser, args): + """Lists idle slaves.""" + return find_idle_busy_slaves(parser, args, True) + + +@need_buildbot +def CMDbusy(parser, args): + """Lists idle slaves.""" + return find_idle_busy_slaves(parser, args, False) + + +@need_buildbot +def CMDdisconnected(parser, args): + """Lists disconnected slaves.""" + _, args, buildbot = parser.parse_args(args) + if args: + parser.error('Unrecognized parameters: %s' % ' '.join(args)) + for slave in buildbot.slaves: + if not slave.connected: + print(slave.name) + return 0 + + +def find_idle_busy_slaves(parser, args, show_idle): + parser.add_option('-b', + '--builder', + dest='builders', + action='append', + default=[], + help='Builders to filter on') + parser.add_option('-s', + '--slave', + dest='slaves', + action='append', + default=[], + help='Slaves to filter on') + options, args, buildbot = parser.parse_args(args) + if args: + parser.error('Unrecognized parameters: %s' % ' '.join(args)) + if not options.builders: + options.builders = buildbot.builders.keys + for builder in options.builders: + builder = buildbot.builders[builder] + if options.slaves: + # Only the subset of slaves connected to the builder. + slaves = list(set(options.slaves).intersection(set(builder.slaves.names))) + if not slaves: + continue + else: + slaves = builder.slaves.names + busy_slaves = [build.slave.name for build in builder.current_builds] + if show_idle: + slaves = natsorted(set(slaves) - set(busy_slaves)) + else: + slaves = natsorted(set(slaves) & set(busy_slaves)) + if options.quiet: + for slave in slaves: + print(slave) + else: + if slaves: + print('Builder %s: %s' % (builder.name, ', '.join(slaves))) + return 0 + + +def last_failure(buildbot, + builders=None, + slaves=None, + steps=None, + no_cache=False): + """Returns Build object with last failure with the specific filters.""" + builders = builders or buildbot.builders.keys + for builder in builders: + builder = buildbot.builders[builder] + if slaves: + # Only the subset of slaves connected to the builder. + builder_slaves = list(set(slaves).intersection(set(builder.slaves.names))) + if not builder_slaves: + continue + else: + builder_slaves = builder.slaves.names + + if not no_cache and len(builder.slaves) > 2: + # Unless you just want the last few builds, it's often faster to + # fetch the whole thing at once, at the cost of a small hickup on + # the buildbot. + # TODO(maruel): Cache only N last builds or all builds since + # datetime. + builder.builds.cache() + + found = [] + for build in builder.builds: + if build.slave.name not in builder_slaves or build.slave.name in found: + continue + # Only add the slave for the first completed build but still look for + # incomplete builds. + if build.completed: + found.append(build.slave.name) + + if steps: + if any(build.steps[step].simplified_result is False for step in steps): + yield build + elif build.simplified_result is False: + yield build + + if len(found) == len(builder_slaves): + # Found all the slaves, quit. + break + + +@need_buildbot +def CMDlast_failure(parser, args): + """Lists all slaves that failed on that step on their last build. + + Example: to find all slaves where their last build was a compile failure, + run with --step compile + """ + parser.add_option( + '-S', + '--step', + dest='steps', + action='append', + default=[], + help='List all slaves that failed on that step on their last build') + parser.add_option('-b', + '--builder', + dest='builders', + action='append', + default=[], + help='Builders to filter on') + parser.add_option('-s', + '--slave', + dest='slaves', + action='append', + default=[], + help='Slaves to filter on') + parser.add_option('-n', + '--no_cache', + action='store_true', + help='Don\'t load all builds at once') + options, args, buildbot = parser.parse_args(args) + if args: + parser.error('Unrecognized parameters: %s' % ' '.join(args)) + print_builders = not options.quiet and len(options.builders) != 1 + last_builder = None + for build in last_failure(buildbot, + builders=options.builders, + slaves=options.slaves, + steps=options.steps, + no_cache=options.no_cache): + + if print_builders and last_builder != build.builder: + print(build.builder.name) + last_builder = build.builder + + if options.quiet: + if options.slaves: + print('%s: %s' % (build.builder.name, build.slave.name)) + else: + print(build.slave.name) + else: + out = '%d on %s: blame:%s' % (build.number, build.slave.name, + ', '.join(build.blame)) + if print_builders: + out = ' ' + out + print(out) + + if len(options.steps) != 1: + for step in build.steps: + if step.simplified_result is False: + # Assume the first line is the text name anyway. + summary = ', '.join(step.data['text'][1:])[:40] + out = ' %s: "%s"' % (step.data['name'], summary) + if print_builders: + out = ' ' + out + print(out) + return 0 + + +@need_buildbot +def CMDcurrent(parser, args): + """Lists current jobs.""" + parser.add_option('-b', + '--builder', + dest='builders', + action='append', + default=[], + help='Builders to filter on') + parser.add_option('--blame', + action='store_true', + help='Only print the blame list') + options, args, buildbot = parser.parse_args(args) + if args: + parser.error('Unrecognized parameters: %s' % ' '.join(args)) + if not options.builders: + options.builders = buildbot.builders.keys + + if options.blame: + blame = set() + for builder in options.builders: + for build in buildbot.builders[builder].current_builds: + if build.blame: + for blamed in build.blame: + blame.add(blamed) + print('\n'.join(blame)) + return 0 + + for builder in options.builders: + builder = buildbot.builders[builder] + if not options.quiet and builder.current_builds: + print(builder.name) + for build in builder.current_builds: + if options.quiet: + print(build.slave.name) + else: + out = '%4d: slave=%10s' % (build.number, build.slave.name) + out += ' duration=%5d' % (build.duration or 0) + if build.eta: + out += ' eta=%5.0f' % build.eta + else: + out += ' ' + if build.blame: + out += ' blame=' + ', '.join(build.blame) + print(out) + + return 0 + + +@need_buildbot +def CMDbuilds(parser, args): + """Lists all builds. + + Example: to find all builds on a single slave, run with -b bar -s foo + """ + parser.add_option('-r', + '--result', + type='int', + help='Build result to filter on') + parser.add_option('-b', + '--builder', + dest='builders', + action='append', + default=[], + help='Builders to filter on') + parser.add_option('-s', + '--slave', + dest='slaves', + action='append', + default=[], + help='Slaves to filter on') + parser.add_option('-n', + '--no_cache', + action='store_true', + help='Don\'t load all builds at once') + options, args, buildbot = parser.parse_args(args) + if args: + parser.error('Unrecognized parameters: %s' % ' '.join(args)) + builders = options.builders or buildbot.builders.keys + for builder in builders: + builder = buildbot.builders[builder] + for build in builder.builds: + if not options.slaves or build.slave.name in options.slaves: + if options.quiet: + out = '' + if options.builders: + out += '%s/' % builder.name + if len(options.slaves) != 1: + out += '%s/' % build.slave.name + out += '%d revision:%s result:%s blame:%s' % ( + build.number, build.revision, build.result, ','.join(build.blame)) + print(out) + else: + print(build) + return 0 + + +@need_buildbot +def CMDcount(parser, args): + """Count the number of builds that occured during a specific period.""" + parser.add_option('-o', + '--over', + type='int', + help='Number of seconds to look for') + parser.add_option('-b', + '--builder', + dest='builders', + action='append', + default=[], + help='Builders to filter on') + options, args, buildbot = parser.parse_args(args) + if args: + parser.error('Unrecognized parameters: %s' % ' '.join(args)) + if not options.over: + parser.error( + 'Specify the number of seconds, e.g. --over 86400 for the last 24 ' + 'hours') + builders = options.builders or buildbot.builders.keys + counts = {} + since = time.time() - options.over + for builder in builders: + builder = buildbot.builders[builder] + counts[builder.name] = 0 + if not options.quiet: + print(builder.name) + for build in builder.builds.iterall(): + try: + start_time = build.start_time + except urllib2.HTTPError: + # The build was probably trimmed. + print('Failed to fetch build %s/%d' % (builder.name, build.number), + file=sys.stderr) + continue + if start_time >= since: + counts[builder.name] += 1 + else: + break + if not options.quiet: + print('.. %d' % counts[builder.name]) + + align_name = max(len(b) for b in counts) + align_number = max(len(str(c)) for c in counts.itervalues()) + for builder in sorted(counts): + print('%*s: %*d' % (align_name, builder, align_number, counts[builder])) + print('Total: %d' % sum(counts.itervalues())) + return 0 + + +def gen_parser(): + """Returns an OptionParser instance with default options. + + It should be then processed with gen_usage() before being used. + """ + parser = optparse.OptionParser(version=__version__) + # Remove description formatting + parser.format_description = lambda x: parser.description + # Add common parsing. + old_parser_args = parser.parse_args + + def Parse(*args, **kwargs): + options, args = old_parser_args(*args, **kwargs) + if options.verbose >= 2: + logging.basicConfig(level=logging.DEBUG) + elif options.verbose: + logging.basicConfig(level=logging.INFO) + else: + logging.basicConfig(level=logging.WARNING) + return options, args + + parser.parse_args = Parse + + parser.add_option('-v', + '--verbose', + action='count', + help='Use multiple times to increase logging leve') + parser.add_option( + '-q', + '--quiet', + action='store_true', + help='Reduces the output to be parsed by scripts, independent of -v') + parser.add_option('--throttle', + type='float', + help='Minimum delay to sleep between requests') + return parser + +############################################################################### +## Generic subcommand handling code + + +def Command(name): + return getattr(sys.modules[__name__], 'CMD' + name, None) + + +@usage('<command>') +def CMDhelp(parser, args): + """Print list of commands or use 'help <command>'.""" + _, args = parser.parse_args(args) + if len(args) == 1: + return main(args + ['--help']) + parser.print_help() + return 0 + + +def gen_usage(parser, command): + """Modifies an OptionParser object with the command's documentation. + + The documentation is taken from the function's docstring. + """ + obj = Command(command) + more = getattr(obj, 'func_usage_more') + # OptParser.description prefer nicely non-formatted strings. + parser.description = obj.__doc__ + '\n' + parser.set_usage('usage: %%prog %s %s' % (command, more)) + + +def main(args=None): + # Do it late so all commands are listed. + # pylint: disable=E1101 + CMDhelp.__doc__ += '\n\nCommands are:\n' + '\n'.join( + ' %-12s %s' % (fn[3:], Command(fn[3:]).__doc__.split('\n', 1)[0]) + for fn in dir(sys.modules[__name__]) if fn.startswith('CMD')) + + parser = gen_parser() + if args is None: + args = sys.argv[1:] + if args: + command = Command(args[0]) + if command: + # "fix" the usage and the description now that we know the subcommand. + gen_usage(parser, args[0]) + return command(parser, args[1:]) + + # Not a known command. Default to help. + gen_usage(parser, 'help') + return CMDhelp(parser, args) + + +if __name__ == '__main__': + sys.exit(main()) |