path: root/cros_utils/buildbot_json.py

#!/usr/bin/env python2
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#    * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#    * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

# NOTE: This file is NOT under GPL.  See above.
"""Queries buildbot through the json interface.
"""

from __future__ import print_function

__author__ = 'maruel@chromium.org'
__version__ = '1.2'

import code
import datetime
import functools
import json

# Pylint recommends we use "from chromite.lib import cros_logging as logging".
# Chromite specific policy message, we want to keep using the standard logging.
# pylint: disable=cros-logging-import
import logging

# pylint: disable=deprecated-module
import optparse

import time
import urllib
import urllib2
import sys

try:
  from natsort import natsorted
except ImportError:
  # natsorted is a simple helper to sort "naturally", e.g. "vm40" is sorted
  # after "vm7". Defaults to normal sorting.
  natsorted = sorted

# These values are buildbot constants used for Build and BuildStep.
# This line was copied from master/buildbot/status/builder.py.
SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = range(6)

## Generic node caching code.


class Node(object):
  """Root class for all nodes in the graph.

  Provides base functionality for any node in the graph, independent if it has
  children or not or if its content can be addressed through an url or needs to
  be fetched as part of another node.

  self.printable_attributes is only used for self documentation and for str()
  implementation.
  """
  printable_attributes = []

  def __init__(self, parent, url):
    self.printable_attributes = self.printable_attributes[:]
    if url:
      self.printable_attributes.append('url')
      url = url.rstrip('/')
    if parent is not None:
      self.printable_attributes.append('parent')
    self.url = url
    self.parent = parent

  def __str__(self):
    return self.to_string()

  def __repr__(self):
    """Embeds key if present."""
    key = getattr(self, 'key', None)
    if key is not None:
      return '<%s key=%s>' % (self.__class__.__name__, key)
    cached_keys = getattr(self, 'cached_keys', None)
    if cached_keys is not None:
      return '<%s keys=%s>' % (self.__class__.__name__, cached_keys)
    return super(Node, self).__repr__()

  def to_string(self, maximum=100):
    out = ['%s:' % self.__class__.__name__]
    assert not 'printable_attributes' in self.printable_attributes

    def limit(txt):
      txt = str(txt)
      if maximum > 0:
        if len(txt) > maximum + 2:
          txt = txt[:maximum] + '...'
      return txt

    for k in sorted(self.printable_attributes):
      if k == 'parent':
        # Avoid infinite recursion.
        continue
      out.append(limit('  %s: %r' % (k, getattr(self, k))))
    return '\n'.join(out)

  def refresh(self):
    """Refreshes the data."""
    self.discard()
    return self.cache()

  def cache(self):  # pragma: no cover
    """Caches the data."""
    raise NotImplementedError()

  def discard(self):  # pragma: no cover
    """Discards cached data.

    Pretty much everything is temporary except completed Build.
    """
    raise NotImplementedError()


class AddressableBaseDataNode(Node):  # pylint: disable=W0223
  """A node that contains a dictionary of data that can be fetched with an url.

  The node is directly addressable. It also often can be fetched by the parent.
  """
  printable_attributes = Node.printable_attributes + ['data']

  def __init__(self, parent, url, data):
    super(AddressableBaseDataNode, self).__init__(parent, url)
    self._data = data

  @property
  def cached_data(self):
    return self._data

  @property
  def data(self):
    self.cache()
    return self._data

  def cache(self):
    if self._data is None:
      self._data = self._readall()
      return True
    return False

  def discard(self):
    self._data = None

  def read(self, suburl):
    assert self.url, self.__class__.__name__
    url = self.url
    if suburl:
      url = '%s/%s' % (self.url, suburl)
    return self.parent.read(url)

  def _readall(self):
    return self.read('')


class AddressableDataNode(AddressableBaseDataNode):  # pylint: disable=W0223
  """Automatically encodes the url."""

  def __init__(self, parent, url, data):
    super(AddressableDataNode, self).__init__(parent, urllib.quote(url), data)


class NonAddressableDataNode(Node):  # pylint: disable=W0223
  """A node that cannot be addressed by an unique url.

  The data comes directly from the parent.
  """

  def __init__(self, parent, subkey):
    super(NonAddressableDataNode, self).__init__(parent, None)
    self.subkey = subkey

  @property
  def cached_data(self):
    if self.parent.cached_data is None:
      return None
    return self.parent.cached_data[self.subkey]

  @property
  def data(self):
    return self.parent.data[self.subkey]

  def cache(self):
    self.parent.cache()

  def discard(self):  # pragma: no cover
    """Avoid invalid state when parent recreate the object."""
    raise AttributeError('Call parent discard() instead')


class VirtualNodeList(Node):
  """Base class for every node that has children.

  Adds partial supports for keys and iterator functionality. 'key' can be a
  string or a int. Not to be used directly.
  """
  printable_attributes = Node.printable_attributes + ['keys']

  def __init__(self, parent, url):
    super(VirtualNodeList, self).__init__(parent, url)
    # Keeps the keys independently when ordering is needed.
    self._is_cached = False
    self._has_keys_cached = False

  def __contains__(self, key):
    """Enables 'if i in obj:'."""
    return key in self.keys

  def __iter__(self):
    """Enables 'for i in obj:'. It returns children."""
    self.cache_keys()
    for key in self.keys:
      yield self[key]

  def __len__(self):
    """Enables 'len(obj)' to get the number of childs."""
    return len(self.keys)

  def discard(self):
    """Discards data.

    The default behavior is to not invalidate cached keys. The only place where
    keys need to be invalidated is with Builds.
    """
    self._is_cached = False
    self._has_keys_cached = False

  @property
  def cached_children(self):  # pragma: no cover
    """Returns an iterator over the children that are cached."""
    raise NotImplementedError()

  @property
  def cached_keys(self):  # pragma: no cover
    raise NotImplementedError()

  @property
  def keys(self):  # pragma: no cover
    """Returns the keys for every children."""
    raise NotImplementedError()

  def __getitem__(self, key):  # pragma: no cover
    """Returns a child, without fetching its data.

    The children could be invalid since no verification is done.
    """
    raise NotImplementedError()

  def cache(self):  # pragma: no cover
    """Cache all the children."""
    raise NotImplementedError()

  def cache_keys(self):  # pragma: no cover
    """Cache all children's keys."""
    raise NotImplementedError()


class NodeList(VirtualNodeList):  # pylint: disable=W0223
  """Adds a cache of the keys."""

  def __init__(self, parent, url):
    super(NodeList, self).__init__(parent, url)
    self._keys = []

  @property
  def cached_keys(self):
    return self._keys

  @property
  def keys(self):
    self.cache_keys()
    return self._keys


class NonAddressableNodeList(VirtualNodeList):  # pylint: disable=W0223
  """A node that contains children but retrieves all its data from its parent.

  I.e. there's no url to get directly this data.
  """
  # Child class object for children of this instance. For example, BuildSteps
  # has BuildStep children.
  _child_cls = None

  def __init__(self, parent, subkey):
    super(NonAddressableNodeList, self).__init__(parent, None)
    self.subkey = subkey
    assert (not isinstance(self._child_cls, NonAddressableDataNode) and
            issubclass(self._child_cls, NonAddressableDataNode)), (
                self._child_cls.__name__)

  @property
  def cached_children(self):
    if self.parent.cached_data is not None:
      for i in range(len(self.parent.cached_data[self.subkey])):
        yield self[i]

  @property
  def cached_data(self):
    if self.parent.cached_data is None:
      return None
    return self.parent.data.get(self.subkey, None)

  @property
  def cached_keys(self):
    if self.parent.cached_data is None:
      return None
    return range(len(self.parent.data.get(self.subkey, [])))

  @property
  def data(self):
    return self.parent.data[self.subkey]

  def cache(self):
    self.parent.cache()

  def cache_keys(self):
    self.parent.cache()

  def discard(self):  # pragma: no cover
    """Do not call.

    Avoid infinite recursion by having the caller calls the parent's
    discard() explicitely.
    """
    raise AttributeError('Call parent discard() instead')

  def __iter__(self):
    """Enables 'for i in obj:'. It returns children."""
    if self.data:
      for i in range(len(self.data)):
        yield self[i]

  def __getitem__(self, key):
    """Doesn't cache the value, it's not needed.

    TODO(maruel): Cache?
    """
    if isinstance(key, int) and key < 0:
      key = len(self.data) + key
    # pylint: disable=E1102
    return self._child_cls(self, key)


class AddressableNodeList(NodeList):
  """A node that has children that can be addressed with an url."""

  # Child class object for children of this instance. For example, Builders has
  # Builder children and Builds has Build children.
  _child_cls = None

  def __init__(self, parent, url):
    super(AddressableNodeList, self).__init__(parent, url)
    self._cache = {}
    assert (not isinstance(self._child_cls, AddressableDataNode) and
            issubclass(self._child_cls, AddressableDataNode)), (
                self._child_cls.__name__)

  @property
  def cached_children(self):
    for item in self._cache.itervalues():
      if item.cached_data is not None:
        yield item

  @property
  def cached_keys(self):
    return self._cache.keys()

  def __getitem__(self, key):
    """Enables 'obj[i]'."""
    if self._has_keys_cached and not key in self._keys:
      raise KeyError(key)

    if not key in self._cache:
      # Create an empty object.
      self._create_obj(key, None)
    return self._cache[key]

  def cache(self):
    if not self._is_cached:
      data = self._readall()
      for key in sorted(data):
        self._create_obj(key, data[key])
      self._is_cached = True
      self._has_keys_cached = True

  def cache_partial(self, children):
    """Caches a partial number of children.

    This method is more efficient since it does a single request for all the
    children instead of one request per children.

    It only grab objects not already cached.
    """
    # pylint: disable=W0212
    if not self._is_cached:
      to_fetch = [
          child
          for child in children
          if not (child in self._cache and self._cache[child].cached_data)
      ]
      if to_fetch:
        # Similar to cache(). The only reason to sort is to simplify testing.
        params = '&'.join('select=%s' % urllib.quote(str(v))
                          for v in sorted(to_fetch))
        data = self.read('?' + params)
        for key in sorted(data):
          self._create_obj(key, data[key])

  def cache_keys(self):
    """Implement to speed up enumeration. Defaults to call cache()."""
    if not self._has_keys_cached:
      self.cache()
      assert self._has_keys_cached

  def discard(self):
    """Discards temporary children."""
    super(AddressableNodeList, self).discard()
    for v in self._cache.itervalues():
      v.discard()

  def read(self, suburl):
    assert self.url, self.__class__.__name__
    url = self.url
    if suburl:
      url = '%s/%s' % (self.url, suburl)
    return self.parent.read(url)

  def _create_obj(self, key, data):
    """Creates an object of type self._child_cls."""
    # pylint: disable=E1102
    obj = self._child_cls(self, key, data)
    # obj.key and key may be different.
    # No need to overide cached data with None.
    if data is not None or obj.key not in self._cache:
      self._cache[obj.key] = obj
    if obj.key not in self._keys:
      self._keys.append(obj.key)

  def _readall(self):
    return self.read('')


class SubViewNodeList(VirtualNodeList):  # pylint: disable=W0223
  """A node that shows a subset of children that comes from another structure.

  The node is not addressable.

  E.g. the keys are retrieved from parent but the actual data comes from
  virtual_parent.
  """

  def __init__(self, parent, virtual_parent, subkey):
    super(SubViewNodeList, self).__init__(parent, None)
    self.subkey = subkey
    self.virtual_parent = virtual_parent
    assert isinstance(self.parent, AddressableDataNode)
    assert isinstance(self.virtual_parent, NodeList)

  @property
  def cached_children(self):
    if self.parent.cached_data is not None:
      for item in self.keys:
        if item in self.virtual_parent.keys:
          child = self[item]
          if child.cached_data is not None:
            yield child

  @property
  def cached_keys(self):
    return (self.parent.cached_data or {}).get(self.subkey, [])

  @property
  def keys(self):
    self.cache_keys()
    return self.parent.data.get(self.subkey, [])

  def cache(self):
    """Batch request for each child in a single read request."""
    if not self._is_cached:
      self.virtual_parent.cache_partial(self.keys)
      self._is_cached = True

  def cache_keys(self):
    if not self._has_keys_cached:
      self.parent.cache()
      self._has_keys_cached = True

  def discard(self):
    if self.parent.cached_data is not None:
      for child in self.virtual_parent.cached_children:
        if child.key in self.keys:
          child.discard()
      self.parent.discard()
    super(SubViewNodeList, self).discard()

  def __getitem__(self, key):
    """Makes sure the key is in our key but grab it from the virtual parent."""
    return self.virtual_parent[key]

  def __iter__(self):
    self.cache()
    return super(SubViewNodeList, self).__iter__()

###############################################################################
## Buildbot-specific code


class Slave(AddressableDataNode):
  """Buildbot slave class."""
  printable_attributes = AddressableDataNode.printable_attributes + [
      'name',
      'key',
      'connected',
      'version',
  ]

  def __init__(self, parent, name, data):
    super(Slave, self).__init__(parent, name, data)
    self.name = name
    self.key = self.name
    # TODO(maruel): Add SlaveBuilders and a 'builders' property.
    # TODO(maruel): Add a 'running_builds' property.

  @property
  def connected(self):
    return self.data.get('connected', False)

  @property
  def version(self):
    return self.data.get('version')


class Slaves(AddressableNodeList):
  """Buildbot slaves."""
  _child_cls = Slave
  printable_attributes = AddressableNodeList.printable_attributes + ['names']

  def __init__(self, parent):
    super(Slaves, self).__init__(parent, 'slaves')

  @property
  def names(self):
    return self.keys


class BuilderSlaves(SubViewNodeList):
  """Similar to Slaves but only list slaves connected to a specific builder."""
  printable_attributes = SubViewNodeList.printable_attributes + ['names']

  def __init__(self, parent):
    super(BuilderSlaves, self).__init__(parent, parent.parent.parent.slaves,
                                        'slaves')

  @property
  def names(self):
    return self.keys


class BuildStep(NonAddressableDataNode):
  """Class for a buildbot build step."""
  printable_attributes = NonAddressableDataNode.printable_attributes + [
      'name',
      'number',
      'start_time',
      'end_time',
      'duration',
      'is_started',
      'is_finished',
      'is_running',
      'result',
      'simplified_result',
  ]

  def __init__(self, parent, number):
    """Pre-loaded, since the data is retrieved via the Build object."""
    assert isinstance(number, int)
    super(BuildStep, self).__init__(parent, number)
    self.number = number

  @property
  def start_time(self):
    if self.data.get('times'):
      return int(round(self.data['times'][0]))

  @property
  def end_time(self):
    times = self.data.get('times')
    if times and len(times) == 2 and times[1]:
      return int(round(times[1]))

  @property
  def duration(self):
    if self.start_time:
      return (self.end_time or int(round(time.time()))) - self.start_time

  @property
  def name(self):
    return self.data['name']

  @property
  def is_started(self):
    return self.data.get('isStarted', False)

  @property
  def is_finished(self):
    return self.data.get('isFinished', False)

  @property
  def is_running(self):
    return self.is_started and not self.is_finished

  @property
  def result(self):
    result = self.data.get('results')
    if result is None:
      # results may be 0, in that case with filter=1, the value won't be
      # present.
      if self.data.get('isFinished'):
        result = self.data.get('results', 0)
    while isinstance(result, list):
      result = result[0]
    return result

  @property
  def simplified_result(self):
    """Returns a simplified 3 state value, True, False or None."""
    result = self.result
    if result in (SUCCESS, WARNINGS):
      return True
    elif result in (FAILURE, EXCEPTION, RETRY):
      return False
    assert result in (None, SKIPPED), (result, self.data)
    return None


class BuildSteps(NonAddressableNodeList):
  """Duplicates keys to support lookup by both step number and step name."""
  printable_attributes = NonAddressableNodeList.printable_attributes + [
      'failed',
  ]
  _child_cls = BuildStep

  def __init__(self, parent):
    """Pre-loaded, since the data is retrieved via the Build object."""
    super(BuildSteps, self).__init__(parent, 'steps')

  @property
  def keys(self):
    """Returns the steps name in order."""
    return [i['name'] for i in self.data or []]

  @property
  def failed(self):
    """Shortcuts that lists the step names of steps that failed."""
    return [step.name for step in self if step.simplified_result is False]

  def __getitem__(self, key):
    """Accept step name in addition to index number."""
    if isinstance(key, basestring):
      # It's a string, try to find the corresponding index.
      for i, step in enumerate(self.data):
        if step['name'] == key:
          key = i
          break
      else:
        raise KeyError(key)
    return super(BuildSteps, self).__getitem__(key)


class Build(AddressableDataNode):
  """Buildbot build info."""
  printable_attributes = AddressableDataNode.printable_attributes + [
      'key',
      'number',
      'steps',
      'blame',
      'reason',
      'revision',
      'result',
      'simplified_result',
      'start_time',
      'end_time',
      'duration',
      'slave',
      'properties',
      'completed',
  ]

  def __init__(self, parent, key, data):
    super(Build, self).__init__(parent, str(key), data)
    self.number = int(key)
    self.key = self.number
    self.steps = BuildSteps(self)

  @property
  def blame(self):
    return self.data.get('blame', [])

  @property
  def builder(self):
    """Returns the Builder object.

    Goes up the hierarchy to find the Buildbot.builders[builder] instance.
    """
    return self.parent.parent.parent.parent.builders[self.data['builderName']]

  @property
  def start_time(self):
    if self.data.get('times'):
      return int(round(self.data['times'][0]))

  @property
  def end_time(self):
    times = self.data.get('times')
    if times and len(times) == 2 and times[1]:
      return int(round(times[1]))

  @property
  def duration(self):
    if self.start_time:
      return (self.end_time or int(round(time.time()))) - self.start_time

  @property
  def eta(self):
    return self.data.get('eta', 0)

  @property
  def completed(self):
    return self.data.get('currentStep') is None

  @property
  def properties(self):
    return self.data.get('properties', [])

  @property
  def reason(self):
    return self.data.get('reason')

  @property
  def result(self):
    result = self.data.get('results')
    while isinstance(result, list):
      result = result[0]
    if result is None and self.steps:
      # results may be 0, in that case with filter=1, the value won't be
      # present.
      result = self.steps[-1].result
    return result

  @property
  def revision(self):
    return self.data.get('sourceStamp', {}).get('revision')

  @property
  def simplified_result(self):
    """Returns a simplified 3 state value, True, False or None."""
    result = self.result
    if result in (SUCCESS, WARNINGS, SKIPPED):
      return True
    elif result in (FAILURE, EXCEPTION, RETRY):
      return False
    assert result is None, (result, self.data)
    return None

  @property
  def slave(self):
    """Returns the Slave object.

    Goes up the hierarchy to find the Buildbot.slaves[slave] instance.
    """
    return self.parent.parent.parent.parent.slaves[self.data['slave']]

  def discard(self):
    """Completed Build isn't discarded."""
    if self._data and self.result is None:
      assert not self.steps or not self.steps[-1].data.get('isFinished')
      self._data = None


class CurrentBuilds(SubViewNodeList):
  """Lists of the current builds."""

  def __init__(self, parent):
    super(CurrentBuilds, self).__init__(parent, parent.builds, 'currentBuilds')


class PendingBuilds(AddressableDataNode):
  """List of the pending builds."""

  def __init__(self, parent):
    super(PendingBuilds, self).__init__(parent, 'pendingBuilds', None)


class Builds(AddressableNodeList):
  """Supports iteration.

  Recommends using .cache() to speed up if a significant number of builds are
  iterated over.
  """
  _child_cls = Build

  def __init__(self, parent):
    super(Builds, self).__init__(parent, 'builds')

  def __getitem__(self, key):
    """Support for negative reference and enable retrieving non-cached builds.

    e.g. -1 is the last build, -2 is the previous build before the last one.
    """
    key = int(key)
    if key < 0:
      # Convert negative to positive build number.
      self.cache_keys()
      # Since the negative value can be outside of the cache keys range, use the
      # highest key value and calculate from it.
      key = max(self._keys) + key + 1

    if not key in self._cache:
      # Create an empty object.
      self._create_obj(key, None)
    return self._cache[key]

  def __iter__(self):
    """Returns cached Build objects in reversed order.

    The most recent build is returned first and then in reverse chronological
    order, up to the oldest cached build by the server. Older builds can be
    accessed but will trigger significantly more I/O so they are not included by
    default in the iteration.

    To access the older builds, use self.iterall() instead.
    """
    self.cache()
    return reversed(self._cache.values())

  def iterall(self):
    """Returns Build objects in decreasing order unbounded up to build 0.

    The most recent build is returned first and then in reverse chronological
    order. Older builds can be accessed and will trigger significantly more I/O
    so use this carefully.
    """
    # Only cache keys here.
    self.cache_keys()
    if self._keys:
      for i in range(max(self._keys), -1, -1):
        yield self[i]

  def cache_keys(self):
    """Grabs the keys (build numbers) from the builder."""
    if not self._has_keys_cached:
      for i in self.parent.data.get('cachedBuilds', []):
        i = int(i)
        self._cache.setdefault(i, Build(self, i, None))
        if i not in self._keys:
          self._keys.append(i)
      self._has_keys_cached = True

  def discard(self):
    super(Builds, self).discard()
    # Can't keep keys.
    self._has_keys_cached = False

  def _readall(self):
    return self.read('_all')


class Builder(AddressableDataNode):
  """Builder status."""
  printable_attributes = AddressableDataNode.printable_attributes + [
      'name',
      'key',
      'builds',
      'slaves',
      'pending_builds',
      'current_builds',
  ]

  def __init__(self, parent, name, data):
    super(Builder, self).__init__(parent, name, data)
    self.name = name
    self.key = name
    self.builds = Builds(self)
    self.slaves = BuilderSlaves(self)
    self.current_builds = CurrentBuilds(self)
    self.pending_builds = PendingBuilds(self)

  def discard(self):
    super(Builder, self).discard()
    self.builds.discard()
    self.slaves.discard()
    self.current_builds.discard()


class Builders(AddressableNodeList):
  """Root list of builders."""
  _child_cls = Builder

  def __init__(self, parent):
    super(Builders, self).__init__(parent, 'builders')


class Buildbot(AddressableBaseDataNode):
  """This object should be recreated on a master restart as it caches data."""
  # Throttle fetches to not kill the server.
  auto_throttle = None
  printable_attributes = AddressableDataNode.printable_attributes + [
      'slaves',
      'builders',
      'last_fetch',
  ]

  def __init__(self, url):
    super(Buildbot, self).__init__(None, url.rstrip('/') + '/json', None)
    self._builders = Builders(self)
    self._slaves = Slaves(self)
    self.last_fetch = None

  @property
  def builders(self):
    return self._builders

  @property
  def slaves(self):
    return self._slaves

  def discard(self):
    """Discards information about Builders and Slaves."""
    super(Buildbot, self).discard()
    self._builders.discard()
    self._slaves.discard()

  def read(self, suburl):
    if self.auto_throttle:
      if self.last_fetch:
        delta = datetime.datetime.utcnow() - self.last_fetch
        remaining = (datetime.timedelta(seconds=self.auto_throttle) - delta)
        if remaining > datetime.timedelta(seconds=0):
          logging.debug('Sleeping for %ss', remaining)
          time.sleep(remaining.seconds)
      self.last_fetch = datetime.datetime.utcnow()
    url = '%s/%s' % (self.url, suburl)
    if '?' in url:
      url += '&filter=1'
    else:
      url += '?filter=1'
    logging.info('read(%s)', suburl)
    channel = urllib.urlopen(url)
    data = channel.read()
    try:
      return json.loads(data)
    except ValueError:
      if channel.getcode() >= 400:
        # Convert it into an HTTPError for easier processing.
        raise urllib2.HTTPError(url, channel.getcode(), '%s:\n%s' % (url, data),
                                channel.headers, None)
      raise

  def _readall(self):
    return self.read('project')

###############################################################################
## Controller code


def usage(more):

  def hook(fn):
    fn.func_usage_more = more
    return fn

  return hook


def need_buildbot(fn):
  """Post-parse args to create a buildbot object."""

  @functools.wraps(fn)
  def hook(parser, args, *extra_args, **kwargs):
    old_parse_args = parser.parse_args

    def new_parse_args(args):
      options, args = old_parse_args(args)
      if len(args) < 1:
        parser.error('Need to pass the root url of the buildbot')
      url = args.pop(0)
      if not url.startswith('http'):
        url = 'http://' + url
      buildbot = Buildbot(url)
      buildbot.auto_throttle = options.throttle
      return options, args, buildbot

    parser.parse_args = new_parse_args
    # Call the original function with the modified parser.
    return fn(parser, args, *extra_args, **kwargs)

  hook.func_usage_more = '[options] <url>'
  return hook


@need_buildbot
def CMDpending(parser, args):
  """Lists pending jobs."""
  parser.add_option('-b',
                    '--builder',
                    dest='builders',
                    action='append',
                    default=[],
                    help='Builders to filter on')
  options, args, buildbot = parser.parse_args(args)
  if args:
    parser.error('Unrecognized parameters: %s' % ' '.join(args))
  if not options.builders:
    options.builders = buildbot.builders.keys
  for builder in options.builders:
    builder = buildbot.builders[builder]
    pending_builds = builder.data.get('pendingBuilds', 0)
    if not pending_builds:
      continue
    print('Builder %s: %d' % (builder.name, pending_builds))
    if not options.quiet:
      for pending in builder.pending_builds.data:
        if 'revision' in pending['source']:
          print('  revision: %s' % pending['source']['revision'])
        for change in pending['source']['changes']:
          print('  change:')
          print('    comment: %r' % unicode(change['comments'][:50]))
          print('    who:     %s' % change['who'])
  return 0


@usage('[options] <url> [commands] ...')
@need_buildbot
def CMDrun(parser, args):
  """Runs commands passed as parameters.

  When passing commands on the command line, each command will be run as if it
  was on its own line.
  """
  parser.add_option('-f', '--file', help='Read script from file')
  parser.add_option('-i',
                    dest='use_stdin',
                    action='store_true',
                    help='Read script on stdin')
  # Variable 'buildbot' is not used directly.
  # pylint: disable=W0612
  options, args, buildbot = parser.parse_args(args)
  if (bool(args) + bool(options.use_stdin) + bool(options.file)) != 1:
    parser.error('Need to pass only one of: <commands>, -f <file> or -i')
  if options.use_stdin:
    cmds = sys.stdin.read()
  elif options.file:
    cmds = open(options.file).read()
  else:
    cmds = '\n'.join(args)
  compiled = compile(cmds, '<cmd line>', 'exec')
  # pylint: disable=eval-used
  eval(compiled, globals(), locals())
  return 0


@need_buildbot
def CMDinteractive(parser, args):
  """Runs an interactive shell to run queries."""
  _, args, buildbot = parser.parse_args(args)
  if args:
    parser.error('Unrecognized parameters: %s' % ' '.join(args))
  prompt = (
      'Buildbot interactive console for "%s".\n'
      'Hint: Start with typing: \'buildbot.printable_attributes\' or '
      '\'print str(buildbot)\' to explore.') % buildbot.url[:-len('/json')]
  local_vars = {'buildbot': buildbot, 'b': buildbot}
  code.interact(prompt, None, local_vars)


@need_buildbot
def CMDidle(parser, args):
  """Lists idle slaves."""
  return find_idle_busy_slaves(parser, args, True)


@need_buildbot
def CMDbusy(parser, args):
  """Lists idle slaves."""
  return find_idle_busy_slaves(parser, args, False)


@need_buildbot
def CMDdisconnected(parser, args):
  """Lists disconnected slaves."""
  _, args, buildbot = parser.parse_args(args)
  if args:
    parser.error('Unrecognized parameters: %s' % ' '.join(args))
  for slave in buildbot.slaves:
    if not slave.connected:
      print(slave.name)
  return 0


def find_idle_busy_slaves(parser, args, show_idle):
  parser.add_option('-b',
                    '--builder',
                    dest='builders',
                    action='append',
                    default=[],
                    help='Builders to filter on')
  parser.add_option('-s',
                    '--slave',
                    dest='slaves',
                    action='append',
                    default=[],
                    help='Slaves to filter on')
  options, args, buildbot = parser.parse_args(args)
  if args:
    parser.error('Unrecognized parameters: %s' % ' '.join(args))
  if not options.builders:
    options.builders = buildbot.builders.keys
  for builder in options.builders:
    builder = buildbot.builders[builder]
    if options.slaves:
      # Only the subset of slaves connected to the builder.
      slaves = list(set(options.slaves).intersection(set(builder.slaves.names)))
      if not slaves:
        continue
    else:
      slaves = builder.slaves.names
    busy_slaves = [build.slave.name for build in builder.current_builds]
    if show_idle:
      slaves = natsorted(set(slaves) - set(busy_slaves))
    else:
      slaves = natsorted(set(slaves) & set(busy_slaves))
    if options.quiet:
      for slave in slaves:
        print(slave)
    else:
      if slaves:
        print('Builder %s: %s' % (builder.name, ', '.join(slaves)))
  return 0


def last_failure(buildbot,
                 builders=None,
                 slaves=None,
                 steps=None,
                 no_cache=False):
  """Returns Build object with last failure with the specific filters."""
  builders = builders or buildbot.builders.keys
  for builder in builders:
    builder = buildbot.builders[builder]
    if slaves:
      # Only the subset of slaves connected to the builder.
      builder_slaves = list(set(slaves).intersection(set(builder.slaves.names)))
      if not builder_slaves:
        continue
    else:
      builder_slaves = builder.slaves.names

    if not no_cache and len(builder.slaves) > 2:
      # Unless you just want the last few builds, it's often faster to
      # fetch the whole thing at once, at the cost of a small hickup on
      # the buildbot.
      # TODO(maruel): Cache only N last builds or all builds since
      # datetime.
      builder.builds.cache()

    found = []
    for build in builder.builds:
      if build.slave.name not in builder_slaves or build.slave.name in found:
        continue
      # Only add the slave for the first completed build but still look for
      # incomplete builds.
      if build.completed:
        found.append(build.slave.name)

      if steps:
        if any(build.steps[step].simplified_result is False for step in steps):
          yield build
      elif build.simplified_result is False:
        yield build

      if len(found) == len(builder_slaves):
        # Found all the slaves, quit.
        break


@need_buildbot
def CMDlast_failure(parser, args):
  """Lists all slaves that failed on that step on their last build.

  Example: to find all slaves where their last build was a compile failure,
  run with --step compile
  """
  parser.add_option(
      '-S',
      '--step',
      dest='steps',
      action='append',
      default=[],
      help='List all slaves that failed on that step on their last build')
  parser.add_option('-b',
                    '--builder',
                    dest='builders',
                    action='append',
                    default=[],
                    help='Builders to filter on')
  parser.add_option('-s',
                    '--slave',
                    dest='slaves',
                    action='append',
                    default=[],
                    help='Slaves to filter on')
  parser.add_option('-n',
                    '--no_cache',
                    action='store_true',
                    help='Don\'t load all builds at once')
  options, args, buildbot = parser.parse_args(args)
  if args:
    parser.error('Unrecognized parameters: %s' % ' '.join(args))
  print_builders = not options.quiet and len(options.builders) != 1
  last_builder = None
  for build in last_failure(buildbot,
                            builders=options.builders,
                            slaves=options.slaves,
                            steps=options.steps,
                            no_cache=options.no_cache):

    if print_builders and last_builder != build.builder:
      print(build.builder.name)
      last_builder = build.builder

    if options.quiet:
      if options.slaves:
        print('%s: %s' % (build.builder.name, build.slave.name))
      else:
        print(build.slave.name)
    else:
      out = '%d on %s: blame:%s' % (build.number, build.slave.name,
                                    ', '.join(build.blame))
      if print_builders:
        out = '  ' + out
      print(out)

      if len(options.steps) != 1:
        for step in build.steps:
          if step.simplified_result is False:
            # Assume the first line is the text name anyway.
            summary = ', '.join(step.data['text'][1:])[:40]
            out = '  %s: "%s"' % (step.data['name'], summary)
            if print_builders:
              out = '  ' + out
            print(out)
  return 0


@need_buildbot
def CMDcurrent(parser, args):
  """Lists current jobs."""
  parser.add_option('-b',
                    '--builder',
                    dest='builders',
                    action='append',
                    default=[],
                    help='Builders to filter on')
  parser.add_option('--blame',
                    action='store_true',
                    help='Only print the blame list')
  options, args, buildbot = parser.parse_args(args)
  if args:
    parser.error('Unrecognized parameters: %s' % ' '.join(args))
  if not options.builders:
    options.builders = buildbot.builders.keys

  if options.blame:
    blame = set()
    for builder in options.builders:
      for build in buildbot.builders[builder].current_builds:
        if build.blame:
          for blamed in build.blame:
            blame.add(blamed)
    print('\n'.join(blame))
    return 0

  for builder in options.builders:
    builder = buildbot.builders[builder]
    if not options.quiet and builder.current_builds:
      print(builder.name)
    for build in builder.current_builds:
      if options.quiet:
        print(build.slave.name)
      else:
        out = '%4d: slave=%10s' % (build.number, build.slave.name)
        out += '  duration=%5d' % (build.duration or 0)
        if build.eta:
          out += '  eta=%5.0f' % build.eta
        else:
          out += '           '
        if build.blame:
          out += '  blame=' + ', '.join(build.blame)
        print(out)

  return 0


@need_buildbot
def CMDbuilds(parser, args):
  """Lists all builds.

  Example: to find all builds on a single slave, run with -b bar -s foo
  """
  parser.add_option('-r',
                    '--result',
                    type='int',
                    help='Build result to filter on')
  parser.add_option('-b',
                    '--builder',
                    dest='builders',
                    action='append',
                    default=[],
                    help='Builders to filter on')
  parser.add_option('-s',
                    '--slave',
                    dest='slaves',
                    action='append',
                    default=[],
                    help='Slaves to filter on')
  parser.add_option('-n',
                    '--no_cache',
                    action='store_true',
                    help='Don\'t load all builds at once')
  options, args, buildbot = parser.parse_args(args)
  if args:
    parser.error('Unrecognized parameters: %s' % ' '.join(args))
  builders = options.builders or buildbot.builders.keys
  for builder in builders:
    builder = buildbot.builders[builder]
    for build in builder.builds:
      if not options.slaves or build.slave.name in options.slaves:
        if options.quiet:
          out = ''
          if options.builders:
            out += '%s/' % builder.name
          if len(options.slaves) != 1:
            out += '%s/' % build.slave.name
          out += '%d  revision:%s  result:%s  blame:%s' % (
              build.number, build.revision, build.result, ','.join(build.blame))
          print(out)
        else:
          print(build)
  return 0


@need_buildbot
def CMDcount(parser, args):
  """Count the number of builds that occured during a specific period."""
  parser.add_option('-o',
                    '--over',
                    type='int',
                    help='Number of seconds to look for')
  parser.add_option('-b',
                    '--builder',
                    dest='builders',
                    action='append',
                    default=[],
                    help='Builders to filter on')
  options, args, buildbot = parser.parse_args(args)
  if args:
    parser.error('Unrecognized parameters: %s' % ' '.join(args))
  if not options.over:
    parser.error(
        'Specify the number of seconds, e.g. --over 86400 for the last 24 '
        'hours')
  builders = options.builders or buildbot.builders.keys
  counts = {}
  since = time.time() - options.over
  for builder in builders:
    builder = buildbot.builders[builder]
    counts[builder.name] = 0
    if not options.quiet:
      print(builder.name)
    for build in builder.builds.iterall():
      try:
        start_time = build.start_time
      except urllib2.HTTPError:
        # The build was probably trimmed.
        print('Failed to fetch build %s/%d' % (builder.name, build.number),
              file=sys.stderr)
        continue
      if start_time >= since:
        counts[builder.name] += 1
      else:
        break
    if not options.quiet:
      print('.. %d' % counts[builder.name])

  align_name = max(len(b) for b in counts)
  align_number = max(len(str(c)) for c in counts.itervalues())
  for builder in sorted(counts):
    print('%*s: %*d' % (align_name, builder, align_number, counts[builder]))
  print('Total: %d' % sum(counts.itervalues()))
  return 0


def gen_parser():
  """Returns an OptionParser instance with default options.

  It should be then processed with gen_usage() before being used.
  """
  parser = optparse.OptionParser(version=__version__)
  # Remove description formatting
  parser.format_description = lambda x: parser.description
  # Add common parsing.
  old_parser_args = parser.parse_args

  def Parse(*args, **kwargs):
    options, args = old_parser_args(*args, **kwargs)
    if options.verbose >= 2:
      logging.basicConfig(level=logging.DEBUG)
    elif options.verbose:
      logging.basicConfig(level=logging.INFO)
    else:
      logging.basicConfig(level=logging.WARNING)
    return options, args

  parser.parse_args = Parse

  parser.add_option('-v',
                    '--verbose',
                    action='count',
                    help='Use multiple times to increase logging leve')
  parser.add_option(
      '-q',
      '--quiet',
      action='store_true',
      help='Reduces the output to be parsed by scripts, independent of -v')
  parser.add_option('--throttle',
                    type='float',
                    help='Minimum delay to sleep between requests')
  return parser

###############################################################################
## Generic subcommand handling code


def Command(name):
  return getattr(sys.modules[__name__], 'CMD' + name, None)


@usage('<command>')
def CMDhelp(parser, args):
  """Print list of commands or use 'help <command>'."""
  _, args = parser.parse_args(args)
  if len(args) == 1:
    return main(args + ['--help'])
  parser.print_help()
  return 0


def gen_usage(parser, command):
  """Modifies an OptionParser object with the command's documentation.

  The documentation is taken from the function's docstring.
  """
  obj = Command(command)
  more = getattr(obj, 'func_usage_more')
  # OptParser.description prefer nicely non-formatted strings.
  parser.description = obj.__doc__ + '\n'
  parser.set_usage('usage: %%prog %s %s' % (command, more))


def main(args=None):
  # Do it late so all commands are listed.
  # pylint: disable=E1101
  CMDhelp.__doc__ += '\n\nCommands are:\n' + '\n'.join(
      '  %-12s %s' % (fn[3:], Command(fn[3:]).__doc__.split('\n', 1)[0])
      for fn in dir(sys.modules[__name__]) if fn.startswith('CMD'))

  parser = gen_parser()
  if args is None:
    args = sys.argv[1:]
  if args:
    command = Command(args[0])
    if command:
      # "fix" the usage and the description now that we know the subcommand.
      gen_usage(parser, args[0])
      return command(parser, args[1:])

  # Not a known command. Default to help.
  gen_usage(parser, 'help')
  return CMDhelp(parser, args)


if __name__ == '__main__':
  sys.exit(main())