From 8078daaf7a0370d85546210425bbde7b507cfebf Mon Sep 17 00:00:00 2001 From: Tiancong Wang Date: Fri, 26 Jun 2020 10:36:37 -0700 Subject: cros_utils: clean up unused file/code As an effort to fix up naming in crbug.com/1099035, we can just remove the unused code and variable in these two files. BUG=chromium:1099035 TEST=buildbot_utils_unittest.py pass after the removal. Change-Id: Id4c12aeab9bf44b67e58b11c4297ceafe441421d Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2270723 Reviewed-by: Manoj Gupta Reviewed-by: Caroline Tice Commit-Queue: Tiancong Wang Tested-by: Tiancong Wang --- cros_utils/buildbot_json.py | 1534 --------------------------------- cros_utils/buildbot_utils_unittest.py | 12 - 2 files changed, 1546 deletions(-) delete mode 100755 cros_utils/buildbot_json.py diff --git a/cros_utils/buildbot_json.py b/cros_utils/buildbot_json.py deleted file mode 100755 index 08a8ae05..00000000 --- a/cros_utils/buildbot_json.py +++ /dev/null @@ -1,1534 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# Copyright 2019 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# NOTE: This file is NOT under GPL. See above. - -"""Queries buildbot through the json interface. -""" - -from __future__ import print_function - -__author__ = 'maruel@chromium.org' -__version__ = '1.2' - -import code -import datetime -import functools -import json - -# Pylint recommends we use "from chromite.lib import cros_logging as logging". -# Chromite specific policy message, we want to keep using the standard logging. -# pylint: disable=cros-logging-import -import logging - -# pylint: disable=deprecated-module -import optparse - -import time -import sys -import urllib.error -import urllib.parse -import urllib.request - -try: - from natsort import natsorted -except ImportError: - # natsorted is a simple helper to sort "naturally", e.g. "vm40" is sorted - # after "vm7". Defaults to normal sorting. - natsorted = sorted - -# These values are buildbot constants used for Build and BuildStep. -# This line was copied from master/buildbot/status/builder.py. -SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY = list(range(6)) - -## Generic node caching code. - - -class Node(object): - """Root class for all nodes in the graph. - - Provides base functionality for any node in the graph, independent if it has - children or not or if its content can be addressed through an url or needs to - be fetched as part of another node. - - self.printable_attributes is only used for self documentation and for str() - implementation. - """ - printable_attributes = [] - - def __init__(self, parent, url): - self.printable_attributes = self.printable_attributes[:] - if url: - self.printable_attributes.append('url') - url = url.rstrip('/') - if parent is not None: - self.printable_attributes.append('parent') - self.url = url - self.parent = parent - - def __str__(self): - return self.to_string() - - def __repr__(self): - """Embeds key if present.""" - key = getattr(self, 'key', None) - if key is not None: - return '<%s key=%s>' % (self.__class__.__name__, key) - cached_keys = getattr(self, 'cached_keys', None) - if cached_keys is not None: - return '<%s keys=%s>' % (self.__class__.__name__, cached_keys) - return super(Node, self).__repr__() - - def to_string(self, maximum=100): - out = ['%s:' % self.__class__.__name__] - assert not 'printable_attributes' in self.printable_attributes - - def limit(txt): - txt = str(txt) - if maximum > 0: - if len(txt) > maximum + 2: - txt = txt[:maximum] + '...' - return txt - - for k in sorted(self.printable_attributes): - if k == 'parent': - # Avoid infinite recursion. - continue - out.append(limit(' %s: %r' % (k, getattr(self, k)))) - return '\n'.join(out) - - def refresh(self): - """Refreshes the data.""" - self.discard() - return self.cache() - - def cache(self): # pragma: no cover - """Caches the data.""" - raise NotImplementedError() - - def discard(self): # pragma: no cover - """Discards cached data. - - Pretty much everything is temporary except completed Build. - """ - raise NotImplementedError() - - -class AddressableBaseDataNode(Node): # pylint: disable=W0223 - """A node that contains a dictionary of data that can be fetched with an url. - - The node is directly addressable. It also often can be fetched by the parent. - """ - printable_attributes = Node.printable_attributes + ['data'] - - def __init__(self, parent, url, data): - super(AddressableBaseDataNode, self).__init__(parent, url) - self._data = data - - @property - def cached_data(self): - return self._data - - @property - def data(self): - self.cache() - return self._data - - def cache(self): - if self._data is None: - self._data = self._readall() - return True - return False - - def discard(self): - self._data = None - - def read(self, suburl): - assert self.url, self.__class__.__name__ - url = self.url - if suburl: - url = '%s/%s' % (self.url, suburl) - return self.parent.read(url) - - def _readall(self): - return self.read('') - - -class AddressableDataNode(AddressableBaseDataNode): # pylint: disable=W0223 - """Automatically encodes the url.""" - - def __init__(self, parent, url, data): - super(AddressableDataNode, self).__init__(parent, urllib.parse.quote(url), - data) - - -class NonAddressableDataNode(Node): # pylint: disable=W0223 - """A node that cannot be addressed by an unique url. - - The data comes directly from the parent. - """ - - def __init__(self, parent, subkey): - super(NonAddressableDataNode, self).__init__(parent, None) - self.subkey = subkey - - @property - def cached_data(self): - if self.parent.cached_data is None: - return None - return self.parent.cached_data[self.subkey] - - @property - def data(self): - return self.parent.data[self.subkey] - - def cache(self): - self.parent.cache() - - def discard(self): # pragma: no cover - """Avoid invalid state when parent recreate the object.""" - raise AttributeError('Call parent discard() instead') - - -class VirtualNodeList(Node): - """Base class for every node that has children. - - Adds partial supports for keys and iterator functionality. 'key' can be a - string or a int. Not to be used directly. - """ - printable_attributes = Node.printable_attributes + ['keys'] - - def __init__(self, parent, url): - super(VirtualNodeList, self).__init__(parent, url) - # Keeps the keys independently when ordering is needed. - self._is_cached = False - self._has_keys_cached = False - - def __contains__(self, key): - """Enables 'if i in obj:'.""" - return key in self.keys - - def __iter__(self): - """Enables 'for i in obj:'. It returns children.""" - self.cache_keys() - for key in self.keys: - yield self[key] - - def __len__(self): - """Enables 'len(obj)' to get the number of childs.""" - return len(self.keys) - - def discard(self): - """Discards data. - - The default behavior is to not invalidate cached keys. The only place where - keys need to be invalidated is with Builds. - """ - self._is_cached = False - self._has_keys_cached = False - - @property - def cached_children(self): # pragma: no cover - """Returns an iterator over the children that are cached.""" - raise NotImplementedError() - - @property - def cached_keys(self): # pragma: no cover - raise NotImplementedError() - - @property - def keys(self): # pragma: no cover - """Returns the keys for every children.""" - raise NotImplementedError() - - def __getitem__(self, key): # pragma: no cover - """Returns a child, without fetching its data. - - The children could be invalid since no verification is done. - """ - raise NotImplementedError() - - def cache(self): # pragma: no cover - """Cache all the children.""" - raise NotImplementedError() - - def cache_keys(self): # pragma: no cover - """Cache all children's keys.""" - raise NotImplementedError() - - -class NodeList(VirtualNodeList): # pylint: disable=W0223 - """Adds a cache of the keys.""" - - def __init__(self, parent, url): - super(NodeList, self).__init__(parent, url) - self._keys = [] - - @property - def cached_keys(self): - return self._keys - - @property - def keys(self): - self.cache_keys() - return self._keys - - -class NonAddressableNodeList(VirtualNodeList): # pylint: disable=W0223 - """A node that contains children but retrieves all its data from its parent. - - I.e. there's no url to get directly this data. - """ - # Child class object for children of this instance. For example, BuildSteps - # has BuildStep children. - _child_cls = None - - def __init__(self, parent, subkey): - super(NonAddressableNodeList, self).__init__(parent, None) - self.subkey = subkey - assert (not isinstance(self._child_cls, NonAddressableDataNode) and - issubclass(self._child_cls, NonAddressableDataNode)), ( - self._child_cls.__name__) - - @property - def cached_children(self): - if self.parent.cached_data is not None: - for i in range(len(self.parent.cached_data[self.subkey])): - yield self[i] - - @property - def cached_data(self): - if self.parent.cached_data is None: - return None - return self.parent.data.get(self.subkey, None) - - @property - def cached_keys(self): - if self.parent.cached_data is None: - return None - return list(range(len(self.parent.data.get(self.subkey, [])))) - - @property - def data(self): - return self.parent.data[self.subkey] - - def cache(self): - self.parent.cache() - - def cache_keys(self): - self.parent.cache() - - def discard(self): # pragma: no cover - """Do not call. - - Avoid infinite recursion by having the caller calls the parent's - discard() explicitely. - """ - raise AttributeError('Call parent discard() instead') - - def __iter__(self): - """Enables 'for i in obj:'. It returns children.""" - if self.data: - for i in range(len(self.data)): - yield self[i] - - def __getitem__(self, key): - """Doesn't cache the value, it's not needed. - - TODO(maruel): Cache? - """ - if isinstance(key, int) and key < 0: - key = len(self.data) + key - # pylint: disable=E1102 - return self._child_cls(self, key) - - -class AddressableNodeList(NodeList): - """A node that has children that can be addressed with an url.""" - - # Child class object for children of this instance. For example, Builders has - # Builder children and Builds has Build children. - _child_cls = None - - def __init__(self, parent, url): - super(AddressableNodeList, self).__init__(parent, url) - self._cache = {} - assert (not isinstance(self._child_cls, AddressableDataNode) and - issubclass(self._child_cls, AddressableDataNode)), ( - self._child_cls.__name__) - - @property - def cached_children(self): - for item in self._cache.values(): - if item.cached_data is not None: - yield item - - @property - def cached_keys(self): - return list(self._cache.keys()) - - def __getitem__(self, key): - """Enables 'obj[i]'.""" - if self._has_keys_cached and not key in self._keys: - raise KeyError(key) - - if not key in self._cache: - # Create an empty object. - self._create_obj(key, None) - return self._cache[key] - - def cache(self): - if not self._is_cached: - data = self._readall() - for key in sorted(data): - self._create_obj(key, data[key]) - self._is_cached = True - self._has_keys_cached = True - - def cache_partial(self, children): - """Caches a partial number of children. - - This method is more efficient since it does a single request for all the - children instead of one request per children. - - It only grab objects not already cached. - """ - # pylint: disable=W0212 - if not self._is_cached: - to_fetch = [ - child for child in children - if not (child in self._cache and self._cache[child].cached_data) - ] - if to_fetch: - # Similar to cache(). The only reason to sort is to simplify testing. - params = '&'.join( - 'select=%s' % urllib.parse.quote(str(v)) for v in sorted(to_fetch)) - data = self.read('?' + params) - for key in sorted(data): - self._create_obj(key, data[key]) - - def cache_keys(self): - """Implement to speed up enumeration. Defaults to call cache().""" - if not self._has_keys_cached: - self.cache() - assert self._has_keys_cached - - def discard(self): - """Discards temporary children.""" - super(AddressableNodeList, self).discard() - for v in self._cache.values(): - v.discard() - - def read(self, suburl): - assert self.url, self.__class__.__name__ - url = self.url - if suburl: - url = '%s/%s' % (self.url, suburl) - return self.parent.read(url) - - def _create_obj(self, key, data): - """Creates an object of type self._child_cls.""" - # pylint: disable=E1102 - obj = self._child_cls(self, key, data) - # obj.key and key may be different. - # No need to overide cached data with None. - if data is not None or obj.key not in self._cache: - self._cache[obj.key] = obj - if obj.key not in self._keys: - self._keys.append(obj.key) - - def _readall(self): - return self.read('') - - -class SubViewNodeList(VirtualNodeList): # pylint: disable=W0223 - """A node that shows a subset of children that comes from another structure. - - The node is not addressable. - - E.g. the keys are retrieved from parent but the actual data comes from - virtual_parent. - """ - - def __init__(self, parent, virtual_parent, subkey): - super(SubViewNodeList, self).__init__(parent, None) - self.subkey = subkey - self.virtual_parent = virtual_parent - assert isinstance(self.parent, AddressableDataNode) - assert isinstance(self.virtual_parent, NodeList) - - @property - def cached_children(self): - if self.parent.cached_data is not None: - for item in self.keys: - if item in self.virtual_parent.keys: - child = self[item] - if child.cached_data is not None: - yield child - - @property - def cached_keys(self): - return (self.parent.cached_data or {}).get(self.subkey, []) - - @property - def keys(self): - self.cache_keys() - return self.parent.data.get(self.subkey, []) - - def cache(self): - """Batch request for each child in a single read request.""" - if not self._is_cached: - self.virtual_parent.cache_partial(self.keys) - self._is_cached = True - - def cache_keys(self): - if not self._has_keys_cached: - self.parent.cache() - self._has_keys_cached = True - - def discard(self): - if self.parent.cached_data is not None: - for child in self.virtual_parent.cached_children: - if child.key in self.keys: - child.discard() - self.parent.discard() - super(SubViewNodeList, self).discard() - - def __getitem__(self, key): - """Makes sure the key is in our key but grab it from the virtual parent.""" - return self.virtual_parent[key] - - def __iter__(self): - self.cache() - return super(SubViewNodeList, self).__iter__() - - -# Buildbot-specific code - - -class Slave(AddressableDataNode): - """Buildbot slave class.""" - printable_attributes = AddressableDataNode.printable_attributes + [ - 'name', - 'key', - 'connected', - 'version', - ] - - def __init__(self, parent, name, data): - super(Slave, self).__init__(parent, name, data) - self.name = name - self.key = self.name - # TODO(maruel): Add SlaveBuilders and a 'builders' property. - # TODO(maruel): Add a 'running_builds' property. - - @property - def connected(self): - return self.data.get('connected', False) - - @property - def version(self): - return self.data.get('version') - - -class Slaves(AddressableNodeList): - """Buildbot slaves.""" - _child_cls = Slave - printable_attributes = AddressableNodeList.printable_attributes + ['names'] - - def __init__(self, parent): - super(Slaves, self).__init__(parent, 'slaves') - - @property - def names(self): - return self.keys - - -class BuilderSlaves(SubViewNodeList): - """Similar to Slaves but only list slaves connected to a specific builder.""" - printable_attributes = SubViewNodeList.printable_attributes + ['names'] - - def __init__(self, parent): - super(BuilderSlaves, self).__init__(parent, parent.parent.parent.slaves, - 'slaves') - - @property - def names(self): - return self.keys - - -class BuildStep(NonAddressableDataNode): - """Class for a buildbot build step.""" - printable_attributes = NonAddressableDataNode.printable_attributes + [ - 'name', - 'number', - 'start_time', - 'end_time', - 'duration', - 'is_started', - 'is_finished', - 'is_running', - 'result', - 'simplified_result', - ] - - def __init__(self, parent, number): - """Pre-loaded, since the data is retrieved via the Build object.""" - assert isinstance(number, int) - super(BuildStep, self).__init__(parent, number) - self.number = number - - @property - def start_time(self): - if self.data.get('times'): - return int(round(self.data['times'][0])) - - @property - def end_time(self): - times = self.data.get('times') - if times and len(times) == 2 and times[1]: - return int(round(times[1])) - - @property - def duration(self): - if self.start_time: - return (self.end_time or int(round(time.time()))) - self.start_time - - @property - def name(self): - return self.data['name'] - - @property - def is_started(self): - return self.data.get('isStarted', False) - - @property - def is_finished(self): - return self.data.get('isFinished', False) - - @property - def is_running(self): - return self.is_started and not self.is_finished - - @property - def result(self): - result = self.data.get('results') - if result is None: - # results may be 0, in that case with filter=1, the value won't be - # present. - if self.data.get('isFinished'): - result = self.data.get('results', 0) - while isinstance(result, list): - result = result[0] - return result - - @property - def simplified_result(self): - """Returns a simplified 3 state value, True, False or None.""" - result = self.result - if result in (SUCCESS, WARNINGS): - return True - elif result in (FAILURE, EXCEPTION, RETRY): - return False - assert result in (None, SKIPPED), (result, self.data) - return None - - -class BuildSteps(NonAddressableNodeList): - """Duplicates keys to support lookup by both step number and step name.""" - printable_attributes = NonAddressableNodeList.printable_attributes + [ - 'failed', - ] - _child_cls = BuildStep - - def __init__(self, parent): - """Pre-loaded, since the data is retrieved via the Build object.""" - super(BuildSteps, self).__init__(parent, 'steps') - - @property - def keys(self): - """Returns the steps name in order.""" - return [i['name'] for i in self.data or []] - - @property - def failed(self): - """Shortcuts that lists the step names of steps that failed.""" - return [step.name for step in self if step.simplified_result is False] - - def __getitem__(self, key): - """Accept step name in addition to index number.""" - if isinstance(key, str): - # It's a string, try to find the corresponding index. - for i, step in enumerate(self.data): - if step['name'] == key: - key = i - break - else: - raise KeyError(key) - return super(BuildSteps, self).__getitem__(key) - - -class Build(AddressableDataNode): - """Buildbot build info.""" - printable_attributes = AddressableDataNode.printable_attributes + [ - 'key', - 'number', - 'steps', - 'blame', - 'reason', - 'revision', - 'result', - 'simplified_result', - 'start_time', - 'end_time', - 'duration', - 'slave', - 'properties', - 'completed', - ] - - def __init__(self, parent, key, data): - super(Build, self).__init__(parent, str(key), data) - self.number = int(key) - self.key = self.number - self.steps = BuildSteps(self) - - @property - def blame(self): - return self.data.get('blame', []) - - @property - def builder(self): - """Returns the Builder object. - - Goes up the hierarchy to find the Buildbot.builders[builder] instance. - """ - return self.parent.parent.parent.parent.builders[self.data['builderName']] - - @property - def start_time(self): - if self.data.get('times'): - return int(round(self.data['times'][0])) - - @property - def end_time(self): - times = self.data.get('times') - if times and len(times) == 2 and times[1]: - return int(round(times[1])) - - @property - def duration(self): - if self.start_time: - return (self.end_time or int(round(time.time()))) - self.start_time - - @property - def eta(self): - return self.data.get('eta', 0) - - @property - def completed(self): - return self.data.get('currentStep') is None - - @property - def properties(self): - return self.data.get('properties', []) - - @property - def reason(self): - return self.data.get('reason') - - @property - def result(self): - result = self.data.get('results') - while isinstance(result, list): - result = result[0] - if result is None and self.steps: - # results may be 0, in that case with filter=1, the value won't be - # present. - result = self.steps[-1].result - return result - - @property - def revision(self): - return self.data.get('sourceStamp', {}).get('revision') - - @property - def simplified_result(self): - """Returns a simplified 3 state value, True, False or None.""" - result = self.result - if result in (SUCCESS, WARNINGS, SKIPPED): - return True - elif result in (FAILURE, EXCEPTION, RETRY): - return False - assert result is None, (result, self.data) - return None - - @property - def slave(self): - """Returns the Slave object. - - Goes up the hierarchy to find the Buildbot.slaves[slave] instance. - """ - return self.parent.parent.parent.parent.slaves[self.data['slave']] - - def discard(self): - """Completed Build isn't discarded.""" - if self._data and self.result is None: - assert not self.steps or not self.steps[-1].data.get('isFinished') - self._data = None - - -class CurrentBuilds(SubViewNodeList): - """Lists of the current builds.""" - - def __init__(self, parent): - super(CurrentBuilds, self).__init__(parent, parent.builds, 'currentBuilds') - - -class PendingBuilds(AddressableDataNode): - """List of the pending builds.""" - - def __init__(self, parent): - super(PendingBuilds, self).__init__(parent, 'pendingBuilds', None) - - -class Builds(AddressableNodeList): - """Supports iteration. - - Recommends using .cache() to speed up if a significant number of builds are - iterated over. - """ - _child_cls = Build - - def __init__(self, parent): - super(Builds, self).__init__(parent, 'builds') - - def __getitem__(self, key): - """Support for negative reference and enable retrieving non-cached builds. - - e.g. -1 is the last build, -2 is the previous build before the last one. - """ - key = int(key) - if key < 0: - # Convert negative to positive build number. - self.cache_keys() - # Since the negative value can be outside of the cache keys range, use the - # highest key value and calculate from it. - key = max(self._keys) + key + 1 - - if key not in self._cache: - # Create an empty object. - self._create_obj(key, None) - return self._cache[key] - - def __iter__(self): - """Returns cached Build objects in reversed order. - - The most recent build is returned first and then in reverse chronological - order, up to the oldest cached build by the server. Older builds can be - accessed but will trigger significantly more I/O so they are not included by - default in the iteration. - - To access the older builds, use self.iterall() instead. - """ - self.cache() - return reversed(list(self._cache.values())) - - def iterall(self): - """Returns Build objects in decreasing order unbounded up to build 0. - - The most recent build is returned first and then in reverse chronological - order. Older builds can be accessed and will trigger significantly more I/O - so use this carefully. - """ - # Only cache keys here. - self.cache_keys() - if self._keys: - for i in range(max(self._keys), -1, -1): - yield self[i] - - def cache_keys(self): - """Grabs the keys (build numbers) from the builder.""" - if not self._has_keys_cached: - for i in self.parent.data.get('cachedBuilds', []): - i = int(i) - self._cache.setdefault(i, Build(self, i, None)) - if i not in self._keys: - self._keys.append(i) - self._has_keys_cached = True - - def discard(self): - super(Builds, self).discard() - # Can't keep keys. - self._has_keys_cached = False - - def _readall(self): - return self.read('_all') - - -class Builder(AddressableDataNode): - """Builder status.""" - printable_attributes = AddressableDataNode.printable_attributes + [ - 'name', - 'key', - 'builds', - 'slaves', - 'pending_builds', - 'current_builds', - ] - - def __init__(self, parent, name, data): - super(Builder, self).__init__(parent, name, data) - self.name = name - self.key = name - self.builds = Builds(self) - self.slaves = BuilderSlaves(self) - self.current_builds = CurrentBuilds(self) - self.pending_builds = PendingBuilds(self) - - def discard(self): - super(Builder, self).discard() - self.builds.discard() - self.slaves.discard() - self.current_builds.discard() - - -class Builders(AddressableNodeList): - """Root list of builders.""" - _child_cls = Builder - - def __init__(self, parent): - super(Builders, self).__init__(parent, 'builders') - - -class Buildbot(AddressableBaseDataNode): - """This object should be recreated on a master restart as it caches data.""" - # Throttle fetches to not kill the server. - auto_throttle = None - printable_attributes = AddressableDataNode.printable_attributes + [ - 'slaves', - 'builders', - 'last_fetch', - ] - - def __init__(self, url): - super(Buildbot, self).__init__(None, url.rstrip('/') + '/json', None) - self._builders = Builders(self) - self._slaves = Slaves(self) - self.last_fetch = None - - @property - def builders(self): - return self._builders - - @property - def slaves(self): - return self._slaves - - def discard(self): - """Discards information about Builders and Slaves.""" - super(Buildbot, self).discard() - self._builders.discard() - self._slaves.discard() - - def read(self, suburl): - if self.auto_throttle: - if self.last_fetch: - delta = datetime.datetime.utcnow() - self.last_fetch - remaining = (datetime.timedelta(seconds=self.auto_throttle) - delta) - if remaining > datetime.timedelta(seconds=0): - logging.debug('Sleeping for %ss', remaining) - time.sleep(remaining.seconds) - self.last_fetch = datetime.datetime.utcnow() - url = '%s/%s' % (self.url, suburl) - if '?' in url: - url += '&filter=1' - else: - url += '?filter=1' - logging.info('read(%s)', suburl) - channel = urllib.request.urlopen(url) - data = channel.read() - try: - return json.loads(data) - except ValueError: - if channel.getcode() >= 400: - # Convert it into an HTTPError for easier processing. - raise urllib.error.HTTPError(url, channel.getcode(), - '%s:\n%s' % (url, data), channel.headers, - None) - raise - - def _readall(self): - return self.read('project') - - -# Controller code - - -def usage(more): - - def hook(fn): - fn.func_usage_more = more - return fn - - return hook - - -def need_buildbot(fn): - """Post-parse args to create a buildbot object.""" - - @functools.wraps(fn) - def hook(parser, args, *extra_args, **kwargs): - old_parse_args = parser.parse_args - - def new_parse_args(args): - options, args = old_parse_args(args) - if len(args) < 1: - parser.error('Need to pass the root url of the buildbot') - url = args.pop(0) - if not url.startswith('http'): - url = 'http://' + url - buildbot = Buildbot(url) - buildbot.auto_throttle = options.throttle - return options, args, buildbot - - parser.parse_args = new_parse_args - # Call the original function with the modified parser. - return fn(parser, args, *extra_args, **kwargs) - - hook.func_usage_more = '[options] ' - return hook - - -@need_buildbot -def CMDpending(parser, args): - """Lists pending jobs.""" - parser.add_option( - '-b', - '--builder', - dest='builders', - action='append', - default=[], - help='Builders to filter on') - options, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - if not options.builders: - options.builders = buildbot.builders.keys - for builder in options.builders: - builder = buildbot.builders[builder] - pending_builds = builder.data.get('pendingBuilds', 0) - if not pending_builds: - continue - print('Builder %s: %d' % (builder.name, pending_builds)) - if not options.quiet: - for pending in builder.pending_builds.data: - if 'revision' in pending['source']: - print(' revision: %s' % pending['source']['revision']) - for change in pending['source']['changes']: - print(' change:') - print(' comment: %r' % change['comments'][:50]) - print(' who: %s' % change['who']) - return 0 - - -@usage('[options] [commands] ...') -@need_buildbot -def CMDrun(parser, args): - """Runs commands passed as parameters. - - When passing commands on the command line, each command will be run as if it - was on its own line. - """ - parser.add_option('-f', '--file', help='Read script from file') - parser.add_option( - '-i', dest='use_stdin', action='store_true', help='Read script on stdin') - # Variable 'buildbot' is not used directly. - # pylint: disable=W0612 - options, args, _ = parser.parse_args(args) - if (bool(args) + bool(options.use_stdin) + bool(options.file)) != 1: - parser.error('Need to pass only one of: , -f or -i') - if options.use_stdin: - cmds = sys.stdin.read() - elif options.file: - cmds = open(options.file).read() - else: - cmds = '\n'.join(args) - compiled = compile(cmds, '', 'exec') - # pylint: disable=eval-used - eval(compiled, globals(), locals()) - return 0 - - -@need_buildbot -def CMDinteractive(parser, args): - """Runs an interactive shell to run queries.""" - _, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - prompt = ('Buildbot interactive console for "%s".\n' - "Hint: Start with typing: 'buildbot.printable_attributes' or " - "'print str(buildbot)' to explore.") % buildbot.url[:-len('/json')] - local_vars = {'buildbot': buildbot, 'b': buildbot} - code.interact(prompt, None, local_vars) - - -@need_buildbot -def CMDidle(parser, args): - """Lists idle slaves.""" - return find_idle_busy_slaves(parser, args, True) - - -@need_buildbot -def CMDbusy(parser, args): - """Lists idle slaves.""" - return find_idle_busy_slaves(parser, args, False) - - -@need_buildbot -def CMDdisconnected(parser, args): - """Lists disconnected slaves.""" - _, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - for slave in buildbot.slaves: - if not slave.connected: - print(slave.name) - return 0 - - -def find_idle_busy_slaves(parser, args, show_idle): - parser.add_option( - '-b', - '--builder', - dest='builders', - action='append', - default=[], - help='Builders to filter on') - parser.add_option( - '-s', - '--slave', - dest='slaves', - action='append', - default=[], - help='Slaves to filter on') - options, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - if not options.builders: - options.builders = buildbot.builders.keys - for builder in options.builders: - builder = buildbot.builders[builder] - if options.slaves: - # Only the subset of slaves connected to the builder. - slaves = list(set(options.slaves).intersection(set(builder.slaves.names))) - if not slaves: - continue - else: - slaves = builder.slaves.names - busy_slaves = [build.slave.name for build in builder.current_builds] - if show_idle: - slaves = natsorted(set(slaves) - set(busy_slaves)) - else: - slaves = natsorted(set(slaves) & set(busy_slaves)) - if options.quiet: - for slave in slaves: - print(slave) - else: - if slaves: - print('Builder %s: %s' % (builder.name, ', '.join(slaves))) - return 0 - - -def last_failure(buildbot, - builders=None, - slaves=None, - steps=None, - no_cache=False): - """Returns Build object with last failure with the specific filters.""" - builders = builders or buildbot.builders.keys - for builder in builders: - builder = buildbot.builders[builder] - if slaves: - # Only the subset of slaves connected to the builder. - builder_slaves = list(set(slaves).intersection(set(builder.slaves.names))) - if not builder_slaves: - continue - else: - builder_slaves = builder.slaves.names - - if not no_cache and len(builder.slaves) > 2: - # Unless you just want the last few builds, it's often faster to - # fetch the whole thing at once, at the cost of a small hickup on - # the buildbot. - # TODO(maruel): Cache only N last builds or all builds since - # datetime. - builder.builds.cache() - - found = [] - for build in builder.builds: - if build.slave.name not in builder_slaves or build.slave.name in found: - continue - # Only add the slave for the first completed build but still look for - # incomplete builds. - if build.completed: - found.append(build.slave.name) - - if steps: - if any(build.steps[step].simplified_result is False for step in steps): - yield build - elif build.simplified_result is False: - yield build - - if len(found) == len(builder_slaves): - # Found all the slaves, quit. - break - - -@need_buildbot -def CMDlast_failure(parser, args): - """Lists all slaves that failed on that step on their last build. - - Examples: - To find all slaves where their last build was a compile failure, - run with --step compile - """ - parser.add_option( - '-S', - '--step', - dest='steps', - action='append', - default=[], - help='List all slaves that failed on that step on their last build') - parser.add_option( - '-b', - '--builder', - dest='builders', - action='append', - default=[], - help='Builders to filter on') - parser.add_option( - '-s', - '--slave', - dest='slaves', - action='append', - default=[], - help='Slaves to filter on') - parser.add_option( - '-n', - '--no_cache', - action='store_true', - help="Don't load all builds at once") - options, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - print_builders = not options.quiet and len(options.builders) != 1 - last_builder = None - for build in last_failure( - buildbot, - builders=options.builders, - slaves=options.slaves, - steps=options.steps, - no_cache=options.no_cache): - - if print_builders and last_builder != build.builder: - print(build.builder.name) - last_builder = build.builder - - if options.quiet: - if options.slaves: - print('%s: %s' % (build.builder.name, build.slave.name)) - else: - print(build.slave.name) - else: - out = '%d on %s: blame:%s' % (build.number, build.slave.name, ', '.join( - build.blame)) - if print_builders: - out = ' ' + out - print(out) - - if len(options.steps) != 1: - for step in build.steps: - if step.simplified_result is False: - # Assume the first line is the text name anyway. - summary = ', '.join(step.data['text'][1:])[:40] - out = ' %s: "%s"' % (step.data['name'], summary) - if print_builders: - out = ' ' + out - print(out) - return 0 - - -@need_buildbot -def CMDcurrent(parser, args): - """Lists current jobs.""" - parser.add_option( - '-b', - '--builder', - dest='builders', - action='append', - default=[], - help='Builders to filter on') - parser.add_option( - '--blame', action='store_true', help='Only print the blame list') - options, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - if not options.builders: - options.builders = buildbot.builders.keys - - if options.blame: - blame = set() - for builder in options.builders: - for build in buildbot.builders[builder].current_builds: - if build.blame: - for blamed in build.blame: - blame.add(blamed) - print('\n'.join(blame)) - return 0 - - for builder in options.builders: - builder = buildbot.builders[builder] - if not options.quiet and builder.current_builds: - print(builder.name) - for build in builder.current_builds: - if options.quiet: - print(build.slave.name) - else: - out = '%4d: slave=%10s' % (build.number, build.slave.name) - out += ' duration=%5d' % (build.duration or 0) - if build.eta: - out += ' eta=%5.0f' % build.eta - else: - out += ' ' - if build.blame: - out += ' blame=' + ', '.join(build.blame) - print(out) - - return 0 - - -@need_buildbot -def CMDbuilds(parser, args): - """Lists all builds. - - Examples: - To find all builds on a single slave, run with -b bar -s foo. - """ - parser.add_option( - '-r', '--result', type='int', help='Build result to filter on') - parser.add_option( - '-b', - '--builder', - dest='builders', - action='append', - default=[], - help='Builders to filter on') - parser.add_option( - '-s', - '--slave', - dest='slaves', - action='append', - default=[], - help='Slaves to filter on') - parser.add_option( - '-n', - '--no_cache', - action='store_true', - help="Don't load all builds at once") - options, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - builders = options.builders or buildbot.builders.keys - for builder in builders: - builder = buildbot.builders[builder] - for build in builder.builds: - if not options.slaves or build.slave.name in options.slaves: - if options.quiet: - out = '' - if options.builders: - out += '%s/' % builder.name - if len(options.slaves) != 1: - out += '%s/' % build.slave.name - out += '%d revision:%s result:%s blame:%s' % ( - build.number, build.revision, build.result, ','.join(build.blame)) - print(out) - else: - print(build) - return 0 - - -@need_buildbot -def CMDcount(parser, args): - """Count the number of builds that occured during a specific period.""" - parser.add_option( - '-o', '--over', type='int', help='Number of seconds to look for') - parser.add_option( - '-b', - '--builder', - dest='builders', - action='append', - default=[], - help='Builders to filter on') - options, args, buildbot = parser.parse_args(args) - if args: - parser.error('Unrecognized parameters: %s' % ' '.join(args)) - if not options.over: - parser.error( - 'Specify the number of seconds, e.g. --over 86400 for the last 24 ' - 'hours') - builders = options.builders or buildbot.builders.keys - counts = {} - since = time.time() - options.over - for builder in builders: - builder = buildbot.builders[builder] - counts[builder.name] = 0 - if not options.quiet: - print(builder.name) - for build in builder.builds.iterall(): - try: - start_time = build.start_time - except urllib.error.HTTPError: - # The build was probably trimmed. - print( - 'Failed to fetch build %s/%d' % (builder.name, build.number), - file=sys.stderr) - continue - if start_time >= since: - counts[builder.name] += 1 - else: - break - if not options.quiet: - print('.. %d' % counts[builder.name]) - - align_name = max(len(b) for b in counts) - align_number = max(len(str(c)) for c in counts.values()) - for builder in sorted(counts): - print('%*s: %*d' % (align_name, builder, align_number, counts[builder])) - print('Total: %d' % sum(counts.values())) - return 0 - - -def gen_parser(): - """Returns an OptionParser instance with default options. - - It should be then processed with gen_usage() before being used. - """ - parser = optparse.OptionParser(version=__version__) - # Remove description formatting - parser.format_description = lambda x: parser.description - # Add common parsing. - old_parser_args = parser.parse_args - - def Parse(*args, **kwargs): - options, args = old_parser_args(*args, **kwargs) - if options.verbose >= 2: - logging.basicConfig(level=logging.DEBUG) - elif options.verbose: - logging.basicConfig(level=logging.INFO) - else: - logging.basicConfig(level=logging.WARNING) - return options, args - - parser.parse_args = Parse - - parser.add_option( - '-v', - '--verbose', - action='count', - help='Use multiple times to increase logging leve') - parser.add_option( - '-q', - '--quiet', - action='store_true', - help='Reduces the output to be parsed by scripts, independent of -v') - parser.add_option( - '--throttle', - type='float', - help='Minimum delay to sleep between requests') - return parser - - -# Generic subcommand handling code - - -def Command(name): - return getattr(sys.modules[__name__], 'CMD' + name, None) - - -@usage('') -def CMDhelp(parser, args): - """Print list of commands or use 'help '.""" - _, args = parser.parse_args(args) - if len(args) == 1: - return main(args + ['--help']) - parser.print_help() - return 0 - - -def gen_usage(parser, command): - """Modifies an OptionParser object with the command's documentation. - - The documentation is taken from the function's docstring. - """ - obj = Command(command) - more = getattr(obj, 'func_usage_more') - # OptParser.description prefer nicely non-formatted strings. - parser.description = obj.__doc__ + '\n' - parser.set_usage('usage: %%prog %s %s' % (command, more)) - - -def main(args=None): - # Do it late so all commands are listed. - # pylint: disable=E1101 - CMDhelp.__doc__ += '\n\nCommands are:\n' + '\n'.join( - ' %-12s %s' % (fn[3:], Command(fn[3:]).__doc__.split('\n', 1)[0]) - for fn in dir(sys.modules[__name__]) - if fn.startswith('CMD')) - - parser = gen_parser() - if args is None: - args = sys.argv[1:] - if args: - command = Command(args[0]) - if command: - # "fix" the usage and the description now that we know the subcommand. - gen_usage(parser, args[0]) - return command(parser, args[1:]) - - # Not a known command. Default to help. - gen_usage(parser, 'help') - return CMDhelp(parser, args) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/cros_utils/buildbot_utils_unittest.py b/cros_utils/buildbot_utils_unittest.py index afbdfaef..c615c95f 100755 --- a/cros_utils/buildbot_utils_unittest.py +++ b/cros_utils/buildbot_utils_unittest.py @@ -21,18 +21,6 @@ from cros_utils import command_executer class TrybotTest(unittest.TestCase): """Test for CommandExecuter class.""" - old_tryjob_out = ( - 'Verifying patches...\n' - 'Submitting tryjob...\n' - 'Successfully sent PUT request to [buildbucket_bucket:master.chromiumos.t' - 'ryserver] with [config:success-build] [buildbucket_id:895272114382368817' - '6].\n' - 'Tryjob submitted!\n' - 'To view your tryjobs, visit:\n' - ' http://cros-goldeneye/chromeos/healthmonitoring/buildDetails?buildbuck' - 'etId=8952721143823688176\n' - ' https://uberchromegw.corp.google.com/i/chromiumos.tryserver/waterfall?' - 'committer=laszio@chromium.org&builder=etc\n') tryjob_out = ( '[{"buildbucket_id": "8952721143823688176", "build_config": ' '"cave-llvm-toolchain-tryjob", "url": ' -- cgit v1.2.3