diff options
author | Daniel Peykov <peykov@google.com> | 2023-02-28 10:57:46 -0800 |
---|---|---|
committer | Copybara-Service <copybara-worker@google.com> | 2023-02-28 10:58:24 -0800 |
commit | 94a7d9dd4eb9bcae4f72897708d527cd7f11f22b (patch) | |
tree | 92cb6d7925a2d4606cd41149a38e94507db090ee | |
parent | e5daa7074b20b1b9881c8244070a6e61022b2f92 (diff) | |
download | tradefed_cluster-94a7d9dd4eb9bcae4f72897708d527cd7f11f22b.tar.gz |
Internal change
PiperOrigin-RevId: 512982420
Change-Id: I76ae31fbf2e3ed390ab65c199167ecacc7f88961
90 files changed, 188 insertions, 49 deletions
diff --git a/tradefed_cluster/acl_check_api.py b/tradefed_cluster/acl_check_api.py index e004abe..f672f52 100644 --- a/tradefed_cluster/acl_check_api.py +++ b/tradefed_cluster/acl_check_api.py @@ -25,6 +25,7 @@ from tradefed_cluster.services import acl_service from tradefed_cluster.util import ndb_shim as ndb + _JUMP_GROUP_NAME = "jump" _JUMP_HOST_ACCOUNT = "android-test" _PRINCIPALS_KEY = "principals" diff --git a/tradefed_cluster/acl_check_api_test.py b/tradefed_cluster/acl_check_api_test.py index 5dd274c..214848a 100644 --- a/tradefed_cluster/acl_check_api_test.py +++ b/tradefed_cluster/acl_check_api_test.py @@ -22,6 +22,7 @@ from unittest import mock import endpoints from protorpc import protojson + from tradefed_cluster import api_messages from tradefed_cluster import api_test from tradefed_cluster import datastore_test_util diff --git a/tradefed_cluster/affinity_manager.py b/tradefed_cluster/affinity_manager.py index a046289..64144e1 100644 --- a/tradefed_cluster/affinity_manager.py +++ b/tradefed_cluster/affinity_manager.py @@ -130,7 +130,8 @@ def GetDeviceAffinityInfos( a list of device affinity infos. """ keys = [ - ndb.Key( datastore_entities.DeviceAffinityInfo, + ndb.Key( + datastore_entities.DeviceAffinityInfo, device_serial, namespace=common.NAMESPACE) for device_serial in device_serials ] @@ -225,7 +226,8 @@ def GetTaskAffinityInfos( a list of task affinity infos. """ keys = [ - ndb.Key( datastore_entities.TaskAffinityInfo, + ndb.Key( + datastore_entities.TaskAffinityInfo, task_id, namespace=common.NAMESPACE) for task_id in task_ids ] @@ -254,7 +256,8 @@ def GetAffinityStatuses( the affinity statuses. """ keys = [ - ndb.Key( datastore_entities.AffinityStatus, + ndb.Key( + datastore_entities.AffinityStatus, affinity_tag, namespace=common.NAMESPACE) for affinity_tag in affinity_tags ] diff --git a/tradefed_cluster/api.py b/tradefed_cluster/api.py index e58d8d8..9d74d71 100644 --- a/tradefed_cluster/api.py +++ b/tradefed_cluster/api.py @@ -16,6 +16,7 @@ import endpoints + from tradefed_cluster import cluster_api from tradefed_cluster import cluster_device_api from tradefed_cluster import cluster_host_api diff --git a/tradefed_cluster/api_messages.py b/tradefed_cluster/api_messages.py index c35a9ae..37dc9ea 100644 --- a/tradefed_cluster/api_messages.py +++ b/tradefed_cluster/api_messages.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + # Messages may be used in api config, # and we want to set the api parameter camel case. """Common config and classes for api.""" diff --git a/tradefed_cluster/api_messages_test.py b/tradefed_cluster/api_messages_test.py index 3d16471..4de4ada 100644 --- a/tradefed_cluster/api_messages_test.py +++ b/tradefed_cluster/api_messages_test.py @@ -28,6 +28,7 @@ from six.moves import zip from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import api_messages from tradefed_cluster import api_test from tradefed_cluster import common diff --git a/tradefed_cluster/api_test.py b/tradefed_cluster/api_test.py index 0f6e584..fa7126e 100644 --- a/tradefed_cluster/api_test.py +++ b/tradefed_cluster/api_test.py @@ -17,6 +17,7 @@ import endpoints import webtest + from tradefed_cluster import cluster_api from tradefed_cluster import cluster_device_api from tradefed_cluster import cluster_host_api diff --git a/tradefed_cluster/cluster_api.py b/tradefed_cluster/cluster_api.py index 8f66598..2ba7e74 100644 --- a/tradefed_cluster/cluster_api.py +++ b/tradefed_cluster/cluster_api.py @@ -18,6 +18,7 @@ from protorpc import message_types from protorpc import messages from protorpc import remote + from tradefed_cluster import api_common from tradefed_cluster import api_messages from tradefed_cluster import datastore_entities @@ -112,11 +113,13 @@ class ClusterApi(remote.Service): """ hosts = (datastore_entities.HostInfo.query() .filter(datastore_entities.HostInfo.clusters == cluster_id) - .filter(datastore_entities.HostInfo.hidden == False) .fetch()) + .filter(datastore_entities.HostInfo.hidden == False) + .fetch()) host_msgs = [] for host in hosts: devices = (datastore_entities.DeviceInfo.query(ancestor=host.key) - .filter(datastore_entities.DeviceInfo.hidden == False) .fetch()) + .filter(datastore_entities.DeviceInfo.hidden == False) + .fetch()) host_msgs.append(datastore_entities.ToMessage(host, devices)) return host_msgs diff --git a/tradefed_cluster/cluster_device_api.py b/tradefed_cluster/cluster_device_api.py index af98399..c0654bf 100644 --- a/tradefed_cluster/cluster_device_api.py +++ b/tradefed_cluster/cluster_device_api.py @@ -26,6 +26,7 @@ from protorpc import remote from tradefed_cluster.util import elasticsearch_client from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import api_common from tradefed_cluster import api_messages from tradefed_cluster import common @@ -175,6 +176,7 @@ class ClusterDeviceApi(remote.Service): # in(hostnames), but it was not scalable at all. if not request.include_hidden: query = query.filter(datastore_entities.DeviceInfo.hidden == False) + if request.product: query = query.filter( datastore_entities.DeviceInfo.product == request.product) diff --git a/tradefed_cluster/cluster_device_api_test.py b/tradefed_cluster/cluster_device_api_test.py index fbf4894..04ad041 100644 --- a/tradefed_cluster/cluster_device_api_test.py +++ b/tradefed_cluster/cluster_device_api_test.py @@ -21,6 +21,7 @@ from protorpc import protojson import pytz from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import api_messages from tradefed_cluster import api_test from tradefed_cluster import common diff --git a/tradefed_cluster/cluster_host_api.py b/tradefed_cluster/cluster_host_api.py index 7448061..1270017 100644 --- a/tradefed_cluster/cluster_host_api.py +++ b/tradefed_cluster/cluster_host_api.py @@ -22,6 +22,7 @@ from protorpc import remote from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import api_common from tradefed_cluster import api_messages from tradefed_cluster import common @@ -125,6 +126,7 @@ class ClusterHostApi(remote.Service): if not request.include_hidden: query = query.filter(datastore_entities.HostInfo.hidden == False) + if request.flated_extra_info: query = query.filter(datastore_entities.HostInfo.flated_extra_info == request.flated_extra_info) @@ -212,7 +214,8 @@ class ClusterHostApi(remote.Service): device_query = datastore_entities.DeviceInfo.query(ancestor=host.key) if not request.include_hidden: device_query = device_query.filter( - datastore_entities.DeviceInfo.hidden == False) devices = device_query.fetch() + datastore_entities.DeviceInfo.hidden == False) + devices = device_query.fetch() host_infos.append(datastore_entities.ToMessage( host, devices=devices, host_update_state_entity=host_update_state)) @@ -259,7 +262,8 @@ class ClusterHostApi(remote.Service): device_query = datastore_entities.DeviceInfo.query(ancestor=host.key) if not request.include_hidden: device_query = device_query.filter( - datastore_entities.DeviceInfo.hidden == False) devices = device_query.fetch() + datastore_entities.DeviceInfo.hidden == False) + devices = device_query.fetch() host_update_state = ndb.Key( datastore_entities.HostUpdateState, hostname).get() diff --git a/tradefed_cluster/cluster_host_api_test.py b/tradefed_cluster/cluster_host_api_test.py index eed1721..33483d8 100644 --- a/tradefed_cluster/cluster_host_api_test.py +++ b/tradefed_cluster/cluster_host_api_test.py @@ -23,6 +23,7 @@ from protorpc import protojson from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import api_messages from tradefed_cluster import api_test from tradefed_cluster import common diff --git a/tradefed_cluster/command_attempt_api.py b/tradefed_cluster/command_attempt_api.py index 0cbdbc2..ce57e6a 100644 --- a/tradefed_cluster/command_attempt_api.py +++ b/tradefed_cluster/command_attempt_api.py @@ -19,6 +19,7 @@ from protorpc import message_types from protorpc import messages from protorpc import remote + from tradefed_cluster import api_common from tradefed_cluster import api_messages from tradefed_cluster import common diff --git a/tradefed_cluster/command_attempt_api_test.py b/tradefed_cluster/command_attempt_api_test.py index f860da6..8974339 100644 --- a/tradefed_cluster/command_attempt_api_test.py +++ b/tradefed_cluster/command_attempt_api_test.py @@ -21,6 +21,7 @@ from protorpc import protojson from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import api_messages from tradefed_cluster import api_test from tradefed_cluster import common diff --git a/tradefed_cluster/command_attempt_monitor.py b/tradefed_cluster/command_attempt_monitor.py index 48112fa..7ad3154 100644 --- a/tradefed_cluster/command_attempt_monitor.py +++ b/tradefed_cluster/command_attempt_monitor.py @@ -20,6 +20,7 @@ import logging import flask + from tradefed_cluster import command_event from tradefed_cluster import command_manager from tradefed_cluster import commander diff --git a/tradefed_cluster/command_attempt_monitor_test.py b/tradefed_cluster/command_attempt_monitor_test.py index 9d181c2..972c463 100644 --- a/tradefed_cluster/command_attempt_monitor_test.py +++ b/tradefed_cluster/command_attempt_monitor_test.py @@ -27,13 +27,15 @@ import mock from six.moves import range import webtest + from tradefed_cluster import command_attempt_monitor from tradefed_cluster import command_event_test_util from tradefed_cluster import command_manager from tradefed_cluster import commander from tradefed_cluster import common from tradefed_cluster import datastore_entities -from tradefed_cluster import env_config from tradefed_cluster import request_manager +from tradefed_cluster import env_config +from tradefed_cluster import request_manager from tradefed_cluster import testbed_dependent_test from tradefed_cluster.util import ndb_shim as ndb diff --git a/tradefed_cluster/command_error_type_config_test.py b/tradefed_cluster/command_error_type_config_test.py index 7f9ae31..dec81fb 100644 --- a/tradefed_cluster/command_error_type_config_test.py +++ b/tradefed_cluster/command_error_type_config_test.py @@ -18,6 +18,7 @@ import unittest from absl.testing import parameterized + from tradefed_cluster import command_error_type_config from tradefed_cluster import common from tradefed_cluster import datastore_entities diff --git a/tradefed_cluster/command_event_api.py b/tradefed_cluster/command_event_api.py index 447ca73..81e9034 100644 --- a/tradefed_cluster/command_event_api.py +++ b/tradefed_cluster/command_event_api.py @@ -20,6 +20,7 @@ from protorpc import message_types from protorpc import protojson from protorpc import remote + from tradefed_cluster import api_common from tradefed_cluster import api_messages from tradefed_cluster import command_event_handler diff --git a/tradefed_cluster/command_event_handler.py b/tradefed_cluster/command_event_handler.py index a829f5c..2db883c 100644 --- a/tradefed_cluster/command_event_handler.py +++ b/tradefed_cluster/command_event_handler.py @@ -22,6 +22,7 @@ import zlib import flask import six + from tradefed_cluster import command_event from tradefed_cluster import command_manager from tradefed_cluster import commander @@ -105,7 +106,8 @@ def LogCommandEventMetrics(command, event): run_target=command.run_target, command_action=TIMING_DATA_FIELDS_TO_COMMAND_ACTIONS[key], latency_secs=latency_secs) - except: # Protecting against bad data + except: + # Protecting against bad data logging.warning("Failed to report timing metrics.", exc_info=True) @@ -163,7 +165,8 @@ def HandleCommandEvent(): logging.warning("Ignore unknown state event:\n%s.", event) continue ProcessCommandEvent(event) - except Exception as e: exception = e + except Exception as e: + exception = e logging.warning( "Failed to process event, will retry: obj=%s", _Truncate(obj), exc_info=True) @@ -173,5 +176,6 @@ def HandleCommandEvent(): logging.warning("%d/%d command events failed to process.", len(failed_objs), len(objs)) if len(failed_objs) == len(objs) and exception: - raise exception EnqueueCommandEvents(failed_objs) + raise exception + EnqueueCommandEvents(failed_objs) return common.HTTP_OK diff --git a/tradefed_cluster/command_event_handler_test.py b/tradefed_cluster/command_event_handler_test.py index c14b8e6..1e6c037 100644 --- a/tradefed_cluster/command_event_handler_test.py +++ b/tradefed_cluster/command_event_handler_test.py @@ -26,13 +26,15 @@ import mock from six.moves import range import webtest + from tradefed_cluster import command_event_handler from tradefed_cluster import command_event_test_util from tradefed_cluster import command_manager from tradefed_cluster import command_task_store from tradefed_cluster import common from tradefed_cluster import datastore_entities -from tradefed_cluster import env_config from tradefed_cluster import metric +from tradefed_cluster import env_config +from tradefed_cluster import metric from tradefed_cluster import request_manager from tradefed_cluster import request_sync_monitor from tradefed_cluster import testbed_dependent_test diff --git a/tradefed_cluster/command_manager.py b/tradefed_cluster/command_manager.py index 8192c0b..019d2af 100644 --- a/tradefed_cluster/command_manager.py +++ b/tradefed_cluster/command_manager.py @@ -28,6 +28,7 @@ import six from six.moves import range from six.moves import zip + from tradefed_cluster import api_messages from tradefed_cluster import command_task_store from tradefed_cluster import common diff --git a/tradefed_cluster/command_manager_test.py b/tradefed_cluster/command_manager_test.py index d91f33d..4b6fbfc 100644 --- a/tradefed_cluster/command_manager_test.py +++ b/tradefed_cluster/command_manager_test.py @@ -28,6 +28,7 @@ import mock from protorpc import protojson from six.moves import range + from tradefed_cluster import api_messages from tradefed_cluster import command_event_test_util from tradefed_cluster import command_manager diff --git a/tradefed_cluster/command_monitor.py b/tradefed_cluster/command_monitor.py index 5f35880..2195d95 100644 --- a/tradefed_cluster/command_monitor.py +++ b/tradefed_cluster/command_monitor.py @@ -19,6 +19,7 @@ import logging import flask + from tradefed_cluster import command_manager from tradefed_cluster import common from tradefed_cluster import metric diff --git a/tradefed_cluster/command_monitor_test.py b/tradefed_cluster/command_monitor_test.py index 3126429..d203653 100644 --- a/tradefed_cluster/command_monitor_test.py +++ b/tradefed_cluster/command_monitor_test.py @@ -27,11 +27,13 @@ import mock from six.moves import range import webtest + from tradefed_cluster import command_manager from tradefed_cluster import command_monitor from tradefed_cluster import common from tradefed_cluster import datastore_entities -from tradefed_cluster import env_config from tradefed_cluster import request_manager +from tradefed_cluster import env_config +from tradefed_cluster import request_manager from tradefed_cluster import testbed_dependent_test from tradefed_cluster.services import task_scheduler from tradefed_cluster.util import ndb_shim as ndb diff --git a/tradefed_cluster/command_task_api.py b/tradefed_cluster/command_task_api.py index 4f2e2fc..7abc3a9 100644 --- a/tradefed_cluster/command_task_api.py +++ b/tradefed_cluster/command_task_api.py @@ -23,6 +23,7 @@ from protorpc import remote from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import affinity_manager from tradefed_cluster import api_common from tradefed_cluster import api_messages @@ -135,7 +136,8 @@ class CommandTaskApi(remote.Service): break if matcher.IsEmpty(): break - except: + + except: logging.exception( 'Failed to lease tasks for "%s" cluster. Skipping...', cluster) @@ -264,7 +266,8 @@ class CommandTaskApi(remote.Service): for device in matched_devices: affinity_manager.SetDeviceAffinity( device.device_serial, affinity_tag) - except Exception as e: # Datastore entities can only be written to once per second. If we fail + except Exception as e: + # Datastore entities can only be written to once per second. If we fail # to update the command or task, log the error, and try leasing other # tasks. logging.warning("Error leasing task %s: %s", task.task_id, e) diff --git a/tradefed_cluster/command_task_api_test.py b/tradefed_cluster/command_task_api_test.py index d3a61ba..b6c3272 100644 --- a/tradefed_cluster/command_task_api_test.py +++ b/tradefed_cluster/command_task_api_test.py @@ -23,16 +23,19 @@ from protorpc import protojson from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import affinity_manager from tradefed_cluster import api_messages from tradefed_cluster import api_test from tradefed_cluster import command_manager from tradefed_cluster import command_task_api -from tradefed_cluster import command_task_matcher from tradefed_cluster import command_task_store +from tradefed_cluster import command_task_matcher +from tradefed_cluster import command_task_store from tradefed_cluster import common from tradefed_cluster import datastore_entities from tradefed_cluster import datastore_test_util -from tradefed_cluster import env_config from tradefed_cluster import metric +from tradefed_cluster import env_config +from tradefed_cluster import metric TIMESTAMP = datetime.datetime(2018, 4, 30, 0, 0, 0) diff --git a/tradefed_cluster/command_task_matcher.py b/tradefed_cluster/command_task_matcher.py index 40fe8d1..83ef0af 100644 --- a/tradefed_cluster/command_task_matcher.py +++ b/tradefed_cluster/command_task_matcher.py @@ -18,7 +18,8 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -from collections import defaultdict, namedtuple import logging +from collections import defaultdict, namedtuple +import logging import six from tradefed_cluster import api_messages diff --git a/tradefed_cluster/command_task_store.py b/tradefed_cluster/command_task_store.py index 51aa145..63d3da2 100644 --- a/tradefed_cluster/command_task_store.py +++ b/tradefed_cluster/command_task_store.py @@ -23,6 +23,7 @@ from typing import Dict, Optional, Any from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import affinity_manager from tradefed_cluster import api_messages from tradefed_cluster import common @@ -158,7 +159,8 @@ def GetLeasableTasks(cluster, run_targets): tasks = ( datastore_entities.CommandTask.query( datastore_entities.CommandTask.cluster == cluster, - datastore_entities.CommandTask.leasable == True, namespace=common.NAMESPACE) + datastore_entities.CommandTask.leasable == True, + namespace=common.NAMESPACE) .order(-datastore_entities.CommandTask.priority)) # Because Datastore will always return the tasks from the above query in the diff --git a/tradefed_cluster/commander.py b/tradefed_cluster/commander.py index a7052f4..7f38f7e 100644 --- a/tradefed_cluster/commander.py +++ b/tradefed_cluster/commander.py @@ -24,6 +24,7 @@ import zlib import flask + from tradefed_cluster import command_manager from tradefed_cluster import command_monitor from tradefed_cluster import command_task_store diff --git a/tradefed_cluster/commander_test.py b/tradefed_cluster/commander_test.py index 30afdc7..298b5b9 100644 --- a/tradefed_cluster/commander_test.py +++ b/tradefed_cluster/commander_test.py @@ -21,6 +21,7 @@ import hamcrest import mock import webtest + from tradefed_cluster import command_event_test_util from tradefed_cluster import command_manager from tradefed_cluster import command_monitor @@ -308,7 +309,8 @@ class CommanderTest(testbed_dependent_test.TestbedDependentTest): self, plugin, schedule_tasks, monitor): request_id = "1001" command_infos = [ - datastore_entities.CommandInfo( command_line="command_line %04d" % i, + datastore_entities.CommandInfo( + command_line="command_line %04d" % i, cluster="cluster %04d" % i, run_target="run_target %04d" % i, run_count=1, @@ -338,7 +340,8 @@ class CommanderTest(testbed_dependent_test.TestbedDependentTest): self.assertIsNone(command.shard_count) plugin.OnCreateCommands.assert_has_calls([ mock.call([ - plugin_base.CommandInfo( command_id=int(command.key.id()), + plugin_base.CommandInfo( + command_id=int(command.key.id()), command_line=command.command_line, run_count=command.run_count, shard_count=1, @@ -642,7 +645,8 @@ class CommanderTest(testbed_dependent_test.TestbedDependentTest): # Test ProcessCommandEvent for a non-final state with deletion request_id = "1001" command_infos = [ - datastore_entities.CommandInfo( command_line="command_line %04d" % i, + datastore_entities.CommandInfo( + command_line="command_line %04d" % i, cluster="cluster %04d" % i, run_target="run_target %04d" % i, run_count=1, @@ -749,7 +753,8 @@ class CommanderTest(testbed_dependent_test.TestbedDependentTest): def testCheckPendingCommands(self, schedule_tasks, monitor): request_id = "1001" command_infos = [ - datastore_entities.CommandInfo( command_line="command_line %04d" % i, + datastore_entities.CommandInfo( + command_line="command_line %04d" % i, cluster="cluster %04d" % i, run_target="run_target %04d" % i, run_count=1, @@ -795,7 +800,8 @@ class CommanderTest(testbed_dependent_test.TestbedDependentTest): self, schedule_tasks, monitor): request_id = "1001" command_infos = [ - datastore_entities.CommandInfo( command_line="command_line %04d" % i, + datastore_entities.CommandInfo( + command_line="command_line %04d" % i, cluster="cluster %04d" % i, run_target="run_target %04d" % i, run_count=1, diff --git a/tradefed_cluster/common.py b/tradefed_cluster/common.py index 3158168..9f0cdfa 100644 --- a/tradefed_cluster/common.py +++ b/tradefed_cluster/common.py @@ -68,7 +68,8 @@ def RetryNdbContentionErrors(f): def Wrapper(*args, **kwargs): try: return f(*args, **kwargs) - except Exception as e: exception_message = str(e) + except Exception as e: + exception_message = str(e) if "too much contention" in exception_message: raise TooMuchContentionError(exception_message) # If not matched by the parser will be raised. diff --git a/tradefed_cluster/common_test.py b/tradefed_cluster/common_test.py index b771589..c3c88ab 100644 --- a/tradefed_cluster/common_test.py +++ b/tradefed_cluster/common_test.py @@ -21,6 +21,7 @@ from tradefed_cluster import common # GPyLint does not recognize the ClassProperty in the common module. + class TestIntState(int): def __new__(cls, value): diff --git a/tradefed_cluster/config_syncer_gcs_to_ndb.py b/tradefed_cluster/config_syncer_gcs_to_ndb.py index 6ebab6e..ab3c56a 100644 --- a/tradefed_cluster/config_syncer_gcs_to_ndb.py +++ b/tradefed_cluster/config_syncer_gcs_to_ndb.py @@ -21,6 +21,7 @@ import re import flask + from tradefed_cluster import common from tradefed_cluster import datastore_entities from tradefed_cluster import datastore_util @@ -31,6 +32,7 @@ from tradefed_cluster.services import acl_service from tradefed_cluster.services import file_storage from tradefed_cluster.util import ndb_shim as ndb + # Ansible package depends on certain environment variable. ansible_import_error = None try: @@ -38,7 +40,8 @@ try: from ansible.parsing import dataloader # unified_lab_config depends on ansible, so need to import after ansible. from tradefed_cluster.configs import unified_lab_config as unified_lab_config_util -except Exception as e: logging.warning('Fail to import ansible package:\n%s', e) +except Exception as e: + logging.warning('Fail to import ansible package:\n%s', e) ansible_import_error = e diff --git a/tradefed_cluster/config_syncer_gcs_to_ndb_test.py b/tradefed_cluster/config_syncer_gcs_to_ndb_test.py index 83638c9..257de31 100644 --- a/tradefed_cluster/config_syncer_gcs_to_ndb_test.py +++ b/tradefed_cluster/config_syncer_gcs_to_ndb_test.py @@ -19,6 +19,7 @@ from unittest import mock import six + from tradefed_cluster import config_syncer_gcs_to_ndb from tradefed_cluster import datastore_entities from tradefed_cluster import testbed_dependent_test diff --git a/tradefed_cluster/configs/lab_config.py b/tradefed_cluster/configs/lab_config.py index 631ea1b..fe9b113 100644 --- a/tradefed_cluster/configs/lab_config.py +++ b/tradefed_cluster/configs/lab_config.py @@ -22,11 +22,13 @@ import six import strictyaml as syaml + try: from google.protobuf import json_format except ImportError: from google3.net.proto2.python.public import json_format + from tradefed_cluster.configs import lab_config_pb2 # Ansible package depends on certain environment variable. @@ -35,9 +37,11 @@ try: os.environ['ANSIBLE_LOCAL_TEMP'] = '/tmp' # unified_lab_config depends on ansible. from tradefed_cluster.configs import unified_lab_config as unified_lab_config_util -except Exception as e: logging.warning('Fail to import ansible package:\n%s', e) +except Exception as e: + logging.warning('Fail to import ansible package:\n%s', e) ansible_import_error = e + logger = logging.getLogger(__name__) _DEFAULT_SHUTDOWN_TIMEOUT_SEC = 3600 diff --git a/tradefed_cluster/coordinator_api.py b/tradefed_cluster/coordinator_api.py index 43c5a2e..a9cb05d 100644 --- a/tradefed_cluster/coordinator_api.py +++ b/tradefed_cluster/coordinator_api.py @@ -19,6 +19,7 @@ import logging from protorpc import message_types from protorpc import remote + from tradefed_cluster import api_common from tradefed_cluster import command_attempt_monitor from tradefed_cluster import command_monitor diff --git a/tradefed_cluster/coordinator_api_test.py b/tradefed_cluster/coordinator_api_test.py index 9cd751d..6b45f6d 100644 --- a/tradefed_cluster/coordinator_api_test.py +++ b/tradefed_cluster/coordinator_api_test.py @@ -24,12 +24,14 @@ import hamcrest import mock from six.moves import range + from tradefed_cluster import api_test from tradefed_cluster import command_manager from tradefed_cluster import command_monitor from tradefed_cluster import common from tradefed_cluster import datastore_entities -from tradefed_cluster import env_config from tradefed_cluster import request_manager +from tradefed_cluster import env_config +from tradefed_cluster import request_manager from tradefed_cluster import request_sync_monitor from tradefed_cluster.util import ndb_shim as ndb diff --git a/tradefed_cluster/datastore_entities.py b/tradefed_cluster/datastore_entities.py index eeb1a60..34e78b2 100644 --- a/tradefed_cluster/datastore_entities.py +++ b/tradefed_cluster/datastore_entities.py @@ -32,6 +32,7 @@ from tradefed_cluster.util import ndb_util + # Constant to hold a time of 0 (0 hours, 0 minutes, 0 seconds) ZERO_TIME = datetime.time() @@ -69,9 +70,11 @@ def ToMessage(entity, *args, **kwargs): """A utility method to convert a datastore entity into an API message.""" if entity is None: return None - assert type(entity) in _CONVERTER_DISPATCH_DICT, ( + + assert type(entity) in _CONVERTER_DISPATCH_DICT, ( 'Converter for "%s" not defined.' % type(entity).__name__) - return _CONVERTER_DISPATCH_DICT[type(entity)](entity, *args, **kwargs) + + return _CONVERTER_DISPATCH_DICT[type(entity)](entity, *args, **kwargs) class AffinityStatus(ndb.Model): @@ -560,7 +563,8 @@ def CommandInfoToMessage(entity): test_bench=ToMessage(entity.test_bench)) -def _Request_AddCommandInfo(obj): """Upgrade function for legacy Request objects.""" +def _Request_AddCommandInfo(obj): + """Upgrade function for legacy Request objects.""" obj.command_infos = [ CommandInfo( command_line=obj.depr_command_line, diff --git a/tradefed_cluster/datastore_entities_test.py b/tradefed_cluster/datastore_entities_test.py index b29f0c6..8619b8e 100644 --- a/tradefed_cluster/datastore_entities_test.py +++ b/tradefed_cluster/datastore_entities_test.py @@ -24,6 +24,7 @@ import unittest from six.moves import range + from tradefed_cluster import api_messages from tradefed_cluster import common from tradefed_cluster import datastore_entities diff --git a/tradefed_cluster/datastore_test_util.py b/tradefed_cluster/datastore_test_util.py index 05b0d9e..f1909cb 100644 --- a/tradefed_cluster/datastore_test_util.py +++ b/tradefed_cluster/datastore_test_util.py @@ -15,6 +15,7 @@ import datetime import six + from tradefed_cluster import api_messages from tradefed_cluster import common from tradefed_cluster import datastore_entities diff --git a/tradefed_cluster/datastore_util_test.py b/tradefed_cluster/datastore_util_test.py index 45e8895..40294ae 100644 --- a/tradefed_cluster/datastore_util_test.py +++ b/tradefed_cluster/datastore_util_test.py @@ -26,6 +26,7 @@ import mock from six.moves import range + from tradefed_cluster import api_messages from tradefed_cluster import datastore_entities from tradefed_cluster import datastore_test_util diff --git a/tradefed_cluster/device_blocklist_api.py b/tradefed_cluster/device_blocklist_api.py index 1a896df..ace8c44 100644 --- a/tradefed_cluster/device_blocklist_api.py +++ b/tradefed_cluster/device_blocklist_api.py @@ -19,6 +19,7 @@ from protorpc import message_types from protorpc import messages from protorpc import remote + from tradefed_cluster import api_common from tradefed_cluster import api_messages from tradefed_cluster import datastore_entities diff --git a/tradefed_cluster/device_blocklist_api_test.py b/tradefed_cluster/device_blocklist_api_test.py index 08c88c0..f3c34eb 100644 --- a/tradefed_cluster/device_blocklist_api_test.py +++ b/tradefed_cluster/device_blocklist_api_test.py @@ -16,6 +16,7 @@ import unittest from protorpc import protojson + from tradefed_cluster import api_messages from tradefed_cluster import api_test from tradefed_cluster import datastore_test_util diff --git a/tradefed_cluster/device_history_cleaner.py b/tradefed_cluster/device_history_cleaner.py index b19b563..28b22c3 100644 --- a/tradefed_cluster/device_history_cleaner.py +++ b/tradefed_cluster/device_history_cleaner.py @@ -19,6 +19,7 @@ import logging import flask + from tradefed_cluster import datastore_entities from tradefed_cluster.util import ndb_shim as ndb diff --git a/tradefed_cluster/device_history_cleaner_test.py b/tradefed_cluster/device_history_cleaner_test.py index 66c4959..2b692df 100644 --- a/tradefed_cluster/device_history_cleaner_test.py +++ b/tradefed_cluster/device_history_cleaner_test.py @@ -21,6 +21,7 @@ import mock import six import webtest + from tradefed_cluster import datastore_entities from tradefed_cluster import device_history_cleaner from tradefed_cluster import testbed_dependent_test diff --git a/tradefed_cluster/device_info_reporter.py b/tradefed_cluster/device_info_reporter.py index 73f1c11..fe879b4 100644 --- a/tradefed_cluster/device_info_reporter.py +++ b/tradefed_cluster/device_info_reporter.py @@ -28,6 +28,7 @@ import dateutil.parser import flask import six + from tradefed_cluster import api_messages from tradefed_cluster import common from tradefed_cluster import datastore_entities @@ -198,7 +199,8 @@ def GetHostsToReport(cluster_prefix=None): """ query = (datastore_entities.HostInfo.query() .filter( - datastore_entities.HostInfo.hidden == False)) if cluster_prefix: + datastore_entities.HostInfo.hidden == False)) + if cluster_prefix: query = query.filter( datastore_entities.HostInfo.physical_cluster >= cluster_prefix).filter( datastore_entities.HostInfo.physical_cluster < @@ -215,7 +217,8 @@ def GetDevicesToReport(cluster_prefix=None): A list of physical devices. """ query = (datastore_entities.DeviceInfo.query() - .filter(datastore_entities.DeviceInfo.hidden == False) .filter(datastore_entities.DeviceInfo.device_type == + .filter(datastore_entities.DeviceInfo.hidden == False) + .filter(datastore_entities.DeviceInfo.device_type == api_messages.DeviceTypeMessage.PHYSICAL)) if cluster_prefix: query = ( diff --git a/tradefed_cluster/device_info_reporter_test.py b/tradefed_cluster/device_info_reporter_test.py index 8ab11fa..b3bfbff 100644 --- a/tradefed_cluster/device_info_reporter_test.py +++ b/tradefed_cluster/device_info_reporter_test.py @@ -24,6 +24,7 @@ import mock import six import webtest + from tradefed_cluster import api_messages from tradefed_cluster import datastore_entities from tradefed_cluster import datastore_test_util diff --git a/tradefed_cluster/device_manager.py b/tradefed_cluster/device_manager.py index e6b6957..61bd3ce 100644 --- a/tradefed_cluster/device_manager.py +++ b/tradefed_cluster/device_manager.py @@ -26,6 +26,7 @@ import uuid import six from six.moves import zip + from tradefed_cluster import affinity_manager from tradefed_cluster import api_messages from tradefed_cluster import common @@ -335,7 +336,8 @@ def _CountDeviceForHost(hostname): devices = ( datastore_entities.DeviceInfo .query(ancestor=ndb.Key(datastore_entities.HostInfo, hostname)) - .filter(datastore_entities.DeviceInfo.hidden == False) .fetch(projection=[ + .filter(datastore_entities.DeviceInfo.hidden == False) + .fetch(projection=[ datastore_entities.DeviceInfo.run_target, datastore_entities.DeviceInfo.state])) _DoCountDeviceForHost(host, devices) @@ -672,7 +674,8 @@ def _UpdateGoneDevicesInNDB(hostname, reported_devices, timestamp): device_keys = ( datastore_entities.DeviceInfo .query(ancestor=ndb.Key(datastore_entities.HostInfo, hostname)) - .filter(datastore_entities.DeviceInfo.hidden == False) .fetch(keys_only=True)) + .filter(datastore_entities.DeviceInfo.hidden == False) + .fetch(keys_only=True)) missing_device_keys = [] for device_key in device_keys: if device_key.id() in reported_devices: @@ -779,7 +782,8 @@ def GetDevicesOnHost(hostname): """Get device entities on a host.""" return (datastore_entities.DeviceInfo .query(ancestor=ndb.Key(datastore_entities.HostInfo, hostname)) - .filter(datastore_entities.DeviceInfo.hidden == False) .fetch()) + .filter(datastore_entities.DeviceInfo.hidden == False) + .fetch()) def UpdateGoneHost(hostname): @@ -1192,7 +1196,8 @@ def GetRunTargetsFromNDB(cluster=None): query = datastore_entities.DeviceInfo.query( projection=[datastore_entities.DeviceInfo.run_target], distinct=True).filter( - datastore_entities.DeviceInfo.hidden == False) if cluster: + datastore_entities.DeviceInfo.hidden == False) + if cluster: query = query.filter( datastore_entities.DeviceInfo.clusters == cluster) return (h.run_target for h in query if h.run_target) diff --git a/tradefed_cluster/device_manager_test.py b/tradefed_cluster/device_manager_test.py index 0fc7679..3b8dd28 100644 --- a/tradefed_cluster/device_manager_test.py +++ b/tradefed_cluster/device_manager_test.py @@ -23,6 +23,7 @@ import mock from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import affinity_manager from tradefed_cluster import api_messages from tradefed_cluster import common diff --git a/tradefed_cluster/device_monitor.py b/tradefed_cluster/device_monitor.py index b144678..661991b 100644 --- a/tradefed_cluster/device_monitor.py +++ b/tradefed_cluster/device_monitor.py @@ -28,6 +28,7 @@ from protorpc import protojson import six + from tradefed_cluster import api_messages from tradefed_cluster import common from tradefed_cluster import datastore_entities @@ -239,7 +240,8 @@ def _ScanHosts(): hosts = [] query = ( datastore_entities.HostInfo.query() - .filter(datastore_entities.HostInfo.hidden == False)) # LINT.IfChange(scan_host_projection) + .filter(datastore_entities.HostInfo.hidden == False)) + # LINT.IfChange(scan_host_projection) projection = [ datastore_entities.HostInfo.lab_name, datastore_entities.HostInfo.physical_cluster, diff --git a/tradefed_cluster/device_monitor_test.py b/tradefed_cluster/device_monitor_test.py index 0ec54af..0ee2021 100644 --- a/tradefed_cluster/device_monitor_test.py +++ b/tradefed_cluster/device_monitor_test.py @@ -26,6 +26,7 @@ from protorpc import protojson from six.moves import zip import webtest + from tradefed_cluster import api_messages from tradefed_cluster import common from tradefed_cluster import datastore_entities diff --git a/tradefed_cluster/device_snapshot_api.py b/tradefed_cluster/device_snapshot_api.py index 866d4e7..47aed6b 100644 --- a/tradefed_cluster/device_snapshot_api.py +++ b/tradefed_cluster/device_snapshot_api.py @@ -23,6 +23,7 @@ from protorpc import message_types from protorpc import messages from protorpc import remote + from tradefed_cluster import api_common from tradefed_cluster import api_messages from tradefed_cluster import datastore_entities @@ -34,6 +35,7 @@ ZERO_TIME = datetime.time() # Camel case used on API messages. + class DeviceSnapshot(messages.Message): """Snapshot of all the devices for a given report date.""" date = message_types.DateTimeField(1, required=True) diff --git a/tradefed_cluster/device_snapshot_api_test.py b/tradefed_cluster/device_snapshot_api_test.py index 4505a60..30c0901 100644 --- a/tradefed_cluster/device_snapshot_api_test.py +++ b/tradefed_cluster/device_snapshot_api_test.py @@ -20,6 +20,7 @@ import unittest import mock from protorpc import protojson + from tradefed_cluster import api_messages from tradefed_cluster import api_test from tradefed_cluster import datastore_entities diff --git a/tradefed_cluster/filter_hint_api.py b/tradefed_cluster/filter_hint_api.py index 8a9d593..5744e79 100644 --- a/tradefed_cluster/filter_hint_api.py +++ b/tradefed_cluster/filter_hint_api.py @@ -80,7 +80,8 @@ class FilterHintApi(remote.Service): """Fetches a list of pool.""" entities = datastore_entities.HostInfo.query( projection=[datastore_entities.HostInfo.pools], distinct=True).filter( - datastore_entities.HostInfo.hidden == False).fetch( projection=[datastore_entities.HostInfo.pools]) + datastore_entities.HostInfo.hidden == False).fetch( + projection=[datastore_entities.HostInfo.pools]) mergedlist = [] for item in entities: mergedlist += item.pools @@ -109,7 +110,8 @@ class FilterHintApi(remote.Service): def _ListHosts(self): """Fetches a list of hostnames.""" entities = datastore_entities.HostInfo.query().filter( - datastore_entities.HostInfo.hidden == False).fetch(keys_only=True) infos = [ + datastore_entities.HostInfo.hidden == False).fetch(keys_only=True) + infos = [ api_messages.FilterHintMessage(value=str(item.id())) for item in entities ] @@ -120,7 +122,8 @@ class FilterHintApi(remote.Service): entities = datastore_entities.HostInfo.query( projection=[datastore_entities.HostInfo.test_harness], distinct=True).filter( - datastore_entities.HostInfo.hidden == False).fetch( projection=[datastore_entities.HostInfo.test_harness]) + datastore_entities.HostInfo.hidden == False).fetch( + projection=[datastore_entities.HostInfo.test_harness]) infos = [ api_messages.FilterHintMessage(value=item.test_harness) for item in entities @@ -132,7 +135,8 @@ class FilterHintApi(remote.Service): entities = datastore_entities.HostInfo.query( projection=[datastore_entities.HostInfo.test_harness_version], distinct=True).filter( - datastore_entities.HostInfo.hidden == False).fetch( projection=[datastore_entities.HostInfo.test_harness_version]) + datastore_entities.HostInfo.hidden == False).fetch( + projection=[datastore_entities.HostInfo.test_harness_version]) infos = [ api_messages.FilterHintMessage(value=item.test_harness_version) for item in entities @@ -158,7 +162,8 @@ class FilterHintApi(remote.Service): entities = datastore_entities.HostInfo.query( projection=[datastore_entities.HostInfo.host_group], distinct=True).filter( - datastore_entities.HostInfo.hidden == False).fetch( projection=[datastore_entities.HostInfo.host_group]) + datastore_entities.HostInfo.hidden == False).fetch( + projection=[datastore_entities.HostInfo.host_group]) infos = [ api_messages.FilterHintMessage(value=item.host_group) for item in entities @@ -178,7 +183,8 @@ class FilterHintApi(remote.Service): entities = datastore_entities.DeviceInfo.query( projection=[datastore_entities.DeviceInfo.product], distinct=True).filter( - datastore_entities.DeviceInfo.hidden == False) products = [ + datastore_entities.DeviceInfo.hidden == False) + products = [ api_messages.FilterHintMessage(value=item.product) for item in entities if item.product ] @@ -189,7 +195,8 @@ class FilterHintApi(remote.Service): entities = datastore_entities.DeviceInfo.query( projection=[datastore_entities.DeviceInfo.product_variant], distinct=True).filter( - datastore_entities.DeviceInfo.hidden == False) product_variants = [ + datastore_entities.DeviceInfo.hidden == False) + product_variants = [ api_messages.FilterHintMessage(value=item.product_variant) for item in entities if item.product_variant ] diff --git a/tradefed_cluster/host_event_api.py b/tradefed_cluster/host_event_api.py index f8a85e3..328f2e9 100644 --- a/tradefed_cluster/host_event_api.py +++ b/tradefed_cluster/host_event_api.py @@ -32,6 +32,7 @@ from protorpc import protojson from protorpc import remote from six.moves import range + from tradefed_cluster import api_common from tradefed_cluster import api_messages from tradefed_cluster import common diff --git a/tradefed_cluster/host_event_api_test.py b/tradefed_cluster/host_event_api_test.py index 430fe61..7757f8c 100644 --- a/tradefed_cluster/host_event_api_test.py +++ b/tradefed_cluster/host_event_api_test.py @@ -19,6 +19,7 @@ import unittest import mock + from tradefed_cluster import api_test from tradefed_cluster import datastore_entities from tradefed_cluster import device_manager diff --git a/tradefed_cluster/lab_management_api.py b/tradefed_cluster/lab_management_api.py index 3d9d5c9..d4b235f 100644 --- a/tradefed_cluster/lab_management_api.py +++ b/tradefed_cluster/lab_management_api.py @@ -19,6 +19,7 @@ from protorpc import message_types from protorpc import messages from protorpc import remote + from tradefed_cluster import api_common from tradefed_cluster import api_messages from tradefed_cluster import datastore_entities diff --git a/tradefed_cluster/lab_management_api_test.py b/tradefed_cluster/lab_management_api_test.py index 1a534e7..1329e35 100644 --- a/tradefed_cluster/lab_management_api_test.py +++ b/tradefed_cluster/lab_management_api_test.py @@ -21,6 +21,7 @@ import unittest from protorpc import protojson from six.moves import zip + from tradefed_cluster import api_messages from tradefed_cluster import api_test from tradefed_cluster import datastore_entities diff --git a/tradefed_cluster/lab_resource_monitor.py b/tradefed_cluster/lab_resource_monitor.py index 975618e..a233bf4 100644 --- a/tradefed_cluster/lab_resource_monitor.py +++ b/tradefed_cluster/lab_resource_monitor.py @@ -63,7 +63,8 @@ def _Pull(topic): _ProcessMessage(data) message_count += 1 ack_ids.append(message.get('ackId')) - except Exception: logging.exception('error processing message: %s', message) + except Exception: + logging.exception('error processing message: %s', message) finally: if ack_ids: pubsub_client.AcknowledgeMessages(PUBSUB_SUBSCRIPTION, ack_ids) diff --git a/tradefed_cluster/lab_resource_monitor_test.py b/tradefed_cluster/lab_resource_monitor_test.py index e670733..e476dc9 100644 --- a/tradefed_cluster/lab_resource_monitor_test.py +++ b/tradefed_cluster/lab_resource_monitor_test.py @@ -24,6 +24,7 @@ import unittest import mock + from tradefed_cluster import common from tradefed_cluster import datastore_entities from tradefed_cluster import lab_resource_monitor diff --git a/tradefed_cluster/note_manager.py b/tradefed_cluster/note_manager.py index 470f544..3b3a913 100644 --- a/tradefed_cluster/note_manager.py +++ b/tradefed_cluster/note_manager.py @@ -19,6 +19,7 @@ import logging from protorpc import protojson + from tradefed_cluster import common from tradefed_cluster import datastore_entities from tradefed_cluster import env_config diff --git a/tradefed_cluster/note_manager_test.py b/tradefed_cluster/note_manager_test.py index 8a28ffe..33e5562 100644 --- a/tradefed_cluster/note_manager_test.py +++ b/tradefed_cluster/note_manager_test.py @@ -21,6 +21,7 @@ import mock from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import api_messages from tradefed_cluster import common from tradefed_cluster import datastore_entities diff --git a/tradefed_cluster/notification_handler.py b/tradefed_cluster/notification_handler.py index 63edae0..17c134e 100644 --- a/tradefed_cluster/notification_handler.py +++ b/tradefed_cluster/notification_handler.py @@ -18,6 +18,7 @@ import logging import flask + from tradefed_cluster import common from tradefed_cluster import datastore_entities from tradefed_cluster import request_manager @@ -41,7 +42,8 @@ def HandleNotification(fake=None): def NotifyPendingRequestStateChanges(): """Notify all states that have the notify_test_status flag set.""" keys = (datastore_entities.Request.query(namespace=common.NAMESPACE) - .filter(datastore_entities.Request.notify_state_change == True) + + .filter(datastore_entities.Request.notify_state_change == True) .fetch(keys_only=True)) for k in keys: task_scheduler.AddCallableTask( diff --git a/tradefed_cluster/notification_handler_test.py b/tradefed_cluster/notification_handler_test.py index ec805a0..6b633f1 100644 --- a/tradefed_cluster/notification_handler_test.py +++ b/tradefed_cluster/notification_handler_test.py @@ -21,6 +21,7 @@ import zlib from googleapiclient import errors import mock + from tradefed_cluster import common from tradefed_cluster import datastore_entities from tradefed_cluster import datastore_test_util diff --git a/tradefed_cluster/notifier.py b/tradefed_cluster/notifier.py index e9e2770..f05d6ea 100644 --- a/tradefed_cluster/notifier.py +++ b/tradefed_cluster/notifier.py @@ -20,6 +20,7 @@ import zlib import flask + from tradefed_cluster import common from tradefed_cluster import env_config from tradefed_cluster.services import task_scheduler diff --git a/tradefed_cluster/notifier_test.py b/tradefed_cluster/notifier_test.py index 580c844..dc18813 100644 --- a/tradefed_cluster/notifier_test.py +++ b/tradefed_cluster/notifier_test.py @@ -24,6 +24,7 @@ from protorpc import protojson import six import webtest + from tradefed_cluster import api_messages from tradefed_cluster import common from tradefed_cluster import datastore_entities diff --git a/tradefed_cluster/plugins/appengine.py b/tradefed_cluster/plugins/appengine.py index 7b264fb..8541c4e 100644 --- a/tradefed_cluster/plugins/appengine.py +++ b/tradefed_cluster/plugins/appengine.py @@ -20,6 +20,7 @@ from google.appengine.api import mail from google.appengine.api import taskqueue from google.appengine.api.modules import modules + from tradefed_cluster.plugins import base diff --git a/tradefed_cluster/plugins/appengine_test.py b/tradefed_cluster/plugins/appengine_test.py index aea4248..b87d785 100644 --- a/tradefed_cluster/plugins/appengine_test.py +++ b/tradefed_cluster/plugins/appengine_test.py @@ -21,6 +21,7 @@ from google.appengine.ext import testbed from google3.apphosting.contrib.testing import internal_testbed + from tradefed_cluster.plugins import appengine diff --git a/tradefed_cluster/plugins/google_cloud.py b/tradefed_cluster/plugins/google_cloud.py index 691d5e9..f4a3d51 100644 --- a/tradefed_cluster/plugins/google_cloud.py +++ b/tradefed_cluster/plugins/google_cloud.py @@ -26,6 +26,7 @@ from google.api_core import retry from google.cloud import tasks_v2 from google3.google.protobuf import timestamp_pb2 + from tradefed_cluster.plugins import base DEFAULT_RETRY_OPTION = retry.Retry(deadline=60) diff --git a/tradefed_cluster/plugins/google_cloud_test.py b/tradefed_cluster/plugins/google_cloud_test.py index 1bc5167..ea00c42 100644 --- a/tradefed_cluster/plugins/google_cloud_test.py +++ b/tradefed_cluster/plugins/google_cloud_test.py @@ -23,6 +23,7 @@ import mock from google.cloud import tasks_v2 from google3.google.protobuf import timestamp_pb2 + from tradefed_cluster.plugins import google_cloud diff --git a/tradefed_cluster/plugins/registry.py b/tradefed_cluster/plugins/registry.py index 22fb0e8..1344491 100644 --- a/tradefed_cluster/plugins/registry.py +++ b/tradefed_cluster/plugins/registry.py @@ -17,6 +17,7 @@ from tradefed_cluster.plugins import base from tradefed_cluster.plugins import noop_plugin + def GetPlugin(name, **kwargs): """Returns a plugin instance registered with the given name. diff --git a/tradefed_cluster/predefined_message_api.py b/tradefed_cluster/predefined_message_api.py index cfc9fb6..d9c5c18 100644 --- a/tradefed_cluster/predefined_message_api.py +++ b/tradefed_cluster/predefined_message_api.py @@ -22,6 +22,7 @@ from protorpc import remote from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import api_common from tradefed_cluster import api_messages from tradefed_cluster import datastore_entities diff --git a/tradefed_cluster/report_api.py b/tradefed_cluster/report_api.py index ed3fa88..a7d1de5 100644 --- a/tradefed_cluster/report_api.py +++ b/tradefed_cluster/report_api.py @@ -28,6 +28,7 @@ from protorpc import messages from protorpc import remote import six + from tradefed_cluster import api_common from tradefed_cluster import device_info_reporter @@ -37,6 +38,7 @@ ZERO_TIME = datetime.time() # Disabling invalid-name pylint to allow camel cases. + class RecordGroup(messages.Message): """A count of devices matching the criteria for a given group.""" groupByField = messages.StringField(1, required=True) diff --git a/tradefed_cluster/request_api.py b/tradefed_cluster/request_api.py index 0ee3413..68cd808 100644 --- a/tradefed_cluster/request_api.py +++ b/tradefed_cluster/request_api.py @@ -23,6 +23,7 @@ from protorpc import remote from tradefed_cluster.util import ndb_shim as ndb + from tradefed_cluster import api_common from tradefed_cluster import api_messages from tradefed_cluster import command_manager diff --git a/tradefed_cluster/request_api_test.py b/tradefed_cluster/request_api_test.py index c89705c..8c8e2d9 100644 --- a/tradefed_cluster/request_api_test.py +++ b/tradefed_cluster/request_api_test.py @@ -28,6 +28,7 @@ from protorpc import protojson from six.moves import range from six.moves import zip + from tradefed_cluster import api_messages from tradefed_cluster import api_test from tradefed_cluster import command_manager diff --git a/tradefed_cluster/request_manager.py b/tradefed_cluster/request_manager.py index 3c1ad19..ad96616 100644 --- a/tradefed_cluster/request_manager.py +++ b/tradefed_cluster/request_manager.py @@ -22,6 +22,7 @@ import zlib from protorpc import protojson import six + from tradefed_cluster import api_messages from tradefed_cluster import command_error_type_config from tradefed_cluster import common diff --git a/tradefed_cluster/request_manager_test.py b/tradefed_cluster/request_manager_test.py index b5fa1b2..d736b3f 100644 --- a/tradefed_cluster/request_manager_test.py +++ b/tradefed_cluster/request_manager_test.py @@ -27,6 +27,7 @@ import mock from protorpc import protojson from six.moves import range + from tradefed_cluster import api_messages from tradefed_cluster import common from tradefed_cluster import datastore_entities diff --git a/tradefed_cluster/request_sync_monitor.py b/tradefed_cluster/request_sync_monitor.py index 25de74a..4a9c461 100644 --- a/tradefed_cluster/request_sync_monitor.py +++ b/tradefed_cluster/request_sync_monitor.py @@ -167,7 +167,8 @@ def _ProcessCommandEvents(request_id): try: commander.ProcessCommandEvent(event) raw_events_keys_to_delete.append(raw_event.key) - except Exception as e: logging.warning('Error while processing event: %s', event, exc_info=True) + except Exception as e: + logging.warning('Error while processing event: %s', event, exc_info=True) error = e break diff --git a/tradefed_cluster/run_target_api.py b/tradefed_cluster/run_target_api.py index bd22ccc..ff1f8a0 100644 --- a/tradefed_cluster/run_target_api.py +++ b/tradefed_cluster/run_target_api.py @@ -19,6 +19,7 @@ from protorpc import message_types from protorpc import messages from protorpc import remote + from tradefed_cluster import api_common from tradefed_cluster import api_messages from tradefed_cluster import device_manager diff --git a/tradefed_cluster/server_test.py b/tradefed_cluster/server_test.py index aa2e312..b34ab77 100644 --- a/tradefed_cluster/server_test.py +++ b/tradefed_cluster/server_test.py @@ -16,6 +16,7 @@ import unittest import werkzeug + from tradefed_cluster import server from tradefed_cluster import testbed_dependent_test diff --git a/tradefed_cluster/services/app_manager_test.py b/tradefed_cluster/services/app_manager_test.py index 22bf279..2bd4f41 100644 --- a/tradefed_cluster/services/app_manager_test.py +++ b/tradefed_cluster/services/app_manager_test.py @@ -17,6 +17,7 @@ from absl.testing import absltest import mock + from tradefed_cluster import env_config from tradefed_cluster.plugins import base as plugins_base from tradefed_cluster.services import app_manager diff --git a/tradefed_cluster/services/file_storage_test.py b/tradefed_cluster/services/file_storage_test.py index d7701b6..381fd7d 100644 --- a/tradefed_cluster/services/file_storage_test.py +++ b/tradefed_cluster/services/file_storage_test.py @@ -17,6 +17,7 @@ from absl.testing import absltest import mock + from tradefed_cluster import env_config from tradefed_cluster.plugins import base as plugins_base from tradefed_cluster.services import file_storage diff --git a/tradefed_cluster/services/mailer_test.py b/tradefed_cluster/services/mailer_test.py index 79b24b3..feeb805 100644 --- a/tradefed_cluster/services/mailer_test.py +++ b/tradefed_cluster/services/mailer_test.py @@ -17,6 +17,7 @@ from absl.testing import absltest import mock + from tradefed_cluster import env_config from tradefed_cluster.plugins import base as plugins_base from tradefed_cluster.services import mailer diff --git a/tradefed_cluster/services/task_scheduler.py b/tradefed_cluster/services/task_scheduler.py index 277af8f..ff543e6 100644 --- a/tradefed_cluster/services/task_scheduler.py +++ b/tradefed_cluster/services/task_scheduler.py @@ -26,6 +26,7 @@ import flask from google3.pyglib import stringutil + from tradefed_cluster import common from tradefed_cluster import env_config from tradefed_cluster.plugins import base as plugins_base diff --git a/tradefed_cluster/services/task_scheduler_test.py b/tradefed_cluster/services/task_scheduler_test.py index 1d2ebbf..f85f657 100644 --- a/tradefed_cluster/services/task_scheduler_test.py +++ b/tradefed_cluster/services/task_scheduler_test.py @@ -31,6 +31,7 @@ from tradefed_cluster.util import ndb_shim as ndb from google3.pyglib import stringutil + _object = threading.local() diff --git a/tradefed_cluster/testbed_dependent_test.py b/tradefed_cluster/testbed_dependent_test.py index 1605a61..00f87ba 100644 --- a/tradefed_cluster/testbed_dependent_test.py +++ b/tradefed_cluster/testbed_dependent_test.py @@ -23,6 +23,7 @@ import uuid import mock + from tradefed_cluster import env_config from tradefed_cluster.plugins import base from tradefed_cluster.util import ndb_test_lib diff --git a/tradefed_cluster/util/datastore_emulator.py b/tradefed_cluster/util/datastore_emulator.py index a8ef54c..15cf65f 100644 --- a/tradefed_cluster/util/datastore_emulator.py +++ b/tradefed_cluster/util/datastore_emulator.py @@ -42,7 +42,8 @@ class DatastoreEmulatorFactory(datastore_emulator.DatastoreEmulatorFactory): else tempfile.mkdtemp()) # Disable bad lint cause the class name is the same as the parent class - super(DatastoreEmulatorFactory, self).__init__( + + super(DatastoreEmulatorFactory, self).__init__( working_directory, os.getenv('DATASTORE_EMULATOR_ZIP_PATH'), os.getenv('JAVA_HOME')) diff --git a/tradefed_cluster/util/email_sender_test.py b/tradefed_cluster/util/email_sender_test.py index 2f583e7..3ab0553 100644 --- a/tradefed_cluster/util/email_sender_test.py +++ b/tradefed_cluster/util/email_sender_test.py @@ -19,6 +19,7 @@ import unittest import jinja2 + from tradefed_cluster import testbed_dependent_test from tradefed_cluster.util import email_sender diff --git a/tradefed_cluster/util/ndb_shim.py b/tradefed_cluster/util/ndb_shim.py index a8ee21e..02a0952 100644 --- a/tradefed_cluster/util/ndb_shim.py +++ b/tradefed_cluster/util/ndb_shim.py @@ -19,6 +19,7 @@ from __future__ import print_function import functools from six.moves import map + from google import auth from google.cloud import ndb from google.cloud.ndb import context as context_module diff --git a/tradefed_cluster/util/pubsub_client_test.py b/tradefed_cluster/util/pubsub_client_test.py index b8abd31..1209177 100644 --- a/tradefed_cluster/util/pubsub_client_test.py +++ b/tradefed_cluster/util/pubsub_client_test.py @@ -21,6 +21,7 @@ from googleapiclient import errors import mock + from tradefed_cluster import testbed_dependent_test from tradefed_cluster.util import pubsub_client |