diff options
Diffstat (limited to 'harnesses/host_controller/command_processor/command_config.py')
| -rw-r--r-- | harnesses/host_controller/command_processor/command_config.py | 440 |
1 files changed, 0 insertions, 440 deletions
diff --git a/harnesses/host_controller/command_processor/command_config.py b/harnesses/host_controller/command_processor/command_config.py deleted file mode 100644 index a8b1a24..0000000 --- a/harnesses/host_controller/command_processor/command_config.py +++ /dev/null @@ -1,440 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import httplib2 -import itertools -import logging -import os -import socket -import threading -import time - -from googleapiclient import errors -from google.protobuf import text_format - -from host_controller import common -from host_controller.command_processor import base_command_processor -from host_controller.console_argument_parser import ConsoleArgumentError -from host_controller.tradefed import remote_operation - -from vti.test_serving.proto import TestLabConfigMessage_pb2 as LabCfgMsg -from vti.test_serving.proto import TestScheduleConfigMessage_pb2 as SchedCfgMsg - - -class CommandConfig(base_command_processor.BaseCommandProcessor): - """Command processor for config command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - schedule_thread: dict containing threading.Thread instances(s) that - update schedule info regularly. - """ - - command = "config" - command_detail = "Specifies a global config type to monitor." - - def UpdateConfig(self, account_id, branch, targets, config_type, method, - update_build, clear_schedule, clear_labinfo): - """Updates the global configuration data. - - Args: - account_id: string, Partner Android Build account_id to use. - branch: string, branch to grab the artifact from. - targets: string, a comma-separate list of build target product(s). - config_type: string, config type (`prod` or `test'). - method: string, HTTP method for fetching. - update_build: boolean, indicating whether to upload build info. - clear_schedule: bool, True to clear all schedule data exist on the - scheduler - clear_labinfo: bool, True to clear all lab data exist on the - scheduler - """ - for target in targets.split(","): - fetch_path = self.FetchConfig( - account_id=account_id, - branch=branch, - target=target, - config_type=config_type, - method=method) - if fetch_path: - self.UploadConfig( - path=fetch_path, - update_build=update_build, - clear_schedule=clear_schedule, - clear_labinfo=clear_labinfo) - - def FetchConfig(self, account_id, branch, target, config_type, method): - """Fetches config files from the PAB build provider. - - Args: - account_id: string, Partner Android Build account_id to use. - branch: string, branch to grab the artifact from. - target: string, build target. - config_type: string, config type (`prod` or `test'). - method: string, HTTP method for fetching. - - Returns: - string, a path to the temp directory where config files are stored. - """ - path = "" - self.console._build_provider["pab"].Authenticate() - try: - listed_builds = self.console._build_provider["pab"].GetBuildList( - account_id=account_id, - branch=branch, - target=target, - page_token="", - max_results=1, - method="GET") - except ValueError as e: - logging.exception(e) - return path - - if listed_builds and len(listed_builds) > 0: - listed_build = listed_builds[0] - if listed_build["successful"]: - device_images, test_suites, artifacts, configs = ( - self.console._build_provider["pab"].GetArtifact( - account_id=account_id, - branch=branch, - target=target, - artifact_name=( - "vti-global-config-%s.zip" % config_type), - build_id=listed_build["build_id"], - method=method)) - path = os.path.dirname(configs[config_type]) - - return path - - def UploadConfig(self, path, update_build, clear_schedule, clear_labinfo): - """Uploads configs to VTI server. - - Args: - path: string, a path where config files are stored. - update_build: boolean, indicating whether to upload build info. - clear_schedule: bool, True to clear all schedule data exist on the - scheduler - clear_labinfo: bool, True to clear all lab data exist on the - scheduler - """ - schedules_pbs = [] - lab_pbs = [] - for root, dirs, files in os.walk(path): - for config_file in files: - full_path = os.path.join(root, config_file) - try: - if config_file.endswith(".schedule_config"): - with open(full_path, "r") as fd: - context = fd.read() - sched_cfg_msg = SchedCfgMsg.ScheduleConfigMessage() - text_format.Merge(context, sched_cfg_msg) - schedules_pbs.append(sched_cfg_msg) - logging.info(sched_cfg_msg.manifest_branch) - elif config_file.endswith(".lab_config"): - with open(full_path, "r") as fd: - context = fd.read() - lab_cfg_msg = LabCfgMsg.LabConfigMessage() - text_format.Merge(context, lab_cfg_msg) - lab_pbs.append(lab_cfg_msg) - except text_format.ParseError as e: - logging.error("ERROR: Config parsing error %s", e) - if update_build: - commands = self.GetBuildCommands(schedules_pbs) - if commands: - for command in commands: - ret = self.console.onecmd(command) - if ret == False: - break - self.console._vti_endpoint_client.UploadScheduleInfo( - schedules_pbs, clear_schedule) - self.console._vti_endpoint_client.UploadLabInfo(lab_pbs, clear_labinfo) - - def UpdateConfigLoop(self, account_id, branch, target, config_type, method, - update_build, update_interval, clear_schedule, - clear_labinfo): - """Regularly updates the global configuration. - - Args: - account_id: string, Partner Android Build account_id to use. - branch: string, branch to grab the artifact from. - targets: string, a comma-separate list of build target product(s). - config_type: string, config type (`prod` or `test'). - method: string, HTTP method for fetching. - update_build: boolean, indicating whether to upload build info. - update_interval: int, number of seconds before repeating - clear_schedule: bool, True to clear all schedule data exist on the - scheduler - clear_labinfo: bool, True to clear all lab data exist on the - scheduler - """ - thread = threading.currentThread() - while getattr(thread, 'keep_running', True): - try: - self.UpdateConfig(account_id, branch, target, config_type, - method, update_build, clear_schedule, - clear_labinfo) - except (socket.error, remote_operation.RemoteOperationException, - httplib2.HttpLib2Error, errors.HttpError) as e: - logging.exception(e) - time.sleep(update_interval) - - def GetBuildCommands(self, schedule_pbs): - """Generates a list of build commands with given schedules. - - Args: - schedule_pbs: a list of TestScheduleConfig protobuf messages. - - Returns: - a list of build command strings - """ - attrs = {} - attrs["device"] = [ - "build_storage_type", "manifest_branch", "pab_account_id", - "require_signed_device_build", "name" - ] - attrs["gsi"] = [ - "gsi_storage_type", "gsi_branch", "gsi_pab_account_id", - "gsi_build_target" - ] - attrs["test"] = [ - "test_storage_type", "test_branch", "test_pab_account_id", - "test_build_target" - ] - - class BuildInfo(object): - """A build information class.""" - - def __init__(self, _build_type): - if _build_type in attrs: - for attribute in attrs[_build_type]: - setattr(self, attribute, "") - - def __eq__(self, compare): - return self.__dict__ == compare.__dict__ - - build_commands = [] - if not schedule_pbs: - return build_commands - - # parses the given protobuf and stores as BuildInfo object. - builds = {"device": [], "gsi": [], "test": []} - for pb in schedule_pbs: - for build_target in pb.build_target: - build_type = "device" - device = BuildInfo(build_type) - for attr in attrs[build_type]: - if hasattr(pb, attr): - setattr(device, attr, getattr(pb, attr, None)) - elif hasattr(build_target, attr): - setattr(device, attr, getattr(build_target, attr, - None)) - if not [x for x in builds[build_type] if x == device]: - builds[build_type].append(device) - for test_schedule in build_target.test_schedule: - build_type = "gsi" - gsi = BuildInfo(build_type) - for attr in attrs[build_type]: - if hasattr(test_schedule, attr): - setattr(gsi, attr, - getattr(test_schedule, attr, None)) - if not [x for x in builds[build_type] if x == gsi]: - builds[build_type].append(gsi) - - build_type = "test" - test = BuildInfo(build_type) - for attr in attrs[build_type]: - if hasattr(test_schedule, attr): - setattr(test, attr, - getattr(test_schedule, attr, None)) - if not [x for x in builds[build_type] if x == test]: - builds[build_type].append(test) - - # groups by artifact, branch, and account id, and builds a command. - for artifact in attrs: - load_attrs = attrs[artifact] - if artifact == "device": - storage_type_text = "build_storage_type" - else: - storage_type_text = "" + artifact + "_storage_type" - pab_builds = [ - x for x in builds[artifact] - if getattr(x, storage_type_text) == - SchedCfgMsg.BUILD_STORAGE_TYPE_PAB - ] - pab_builds.sort(key=lambda x: tuple([getattr(x, attribute) - for attribute in load_attrs])) - groups = [list(g) for k, g in itertools.groupby( - pab_builds, lambda x: tuple([getattr(x, attribute) - for attribute - in load_attrs[1:-1]]))] - for group in groups: - command = ("build --artifact-type={} --method=GET " - "--noauth_local_webserver=True --update=single". - format(artifact)) - if artifact == "device": - if group[0].manifest_branch: - command += " --branch={}".format( - group[0].manifest_branch) - else: - logging.debug( - "Device manifest branch is a mandatory field.") - continue - if group[0].pab_account_id: - command += " --account_id={}".format( - group[0].pab_account_id) - if group[0].require_signed_device_build: - command += " --verify-signed-build=True" - targets = ",".join([x.name for x in group if x.name]) - if targets: - command += " --target={}".format(targets) - build_commands.append(command) - else: - if getattr(group[0], "" + artifact + "_branch"): - command += " --branch={}".format( - getattr(group[0], "" + artifact + "_branch")) - else: - logging.debug( - "{} branch is a mandatory field.".format(artifact)) - continue - if getattr(group[0], "" + artifact + "_pab_account_id"): - command += " --account_id={}".format( - getattr(group[0], - "" + artifact + "_pab_account_id")) - targets = ",".join([ - getattr(x, "" + artifact + "_build_target") - for x in group - if getattr(x, "" + artifact + "_build_target") - ]) - if targets: - command += " --target={}".format(targets) - build_commands.append(command) - - return build_commands - - # @Override - def SetUp(self): - """Initializes the parser for config command.""" - self.schedule_thread = {} - self.arg_parser.add_argument( - "--update", - choices=("single", "start", "stop", "list"), - default="start", - help="Update build info") - self.arg_parser.add_argument( - "--id", - default=None, - help="session ID only required for 'stop' update command") - self.arg_parser.add_argument( - "--interval", - type=int, - default=60, - help="Interval (seconds) to repeat build update.") - self.arg_parser.add_argument( - "--config-type", - choices=("prod", "test"), - default="prod", - help="Whether it's for prod") - self.arg_parser.add_argument( - "--branch", - required=True, - help="Branch to grab the artifact from.") - self.arg_parser.add_argument( - "--target", - required=True, - help="a comma-separate list of build target product(s).") - self.arg_parser.add_argument( - "--account_id", - default=common._DEFAULT_ACCOUNT_ID, - help="Partner Android Build account_id to use.") - self.arg_parser.add_argument( - '--method', - default='GET', - choices=('GET', 'POST'), - help='Method for fetching') - self.arg_parser.add_argument( - '--update_build', - dest='update_build', - action='store_true', - help='A boolean value indicating whether to upload build info.') - self.arg_parser.add_argument( - "--clear_schedule", - default=False, - help="True to clear all schedule data on the scheduler cloud") - self.arg_parser.add_argument( - "--clear_labinfo", - default=False, - help="True to clear all lab info data on the scheduler cloud") - - # @Override - def Run(self, arg_line): - """Updates global config.""" - args = self.arg_parser.ParseLine(arg_line) - if args.update == "single": - self.UpdateConfig(args.account_id, args.branch, args.target, - args.config_type, args.method, args.update_build, - args.clear_schedule, args.clear_labinfo) - elif args.update == "list": - logging.info("Running config update sessions:") - for id in self.schedule_thread: - logging.info(" ID %d", id) - elif args.update == "start": - if args.interval <= 0: - raise ConsoleArgumentError("update interval must be positive") - # do not allow user to create new - # thread if one is currently running - if args.id is None: - if not self.schedule_thread: - args.id = 1 - else: - args.id = max(self.schedule_thread) + 1 - else: - args.id = int(args.id) - if args.id in self.schedule_thread and not hasattr( - self.schedule_thread[args.id], 'keep_running'): - logging.warning('config update already running. ' - 'run config --update=stop --id=%s first.', - args.id) - return - self.schedule_thread[args.id] = threading.Thread( - target=self.UpdateConfigLoop, - args=( - args.account_id, - args.branch, - args.target, - args.config_type, - args.method, - args.update_build, - args.interval, - args.clear_schedule, - args.clear_labinfo, - )) - self.schedule_thread[args.id].daemon = True - self.schedule_thread[args.id].start() - elif args.update == "stop": - if args.id is None: - logging.error("--id must be set for stop") - else: - self.schedule_thread[int(args.id)].keep_running = False - - def Help(self): - base_command_processor.BaseCommandProcessor.Help(self) - logging.info("Sample: config --branch=<branch name> " - "--target=<build target> " - "--account_id=<account id> --config-type=[prod|test] " - "--update=single") |