diff options
Diffstat (limited to 'harnesses/host_controller/command_processor/command_upload.py')
| -rw-r--r-- | harnesses/host_controller/command_processor/command_upload.py | 346 |
1 files changed, 0 insertions, 346 deletions
diff --git a/harnesses/host_controller/command_processor/command_upload.py b/harnesses/host_controller/command_processor/command_upload.py deleted file mode 100644 index db57089..0000000 --- a/harnesses/host_controller/command_processor/command_upload.py +++ /dev/null @@ -1,346 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os -import shutil -import socket -import time - -from host_controller import common -from host_controller.command_processor import base_command_processor -from host_controller.utils.gcp import gcs_utils -from host_controller.utils.parser import xml_utils - -from vts.utils.python.common import cmd_utils - -from vti.dashboard.proto import TestSuiteResultMessage_pb2 as SuiteResMsg -from vti.test_serving.proto import TestScheduleConfigMessage_pb2 as SchedCfgMsg - - -class CommandUpload(base_command_processor.BaseCommandProcessor): - """Command processor for upload command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "upload" - command_detail = "Upload <src> file to <dest> Google Cloud Storage. In <src> and <dest>, variables enclosed in {} are replaced with the values stored in the console." - - # @Override - def SetUp(self): - """Initializes the parser for upload command.""" - self.arg_parser.add_argument( - "--src", - required=True, - default="latest-system.img", - help="Path to a source file to upload. Only single file can be " - "uploaded per once. Use 'latest- prefix to upload the latest " - "fetch images. e.g. --src=latest-system.img If argument " - "value is not given, the recently fetched system.img will be " - "uploaded.") - self.arg_parser.add_argument( - "--dest", - required=True, - help="Google Cloud Storage URL to which the file is uploaded.") - self.arg_parser.add_argument( - "--report_path", - help="Google Cloud Storage URL, the dest path of a report file") - self.arg_parser.add_argument( - "--clear_dest", - action="store_true", - help="Delete dest recursively before the upload.") - self.arg_parser.add_argument( - "--clear_results", - default=False, - help="True to clear all the results after the upload.") - self.arg_parser.add_argument( - "--result_from_suite", - default="", - choices=("", "vts", "cts", "gts", "sts"), - help="To specify the type of a test suite report, since there can " - "be multiple numbers of result sets from different test " - "suites. If not specified, the HC will upload the report " - "from last run suite and plan.") - self.arg_parser.add_argument( - "--result_from_plan", - default="", - help="To specify the type of the plan name from which " - "the report is generated.") - - # @Override - def Run(self, arg_line): - """Upload args.src file to args.dest Google Cloud Storage.""" - args = self.arg_parser.ParseLine(arg_line) - - gsutil_path = gcs_utils.GetGsutilPath() - if not gsutil_path: - logging.error("Please check gsutil is installed and on your PATH") - return False - - if args.src.startswith("latest-"): - src_name = args.src[7:] - if src_name in self.console.device_image_info: - src_paths = self.console.device_image_info[src_name] - else: - logging.error( - "Unable to find {} in device_image_info".format(src_name)) - return False - else: - try: - src_paths = self.console.FormatString(args.src) - except KeyError as e: - logging.error("Unknown or uninitialized variable in src: %s", - e) - return False - - src_path_list_tmp = src_paths.split(" ") - src_path_list = [] - if src_path_list_tmp: - for src_path in src_path_list_tmp: - file_path = src_path.strip() - if os.path.isfile(file_path): - src_path_list.append(file_path) - else: - logging.error("Cannot find a file: {}".format(file_path)) - src_paths = " ".join(src_path_list) - - try: - dest_path = self.console.FormatString(args.dest) - except KeyError as e: - logging.error("Unknown or uninitialized variable in dest: %s", e) - return False - - if not dest_path.startswith("gs://"): - logging.error("{} is not correct GCS url.".format(dest_path)) - return False - """ TODO(jongmok) : Before upload, login status, authorization, - and dest check are required. """ - if args.clear_dest: - if not gcs_utils.Remove(gsutil_path, dest_path, recursive=True): - logging.error("Fail to remove %s", dest_path) - - if not gcs_utils.Copy(gsutil_path, src_paths, dest_path): - logging.error("Fail to copy %s to %s", src_paths, dest_path) - - if args.report_path or args.clear_results: - tools_path = "" - if args.result_from_suite: - tools_path = os.path.dirname( - self.console.test_suite_info[args.result_from_suite]) - else: - try: - tools_path = os.path.dirname(self.console.test_suite_info[ - self.console.FormatString("{suite_name}")]) - except KeyError: - if self.console.vti_endpoint_client.CheckBootUpStatus(): - logging.error( - "No test results found from any fetched test suite." - " Please fetch a test suite and run 'test' command," - " then try running 'upload' command again.") - return False - results_base_path = os.path.join(tools_path, - common._RESULTS_BASE_PATH) - - if args.report_path: - report_path = self.console.FormatString(args.report_path) - if not report_path.startswith("gs://"): - logging.error( - "{} is not correct GCS url.".format(report_path)) - else: - self.UploadReport( - gsutil_path, report_path, dest_path, results_base_path, - args.result_from_suite, args.result_from_plan) - - if args.clear_results: - shutil.rmtree(results_base_path, ignore_errors=True) - - def UploadReport(self, gsutil_path, report_path, log_path, results_path, - suite_name, plan_name): - """Uploads report summary file to the given path. - - Args: - gsutil_path: string, the path of a gsutil binary. - report_path: string, the dest GCS URL to which the summarized report - file will be uploaded. - log_path: string, GCS URL where the log files from the test run - have been uploaded. - results_path: string, the base path for the results. - """ - suite_res_msg = SuiteResMsg.TestSuiteResultMessage() - suite_res_msg.result_path = log_path - suite_res_msg.branch = self.console.FormatString("{branch}") - suite_res_msg.target = self.console.FormatString("{target}") - vti = self.console.vti_endpoint_client - suite_res_msg.boot_success = vti.CheckBootUpStatus() - suite_res_msg.test_type = vti.GetJobTestType() - - device_fetch_info = self.console.detailed_fetch_info[ - common._ARTIFACT_TYPE_DEVICE] - gsi_fetch_info = None - if common._ARTIFACT_TYPE_GSI in self.console.detailed_fetch_info: - gsi_fetch_info = self.console.detailed_fetch_info[ - common._ARTIFACT_TYPE_GSI] - - if vti.CheckBootUpStatus(): - former_results = [ - result for result in os.listdir(results_path) - if os.path.isdir(os.path.join(results_path, result)) - and not os.path.islink(os.path.join(results_path, result)) - ] - - if not former_results: - logging.error("No test result found.") - return False - - former_results.sort() - latest_result = former_results[-1] - latest_result_xml_path = os.path.join(results_path, latest_result, - common._TEST_RESULT_XML) - - result_attrs = xml_utils.GetAttributes( - latest_result_xml_path, common._RESULT_TAG, [ - common._SUITE_NAME_ATTR_KEY, common._SUITE_PLAN_ATTR_KEY, - common._SUITE_VERSION_ATTR_KEY, - common._SUITE_BUILD_NUM_ATTR_KEY, - common._START_TIME_ATTR_KEY, common._END_TIME_ATTR_KEY, - common._HOST_NAME_ATTR_KEY - ]) - build_attrs = xml_utils.GetAttributes( - latest_result_xml_path, common._BUILD_TAG, [ - common._FINGERPRINT_ATTR_KEY, - common._SYSTEM_FINGERPRINT_ATTR_KEY, - common._VENDOR_FINGERPRINT_ATTR_KEY - ]) - summary_attrs = xml_utils.GetAttributes( - latest_result_xml_path, common._SUMMARY_TAG, [ - common._PASSED_ATTR_KEY, common._FAILED_ATTR_KEY, - common._MODULES_TOTAL_ATTR_KEY, - common._MODULES_DONE_ATTR_KEY - ]) - - suite_res_msg.build_id = result_attrs[ - common._SUITE_BUILD_NUM_ATTR_KEY] - suite_res_msg.suite_name = result_attrs[ - common._SUITE_NAME_ATTR_KEY] - suite_res_msg.suite_plan = result_attrs[ - common._SUITE_PLAN_ATTR_KEY] - suite_res_msg.suite_version = result_attrs[ - common._SUITE_VERSION_ATTR_KEY] - suite_res_msg.suite_build_number = result_attrs[ - common._SUITE_BUILD_NUM_ATTR_KEY] - suite_res_msg.start_time = long( - result_attrs[common._START_TIME_ATTR_KEY]) - suite_res_msg.end_time = long( - result_attrs[common._END_TIME_ATTR_KEY]) - suite_res_msg.host_name = result_attrs[common._HOST_NAME_ATTR_KEY] - if common._SYSTEM_FINGERPRINT_ATTR_KEY in build_attrs: - suite_res_msg.build_system_fingerprint = build_attrs[ - common._SYSTEM_FINGERPRINT_ATTR_KEY] - else: - suite_res_msg.build_system_fingerprint = build_attrs[ - common._FINGERPRINT_ATTR_KEY] - if common._VENDOR_FINGERPRINT_ATTR_KEY in build_attrs: - suite_res_msg.build_vendor_fingerprint = build_attrs[ - common._VENDOR_FINGERPRINT_ATTR_KEY] - else: - suite_res_msg.build_vendor_fingerprint = build_attrs[ - common._FINGERPRINT_ATTR_KEY] - suite_res_msg.passed_test_case_count = int( - summary_attrs[common._PASSED_ATTR_KEY]) - suite_res_msg.failed_test_case_count = int( - summary_attrs[common._FAILED_ATTR_KEY]) - suite_res_msg.modules_done = int( - summary_attrs[common._MODULES_DONE_ATTR_KEY]) - suite_res_msg.modules_total = int( - summary_attrs[common._MODULES_TOTAL_ATTR_KEY]) - else: - suite_res_msg.build_id = self.console.fetch_info["build_id"] - suite_res_msg.suite_name = suite_name - suite_res_msg.suite_plan = plan_name - suite_res_msg.suite_version = "" - suite_res_msg.suite_build_number = suite_res_msg.build_id - suite_res_msg.start_time = long(time.time() * 1000) - suite_res_msg.end_time = suite_res_msg.start_time - suite_res_msg.host_name = socket.gethostname() - suite_res_msg.build_vendor_fingerprint = "%s/%s/%s" % ( - device_fetch_info["branch"], device_fetch_info["target"], - device_fetch_info["build_id"]) - if gsi_fetch_info: - suite_res_msg.build_system_fingerprint = "%s/%s/%s" % ( - gsi_fetch_info["branch"], gsi_fetch_info["target"], - gsi_fetch_info["build_id"]) - else: - suite_res_msg.build_system_fingerprint = suite_res_msg.build_vendor_fingerprint - suite_res_msg.passed_test_case_count = 0 - suite_res_msg.failed_test_case_count = 0 - suite_res_msg.modules_done = 0 - suite_res_msg.modules_total = 0 - - suite_res_msg.infra_log_path = self.console.FormatString( - "{hc_log_upload_path}") - repack_path_list = [] - repack_path_list.append(self.console.FormatString("{repack_path}")) - suite_res_msg.repacked_image_path.extend(repack_path_list) - - suite_res_msg.schedule_config.build_target.extend( - [SchedCfgMsg.BuildScheduleConfigMessage()]) - build_target_msg = suite_res_msg.schedule_config.build_target[0] - build_target_msg.test_schedule.extend( - [SchedCfgMsg.TestScheduleConfigMessage()]) - test_schedule_msg = build_target_msg.test_schedule[0] - - suite_res_msg.vendor_build_id = device_fetch_info["build_id"] - suite_res_msg.schedule_config.manifest_branch = str( - device_fetch_info["branch"]) - build_target_msg.name = str(device_fetch_info["target"]) - if device_fetch_info["account_id"]: - suite_res_msg.schedule_config.pab_account_id = str( - device_fetch_info["account_id"]) - if device_fetch_info["fetch_signed_build"]: - build_target_msg.require_signed_device_build = device_fetch_info[ - "fetch_signed_build"] - if gsi_fetch_info: - test_schedule_msg.gsi_branch = str(gsi_fetch_info["branch"]) - test_schedule_msg.gsi_build_target = str(gsi_fetch_info["target"]) - suite_res_msg.gsi_build_id = str(gsi_fetch_info["build_id"]) - if gsi_fetch_info["account_id"]: - test_schedule_msg.gsi_pab_account_id = str( - gsi_fetch_info["account_id"]) - test_schedule_msg.gsi_vendor_version = str( - self.console.FormatString("{gsispl.vendor_version}")) - test_schedule_msg.test_pab_account_id = str( - self.console.FormatString("{account_id}")) - build_target_msg.has_bootloader_img = "bootloader.img" in self.console.device_image_info - build_target_msg.has_radio_img = "radio.img" in self.console.device_image_info - - report_file_path = os.path.join( - self.console.tmp_logdir, - self.console.FormatString("{timestamp_time}.bin")) - with open(report_file_path, "w") as fd: - fd.write(suite_res_msg.SerializeToString()) - fd.close() - - copy_command = "{} cp {} {}".format( - gsutil_path, report_file_path, - os.path.join(report_path, os.path.basename(report_file_path))) - _, stderr, err_code = cmd_utils.ExecuteOneShellCommand(copy_command) - if err_code: - logging.error(stderr) |