diff options
| author | Dan Shi <dshi@google.com> | 2020-10-06 22:18:35 +0000 |
|---|---|---|
| committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2020-10-06 22:18:35 +0000 |
| commit | 2b68c2c808eb5561738114f1e7a4dcfba11f0dff (patch) | |
| tree | 44844efda6427314e40c57463b95f55c6c7f22c2 | |
| parent | 94c0c115c2049a6cebb79318d3a685312d8e82d0 (diff) | |
| parent | 25d86869b0ecbb4d2f042e12705ab3622c999cd8 (diff) | |
| download | framework-2b68c2c808eb5561738114f1e7a4dcfba11f0dff.tar.gz | |
Remove vtf related configs am: 376757b0ba am: 647713d162 am: 43e5702f3d am: 25d86869b0
Original change: https://android-review.googlesource.com/c/platform/test/framework/+/1449375
Change-Id: Ia1020dfb78438091861e16d87a551f4262c769dd
120 files changed, 0 insertions, 16195 deletions
diff --git a/PREUPLOAD.cfg b/PREUPLOAD.cfg deleted file mode 100644 index 1204431..0000000 --- a/PREUPLOAD.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[Builtin Hooks] -clang_format = true - -[Builtin Hooks Options] -clang_format = --commit ${PREUPLOAD_COMMIT} --style file --extensions c,h,cc,cpp,java - -[Hook Scripts] -test_framework_unittests = ${REPO_ROOT}/test/framework/script/run-unittest.sh diff --git a/README.md b/README.md deleted file mode 100644 index 60cfc7e..0000000 --- a/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# Android Vendor Test Suite (VTS) Lab - -VTS Lab is an open source test serving infrastructure that can be used -to streamline VTS and CTS-on-GSI (General System Image) tests. diff --git a/build/Android.mk b/build/Android.mk deleted file mode 100644 index 82b1883..0000000 --- a/build/Android.mk +++ /dev/null @@ -1,152 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -LOCAL_PATH := $(call my-dir) -VTF_MK := $(LOCAL_PATH)/vtf.mk -VTF_PACKAGE_MK := test/vts/tools/build/tasks/framework/vtf_package.mk - -include $(CLEAR_VARS) -include $(LOCAL_PATH)/tasks/list/vtslab_apk_package_list.mk -include $(LOCAL_PATH)/tasks/list/vtslab_bin_package_list.mk -include $(LOCAL_PATH)/tasks/list/vtslab_lib_package_list.mk - -VTSLAB_OUT_ROOT := $(HOST_OUT)/vtslab -VTSLAB_TESTCASES_OUT := $(VTSLAB_OUT_ROOT)/android-vtslab/testcases -VTSLAB_TOOLS_OUT := $(VTSLAB_OUT_ROOT)/android-vtslab/tools -VTSLAB_BIN_LIB_OUT := $(VTSLAB_OUT_ROOT)/android-vtslab/ -VTSLAB_TIMESTAMP := $(shell git -C $(LOCAL_PATH) log -s -n 1 --format="%cd" \ - --date=format:"%Y%m%d_%H%M%S" 2>/dev/null) -VTSLAB_SHORTHASH := $(shell git -C $(LOCAL_PATH) rev-parse --short HEAD 2>/dev/null) -VTSLAB_VERSION := $(VTSLAB_TIMESTAMP):$(VTSLAB_SHORTHASH) - -# Packaging rule for android-vtslab.zip -test_suite_name := vtslab -test_suite_readme := test/framework/README.md - -include $(LOCAL_PATH)/package.mk -include $(LOCAL_PATH)/utils/vtslab_package_utils.mk - -# TODO: instead of an alias, deprecate vtslab build target -.PHONY: lab -lab: vtslab -.PHONY: vtslab -vtslab: $(compatibility_zip) -$(call dist-for-goals, vtslab, $(compatibility_zip)) - -# Packaging rule for android-vtslab.zip's testcases dir (DATA subdir). -vtslab_apk_modules := \ - $(vtslab_apk_packages) \ - -vtslab_apk_modules_copy_pairs := \ - $(call target-native-copy-pairs,$(vtslab_apk_modules),$(VTSLAB_TESTCASES_OUT)) - -# host controller files. -host_hc_files := \ - $(call find-files-in-subdirs,test/framework/harnesses/host_controller,"*.py" -and -type f,.) \ - -host_hc_copy_pairs := \ - $(foreach f,$(host_hc_files),\ - test/framework/harnesses/host_controller/$(f):$(VTSLAB_TESTCASES_OUT)/host_controller/$(f)) - -# gsi security patch scripts. -host_hc_gsispl_files := \ - $(call find-files-in-subdirs,test/framework/harnesses/host_controller/gsi,"*.sh" -and -type f,.) \ - -host_hc_gsispl_copy_pairs := \ - $(foreach f,$(host_hc_gsispl_files),\ - test/framework/harnesses/host_controller/gsi/$(f):$(VTSLAB_BIN_LIB_OUT)/bin/$(f)) - -# host controller scripts. -host_hc_extra_copy_pairs := \ - test/framework/tools/host_controller/run:$(VTSLAB_TOOLS_OUT)/run \ - test/framework/tools/host_controller/make_screen:$(VTSLAB_TOOLS_OUT)/make_screen \ - test/vts/script/diagnose.py:$(VTSLAB_BIN_LIB_OUT)/bin/diagnose.py \ - test/vts/script/pip_requirements.txt:$(VTSLAB_BIN_LIB_OUT)/bin/pip_requirements.txt \ - test/vts/script/setup.sh:$(VTSLAB_BIN_LIB_OUT)/bin/setup.sh \ - -host_acloud_files := \ - $(call find-files-in-subdirs,tools/acloud,"*.py" -and -type f,.) \ - $(call find-files-in-subdirs,tools/acloud,"*.config" -and -type f,.) - -host_acloud_copy_pairs := \ - $(foreach f,$(host_acloud_files),\ - tools/acloud/$(f):$(VTSLAB_TESTCASES_OUT)/acloud/$(f)) - -host_vti_dashboard_proto_files := \ - $(call find-files-in-subdirs,test/vti/dashboard/proto,"*.py" -and -type f,.) - -host_vti_test_serving_proto_files := \ - $(call find-files-in-subdirs,test/vti/test_serving/proto,"*.py" -and -type f,.) - -host_vti_copy_pairs := \ - $(foreach f,$(host_vti_test_serving_proto_files),\ - test/vti/test_serving/proto/$(f):$(VTSLAB_TESTCASES_OUT)/vti/test_serving/proto/$(f)) \ - $(foreach f,$(host_vti_dashboard_proto_files),\ - test/vti/dashboard/proto/$(f):$(VTSLAB_TESTCASES_OUT)/vti/dashboard/proto/$(f)) \ - test/vti/dashboard/__init__.py:$(VTSLAB_TESTCASES_OUT)/vti/dashboard/__init__.py \ - test/vti/test_serving/__init__.py:$(VTSLAB_TESTCASES_OUT)/vti/test_serving/__init__.py \ - -$(VTSLAB_TESTCASES_OUT)/vti/__init__.py: - @mkdir -p $(VTSLAB_TESTCASES_OUT)/vti - @touch $(VTSLAB_TESTCASES_OUT)/vti/__init__.py - -host_vti_extra_copy_pairs := \ - $(VTSLAB_TESTCASES_OUT)/vti/__init__.py \ - -vts_host_python_files := \ - $(call find-files-in-subdirs,test/vts,"*.py" -and -type f,.) - -vts_host_python_copy_pairs := \ - $(foreach f,$(vts_host_python_files),\ - test/vts/$(f):$(VTSLAB_TESTCASES_OUT)/vts/$(f)) - -# Packaging rule for host-controller's dependencies -host_hc_bin_lib := \ - $(vtslab_bin_packages) \ - $(vtslab_lib_packages) \ - -host_hc_bin_lib_copy_pairs := \ - $(call host-native-copy-pairs,$(host_hc_bin_lib),$(VTSLAB_BIN_LIB_OUT)) - -vtslab_copy_pairs := \ - $(call copy-many-files,$(vtslab_apk_modules_copy_pairs)) \ - $(call copy-many-files,$(host_hc_copy_pairs)) \ - $(call copy-many-files,$(host_hc_gsispl_copy_pairs)) \ - $(call copy-many-files,$(host_hc_extra_copy_pairs)) \ - $(call copy-many-files,$(host_acloud_copy_pairs)) \ - $(call copy-many-files,$(host_vti_copy_pairs)) \ - $(call copy-many-files,$(vts_host_python_copy_pairs)) \ - $(call copy-many-files,$(host_hc_bin_lib_copy_pairs)) \ - $(host_vti_extra_copy_pairs) \ - -$(compatibility_zip): $(vtslab_copy_pairs) $(VTSLAB_TESTCASES_OUT)/version.txt - -$(VTSLAB_TESTCASES_OUT)/version.txt: - @rm -f $@ - @echo $(VTSLAB_VERSION) > $@ - -# for VTF (Vendor Test Framework) -VTF_OUT_ROOT := $(HOST_OUT)/vtf -VTF_TESTCASES_OUT := $(VTF_OUT_ROOT)/android-vtf/testcases -VTF_TOOLS_OUT := $(VTF_OUT_ROOT)/android-vtf/tools -VTF_EXTRA_SCRIPTS := vtf - -include $(VTF_PACKAGE_MK) -include $(VTF_MK) - -# clears local vars -VTF_MK := -VTF_PACKAGE_MK := diff --git a/build/package.mk b/build/package.mk deleted file mode 100644 index d04b159..0000000 --- a/build/package.mk +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Package up a compatibility test suite in a zip file. -# -# Input variables: -# test_suite_name: the name of this test suite eg. cts -# test_suite_readme: the path to a README file for this test suite -# test_suite_prebuilt_tools: the set of prebuilt tools to be included directly -# in the 'tools' subdirectory of the test suite. -# test_suite_tools: the set of tools for this test suite -# -# Output variables: -# compatibility_zip: the path to the output zip file. - -out_dir := $(HOST_OUT)/$(test_suite_name)/android-$(test_suite_name) -test_artifacts := $(COMPATIBILITY.$(test_suite_name).FILES) -test_tools := $(test_suite_readme) -test_tools += $(test_suite_tools) - -compatibility_zip := $(out_dir).zip -$(compatibility_zip): PRIVATE_NAME := android-$(test_suite_name) -$(compatibility_zip): PRIVATE_OUT_DIR := $(out_dir) -$(compatibility_zip): PRIVATE_TOOLS := $(test_tools) $(test_suite_prebuilt_tools) -$(compatibility_zip): PRIVATE_SUITE_NAME := $(test_suite_name) -$(compatibility_zip): $(test_artifacts) $(test_tools) $(test_suite_prebuilt_tools) $(SOONG_ZIP) | $(ADB) $(ACP) -# Make dir structure - $(hide) mkdir -p $(PRIVATE_OUT_DIR)/tools $(PRIVATE_OUT_DIR)/testcases -# Copy tools - $(hide) $(ACP) -fp $(PRIVATE_TOOLS) $(PRIVATE_OUT_DIR)/tools - $(hide) find $(dir $@)/$(PRIVATE_NAME) | sort >$@.list - $(hide) $(SOONG_ZIP) -d -o $@ -C $(dir $@) -l $@.list - -# Reset all input variables -test_suite_name := -test_suite_dynamic_config := -test_suite_readme := -test_suite_prebuilt_tools := -test_suite_tools := diff --git a/build/tasks/list/vtslab_apk_package_list.mk b/build/tasks/list/vtslab_apk_package_list.mk deleted file mode 100644 index fc842aa..0000000 --- a/build/tasks/list/vtslab_apk_package_list.mk +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -vtslab_apk_packages := \ - WifiUtil \ diff --git a/build/tasks/list/vtslab_bin_package_list.mk b/build/tasks/list/vtslab_bin_package_list.mk deleted file mode 100644 index 664c7fe..0000000 --- a/build/tasks/list/vtslab_bin_package_list.mk +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -vtslab_bin_packages := \ - aapt \ - adb \ - fastboot \ - img2simg \ - mke2fs \ - mkuserimg_mke2fs \ - simg2img \ diff --git a/build/tasks/list/vtslab_lib_package_list.mk b/build/tasks/list/vtslab_lib_package_list.mk deleted file mode 100644 index 5f4740c..0000000 --- a/build/tasks/list/vtslab_lib_package_list.mk +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -vtslab_lib_packages := \ - libc++ \ diff --git a/build/utils/vtslab_package_utils.mk b/build/utils/vtslab_package_utils.mk deleted file mode 100644 index e4ea608..0000000 --- a/build/utils/vtslab_package_utils.mk +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# $(1): List of target native files to copy. -# $(2): Copy destination directory. -# Evaluates to a list of ":"-separated pairs src:dst. -define target-native-copy-pairs -$(foreach m,$(1),\ - $(eval _built_files := $(strip $(ALL_MODULES.$(m).BUILT_INSTALLED)\ - $(ALL_MODULES.$(m)$(TARGET_2ND_ARCH_MODULE_SUFFIX).BUILT_INSTALLED)))\ - $(foreach i, $(_built_files),\ - $(eval bui_ins := $(subst :,$(space),$(i)))\ - $(eval ins := $(word 2,$(bui_ins)))\ - $(if $(filter $(TARGET_OUT_ROOT)/%,$(ins)),\ - $(eval bui := $(word 1,$(bui_ins)))\ - $(eval my_copy_dest := $(patsubst data/%,DATA/%,\ - $(patsubst system/%,DATA/%,\ - $(patsubst $(PRODUCT_OUT)/%,%,$(ins)))))\ - $(bui):$(2)/$(my_copy_dest)))) -endef - -# $(1): List of host native files to copy. -# $(2): Copy destination directory. -# Evaluates to a list of ":"-separated pairs src:dst. -define host-native-copy-pairs -$(foreach m,$(1),\ - $(eval _built_files := $(strip $(ALL_MODULES.$(m).BUILT_INSTALLED)\ - $(ALL_MODULES.$(m)$(HOST_2ND_ARCH_MODULE_SUFFIX).BUILT_INSTALLED)))\ - $(foreach i, $(_built_files),\ - $(eval bui_ins := $(subst :,$(space),$(i)))\ - $(eval ins := $(word 2,$(bui_ins)))\ - $(if $(filter $(HOST_OUT)/% $(HOST_CROSS_OUT)/%,$(ins)),\ - $(eval bui := $(word 1,$(bui_ins)))\ - $(eval my_copy_dest := $(patsubst $(HOST_OUT)/%,%,\ - $(patsubst $(HOST_CROSS_OUT)/%,%,$(ins))))\ - $(bui):$(2)/$(my_copy_dest)))) -endef diff --git a/build/vtf.mk b/build/vtf.mk deleted file mode 100644 index 069b6b2..0000000 --- a/build/vtf.mk +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Build rules for VTF (Vendor Test Framework). -LOCAL_PATH := $(call my-dir) - -vtf_tradefed_modules := \ - compatibility-common-util-tests \ - compatibility-host-util \ - compatibility-host-util-tests \ - compatibility-tradefed-tests \ - host-libprotobuf-java-full \ - loganalysis \ - tradefed \ - vts-tradefed \ - vts10-tradefed \ - vts10-tradefed-tests \ - -vtf_tradefed_copy_pairs := \ - $(foreach f,$(vtf_tradefed_modules),\ - $(HOST_OUT)/framework/$(f).jar:$(VTF_TOOLS_OUT)/$(f).jar) - -vtf_tradefed_additional_deps_copy_pairs := \ - test/vts/tools/vts-tradefed/etc/vts10-tradefed:$(VTF_TOOLS_OUT)/vts10-tradefed - -vtf_tradefed_additional_deps_copy_pairs += \ - $(foreach f,$(VTF_COPY_VTF_BINARY),\ - test/vts/tools/vts-tradefed/etc/$(f):$(VTF_TESTCASES_OUT)/$(f)) - -vtf_package_copy_pairs := \ - $(call copy-many-files,$(vtf_tradefed_copy_pairs)) \ - $(call copy-many-files,$(vtf_tradefed_additional_deps_copy_pairs)) \ - -.PHONY: vtf -vtf: $(vtf_copy_pairs) $(vtf_package_copy_pairs) diff --git a/harnesses/Android.bp b/harnesses/Android.bp deleted file mode 100644 index 93e059b..0000000 --- a/harnesses/Android.bp +++ /dev/null @@ -1,20 +0,0 @@ -// -// Copyright (C) 2017 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -java_import_host { - name: "tradefed-cts-prebuilt", - jars: ["cts-tradefed/tradefed-cts-prebuilt.jar"], -} diff --git a/harnesses/README.md b/harnesses/README.md deleted file mode 100644 index a8c5fde..0000000 --- a/harnesses/README.md +++ /dev/null @@ -1,15 +0,0 @@ -To build a new CTS TF prebuilt binary - -`$ lunch aosp_arm64` -`$ make cts-tradefed -j - -To release to TEST -`$ cp out/host/linux-x86/testcases/cts-tradefed/cts-tradefed.jar test/framework/harnesses/cts-tradefed/tradefed-cts-prebuilt-staging.jar` - -To release to PROD -`$ cp test/framework/harnesses/cts-tradefed/tradefed-cts-prebuilt-staging.jar test/framework/harnesses/cts-tradefed/tradefed-cts-prebuilt.jar` - -To test a test suite which uses that prebuilt binary - -`$ make vts -j32` - diff --git a/harnesses/cts-tradefed/tradefed-cts-prebuilt.jar b/harnesses/cts-tradefed/tradefed-cts-prebuilt.jar Binary files differdeleted file mode 100644 index 7e728bf..0000000 --- a/harnesses/cts-tradefed/tradefed-cts-prebuilt.jar +++ /dev/null diff --git a/harnesses/host_controller/__init__.py b/harnesses/host_controller/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/acloud/__init__.py b/harnesses/host_controller/acloud/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/acloud/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/acloud/acloud_client.py b/harnesses/host_controller/acloud/acloud_client.py deleted file mode 100644 index 5bcd85e..0000000 --- a/harnesses/host_controller/acloud/acloud_client.py +++ /dev/null @@ -1,203 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import getpass -import json -import logging -import os -from os.path import expanduser -import re -import shutil -import tempfile - -from acloud.public import acloud_main -from host_controller.acloud import acloud_config -from vts.utils.python.common import cmd_utils - -DEFAULT_BRANCH = 'git_master' -DEFAULT_BUILD_TARGET = 'gce_x86_phone-userdebug_fastbuild3c_linux' - -#TODO(yuexima): add full support to multiple instances per host - - -class ACloudClient(object): - '''Helper class to manage access to the acloud module.''' - - def __init__(self): - tmpdir_base = os.path.join(os.getcwd(), "tmp") - if not os.path.exists(tmpdir_base): - os.mkdir(tmpdir_base) - self._tmpdir = tempfile.mkdtemp(dir=tmpdir_base) - - def __del__(self): - """Deletes the temp dir if still set.""" - if self._tmpdir: - shutil.rmtree(self._tmp_dirpath) - self._tmpdir = None - - def GetCreateCmd(self, - build_id, - branch=None, - build_target=None, - num=1): - '''Get acould create command with given build id. - - Args: - build_id: string, build_id. - branch: string, build branch - build_target: string, build target - num: int, number of instances to build - - Returns: - string, acloud create command. - ''' - if not branch: - branch = DEFAULT_BRANCH - - if not build_target: - build_target = DEFAULT_BUILD_TARGET - - #TODO latest build id (in the caller class of this function). - #TODO use unique log and tmp file location - cmd = ('create ' - '--branch {branch} ' - '--build_target {build_target} ' - '--build_id {build_id} ' - '--config_file {config_file} ' - '--report_file {report_file} ' - '--log_file {log_file} ' - '--email {email} ' - '--num {num}').format( - branch = branch, - build_target = build_target, - build_id = build_id, - config_file = os.path.join(self._tmpdir, 'acloud.config'), - report_file = os.path.join(self._tmpdir, 'acloud_report.json'), - log_file = os.path.join(self._tmpdir, 'acloud.log'), - #TODO use host email address - email = getpass.getuser() + '@google.com', - num = num - ) - return cmd - - def GetDeleteCmd(self, instance_names): - '''Get Acould delete command with given instance names. - - Args: - instance_names: list of string, instance names. - - Returns: - string, acloud create command. - ''' - cmd = ('delete ' - '--instance_names {instance_names} ' - '--config_file {config_file} ' - '--report_file {report_file} ' - '--log_file {log_file} ' - '--email {email}').format( - instance_names = ' '.join(instance_names), - config_file = os.path.join(self._tmpdir, 'acloud.config'), - report_file = os.path.join(self._tmpdir, 'acloud_report.json'), - log_file = os.path.join(self._tmpdir, 'acloud.log'), - email = getpass.getuser() + '@google.com' - ) - return cmd - - def CreateInstance(self, build_id): - '''Create Acould instance with given build id. - - Args: - build_id: string, build_id. - ''' - cmd = self.GetCreateCmd(build_id) - acloud_main.main(cmd.split()) - - report_file = os.path.join(self._tmpdir, 'acloud_report.json') - with open(report_file, 'r') as f: - report = json.load(f) - - return report['status']=='SUCCESS' - - def PrepareConfig(self, file_path): - '''Prepare acloud Acloud config file. - - Args: - file_path: string, path to acloud config file. - ''' - config = acloud_config.ACloudConfig() - config.Load(file_path) - if not config.Validate(): - logging.error('Failed to prepare acloud config.') - return - - config.Save(os.path.join(self._tmpdir, 'acloud.config')) - - def GetInstanceIP(self): - '''Get an Acloud instance ip''' - #TODO support ip addresses when num > 1 (json parser) - report_file = os.path.join(self._tmpdir, 'acloud_report.json') - - with open(report_file, 'r') as f: - report = json.load(f) - - return report['data']['devices'][0]['ip'] - - def GetInstanceName(self): - '''Get an Acloud instance name''' - report_file = os.path.join(self._tmpdir, 'acloud_report.json') - - with open(report_file, 'r') as f: - report = json.load(f) - - return report['data']['devices'][0]['instance_name'] - - def ConnectInstanceToAdb(self, ip=None): - '''Connect an Acloud instance to adb - - Args: - ip: string, ip address - ''' - if not ip: - ip = self.GetInstanceIP() - - cmds = [('ssh -i ~/.ssh/acloud_rsa -o UserKnownHostsFile=/dev/null ' - '-o StrictHostKeyChecking=no -L 40000:127.0.0.1:6444 -L ' - '30000:127.0.0.1:5555 -N -f -l root %s') % ip, - 'adb connect 127.0.0.1:30000'] - - for cmd in cmds: - print cmd - results = cmd_utils.ExecuteShellCommand(cmd) - if any(results[cmd_utils.EXIT_CODE]): - logging.error("Fail to execute command: %s\nResult:%s" % (cmd, - results)) - return - print 'Acloud instance created and connected to local port 127.0.0.1:30000' - - def DeleteInstance(self, instance_names=None): - '''Delete an Acloud instance - - Args: - instance_names: string, instance name - ''' - cmd = self.GetDeleteCmd(instance_names) - acloud_main.main(cmd.split()) - - report_file = os.path.join(self._tmpdir, 'acloud_report.json') - with open(report_file, 'r') as f: - report = json.load(f) - - return report['status']=='SUCCESS' diff --git a/harnesses/host_controller/acloud/acloud_config.py b/harnesses/host_controller/acloud/acloud_config.py deleted file mode 100644 index 0ceab47..0000000 --- a/harnesses/host_controller/acloud/acloud_config.py +++ /dev/null @@ -1,103 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os - - -# In fact, all fields are required. The fields listed below are used -# to check whether the config class has been properly initialized before -# generating config file -REQUIRED_KEYS = [ - 'ssh_public_key_path', - 'ssh_private_key_path', - 'project', - 'client_id', - 'client_secret' - ] - - -class ACloudConfig(object): - '''For ACloud configuration file operations. - - Attributes: - configs: dict of string:string, configuration keys and values. - has_error: bool, whether error occurred. - ''' - configs = {} - has_error = False - - def Validate(self): - '''Validate config class. - - Check whether required fields has been set. - Check whether loading configuration file is success. - - Returns: - bool, True if validated. - ''' - for key in REQUIRED_KEYS: - if key not in self.configs: - logging.error('Required key %s is not ' - 'set for acloud config' % key) - return False - - return not self.has_error - - def Load(self, file_path): - '''Load configs from a file. - - Args: - file_path: string, path to config file. - ''' - if not os.path.isfile(file_path): - logging.error('Failed to read acloud config file %s' % file_path) - self.has_error = True - return - - separator = ': "' - - with open(file_path, 'r') as f: - for line in f: - line = line.strip() - # Skip empty line and comments - if not line or line.startswith('#'): - continue - - idx = line.find(separator) - - if idx < 1 or not line.endswith('"'): - logging.error('Error parsing line %s from ' - 'acloud config file %s' % (line, file_path)) - self.has_error = True - return - - key = line[:idx] - val = line[len(separator) + idx : -1] - - self.configs[key] = val - - def Save(self, file_path): - '''Save config to a file. - - Args: - file_path: string, path to config file. - ''' - separator = ':' - - with open(file_path, 'w') as f: - for key in self.configs: - f.write(key + separator + '"%s"' % self.configs[key])
\ No newline at end of file diff --git a/harnesses/host_controller/build/README.md b/harnesses/host_controller/build/README.md deleted file mode 100644 index f97f372..0000000 --- a/harnesses/host_controller/build/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# To set up client_secrets.json (once only) - -* Go to https://console.cloud.google.com/projectselector/apis/credentials/consent - and create a new project if needed. -* Once on the consent screen, set the product name to anything and save. -* Click "Create credentials" and select "OAuth client ID" -* Under "Application type", select "Other". Click "Create". -* Click the download button for the client you just created, - and save the resulting file at client_secrets.json - -# Running unit tests - python build_provider_pab_test.py
\ No newline at end of file diff --git a/harnesses/host_controller/build/__init__.py b/harnesses/host_controller/build/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/build/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/build/build_flasher.py b/harnesses/host_controller/build/build_flasher.py deleted file mode 100644 index a920d93..0000000 --- a/harnesses/host_controller/build/build_flasher.py +++ /dev/null @@ -1,364 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""Class to flash build artifacts onto devices""" - -import hashlib -import logging -import os -import resource -import sys -import tempfile -import time - -from host_controller import common -from vts.utils.python.common import cmd_utils -from vts.utils.python.controllers import android_device - - -class BuildFlasher(object): - """Client that manages build flashing. - - Attributes: - device: AndroidDevice, the device associated with the client. - """ - - def __init__(self, serial="", customflasher_path=""): - """Initialize the client. - - If serial not provided, find single device connected. Error if - zero or > 1 devices connected. - - Args: - serial: optional string, serial number for the device. - customflasher_path: optional string, set device to use specified - binary to flash a device - """ - if serial != "": - self.device = android_device.AndroidDevice( - serial, device_callback_port=-1) - else: - serials = android_device.list_adb_devices() - if len(serials) == 0: - serials = android_device.list_fastboot_devices() - if len(serials) == 0: - raise android_device.AndroidDeviceError( - "ADB and fastboot could not find any target devices.") - if len(serials) > 1: - logging.info("ADB or fastboot found more than one device: %s", - serials) - self.device = android_device.AndroidDevice( - serials[0], device_callback_port=-1) - if customflasher_path: - self.device.SetCustomFlasherPath(customflasher_path) - - def SetSerial(self, serial): - """Sets device serial. - - Args: - serial: string, a device serial. - - Returns: - True if successful; False otherwise. - """ - if not serial: - logging.error("no serial is given to BuildFlasher.SetSerial.") - return False - - self.device = android_device.AndroidDevice( - serial, device_callback_port=-1) - return True - - def FlashGSI(self, - system_img, - vbmeta_img=None, - skip_check=False, - skip_vbmeta=False): - """Flash the Generic System Image to the device. - - Args: - system_img: string, path to GSI - vbmeta_img: string, optional, path to vbmeta image for new devices - skip_check: boolean, set True to skip adb-based checks when - the DUT is already running its bootloader. - skip_vbmeta: bool, whether to skip flashing the vbmeta.img or not. - If the device has the vbmeta slot then flash vbmeta.img - even if the skip_vbmeta is set to True. - """ - if not os.path.exists(system_img): - raise ValueError("Couldn't find system image at %s" % system_img) - if not skip_check: - self.device.adb.wait_for_device() - if not self.device.isBootloaderMode: - self.device.log.info(self.device.adb.reboot_bootloader()) - if vbmeta_img is not None: - if skip_vbmeta == False or self.device.hasVbmetaSlot: - self.device.log.info( - self.device.fastboot.flash('vbmeta', vbmeta_img)) - self.device.log.info(self.device.fastboot.erase('system')) - self.device.log.info(self.device.fastboot.flash('system', system_img)) - self.device.log.info(self.device.fastboot.erase('metadata')) - self.device.log.info(self.device.fastboot._w()) - self.device.log.info(self.device.fastboot.reboot()) - - def Flashall(self, directory): - """Flash all images in a directory to the device using flashall. - - Generally the directory is the result of unzipping the .zip from AB. - Args: - directory: string, path to directory containing images - """ - # fastboot flashall looks for imgs in $ANDROID_PRODUCT_OUT - os.environ['ANDROID_PRODUCT_OUT'] = directory - self.device.adb.wait_for_device() - if not self.device.isBootloaderMode: - self.device.log.info(self.device.adb.reboot_bootloader()) - self.device.log.info(self.device.fastboot.flashall()) - - def Flash(self, device_images, skip_vbmeta=False): - """Flash the Generic System Image to the device. - - Args: - device_images: dict, where the key is partition name and value is - image file path. - skip_vbmeta: bool, whether to skip flashing the vbmeta.img or not. - - Returns: - True if succesful; False otherwise - """ - if not device_images: - logging.warn("Flash skipped because no device image is given.") - return False - - if not self.device.isBootloaderMode: - self.device.adb.wait_for_device() - logging.info("rebooting to bootloader") - self.device.log.info(self.device.adb.reboot_bootloader()) - - logging.info("checking to flash bootloader.img and radio.img") - for partition in ["bootloader", "radio"]: - if partition in device_images: - image_path = device_images[partition] - self.device.log.info("fastboot flash %s %s", partition, - image_path) - self.device.log.info( - self.device.fastboot.flash(partition, image_path)) - self.device.log.info("fastboot reboot_bootloader") - self.device.log.info(self.device.fastboot.reboot_bootloader()) - - logging.info("starting to flash vendor and other images...") - full_zipfile = False - if common.FULL_ZIPFILE in device_images: - logging.info("fastboot update %s --skip-reboot", - (device_images[common.FULL_ZIPFILE])) - self.device.log.info( - self.device.fastboot.update(device_images[common.FULL_ZIPFILE], - "--skip-reboot")) - full_zipfile = True - - for partition, image_path in device_images.iteritems(): - if partition in (common.FULL_ZIPFILE, common.FULL_ZIPFILE_DIR, - "system", "vbmeta", "bootloader", "radio", - "metadata", "userdata"): - continue - if full_zipfile and partition in ("vendor", "boot"): - logging.info("%s skipped because full zipfile was updated.", - partition) - continue - if not image_path: - self.device.log.warning("%s image is empty", partition) - continue - self.device.log.info("fastboot flash %s %s", partition, image_path) - self.device.log.info( - self.device.fastboot.flash(partition, image_path)) - - logging.info("starting to flash system and other images...") - if "system" in device_images and device_images["system"]: - system_img = device_images["system"] - vbmeta_img = device_images["vbmeta"] if ( - "vbmeta" in device_images - and device_images["vbmeta"]) else None - self.FlashGSI( - system_img, - vbmeta_img, - skip_check=True, - skip_vbmeta=skip_vbmeta) - else: - self.device.log.info(self.device.fastboot.reboot()) - return True - - def FlashImage(self, device_images, image_partition=None, reboot=False): - """Flash specified image(s) to the device. - - Args: - device_images: dict, where the key is partition name and value is - image file path. - image_partition: string, set to flash only an image in a specified - partition. - reboot: boolean, true to reboot the device. - - Returns: - True if successful, False otherwise - """ - if not device_images: - logging.warn("Flash skipped because no device image is given.") - return False - - if not self.device.isBootloaderMode: - self.device.adb.wait_for_device() - self.device.log.info(self.device.adb.reboot_bootloader()) - - for partition, image_path in device_images.iteritems(): - if image_partition and image_partition != partition: - continue - if partition.endswith(".img"): - partition = partition[:-4] - self.device.log.info( - self.device.fastboot.flash(partition, image_path)) - if reboot: - self.device.log.info(self.device.fastboot.reboot()) - return True - - def WaitForDevice(self, timeout_secs=600): - """Waits for the device to boot completely. - - Args: - timeout_secs: integer, the maximum timeout value for this - operation (unit: seconds). - - Returns: - True if device is booted successfully; False otherwise. - """ - return self.device.waitForBootCompletion(timeout=timeout_secs) - - def FlashUsingCustomBinary(self, - device_images, - reboot_mode, - flasher_args, - timeout_secs_for_reboot=900): - """Flash the customized image to the device. - - Args: - device_images: dict, where the key is partition name and value is - image file path. - reboot_mode: string, decides which mode device will reboot into. - ("bootloader"/"download"). - flasher_args: list of strings, arguments that will be passed to the - flash binary. - timeout_secs_for_reboot: integer, the maximum timeout value for - reboot to flash-able mode(unit: seconds). - - Returns: - True if successful; False otherwise. - """ - if not device_images: - logging.warn("Flash skipped because no device image is given.") - return False - - if not flasher_args: - logging.error("No arguments.") - return False - - if not self.device.isBootloaderMode: - self.device.adb.wait_for_device() - logging.info("rebooting to %s mode", reboot_mode) - self.device.log.info(self.device.adb.reboot(reboot_mode)) - - start = time.time() - while not self.device.customflasher._l(): - if time.time() - start >= timeout_secs_for_reboot: - logging.error( - "Timeout while waiting for %s mode boot completion.", - reboot_mode) - return False - time.sleep(1) - - flasher_output = self.device.customflasher.ExecCustomFlasherCmd( - flasher_args[0], - " ".join(flasher_args[1:] + [device_images["img"]])) - self.device.log.info(flasher_output) - - return True - - def RepackageArtifacts(self, device_images, repackage_form): - """Repackage artifacts into a given format. - - Once repackaged, device_images becomes - {"img": "path_to_repackaged_image"} - - Args: - device_images: dict, where the key is partition name and value is - image file path. - repackage_form: string, format to repackage. - - Returns: - True if succesful; False otherwise. - """ - if not device_images: - logging.warn("Repackage skipped because no device image is given.") - return False - - if repackage_form == "tar.md5": - tmp_file_name = next(tempfile._get_candidate_names()) + ".tar" - tmp_dir_path = os.path.dirname( - device_images[device_images.keys()[0]]) - for img in device_images: - if os.path.dirname(device_images[img]) != tmp_dir_path: - os.rename(device_images[img], - os.path.join(tmp_dir_path, img)) - device_images[img] = os.path.join(tmp_dir_path, img) - - current_dir = os.getcwd() - os.chdir(tmp_dir_path) - - if sys.platform == "linux2": - tar_cmd = "tar -cf %s %s" % (tmp_file_name, ' '.join( - (device_images.keys()))) - else: - logging.error("Unsupported OS for the given repackage form.") - return False - logging.info(tar_cmd) - std_out, std_err, err_code = cmd_utils.ExecuteOneShellCommand( - tar_cmd) - if err_code: - logging.error(std_err) - return False - - hash_md5 = hashlib.md5() - try: - with open(tmp_file_name, "rb") as file: - data_chunk = 0 - chunk_size = resource.getpagesize() - while data_chunk != b'': - data_chunk = file.read(chunk_size) - hash_md5.update(data_chunk) - hash_ret = hash_md5.hexdigest() - with open(tmp_file_name, "a") as file: - file.write("%s %s" % (hash_ret, tmp_file_name)) - except IOError as e: - logging.error(e.strerror) - return False - - device_images.clear() - device_images["img"] = os.path.join(tmp_dir_path, tmp_file_name) - - os.chdir(current_dir) - else: - logging.error( - "Please specify correct repackage form: --repackage=%s", - repackage_form) - return False - - return True diff --git a/harnesses/host_controller/build/build_flasher_test.py b/harnesses/host_controller/build/build_flasher_test.py deleted file mode 100644 index 21e81b3..0000000 --- a/harnesses/host_controller/build/build_flasher_test.py +++ /dev/null @@ -1,135 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import sys -import unittest - -try: - from unittest import mock -except ImportError: - import mock - -from host_controller.build import build_flasher - - -class BuildFlasherTest(unittest.TestCase): - """Tests for Build Flasher""" - - @mock.patch( - "host_controller.build.build_flasher.android_device") - @mock.patch("host_controller.build.build_flasher.os") - def testFlashGSIBadPath(self, mock_os, mock_class): - flasher = build_flasher.BuildFlasher("thisismyserial") - mock_os.path.exists.return_value = False - with self.assertRaises(ValueError) as cm: - flasher.FlashGSI("notexists.img") - self.assertEqual("Couldn't find system image at notexists.img", - str(cm.exception)) - - @mock.patch( - "host_controller.build.build_flasher.android_device") - @mock.patch("host_controller.build.build_flasher.os") - def testFlashGSISystemOnly(self, mock_os, mock_class): - mock_device = mock.Mock() - mock_class.AndroidDevice.return_value = mock_device - flasher = build_flasher.BuildFlasher("thisismyserial") - mock_os.path.exists.return_value = True - flasher.FlashGSI("exists.img") - mock_device.fastboot.erase.assert_any_call('system') - mock_device.fastboot.flash.assert_any_call('system', 'exists.img') - mock_device.fastboot.erase.assert_any_call('metadata') - - @mock.patch( - "host_controller.build.build_flasher.android_device") - def testFlashall(self, mock_class): - mock_device = mock.Mock() - mock_class.AndroidDevice.return_value = mock_device - flasher = build_flasher.BuildFlasher("thisismyserial") - flasher.Flashall("path/to/dir") - mock_device.fastboot.flashall.assert_called_with() - - @mock.patch( - "host_controller.build.build_flasher.android_device") - def testEmptySerial(self, mock_class): - mock_class.list_adb_devices.return_value = ['oneserial'] - flasher = build_flasher.BuildFlasher(serial="") - mock_class.AndroidDevice.assert_called_with( - "oneserial", device_callback_port=-1) - - @mock.patch( - "host_controller.build.build_flasher.android_device") - def testFlashUsingCustomBinary(self, mock_class): - """Test for FlashUsingCustomBinary(). - - Tests if the method passes right args to customflasher - and the execution path of the method. - """ - mock_device = mock.Mock() - mock_class.AndroidDevice.return_value = mock_device - flasher = build_flasher.BuildFlasher("mySerial", "myCustomBinary") - - mock_device.isBootloaderMode = False - device_images = {"img": "my/image/path"} - flasher.FlashUsingCustomBinary(device_images, "reboottowhatever", - ["myarg"]) - mock_device.adb.reboot.assert_called_with("reboottowhatever") - mock_device.customflasher.ExecCustomFlasherCmd.assert_called_with( - "myarg", "my/image/path") - - mock_device.reset_mock() - mock_device.isBootloaderMode = True - device_images["img"] = "my/other/image/path" - flasher.FlashUsingCustomBinary(device_images, "reboottowhatever", - ["myarg"]) - mock_device.adb.reboot.assert_not_called() - mock_device.customflasher.ExecCustomFlasherCmd.assert_called_with( - "myarg", "my/other/image/path") - - @mock.patch("host_controller.build.build_flasher.android_device") - @mock.patch("host_controller.build.build_flasher.logging") - @mock.patch("host_controller.build.build_flasher.cmd_utils") - @mock.patch("host_controller.build.build_flasher.os") - @mock.patch("host_controller.build.build_flasher.open", - new_callable=mock.mock_open) - def testRepackageArtifacts(self, mock_open, mock_os, mock_cmd_utils, - mock_logger, mock_class): - """Test for RepackageArtifacts(). - - Tests if the method executes in correct path regarding - given arguments. - """ - mock_device = mock.Mock() - mock_class.AndroidDevice.return_value = mock_device - flasher = build_flasher.BuildFlasher("serial") - device_images = { - "system.img": "/my/tmp/path/system.img", - "vendor.img": "/my/tmp/path/vendor.img" - } - mock_cmd_utils.ExecuteOneShellCommand.return_value = "", "", 0 - ret = flasher.RepackageArtifacts(device_images, "tar.md5") - self.assertEqual(ret, True) - self.assertIsNotNone(device_images["img"]) - - ret = flasher.RepackageArtifacts(device_images, "incorrect") - self.assertFalse(ret) - mock_logger.error.assert_called_with( - "Please specify correct repackage form: --repackage=%s" ,"incorrect") - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/build/build_info.py b/harnesses/host_controller/build/build_info.py deleted file mode 100644 index 6be9e95..0000000 --- a/harnesses/host_controller/build/build_info.py +++ /dev/null @@ -1,72 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import collections -import logging -import os -import shutil - - -class BuildInfo(dict): - """dict class for fetched device imgs, test suites, etc.""" - - def __init__(self): - super(BuildInfo, self).__init__() - - def __setitem__(self, key, value): - """__setitem__ for BuildInfo dict. - - Remove pre-fetched file which has the same use in HC - if the old one has different file name from the new one. - - Args: - key: string, key for the path to the fetched file. - value: string, path to the newly fetched file. - """ - if key in self and value != self[key]: - logging.info("Removing pre-fetched item: %s", self[key]) - try: - if os.path.isfile(self[key]): - os.remove(self[key]) - elif os.path.isdir(self[key]): - shutil.rmtree(self[key]) - else: - logging.error("%s is not found", self[key]) - except OSError as e: - logging.error("ERROR: error on file remove %s", e) - - super(BuildInfo, self).__setitem__(key, value) - - def update(self, other=None, **kwargs): - """Overrides update() in order to call BuildInfo.__setitem__(). - - Args: - other: dict or iterable of key/value pairs. Update self - using this argument - **kwargs: The optional attributes. - """ - if other is not None: - for k, v in other.items() if isinstance( - other, collections.Mapping) else other: - self[k] = v - for k, v in kwargs.items(): - self[k] = v - - dict_keys = self.keys() - for key in dict_keys: - if not os.path.exists(self[key]): - logging.info( - "Removing path info %s from build info", self.pop(key)) diff --git a/harnesses/host_controller/build/build_provider.py b/harnesses/host_controller/build/build_provider.py deleted file mode 100644 index 17ba0bd..0000000 --- a/harnesses/host_controller/build/build_provider.py +++ /dev/null @@ -1,323 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os -import re -import shutil -import tempfile -import zipfile - -from host_controller import common -from vts.runners.host import utils - - -class BuildProvider(object): - """The base class for build provider. - - Attributes: - _IMAGE_FILE_EXTENSIONS: a list of strings which are common image file - extensions. - _BASIC_IMAGE_FILE_NAMES: a list of strings which are the image names in - an artifact zip. - _CONFIG_FILE_EXTENSION: string, the config file extension. - _additional_files: a dict containing additionally fetched files that - custom features may need. The key is the path - relative to temporary directory and the value is the - full path. - _configs: dict where the key is config type and value is the config file - path. - _device_images: dict where the key is image file name and value is the - path. - _test_suites: dict where the key is test suite type and value is the - test suite package file path. - _host_controller_package: dict where the key is a host controller - package and the value is the path - to a package file. - _tmp_dirpath: string, the temp dir path created to keep artifacts. - _last_fetched_artifact_type: string, stores the type of the last - artifact fetched. - """ - _CONFIG_FILE_EXTENSION = ".zip" - _IMAGE_FILE_EXTENSIONS = [".img", ".bin"] - _BASIC_IMAGE_FILE_NAMES = ["boot.img", "system.img", "vendor.img"] - - def __init__(self): - self._additional_files = {} - self._device_images = {} - self._test_suites = {} - self._host_controller_package = {} - self._configs = {} - self._last_fetched_artifact_type = None - tempdir_base = os.path.join(os.getcwd(), "tmp") - if not os.path.exists(tempdir_base): - os.mkdir(tempdir_base) - self._tmp_dirpath = tempfile.mkdtemp(dir=tempdir_base) - - def __del__(self): - """Deletes the temp dir if still set.""" - if self._tmp_dirpath: - shutil.rmtree(self._tmp_dirpath) - self._tmp_dirpath = None - - @property - def tmp_dirpath(self): - return self._tmp_dirpath - - def CreateNewTmpDir(self): - return tempfile.mkdtemp(dir=self._tmp_dirpath) - - def SetDeviceImage(self, name, path): - """Sets device image `path` for the specified `name`.""" - self._device_images[name] = path - self._last_fetched_artifact_type = common._ARTIFACT_TYPE_DEVICE - - def _IsFullDeviceImage(self, namelist): - """Returns true if given namelist list has all common device images.""" - for image_file in self._BASIC_IMAGE_FILE_NAMES: - if image_file not in namelist: - return False - return True - - def _IsImageFile(self, file_path): - """Returns whether a file is an image. - - Args: - file_path: string, the file path. - - Returns: - boolean, whether the file is an image. - """ - return any(file_path.endswith(ext) - for ext in self._IMAGE_FILE_EXTENSIONS) - - def SetDeviceImageZip(self, path, full_device_images=False): - """Sets device image(s) using files in a given zip file. - - It extracts image files inside the given zip file and selects - known Android image files. - - Args: - path: string, the path to a zip file. - """ - dest_path = path + ".dir" - fetch_type = None - with zipfile.ZipFile(path, 'r') as zip_ref: - if full_device_images or self._IsFullDeviceImage(zip_ref.namelist()): - self.SetDeviceImage(common.FULL_ZIPFILE, path) - dir_key = common.FULL_ZIPFILE_DIR - fetch_type = common._ARTIFACT_TYPE_DEVICE - else: - self.SetDeviceImage("gsi-zipfile", path) - dir_key = common.GSI_ZIPFILE_DIR # "gsi-zipfile-dir" - fetch_type = common._ARTIFACT_TYPE_GSI - if os.path.exists(dest_path): - shutil.rmtree(dest_path) - logging.info("%s %s deleted", dir_key, dest_path) - zip_ref.extractall(dest_path) - self.SetFetchedDirectory(dest_path) - self.SetDeviceImage(dir_key, dest_path) - - self._last_fetched_artifact_type = fetch_type - - def GetDeviceImage(self, name=None): - """Returns device image info.""" - if name is None: - return self._device_images - return self._device_images[name] - - def RemoveDeviceImage(self, name): - """Removes certain device image info. - - Args: - name: string, the name of the device image file - that needs to be removed. - """ - if name in self._device_images: - self._device_images.pop(name) - - def SetTestSuitePackage(self, test_suite, path): - """Sets test suite package `path` for the specified `type`. - - Args: - test_suite: string, test suite type such as 'vts' or 'cts', etc. - path: string, the path of a file. if a file is a zip file, - it's unziped and its main binary is set. - """ - if re.match("[vcgs]ts", test_suite): - suite_name = "android-%s" % test_suite - tradefed_name = "%s-tradefed" % test_suite - dest_path = os.path.join(self.tmp_dirpath, suite_name) - if os.path.exists(dest_path): - shutil.rmtree(dest_path) - logging.info("test suite %s deleted", dest_path) - with zipfile.ZipFile(path, 'r') as zip_ref: - zip_ref.extractall(dest_path) - bin_path = os.path.join(dest_path, suite_name, - "tools", tradefed_name) - os.chmod(bin_path, 0766) - path = bin_path - else: - logging.info("unsupported zip file %s", path) - self._test_suites[test_suite] = path - self._last_fetched_artifact_type = common._ARTIFACT_TYPE_TEST_SUITE - - def GetTestSuitePackage(self, type=None): - """Returns test suite package info.""" - if type is None: - return self._test_suites - return self._test_suites[type] - - def SetHostControllerPackage(self, package_type, path): - """Sets host controller package `path` for the specified `type`. - - Args: - package_type: string, host controller type such as 'vtslab'. - path: string, the path of a package file. - """ - self._host_controller_package[package_type] = path - self._last_fetched_artifact_type = common._ARTIFACT_TYPE_INFRA - - def GetHostControllerPackage(self, package_type=None): - """Returns host controller package info. - - Args: - package_type: string, key value to self._host_controller_package - dict. - - Returns: - the whole dict if package_type is None, otherwise a string which is - the path to the fetched host controller package. - """ - if package_type is None: - return self._host_controller_package - return self._host_controller_package[package_type] - - def SetConfigPackage(self, config_type, path): - """Sets test suite package `path` for the specified `type`. - - All valid config files have .zip extension. - - Args: - config_type: string, config type such as 'prod' or 'test'. - path: string, the path of a config file. - """ - if path.endswith(self._CONFIG_FILE_EXTENSION): - dest_path = os.path.join( - self.tmp_dirpath, os.path.basename(path) + ".dir") - with zipfile.ZipFile(path, 'r') as zip_ref: - zip_ref.extractall(dest_path) - path = dest_path - else: - logging.info("unsupported config package file %s", path) - self._configs[config_type] = path - self._last_fetched_artifact_type = common._ARTIFACT_TYPE_INFRA - - def GetConfigPackage(self, config_type=None): - """Returns config package info.""" - if config_type is None: - return self._configs - return self._configs[config_type] - - def SetAdditionalFile(self, rel_path, full_path): - """Sets the key and value of additionally fetched files. - - Args: - rel_path: the file path relative to temporary directory. - abs_path: the file path that this process can access. - """ - self._additional_files[rel_path] = full_path - self._last_fetched_artifact_type = common._ARTIFACT_TYPE_INFRA - - def GetAdditionalFile(self, rel_path=None): - """Returns the paths to fetched files.""" - if rel_path is None: - return self._additional_files - return self._additional_files[rel_path] - - def SetFetchedDirectory(self, - dir_path, - root_path=None, - full_device_images=False): - """Adds every file in a directory to one of the dictionaries. - - This method follows symlink to file, but skips symlink to directory. - - Args: - dir_path: string, the directory to find files in. - root_path: string, the temporary directory that dir_path is in. - The default value is dir_path. - """ - for dir_name, file_name in utils.iterate_files(dir_path): - full_path = os.path.join(dir_name, file_name) - self.SetFetchedFile(full_path, (root_path - if root_path else dir_path), - full_device_images) - - def SetFetchedFile(self, - file_path, - root_dir=None, - full_device_images=False, - set_suite_as=None): - """Adds a file to one of the dictionaries. - - Args: - file_path: string, the path to the file. - root_dir: string, the temporary directory that file_path is in. - The default value is file_path if file_path is a - directory. Otherwise, the default value is file_path's - parent directory. - set_suite_as: string, the test suite name to use for the given - artifact. Used when the file name does not follow - the standard "android-*ts.zip" file name pattern. - """ - file_name = os.path.basename(file_path) - if os.path.isdir(file_path): - self.SetFetchedDirectory(file_path, root_dir, full_device_images) - elif self._IsImageFile(file_path): - self.SetDeviceImage(file_name, file_path) - elif re.match("android-[vcgs]ts.zip", file_name): - test_suite = (file_name.split("-")[-1]).split(".")[0] - self.SetTestSuitePackage(test_suite, file_path) - elif file_name == "android-vtslab.zip": - self.SetHostControllerPackage("vtslab", file_path) - elif file_name.startswith("vti-global-config"): - self.SetConfigPackage( - "prod" if "prod" in file_name else "test", file_path) - elif set_suite_as: - self.SetTestSuitePackage(set_suite_as, file_path) - elif file_path.endswith(".zip"): - self.SetDeviceImageZip(file_path, full_device_images) - else: - rel_path = (os.path.relpath(file_path, root_dir) if root_dir else - os.path.basename(file_path)) - self.SetAdditionalFile(rel_path, file_path) - - def PrintDeviceImageInfo(self): - """Prints device image info.""" - logging.info(self.GetDeviceImage()) - - def PrintGetTestSuitePackageInfo(self): - """Prints test suite package info.""" - logging.info(self.GetTestSuitePackage()) - - def GetFetchedArtifactType(self): - """Gets the most recently fetched artifact type. - - Returns: - string, type of the artifact. - """ - return self._last_fetched_artifact_type diff --git a/harnesses/host_controller/build/build_provider_ab.py b/harnesses/host_controller/build/build_provider_ab.py deleted file mode 100644 index 3debdb4..0000000 --- a/harnesses/host_controller/build/build_provider_ab.py +++ /dev/null @@ -1,94 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os - -from host_controller.build import build_provider -from vts.utils.python.build.api import artifact_fetcher - - -class BuildProviderAB(build_provider.BuildProvider): - """A build provider for Android Build (AB).""" - - def __init__(self): - super(BuildProviderAB, self).__init__() - if 'run_ab_key' in os.environ: - logging.info( - "For AB, use the key at %s", os.environ['run_ab_key']) - self._artifact_fetcher = artifact_fetcher.AndroidBuildClient( - os.environ['run_ab_key']) - else: - self._artifact_fetcher = None - - def GetLatestBuildId(self, branch, target): - """Get the latest build id. - - Args: - branch: string, android branch to pull resource from. - target: string, build target name. - - Returns: - string, latest build id. None if _artifact_fetcher is not initialized. - """ - if not self._artifact_fetcher: - return None - - recent_build_ids = self._artifact_fetcher.ListBuildIds( - branch, target) - - return recent_build_ids[0] - - def Fetch(self, - branch, - target, - artifact_name, - build_id="latest", - full_device_images=False): - """Fetches Android device artifact file(s) from Android Build. - - Args: - branch: string, android branch to pull resource from. - target: string, build target name. - artifact_name: string, file name. - build_id: string, ID of the build or latest. - - Returns: - a dict containing the device image info. - a dict containing the test suite package info. - a dict containing the artifact info. - """ - fetch_info = {} - fetch_info["build_id"] = None - - if not self._artifact_fetcher: - return self.GetDeviceImage(), self.GetTestSuitePackage(), fetch_info - - if build_id == "latest": - build_id = self.GetLatestBuildId(branch, target) - fetch_info["build_id"] = build_id - - if "{build_id}" in artifact_name: - artifact_name = artifact_name.replace("{build_id}", build_id) - - dest_filepath = os.path.join(self.tmp_dirpath, artifact_name) - self._artifact_fetcher.DownloadArtifactToFile( - branch, target, build_id, artifact_name, - dest_filepath=dest_filepath) - - self.SetFetchedFile(dest_filepath, full_device_images) - - return self.GetDeviceImage(), self.GetTestSuitePackage(), fetch_info diff --git a/harnesses/host_controller/build/build_provider_gcs.py b/harnesses/host_controller/build/build_provider_gcs.py deleted file mode 100644 index 0dc8f5a..0000000 --- a/harnesses/host_controller/build/build_provider_gcs.py +++ /dev/null @@ -1,113 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os -import re -import zipfile - -from host_controller.build import build_provider -from host_controller.utils.gcp import gcs_utils -from vts.utils.python.common import cmd_utils - -_GCLOUD_AUTH_ENV_KEY = "run_gcs_key" - - -class BuildProviderGCS(build_provider.BuildProvider): - """A build provider for GCS (Google Cloud Storage).""" - - def __init__(self): - super(BuildProviderGCS, self).__init__() - if _GCLOUD_AUTH_ENV_KEY in os.environ: - gcloud_path = BuildProviderGCS.GetGcloudPath() - if gcloud_path is not None: - auth_cmd = "%s auth activate-service-account --key-file=%s" % ( - gcloud_path, os.environ[_GCLOUD_AUTH_ENV_KEY]) - _, stderr, ret_code = cmd_utils.ExecuteOneShellCommand( - auth_cmd) - if ret_code == 0: - logging.info(stderr) - else: - logging.error(stderr) - - @staticmethod - def GetGcloudPath(): - """Returns the gcloud file path if found; None otherwise.""" - sh_stdout, _, ret_code = cmd_utils.ExecuteOneShellCommand( - "which gcloud") - if ret_code == 0: - return sh_stdout.strip() - else: - logging.error("`gcloud` doesn't exist on the host; " - "please install Google Cloud SDK before retrying.") - return None - - def Fetch(self, path, full_device_images=False, set_suite_as=None): - """Fetches Android device artifact file(s) from GCS. - - Args: - path: string, the path of a directory which keeps artifacts. - set_suite_as: string, the test suite name to use for the given - artifact. Used when the file name does not follow - the standard "android-*ts.zip" file name pattern. - - Returns: - a dict containing the device image info. - a dict containing the test suite package info. - a dict containing the info about custom tool files. - """ - if not path.startswith("gs://"): - path = "gs://" + re.sub("^/*", "", path) - path = re.sub("/*$", "", path) - gsutil_path = gcs_utils.GetGsutilPath() - if gsutil_path: - temp_dir_path = self.CreateNewTmpDir() - # IsGcsFile returns False if path is directory or doesn't exist. - # cp command returns non-zero if path doesn't exist. - if not gcs_utils.IsGcsFile(gsutil_path, path): - dest_path = temp_dir_path - if "latest.zip" in path: - gsutil_ls_path = re.sub("latest.zip", "*.zip", path) - lines_gsutil_ls = gcs_utils.List(gsutil_path, gsutil_ls_path) - if lines_gsutil_ls: - lines_gsutil_ls.sort() - path = lines_gsutil_ls[-1] - copy_command = "%s cp %s %s" % (gsutil_path, path, - temp_dir_path) - else: - logging.error( - "There is no file(s) that matches the URL %s.", - path) - return (self.GetDeviceImage(), - self.GetTestSuitePackage(), - self.GetAdditionalFile()) - else: - copy_command = "%s cp -r %s/* %s" % (gsutil_path, path, - temp_dir_path) - else: - dest_path = os.path.join(temp_dir_path, os.path.basename(path)) - copy_command = "%s cp %s %s" % (gsutil_path, path, - temp_dir_path) - - _, _, ret_code = cmd_utils.ExecuteOneShellCommand(copy_command) - if ret_code == 0: - self.SetFetchedFile(dest_path, temp_dir_path, - full_device_images, set_suite_as) - else: - logging.error("Error in copy file from GCS (code %s).", - ret_code) - return (self.GetDeviceImage(), self.GetTestSuitePackage(), - self.GetAdditionalFile()) diff --git a/harnesses/host_controller/build/build_provider_local_fs.py b/harnesses/host_controller/build/build_provider_local_fs.py deleted file mode 100644 index afcf429..0000000 --- a/harnesses/host_controller/build/build_provider_local_fs.py +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from host_controller.build import build_provider - - -class BuildProviderLocalFS(build_provider.BuildProvider): - """A build provider for local file system (fs).""" - - def __init__(self): - super(BuildProviderLocalFS, self).__init__() - - def Fetch(self, path, full_device_images=False): - """Fetches Android device artifact file(s) from a local path. - - Args: - path: string, the path of the artifacts. - - Returns: - a dict containing the device image info. - a dict containing the test suite package info. - """ - if path.endswith(".tar.md5"): - self.SetDeviceImage("img", path) - else: - self.SetFetchedFile(path, full_device_images=full_device_images) - return self.GetDeviceImage(), self.GetTestSuitePackage() diff --git a/harnesses/host_controller/build/build_provider_pab.py b/harnesses/host_controller/build/build_provider_pab.py deleted file mode 100644 index 9d932b6..0000000 --- a/harnesses/host_controller/build/build_provider_pab.py +++ /dev/null @@ -1,728 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -"""Module to fetch artifacts from Partner Android Build server.""" - -import argparse -import getpass -import httplib2 -import json -import logging -import os -import requests -import urlparse -from posixpath import join as path_urljoin - -from oauth2client.client import flow_from_clientsecrets -from oauth2client.file import Storage -from oauth2client.tools import argparser -from oauth2client.tools import run_flow - -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.common.exceptions import TimeoutException -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.support.ui import WebDriverWait - -from host_controller.build import build_provider - -# constants for GET and POST endpoints -GET = 'GET' -POST = 'POST' - -# timeout seconds for requests -REQUESTS_TIMEOUT_SECONDS = 60 - - -class BuildProviderPAB(build_provider.BuildProvider): - """Client that manages Partner Android Build downloading. - - Attributes: - BAD_XSRF_CODE: int, error code for bad XSRF token error - BASE_URL: string, path to PAB entry point - BUILDARTIFACT_NAME_KEY: string, index in artifact containing name - BUILD_BUILDID_KEY: string, index in build containing build_id - BUILD_COMPLETED_STATUS: int, value of 'complete' build - BUILD_STATUS_KEY: string, index in build object containing status. - CHROME_DRIVER_LOCATION: string, path to chromedriver - CHROME_LOCATION: string, path to Chrome browser - CLIENT_STORAGE: string, path to store credentials. - DEFAULT_CHUNK_SIZE: int, number of bytes to download at a time. - DOWNLOAD_URL_KEY: string, index in downloadBuildArtifact containing url - EMAIL: string, email constant for userinfo JSON - EXPIRED_XSRF_CODE: int, error code for expired XSRF token error - GETBUILD_ARTIFACTS_KEY, string, index in build obj containing artifacts - GMS_DOWNLOAD_URL: string, base url for downloading artifacts. - LISTBUILD_BUILD_KEY: string, index in listBuild containing builds - PAB_URL: string, redirect url from Google sign-in to PAB - PASSWORD: string, password constant for userinfo JSON - SCOPE: string, URL for which to request access via oauth2. - SVC_URL: string, path to buildsvc RPC - XSRF_STORE: string, path to store xsrf token - _credentials : oauth2client credentials object - _userinfo_file: location of file containing email and password - _xsrf : string, XSRF token from PAB website. expires after 7 days. - """ - _credentials = None - _userinfo_file = None - _xsrf = None - BAD_XSRF_CODE = -32000 - BASE_URL = 'https://partner.android.com' - BUILDARTIFACT_NAME_KEY = '1' - BUILD_BUILDID_KEY = '1' - BUILD_COMPLETED_STATUS = 7 - BUILD_STATUS_KEY = '7' - CHROME_DRIVER_LOCATION = '/usr/local/bin/chromedriver' - CHROME_LOCATION = '/usr/bin/google-chrome' - CLIENT_SECRETS = os.path.join( - os.path.dirname(__file__), 'client_secrets.json') - CLIENT_STORAGE = os.path.join(os.path.dirname(__file__), 'credentials') - DEFAULT_CHUNK_SIZE = 1024 - DOWNLOAD_URL_KEY = '1' - EMAIL = 'email' - EXPIRED_XSRF_CODE = -32001 - GETBUILD_ARTIFACTS_KEY = '2' - GMS_DOWNLOAD_URL = 'https://partnerdash.google.com/build/gmsdownload' - LISTBUILD_BUILD_KEY = '1' - PAB_URL = ('https://www.google.com/accounts/Login?&continue=' - 'https://partner.android.com/build/') - PASSWORD = 'password' - # need both of these scopes to access PAB downloader - scopes = ('https://www.googleapis.com/auth/partnerdash', - 'https://www.googleapis.com/auth/alkali-base') - SCOPE = ' '.join(scopes) - SVC_URL = urlparse.urljoin(BASE_URL, 'build/u/0/_gwt/_rpc/buildsvc') - XSRF_STORE = os.path.join(os.path.dirname(__file__), 'xsrf') - - def __init__(self): - """Creates a temp dir.""" - super(BuildProviderPAB, self).__init__() - - def Authenticate(self, userinfo_file=None, noauth_local_webserver=False, - scopes=SCOPE): - """Authenticate using OAuth2. - - Args: - userinfo_file: (optional) the path of a JSON file which has - "email" and "password" string fields. - noauth_local_webserver: boolean, True if do not (or can not) use - a local web server. - scopes: string or iterable of strings, the scopes to request. - """ - # this should be a JSON file with "email" and "password" string fields - self._userinfo_file = userinfo_file - logging.info('Parsing flags, use --noauth_local_webserver' - ' if running on remote machine') - - parser = argparse.ArgumentParser(parents=[argparser]) - flags, unknown = parser.parse_known_args() - flags.noauth_local_webserver = noauth_local_webserver - logging.info('Preparing OAuth token') - flow = flow_from_clientsecrets(self.CLIENT_SECRETS, scope=scopes) - storage = Storage(self.CLIENT_STORAGE) - if self._credentials is None: - self._credentials = storage.get() - if self._credentials is None or self._credentials.invalid: - logging.info('Credentials not found, authenticating.') - self._credentials = run_flow(flow, storage, flags) - - if self._credentials.access_token_expired: - logging.info('Access token expired, refreshing.') - self._credentials.refresh(http=httplib2.Http()) - - if self.XSRF_STORE is not None and os.path.isfile(self.XSRF_STORE): - with open(self.XSRF_STORE, 'r') as handle: - self._xsrf = handle.read() - - def GetXSRFToken(self, email=None, password=None): - """Get XSRF token. Prompt if email/password not provided. - - Args: - email: string, optional. Gmail account of user logging in - password: string, optional. Password of user logging in - - Returns: - boolean, whether the token was accessed and stored - - Raises: - ValueError if login fails or userinfo file is malformed. - """ - if self._userinfo_file is not None: - with open(self._userinfo_file, 'r') as handle: - userinfo = json.load(handle) - - if self.EMAIL not in userinfo or self.PASSWORD not in userinfo: - raise ValueError( - 'Malformed userinfo file: needs email and password') - - email = userinfo[self.EMAIL] - password = userinfo[self.PASSWORD] - - chrome_options = Options() - chrome_options.add_argument("--headless") - - driver = webdriver.Chrome( - chrome_options=chrome_options) - - driver.set_window_size(1080, 800) - wait = WebDriverWait(driver, 10) - - driver.get(self.PAB_URL) - - query = driver.find_element_by_id("identifierId") - if email is None: - email = raw_input("Email: ") - query.send_keys(email) - driver.find_element_by_id("identifierNext").click() - - pw = wait.until(EC.element_to_be_clickable((By.NAME, "password"))) - pw.clear() - - if password is None: - pw.send_keys(getpass.getpass("Password: ")) - else: - pw.send_keys(password) - - driver.find_element_by_id("passwordNext").click() - - try: - wait.until(EC.title_contains("Partner Android Build")) - except TimeoutException as e: - logging.exception(e) - raise ValueError('Wrong password or non-standard login flow') - - self._xsrf = driver.execute_script("return clientConfig.XSRF_TOKEN;") - with open(self.XSRF_STORE, 'w') as handle: - handle.write(self._xsrf) - - return True - - def CallBuildsvc(self, method, params, account_id): - """Call the buildsvc RPC with given parameters. - - Args: - method: string, name of method to be called in buildsvc - params: dict, parameters to RPC call - account_id: int, ID associated with the PAB account. - - Returns: - dict, result from RPC call - - Raises: - ValueError if RPC call returns an error or an unknown response. - """ - if self._xsrf is None: - self.GetXSRFToken() - params = json.dumps(params) - - data = {"method": method, "params": params, "xsrf": self._xsrf} - data = json.dumps(data) - headers = {} - self._credentials.apply(headers) - headers['Content-Type'] = 'application/json' - headers['x-alkali-account'] = account_id - - try: - response = requests.post(self.SVC_URL, data=data, headers=headers, - timeout=REQUESTS_TIMEOUT_SECONDS) - except requests.exceptions.Timeout as e: - logging.exception(e) - raise ValueError("Request timeout.") - - responseJSON = {} - - try: - responseJSON = response.json() - except ValueError: - raise ValueError("Backend error -- check your account ID") - - if 'result' in responseJSON: - return responseJSON['result'] - - if 'error' in responseJSON and 'code' in responseJSON['error']: - if responseJSON['error']['code'] == self.BAD_XSRF_CODE: - raise ValueError( - "Bad XSRF token -- must be for the same account as your credentials") - if responseJSON['error']['code'] == self.EXPIRED_XSRF_CODE: - raise ValueError("Expired XSRF token -- please refresh") - - raise ValueError("Unknown response from server -- %s" % - json.dumps(responseJSON)) - - def GetBuildList(self, - account_id, - branch, - target, - page_token="", - max_results=10, - internal=True, - method=GET, - verify_signed=False): - """Get the list of builds for a given account, branch and target - Args: - account_id: int, ID associated with the PAB account. - branch: string, branch to pull resource from. - target: string, "latest" or a specific version. - page_token: string, token used for pagination - max_results: maximum build results the build list contains, e.g. 25 - internal: bool, whether to query internal build - method: 'GET' or 'POST', which endpoint to query - verify_signed: bool, whether to verify signed build. - - Returns: - list of dicts representing the builds, descending in time - - Raises: - ValueError if build request returns an error or builds not found. - """ - if method == POST: - params = { - "1": branch, - "2": target, - "3": page_token, - "4": max_results, - "7": int(internal) - } - - result = self.CallBuildsvc("listBuild", params, account_id) - # in listBuild response, index '1' contains builds - if self.LISTBUILD_BUILD_KEY in result: - return result[self.LISTBUILD_BUILD_KEY] - - if verify_signed: - logging.error("verify_signed does not support POST method.") - - raise ValueError("Build list not found -- %s" % params) - elif method == GET: - headers = {} - self._credentials.apply(headers) - - action = 'list-internal' if internal else 'list' - # PAB URL format expects something (anything) to be given as buildid - # and resource, even for action list - dummy = 'DUMMY' - url = path_urljoin(self.BASE_URL, 'build', 'builds', action, - branch, target, dummy, - dummy) + '?a=' + str(account_id) - try: - response = requests.get(url, headers=headers, - timeout=REQUESTS_TIMEOUT_SECONDS) - responseJSON = response.json() - builds = responseJSON['build'] - except requests.exceptions.Timeout as e: - logging.exception(e) - raise ValueError("Request timeout.") - except ValueError as e: - logging.exception(e) - raise ValueError("Backend error -- check your account ID") - - if verify_signed: - for build in builds: - artifact_name = "signed%2Fsigned-{}-img-{}.zip".format( - target.split("-")[0], build["build_id"]) - logging.debug("Checking whether the build is signed for " - "build_target {} and build_id {}".format( - target, build["build_id"])) - signed_build_url = self.GetArtifactURL( - account_id=account_id, - build_id=build["build_id"], - target=target, - artifact_name=artifact_name, - branch=branch, - internal=False, - method=method) - try: - self.GetResponseWithURL(signed_build_url) - logging.debug("The build is signed.") - build["signed"] = True - except requests.HTTPError: - logging.debug("The build is not signed.") - build["signed"] = False - except requests.exceptions.Timeout as e: - logging.debug("Server is not responding.") - logging.exception(e) - build["signed"] = False - return builds - - def GetLatestBuildId(self, account_id, branch, target, method=GET): - """Get the most recent build_id for a given account, branch and target - Args: - account_id: int, ID associated with the PAB account. - branch: string, branch to pull resource from. - target: string, "latest" or a specific version. - method: 'GET' or 'POST', which endpoint to query - - Returns: - string, most recent build id - - Raises: - ValueError if complete builds are not found. - """ - # TODO: support pagination, maybe? - build_list = self.GetBuildList(account_id=account_id, - branch=branch, - target=target, - method=method) - if len(build_list) == 0: - raise ValueError( - 'No builds found for account_id=%s, branch=%s, target=%s' % - (account_id, branch, target)) - for build in build_list: - if method == POST: - # get build status: 7 = completed build - if build.get(self.BUILD_STATUS_KEY, - None) == self.BUILD_COMPLETED_STATUS: - # return build id (index '1') - return build[self.BUILD_BUILDID_KEY] - elif method == GET: - if build['build_attempt_status'] == "COMPLETE" and build[ - "successful"]: - return build['build_id'] - raise ValueError( - 'No complete builds found: %s failed or incomplete builds found' % - len(build_list)) - - def GetBuildArtifacts( - self, account_id, build_id, branch, target, method=POST): - """Get the list of build artifacts. - - For an account, build, target, branch. - - Args: - account_id: int, ID associated with the PAB account. - build_id: string, ID of the build - branch: string, branch to pull resource from. - target: string, "latest" or a specific version. - method: 'GET' or 'POST', which endpoint to query - - Returns: - list of build artifact objects - - Raises: - NotImplementedError if method is 'GET', which is not supported yet. - ValueError if build artifacts are not found. - """ - if method == GET: - raise NotImplementedError( - "GetBuildArtifacts not supported with GET") - params = {"1": build_id, "2": target, "3": branch} - - result = self.CallBuildsvc("getBuild", params, account_id) - # in getBuild response, index '2' contains the artifacts - if self.GETBUILD_ARTIFACTS_KEY in result: - return result[self.GETBUILD_ARTIFACTS_KEY] - if len(result) == 0: - raise ValueError("Build artifacts not found -- %s" % params) - - def GetArtifactURL(self, - account_id, - build_id, - target, - artifact_name, - branch, - internal, - method=GET): - """Get the URL for an artifact on the PAB server, using buildsvc. - - Args: - account_id: int, ID associated with the PAB account. - build_id: string/int, id of the build. - target: string, "latest" or a specific version. - artifact_name: string, simple file name (no parent dir or path). - branch: string, branch to pull resource from. - internal: int, whether the request is for an internal build artifact - method: 'GET' or 'POST', which endpoint to query - - Returns: - string, The URL for the resource specified by the parameters - - Raises: - ValueError if given parameters are incorrect or resource not found. - """ - if method == POST: - params = { - "1": str(build_id), - "2": target, - "3": artifact_name, - "4": branch, - "5": "", # release_candidate_name - "6": internal - } - - result = self.CallBuildsvc(method='downloadBuildArtifact', - params=params, - account_id=account_id) - - # in downloadBuildArtifact response, index '1' contains the url - if self.DOWNLOAD_URL_KEY in result: - return result[self.DOWNLOAD_URL_KEY] - if len(result) == 0: - raise ValueError("Resource not found -- %s" % params) - elif method == GET: - headers = {} - self._credentials.apply(headers) - - action = 'get-internal' if internal else 'get' - get_url = path_urljoin(self.BASE_URL, 'build', 'builds', action, - branch, target, build_id, - artifact_name) + '?a=' + str(account_id) - - try: - response = requests.get(get_url, headers=headers, - timeout=REQUESTS_TIMEOUT_SECONDS) - responseJSON = response.json() - return responseJSON['url'] - except requests.exceptions.Timeout as e: - logging.exception(e) - raise ValueError("Request timeout.") - except ValueError: - raise ValueError("Backend error -- check your account ID") - - def DownloadArtifact(self, download_url, filename): - """Get artifact from Partner Android Build server. - - Args: - download_url: location of resource that we want to download - filename: where the artifact gets downloaded locally. - - Returns: - boolean, whether the file was successfully downloaded - """ - try: - response = self.GetResponseWithURL(download_url) - except (requests.HTTPError, requests.exceptions.Timeout) as error: - logging.exception(error) - return False - logging.info('%s now downloading...', download_url) - with open(filename, 'wb') as handle: - for block in response.iter_content(self.DEFAULT_CHUNK_SIZE): - handle.write(block) - return True - - def GetArtifact(self, - account_id, - branch, - target, - artifact_name, - build_id='latest', - method=GET, - full_device_images=False): - """Get an artifact for an account, branch, target and name and build id. - - If build_id not given, get latest. - - Args: - account_id: int, ID associated with the PAB account. - branch: string, branch to pull resource from. - target: string, "latest" or a specific version. - artifact_name: name of artifact, e.g. aosp_arm64_ab-img-4353141.zip - ({id} will automatically get replaced with build ID) - build_id: string, build ID of an artifact to fetch (or 'latest'). - method: 'GET' or 'POST', which endpoint to query. - - Returns: - a dict containing the device image info. - a dict containing the test suite package info. - a dict containing the artifact info. - a dict containing the global config info. - - Raises: - ValueError if artifacts are not found. - """ - artifact_info = {} - if build_id == 'latest': - build_id = self.GetLatestBuildId(account_id=account_id, - branch=branch, - target=target, - method=method) - logging.info("latest build ID = %s", build_id) - artifact_info["build_id"] = build_id - - if "build_id" in artifact_name: - artifact_name = artifact_name.format(build_id=build_id) - - if method == POST: - artifacts = self.GetBuildArtifacts(account_id=account_id, - build_id=build_id, - branch=branch, - target=target, - method=method) - - if len(artifacts) == 0: - raise ValueError( - "No artifacts found for build_id=%s, branch=%s, target=%s" - % (build_id, branch, target)) - - # in build artifact response, index '1' contains the name - artifact_names = [ - artifact[self.BUILDARTIFACT_NAME_KEY] for artifact in artifacts - ] - if artifact_name not in artifact_names: - raise ValueError("%s not found in artifact list" % - artifact_name) - - url = self.GetArtifactURL(account_id=account_id, - build_id=build_id, - target=target, - artifact_name=artifact_name, - branch=branch, - internal=False, - method=method) - - if self.tmp_dirpath: - artifact_path = os.path.join(self.tmp_dirpath, artifact_name) - else: - artifact_path = artifact_name - self.DownloadArtifact(url, artifact_path) - - self.SetFetchedFile( - artifact_path, full_device_images=full_device_images) - - return (self.GetDeviceImage(), self.GetTestSuitePackage(), - artifact_info, self.GetConfigPackage()) - - def GetSignedBuildArtifact(self, - account_id, - branch, - target, - artifact_name, - build_id='latest', - method=GET, - full_device_images=False): - """Get an signed build artifact from the PAB bulid list. - - Args: - account_id: int, ID associated with the PAB account. - branch: string, branch to pull resource from. - target: string, "latest" or a specific version. - artifact_name: name of artifact, e.g. aosp_arm64_ab-img-4353141.zip - ({id} will automatically get replaced with build ID) - build_id: string, build ID of an artifact to fetch (or 'latest'). - method: 'GET' or 'POST', which endpoint to query. - - Returns: - a dict containing the device image info. - a dict containing the test suite package info. - a dict containing the artifact info. - a dict containing the global config info. - """ - artifact_info = {} - build_ids = [] - artifact_path = "" - if build_id == 'latest': - try: - build_list = self.GetBuildList( - account_id=account_id, - branch=branch, - target=target, - method=method) - for build in build_list: - build_ids.append(build["build_id"]) - except ValueError as e: - logging.exception(e) - else: - build_ids.append(build_id) - - for build_id in build_ids: - _artifact_name = artifact_name - if "build_id" in _artifact_name: - _artifact_name = _artifact_name.format(build_id=build_id) - _artifact_name = "signed%2Fsigned-" + _artifact_name - try: - url = self.GetArtifactURL( - account_id=account_id, - build_id=build_id, - target=target, - artifact_name=_artifact_name, - branch=branch, - internal=False, - method=method) - except ValueError as e: - logging.exception(e) - continue - - if self.tmp_dirpath: - artifact_path = os.path.join(self.tmp_dirpath, _artifact_name) - else: - artifact_path = _artifact_name - ret = self.DownloadArtifact(url, artifact_path) - - if ret: - artifact_info["build_id"] = build_id - break - - self.SetFetchedFile( - artifact_path, full_device_images=full_device_images) - - return (self.GetDeviceImage(), self.GetTestSuitePackage(), - artifact_info, self.GetConfigPackage()) - - def GetResponseWithURL(self, url): - """Gets the response content from the server connected with the url. - - Args: - url: A string representing the server url. - - Returns: - A Response object received from the server. - - Raises: - requests.HTTPError if response.status_code is not 200. - requests.exceptions.Timeout if the server does not respond. - """ - headers = {} - self._credentials.apply(headers) - response = requests.get(url, headers=headers, stream=True, - timeout=REQUESTS_TIMEOUT_SECONDS) - response.raise_for_status() - - return response - - def FetchLatestBuiltHCPackage(self, account_id, branch, target): - """Fetchs the latest <artifact_name> file and return the path. - - Args: - account_id: string, Partner Android Build account_id to use. - branch: string, branch to grab the artifact from. - targets: string, a comma-separate list of build target product(s). - - Returns: - path to the fetched file. None if the fetching has failed. - """ - try: - listed_builds = self.GetBuildList( - account_id=account_id, - branch=branch, - target=target, - page_token="", - max_results=1, - method="GET") - - if listed_builds and len(listed_builds) > 0: - for listed_build in listed_builds: - if listed_build["successful"]: - self.GetArtifact( - account_id=account_id, - branch=branch, - target=target, - artifact_name="android-vtslab.zip", - build_id=listed_build["build_id"], - method="GET") - - return self.GetHostControllerPackage("vtslab") - except ValueError as e: - logging.exception(e) diff --git a/harnesses/host_controller/build/build_provider_pab_test.py b/harnesses/host_controller/build/build_provider_pab_test.py deleted file mode 100644 index 95e83c9..0000000 --- a/harnesses/host_controller/build/build_provider_pab_test.py +++ /dev/null @@ -1,317 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest -from host_controller.build import build_provider_pab - -try: - from unittest import mock -except ImportError: - import mock - -from requests.models import Response - - -class BuildProviderPABTest(unittest.TestCase): - """Tests for Partner Android Build client.""" - - def setUp(self): - self.client = build_provider_pab.BuildProviderPAB() - self.client.XSRF_STORE = None - - def tearDown(self): - del self.client - - @mock.patch("build_provider_pab.flow_from_clientsecrets") - @mock.patch("build_provider_pab.run_flow") - @mock.patch("build_provider_pab.Storage.get") - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - def testAuthenticationNew(self, mock_creds, mock_storage_get, mock_rf, - mock_ffc): - mock_creds.invalid = True - build_provider_pab.flow_from_clientsecrets = mock_ffc - build_provider_pab.run_flow = mock_rf - self.client.Authenticate() - mock_ffc.assert_called_once() - mock_storage_get.assert_called_once() - mock_rf.assert_called_once() - - @mock.patch("build_provider_pab.flow_from_clientsecrets") - @mock.patch("build_provider_pab.run_flow") - @mock.patch("build_provider_pab.Storage.get") - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - def testAuthenticationStale(self, mock_creds, mock_storage_get, mock_rf, - mock_ffc): - mock_creds.invalid = False - mock_creds.access_token_expired = True - build_provider_pab.flow_from_clientsecrets = mock_ffc - build_provider_pab.run_flow = mock_rf - mock_storage_get.return_value = mock_creds - self.client.Authenticate() - mock_ffc.assert_called_once() - mock_storage_get.assert_called_once() - mock_rf.assert_not_called() - mock_creds.refresh.assert_called_once() - - @mock.patch("build_provider_pab.flow_from_clientsecrets") - @mock.patch("build_provider_pab.run_flow") - @mock.patch("build_provider_pab.Storage.get") - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - def testAuthenticationFresh(self, mock_creds, mock_storage_get, mock_rf, - mock_ffc): - mock_creds.invalid = False - mock_creds.access_token_expired = False - build_provider_pab.flow_from_clientsecrets = mock_ffc - build_provider_pab.run_flow = mock_rf - mock_storage_get.return_value = mock_creds - self.client.Authenticate() - mock_ffc.assert_called_once() - mock_storage_get.assert_called_once() - mock_rf.assert_not_called() - mock_creds.refresh.assert_not_called() - - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - @mock.patch('requests.get') - @mock.patch('__builtin__.open') - def testDownloadArtifact(self, mock_open, mock_get, mock_creds): - self.client._credentials = mock_creds - artifact_url = ( - "https://partnerdash.google.com/build/gmsdownload/" - "f_companion/label/clockwork.companion_20170906_211311_RC00/" - "ClockworkCompanionGoogleWithGmsRelease_signed.apk?a=100621237") - self.client.DownloadArtifact( - artifact_url, 'ClockworkCompanionGoogleWithGmsRelease_signed.apk') - self.client._credentials.apply.assert_called_with({}) - mock_get.assert_called_with( - artifact_url, headers={}, stream=True) - mock_open.assert_called_with( - 'ClockworkCompanionGoogleWithGmsRelease_signed.apk', 'wb') - - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - @mock.patch('requests.post') - def testGetArtifactURL(self, mock_post, mock_creds): - self.client._xsrf = 'disable' - response = Response() - response.status_code = 200 - response._content = b'{ "result" : {"1": "this_url"}}' - mock_post.return_value = response - self.client._credentials = mock_creds - url = self.client.GetArtifactURL( - 100621237, - "4331445", - "darwin_mac", - "android-ndk-43345-darwin-x86_64.tar.bz2", - "aosp-master-ndk", - 0, - method='POST') - mock_post.assert_called_with( - 'https://partner.android.com/build/u/0/_gwt/_rpc/buildsvc', - data=mock.ANY, - headers={ - 'Content-Type': 'application/json', - 'x-alkali-account': 100621237, - }) - self.assertEqual(url, "this_url") - - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - @mock.patch('requests.post') - def testGetArtifactURLBackendError(self, mock_post, mock_creds): - self.client._xsrf = 'disable' - response = Response() - response.status_code = 200 - response._content = b'not JSON' - mock_post.return_value = response - self.client._credentials = mock_creds - with self.assertRaises(ValueError) as cm: - self.client.GetArtifactURL( - 100621237, - "4331445", - "darwin_mac", - "android-ndk-43345-darwin-x86_64.tar.bz2", - "aosp-master-ndk", - 0, - method='POST') - expected = "Backend error -- check your account ID" - self.assertEqual(str(cm.exception), expected) - - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - @mock.patch('requests.post') - def testGetArtifactURLMissingResultError(self, mock_post, mock_creds): - self.client._xsrf = 'disable' - response = Response() - response.status_code = 200 - response._content = b'{"result": {}}' - mock_post.return_value = response - self.client._credentials = mock_creds - with self.assertRaises(ValueError) as cm: - self.client.GetArtifactURL( - 100621237, - "4331445", - "darwin_mac", - "android-ndk-43345-darwin-x86_64.tar.bz2", - "aosp-master-ndk", - 0, - method='POST') - expected = "Resource not found" - self.assertIn(expected, str(cm.exception)) - - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - @mock.patch('requests.post') - def testGetArtifactURLInvalidXSRFError(self, mock_post, mock_creds): - self.client._xsrf = 'disable' - response = Response() - response.status_code = 200 - response._content = b'{"error": {"code": -32000, "message":"Invalid"}}' - mock_post.return_value = response - self.client._credentials = mock_creds - with self.assertRaises(ValueError) as cm: - self.client.GetArtifactURL( - 100621237, - "4331445", - "darwin_mac", - "android-ndk-43345-darwin-x86_64.tar.bz2", - "aosp-master-ndk", - 0, - method='POST') - self.assertIn('Bad XSRF token', str(cm.exception)) - - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - @mock.patch('requests.post') - def testGetArtifactURLExpiredXSRFError(self, mock_post, mock_creds): - self.client._xsrf = 'disable' - response = Response() - response.status_code = 200 - response._content = b'{"error": {"code": -32001, "message":"Expired"}}' - mock_post.return_value = response - self.client._credentials = mock_creds - with self.assertRaises(ValueError) as cm: - self.client.GetArtifactURL( - 100621237, - "4331445", - "darwin_mac", - "android-ndk-43345-darwin-x86_64.tar.bz2", - "aosp-master-ndk", - 0, - method='POST') - self.assertIn('Expired XSRF token', str(cm.exception)) - - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - @mock.patch('requests.post') - def testGetArtifactURLUnknownError(self, mock_post, mock_creds): - self.client._xsrf = 'disable' - response = Response() - response.status_code = 200 - response._content = b'{"some_other_json": "foo"}' - mock_post.return_value = response - self.client._credentials = mock_creds - with self.assertRaises(ValueError) as cm: - self.client.GetArtifactURL( - 100621237, - "4331445", - "darwin_mac", - "android-ndk-43345-darwin-x86_64.tar.bz2", - "aosp-master-ndk", - 0, - method='POST') - self.assertIn('Unknown response from server', str(cm.exception)) - - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - @mock.patch('requests.post') - def testGetBuildListSuccess(self, mock_post, mock_creds): - self.client._xsrf = 'disable' - response = Response() - response.status_code = 200 - response._content = b'{"result": {"1": "foo"}}' - mock_post.return_value = response - self.client._credentials = mock_creds - result = self.client.GetBuildList( - 100621237, - "git_oc-treble-dev", - "aosp_arm64_ab-userdebug", - method='POST') - self.assertEqual(result, "foo") - mock_post.assert_called_with( - 'https://partner.android.com/build/u/0/_gwt/_rpc/buildsvc', - data=mock.ANY, - headers={ - 'Content-Type': 'application/json', - 'x-alkali-account': 100621237, - }) - - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - @mock.patch('requests.post') - def testGetBuildListError(self, mock_post, mock_creds): - self.client._xsrf = 'disable' - response = Response() - response.status_code = 200 - response._content = b'{"result": {"3": "foo"}}' - mock_post.return_value = response - self.client._credentials = mock_creds - with self.assertRaises(ValueError) as cm: - self.client.GetBuildList( - 100621237, - "git_oc-treble-dev", - "aosp_arm64_ab-userdebug", - method='POST') - self.assertIn('Build list not found', str(cm.exception)) - - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - @mock.patch('build_provider_pab.BuildProviderPAB.GetBuildList') - def testGetLatestBuildIdSuccess(self, mock_gbl, mock_creds): - self.client._xsrf = 'disable' - mock_gbl.return_value = [{'7': 5, '1': 'bad'}, {'7': 7, '1': 'good'}] - self.client.GetBuildList = mock_gbl - result = self.client.GetLatestBuildId( - 100621237, - "git_oc-treble-dev", - "aosp_arm64_ab-userdebug", - method='POST') - self.assertEqual(result, 'good') - - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - @mock.patch('build_provider_pab.BuildProviderPAB.GetBuildList') - def testGetLatestBuildIdEmpty(self, mock_gbl, mock_creds): - self.client._xsrf = 'disable' - mock_gbl.return_value = [] - self.client.GetBuildList = mock_gbl - with self.assertRaises(ValueError) as cm: - result = self.client.GetLatestBuildId( - 100621237, - "git_oc-treble-dev", - "aosp_arm64_ab-userdebug", - method='POST') - self.assertIn("No builds found for", str(cm.exception)) - - @mock.patch('build_provider_pab.BuildProviderPAB._credentials') - @mock.patch('build_provider_pab.BuildProviderPAB.GetBuildList') - def testGetLatestBuildIdAllBad(self, mock_gbl, mock_creds): - self.client._xsrf = 'disable' - mock_gbl.return_value = [{'7': 0}, {'7': 0}] - self.client.GetBuildList = mock_gbl - with self.assertRaises(ValueError) as cm: - result = self.client.GetLatestBuildId( - 100621237, - "git_oc-treble-dev", - "aosp_arm64_ab-userdebug", - method='POST') - self.assertEqual( - "No complete builds found: 2 failed or incomplete builds found", - str(cm.exception)) - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/build/build_provider_test.py b/harnesses/host_controller/build/build_provider_test.py deleted file mode 100644 index 71b96d1..0000000 --- a/harnesses/host_controller/build/build_provider_test.py +++ /dev/null @@ -1,156 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import shutil -import tempfile -import unittest -import zipfile - -from host_controller import common -from host_controller.build import build_provider - -try: - from unittest import mock -except ImportError: - import mock - - -class BuildProviderTest(unittest.TestCase): - """Tests for build_provider. - - Attributes: - _build_provider: The BuildProvider object under test. - _temp_dir: The path to the temporary directory for test files. - """ - - def setUp(self): - """Creates temporary directory.""" - self._build_provider = build_provider.BuildProvider() - self._temp_dir = tempfile.mkdtemp() - - def tearDown(self): - """Deletes temporary directory.""" - shutil.rmtree(self._temp_dir) - - def _CreateFile(self, name): - """Creates an empty file as test data. - - Args: - name: string, the name of the file. - - Returns: - string, the path to the file. - """ - path = os.path.join(self._temp_dir, name) - with open(path, "w"): - pass - return path - - def _CreateZip(self, name, *paths): - """Creates a zip file containing empty files. - - Args: - name: string, the name of the zip file. - *paths: strings, the file paths to create in the zip file. - - Returns: - string, the path to the zip file. - """ - empty_path = self._CreateFile("empty") - zip_path = os.path.join(self._temp_dir, name) - with zipfile.ZipFile(zip_path, "w") as zip_file: - for path in paths: - zip_file.write(empty_path, path) - return zip_path - - def _CreateVtsPackage(self): - """Creates an android-vts.zip containing vts-tradefed. - - Returns: - string, the path to the zip file. - """ - return self._CreateZip( - "android-vts.zip", - os.path.join("android-vts", "tools", "vts-tradefed")) - - def _CreateDeviceImageZip(self): - """Creates a zip containing common device images. - - Returns: - string, the path to the zip file. - """ - return self._CreateZip( - "img.zip", "system.img", "vendor.img", "boot.img") - - def _CreateProdConfig(self): - """Creates a zip containing config files. - - Returns: - string, the path to the zip file. - """ - return self._CreateZip( - "vti-global-config-prod.zip", os.path.join("test", "prod.config")) - - def testSetTestSuitePackage(self): - """Tests setting a VTS package.""" - vts_path = self._CreateVtsPackage() - self._build_provider.SetTestSuitePackage("vts", vts_path) - - self.assertTrue( - os.path.exists(self._build_provider.GetTestSuitePackage("vts"))) - - def testSetDeviceImageZip(self): - """Tests setting a device image zip.""" - img_path = self._CreateDeviceImageZip() - self._build_provider.SetDeviceImageZip(img_path) - - self.assertEqual( - img_path, - self._build_provider.GetDeviceImage(common.FULL_ZIPFILE)) - - def testSetConfigPackage(self): - """Tests setting a config package.""" - config_path = self._CreateProdConfig() - self._build_provider.SetConfigPackage("prod", config_path) - - self.assertTrue( - os.path.exists(self._build_provider.GetConfigPackage("prod"))) - - def testSetFetchedDirectory(self): - """Tests setting all files in a directory.""" - self._CreateVtsPackage() - self._CreateProdConfig() - img_zip = self._CreateDeviceImageZip() - img_file = self._CreateFile("userdata.img") - txt_file = self._CreateFile("additional.txt") - self._build_provider.SetFetchedDirectory(self._temp_dir) - - self.assertDictContainsSubset( - {common.FULL_ZIPFILE: img_zip, "userdata.img": img_file}, - self._build_provider.GetDeviceImage()) - self.assertTrue( - os.path.exists(self._build_provider.GetTestSuitePackage("vts"))) - self.assertTrue( - os.path.exists(self._build_provider.GetConfigPackage("prod"))) - self.assertDictContainsSubset( - {"additional.txt": txt_file}, - self._build_provider.GetAdditionalFile()) - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/campaigns/__init__.py b/harnesses/host_controller/campaigns/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/campaigns/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/campaigns/campaign_common.py b/harnesses/host_controller/campaigns/campaign_common.py deleted file mode 100644 index 82bb726..0000000 --- a/harnesses/host_controller/campaigns/campaign_common.py +++ /dev/null @@ -1,755 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging - -from host_controller import common -from vti.test_serving.proto import TestScheduleConfigMessage_pb2 as pb - -# The list of the kwargs key. can retrieve informations on the leased job. -_JOB_ATTR_LIST = [ - "build_id", - "test_name", - "shards", - "serial", - "build_target", - "manifest_branch", - "gsi_branch", - "gsi_build_target", - "test_branch", - "test_build_target", -] - - -def HasAttr(attr, **kwargs): - """Returns True if 'attr' is in 'kwargs' as an arg.""" - return True if attr in kwargs and kwargs[attr] else False - - -def GetVersion(branch): - """Returns the API level (integer) for the given branch.""" - branch = str(branch.lower()) - if branch.startswith("git_"): - branch = branch[4:] - if branch.startswith("aosp-"): - branch = branch[5:] - - if "-treble-" in branch: - branch = branch.replace("-treble-", "-") - - if branch.endswith("-dev"): - branch = branch[:-4] - elif branch.endswith("-release"): - branch = branch[:-8] - - if (branch.startswith("o") and branch.endswith( - ("mr1", "m2", "m3", "m4", "m5", "m6"))): - return 8.1 - elif branch.startswith("o"): - return 8.0 - elif branch.startswith("p"): - return 9.0 - elif branch.startswith("gs://"): - if "v8.0" in branch: - return 8.0 - elif "v8.1" in branch: - return 8.1 - elif "v9.0" in branch: - return 9.0 - return 9.0 - - -def EmitFetchCommands(**kwargs): - """Returns a list of common fetch commands. - - This uses a given device branch information and automatically - selects a GSI branch and a test branch. - - Args: - kwargs: keyword argument, contains data about the leased job. - Returns: - list of command string. - bool, True if GSI image is fetched. False otherwise - """ - result = [] - if isinstance(kwargs["build_target"], list): - build_target = kwargs["build_target"][0] - else: - build_target = kwargs["build_target"] - shards = int(kwargs["shards"]) - suite_name, _ = kwargs["test_name"].split("/") - serials = kwargs["serial"] - - if HasAttr("pab_account_id", **kwargs): - pab_account_id = kwargs["pab_account_id"] - else: - pab_account_id = common._DEFAULT_ACCOUNT_ID_INTERNAL - - manifest_branch = kwargs["manifest_branch"] - build_id = kwargs["build_id"] - build_storage_type = pb.BUILD_STORAGE_TYPE_PAB - if HasAttr("build_storage_type", **kwargs): - build_storage_type = int(kwargs["build_storage_type"]) - - if build_storage_type == pb.BUILD_STORAGE_TYPE_PAB: - result.append( - "fetch --type=pab --branch=%s --target=%s --artifact_name=%s-img-%s.zip " - "--build_id=%s --account_id=%s" % - (manifest_branch, build_target, build_target.split("-")[0], - build_id if build_id != "latest" else "{build_id}", build_id, - pab_account_id)) - if HasAttr("require_signed_device_build", **kwargs): - result[-1] += " --fetch_signed_build=True" - if common.UNIVERSAL9810 in build_target: - result[-1] += " --full_device_images=True" - - if HasAttr("has_bootloader_img", **kwargs): - result.append("fetch --type=pab --branch=%s --target=%s " - "--artifact_name=bootloader.img --build_id=%s " - "--account_id=%s" % (manifest_branch, build_target, - build_id, pab_account_id)) - - if HasAttr("has_radio_img", **kwargs): - result.append("fetch --type=pab --branch=%s --target=%s " - "--artifact_name=radio.img --build_id=%s " - "--account_id=%s" % (manifest_branch, build_target, - build_id, pab_account_id)) - - elif build_storage_type == pb.BUILD_STORAGE_TYPE_GCS: - result.append("fetch --type=gcs --path=%s" % (manifest_branch)) - if common.UNIVERSAL9810 in build_target: - result[-1] += " --full_device_images=True" - else: - logging.error("unknown build storage type is given: %d", - build_storage_type) - return None - - if HasAttr("gsi_branch", **kwargs): - gsi = True - else: - gsi = False - - if HasAttr("gsi_vendor_version", **kwargs): - gsi_vendor_version = kwargs["gsi_vendor_version"] - else: - gsi_vendor_version = None - - if gsi: - if common.SDM845 in build_target: - if shards > 1: - sub_commands = [] - if shards <= len(serials): - for shard_index in range(shards): - sub_commands.append( - GenerateSdm845SetupCommands(serials[shard_index])) - result.append(sub_commands) - else: - result.extend(GenerateSdm845SetupCommands(serials[0])) - - if HasAttr("gsi_build_id", **kwargs): - gsi_build_id = kwargs["gsi_build_id"] - else: - gsi_build_id = "latest" - gsi_storage_type = pb.BUILD_STORAGE_TYPE_PAB - if HasAttr("gsi_storage_type", **kwargs): - gsi_storage_type = int(kwargs["gsi_storage_type"]) - - if gsi_storage_type == pb.BUILD_STORAGE_TYPE_PAB: - result.append( - "fetch --type=pab --branch=%s --target=%s --gsi=True " - "--artifact_name=%s-img-{build_id}.zip --build_id=%s" % - (kwargs["gsi_branch"], kwargs["gsi_build_target"], - kwargs["gsi_build_target"].split("-")[0], gsi_build_id)) - elif gsi_storage_type == pb.BUILD_STORAGE_TYPE_GCS: - result.append("fetch --type=gcs --path=%s/%s-img-%s.zip " - "--gsi=True" % - (kwargs["gsi_branch"], - kwargs["gsi_build_target"].split("-")[0], - gsi_build_id)) - else: - logging.error("unknown gsi storage type is given: %d", - gsi_storage_type) - return None - - if HasAttr("gsi_pab_account_id", **kwargs): - result[-1] += " --account_id=%s" % kwargs["gsi_pab_account_id"] - - if HasAttr("test_build_id", **kwargs): - test_build_id = kwargs["test_build_id"] - else: - test_build_id = "latest" - test_storage_type = pb.BUILD_STORAGE_TYPE_PAB - if HasAttr("test_storage_type", **kwargs): - test_storage_type = int(kwargs["test_storage_type"]) - - if test_storage_type == pb.BUILD_STORAGE_TYPE_PAB: - result.append("fetch --type=pab --branch=%s --target=%s " - "--artifact_name=android-%s.zip --build_id=%s" % - (kwargs["test_branch"], kwargs["test_build_target"], - suite_name, test_build_id)) - elif test_storage_type == pb.BUILD_STORAGE_TYPE_GCS: - result.append("fetch --type=gcs --path=%s/%s.zip --set_suite_as=%s" % - (kwargs["test_branch"], kwargs["test_build_target"], - suite_name)) - else: - logging.error("unknown test storage type is given: %d", - test_storage_type) - return None - - if HasAttr("test_pab_account_id", **kwargs): - result[-1] += " --account_id=%s" % kwargs["test_pab_account_id"] - - result.append("info") - if gsi: - gsispl_command = "gsispl --version_from_path=boot.img" - if gsi_vendor_version: - gsispl_command += " --vendor_version=%s" % gsi_vendor_version - result.append(gsispl_command) - result.append("info") - - return result, gsi - - -def EmitFlashCommands(gsi, **kwargs): - """Returns a list of common flash commands. - - This uses a given device branch information and automatically - selects a GSI branch and a test branch. - - Args: - gsi: bool, whether to flash GSI over vendor images or not. - kwargs: keyword argument, contains data about the leased job. - Returns: - list of command string. - """ - result = [] - - if isinstance(kwargs["build_target"], list): - build_target = kwargs["build_target"][0] - else: - build_target = kwargs["build_target"] - shards = int(kwargs["shards"]) - serials = kwargs["serial"] - if gsi: - system_version = GetVersion(kwargs["gsi_branch"]) - else: - system_version = GetVersion(kwargs["manifest_branch"]) - - repack_command = "repack" - if HasAttr("image_package_repo_base", **kwargs): - repack_command += " --dest=%s" % kwargs["image_package_repo_base"] - if common.SDM845 in build_target and gsi: - repack_command += (" --additional_files") - for lib_file in common.SDM845_LIB_LIST: - repack_command += (" {tmp_dir}/%s/%s" % (serials[0], lib_file)) - # TODO: verify this before re-enabling. - # result.append(repack_command) - - if shards > 1: - sub_commands = [] - - if shards <= len(serials): - for shard_index in range(shards): - new_cmd_list = [] - if (common.K39TV1_BSP in build_target - or common.K39TV1_BSP_1G in build_target): - new_cmd_list.extend( - GenerateMt6739GsiFlashingCommands( - serials[shard_index], gsi)) - elif common.SDM845 in build_target and gsi: - new_cmd_list.extend( - GenerateSdm845GsiFlashingCommands( - serials[shard_index])) - elif common.UNIVERSAL9810 in build_target: - new_cmd_list.extend( - GenerateUniversal9810GsiFlashingCommands( - serials[shard_index], gsi)) - else: - new_cmd_list.append( - "flash --current --serial %s --skip-vbmeta=True " % - serials[shard_index]) - new_cmd_list.append("adb -s %s root" % serials[shard_index]) - if common.SDM845 not in build_target: # b/78487061 - new_cmd_list.append( - "dut --operation=wifi_on --serial=%s --ap=%s" % - (serials[shard_index], common._DEFAULT_WIFI_AP)) - new_cmd_list.append( - "dut --operation=volume_mute --serial=%s --version=%s" - % (serials[shard_index], system_version)) - sub_commands.append(new_cmd_list) - result.append(sub_commands) - else: - if (common.K39TV1_BSP in build_target - or common.K39TV1_BSP_1G in build_target): - result.extend(GenerateMt6739GsiFlashingCommands(serials[0], gsi)) - elif common.SDM845 in build_target and gsi: - result.extend(GenerateSdm845GsiFlashingCommands(serials[0])) - elif common.UNIVERSAL9810 in build_target: - result.extend( - GenerateUniversal9810GsiFlashingCommands(serials[0], gsi)) - else: - result.append( - "flash --current --serial %s --skip-vbmeta=True" % serials[0]) - if common.SDM845 not in build_target: # b/78487061 - result.append("dut --operation=wifi_on --serial=%s --ap=%s" % - (serials[0], common._DEFAULT_WIFI_AP)) - result.append( - "dut --operation=volume_mute --serial=%s --version=%s" % - (serials[0], system_version)) - if serials: - serial_arg_list = [] - for serial in serials: - result.append("adb -s %s root" % serial) - serial_arg_list.append("--serial %s" % serial) - - return result - - -def EmitCommonConsoleCommands(**kwargs): - """Runs a common VTS-on-GSI or CTS-on-GSI test. - - This uses a given device branch information and automatically - selects a GSI branch and a test branch. - """ - result = [] - - if not set(_JOB_ATTR_LIST).issubset(kwargs): - missing_keys = [key for key in _JOB_ATTR_LIST if key not in kwargs] - logging.error("Leased job missing attribute(s): {}".format( - ", ".join(missing_keys))) - return None - - if isinstance(kwargs["build_target"], list): - build_target = kwargs["build_target"][0] - else: - build_target = kwargs["build_target"] - shards = int(kwargs["shards"]) - suite_name, plan_name = kwargs["test_name"].split("/") - serials = kwargs["serial"] - - result.append("device --set_serial=%s --from_job_pool --interval=%s" % - (",".join(serials), common.DEFAULT_DEVICE_TIMEOUT_SECS)) - fetch_commands_result, gsi = EmitFetchCommands(**kwargs) - result.extend(fetch_commands_result) - flash_commands_result = EmitFlashCommands(gsi, **kwargs) - result.extend(flash_commands_result) - - param = "" - if HasAttr("param", **kwargs): - param = " ".join(kwargs["param"]) - - test_branch = kwargs["test_branch"] - if (GetVersion(test_branch) >= 9.0 and - (suite_name in ["cts", "gts", "sts"] or plan_name.startswith("cts"))): - shard_option = "--shard-count" - else: - shard_option = "--shards" - - if shards > 1: - test_command = "test --suite %s --keep-result -- %s %s %d %s" % ( - suite_name, plan_name, shard_option, shards, param) - if shards <= len(serials): - for shard_index in range(shards): - test_command += " --serial %s" % serials[shard_index] - result.append(test_command) - else: - if serials: - serial_arg_list = [] - for serial in serials: - serial_arg_list.append("--serial %s" % serial) - result.append("test --suite %s --keep-result -- %s %s %s" % - (suite_name, plan_name, " ".join(serial_arg_list), - param)) - else: - result.append("test --suite %s --keep-result -- %s %s" % - (suite_name, plan_name, param)) - - if "retry_count" in kwargs: - retry_count = int(kwargs["retry_count"]) - result.append( - GenerateRetryCommand(build_target, test_branch, suite_name, - plan_name, serials, retry_count)) - - if HasAttr("test_build_id", **kwargs): - test_build_id = kwargs["test_build_id"] - else: - test_build_id = "latest" - test_storage_type = pb.BUILD_STORAGE_TYPE_PAB - if HasAttr("test_storage_type", **kwargs): - test_storage_type = int(kwargs["test_storage_type"]) - - if HasAttr("report_bucket", **kwargs): - report_buckets = kwargs["report_bucket"] - else: - report_buckets = ["gs://vts-report"] - - upload_dests = [] - upload_commands = [] - for report_bucket in report_buckets: - if test_storage_type == pb.BUILD_STORAGE_TYPE_PAB: - upload_dest = ("%s/{suite_plan}/%s/{branch}/{target}/" - "%s_{build_id}_{timestamp}/" % - (report_bucket, plan_name, build_target)) - elif test_storage_type == pb.BUILD_STORAGE_TYPE_GCS: - upload_dest = ("%s/{suite_plan}/%s/%s/%s/%s_%s_{timestamp}/" % - (report_bucket, plan_name, - kwargs["test_branch"].replace("gs://", "gs_") - if kwargs["test_branch"].startswith("gs://") else - kwargs["test_branch"], kwargs["test_build_target"], - build_target, test_build_id)) - upload_dests.append(upload_dest) - upload_commands.append( - "upload --src={result_full} --dest=%s " - "--report_path=%s/suite_result/{timestamp_year}/{timestamp_month}/" - "{timestamp_day}" % (upload_dest, report_bucket)) - - if HasAttr("report_persistent_url", **kwargs): - for upload_dest in kwargs["report_persistent_url"]: - upload_dests.append(upload_dest) - upload_commands.append("upload --src={result_full} --dest=%s " - "--clear_dest" % upload_dest) - - result.extend(upload_commands) - - if HasAttr("report_reference_url", **kwargs): - ref_urls = kwargs["report_reference_url"] - else: - ref_urls = [] - - extra_rows = " ".join("logs," + x for x in upload_dests) - if HasAttr("report_spreadsheet_id", **kwargs): - for index, sheet_id in enumerate(kwargs["report_spreadsheet_id"]): - sheet_command = ("sheet --src {result_zip} --dest %s " - "--extra_rows %s" % (sheet_id, extra_rows)) - if plan_name == "cts-on-gsi": - sheet_command += " --primary_abi_only" - if index < len(ref_urls): - sheet_command += " --ref " + ref_urls[index] - sheet_command += " --client_secrets DATA/vtslab-gcs.json" - result.append(sheet_command) - - result.append("device --update=stop") - - return result - - -def GenerateRetryCommand(build_target, - test_branch, - suite_name, - plan_name, - serials, - retry_count=None): - """Returns a retry command. - - Args: - build_target: string, build target of the device images - test_branch: string, branch name from which the test suite is fetched - suite_name: string, the name of the test suite - plan_name: string, the name of the test plan that needs to be run - serials: list of strings, serial numbers of the DUTs - retry_count: int, - - Returns: - a string, retry command of the console. - """ - if GetVersion(test_branch) >= 9.0: - retry_option = "" - shard_option = "--shard-count" - if suite_name in ["cts", "gts", "sts"]: - retry_option = "--retry_plan=retry" - elif plan_name.startswith("cts"): - retry_option = "--retry_plan=%s-retry" % plan_name - else: - shard_option = "--shards" - else: - shard_option = "--shards" - retry_option = "" - - if retry_count is None: - retry_count = common.DEFAULT_RETRY_COUNT - retry_command = ("retry --suite %s --count %d %s" % - (suite_name, retry_count, retry_option)) - if serials: - if len(serials) > 1: - retry_command += " %s %d" % (shard_option, len(serials)) - for shard_index in range(len(serials)): - retry_command += " --serial %s" % serials[shard_index] - else: - retry_command += " --serial %s" % serials[0] - if suite_name in ["cts", "gts", "sts"] or plan_name.startswith("cts"): - if common.SDM845 in build_target: - # TODO(vtslab-dev): remove after b/77664643 is resolved - pass - else: - retry_command += " --cleanup_devices=True" - - return retry_command - - -def GenerateSdm845SetupCommands(serial): - """Returns a sequence of console commands to setup a device. - - Args: - serial: string, the target device serial number. - - Returns: - a list of strings, each string is a console command. - """ - result = [] - - result.append( - "fastboot -s %s flash bootloader {device-image[bootloader.img]}" % - serial) - result.append("fastboot -s %s -- reboot bootloader" % serial) - result.append( - "fastboot -s %s flash radio {device-image[radio.img]}" % serial) - result.append("fastboot -s %s -- reboot bootloader" % serial) - result.append( - "fastboot -s %s flash boot {device-image[full-zipfile-dir]}/boot.img" % - serial) - result.append( - "fastboot -s %s flash dtbo {device-image[full-zipfile-dir]}/dtbo.img" % - serial) - result.append( - "fastboot --timeout=900 -s %s flash system {device-image[full-zipfile-dir]}/system.img" - % serial) - result.append( - "fastboot -s %s flash userdata {device-image[full-zipfile-dir]}/userdata.img" - % serial) - result.append( - "fastboot -s %s flash vbmeta {device-image[full-zipfile-dir]}/vbmeta.img" - " -- --disable-verity" % serial) - result.append( - "fastboot -s %s flash vendor {device-image[full-zipfile-dir]}/vendor.img" - % serial) - result.append("fastboot -s %s reboot" % serial) - result.append("sleep 90") # wait for boot_complete (success) - result.append("adb -s %s root" % serial) - # TODO: to make sure {tmp_dir} is unique per session and - # is cleaned up at exit. - result.append("shell -- mkdir -p {tmp_dir}/%s" % serial) - result.extend([ - "adb -s %s pull /system/lib64/%s {tmp_dir}/%s" % (serial, lib_file, - serial) - for lib_file in common.SDM845_LIB_LIST - ]) - - # TODO: remove this paragraph after b/74552817 is fixed. - result.append( - "fetch --type=gcs --path=gs://vts-release/v9.0/sdm845/vbmeta.img " - "--artifact_name=vbmeta.img") - result.append("adb -s %s reboot bootloader" % serial) - result.append("fastboot -s %s flash vbmeta {device-image[vbmeta.img]}" - " -- --disable-verity" % serial) - - return result - - -def GenerateSdm845GsiFlashingCommands(serial, repacked_imageset=False): - """Returns a sequence of console commands to flash GSI to a device. - - Args: - serial: string, the target device serial number. - repacked_imageset: bool, True if this function is called directly from - the console, adjusts the resulting commands for - atomic flashing process. - - Returns: - a list of strings, each string is a console command. - """ - result = [] - - if repacked_imageset: - result.append( - "fastboot -s %s flash bootloader {device-image[bootloader.img]}" % - serial) - result.append("fastboot -s %s -- reboot bootloader" % serial) - result.append( - "fastboot -s %s flash radio {device-image[radio.img]}" % serial) - result.append("fastboot -s %s -- reboot bootloader" % serial) - result.append( - "fastboot -s %s flash boot {device-image[boot.img]}" % serial) - result.append( - "fastboot -s %s flash dtbo {device-image[dtbo.img]}" % serial) - result.append( - "fastboot -s %s flash userdata {device-image[userdata.img]}" % - serial) - result.append( - "fastboot -s %s flash vbmeta {device-image[vbmeta.img]} -- --disable-verity" - % serial) - result.append( - "fastboot -s %s flash vendor {device-image[vendor.img]}" % serial) - - result.append( - "fastboot --timeout=900 -s %s flash system {device-image[system.img]}" - % serial) - # removed -w from below command - result.append("fastboot -s %s -- reboot" % serial) - result.append("sleep 90") # wait until adb shell (not boot complete) - result.append("adb -s %s root" % serial) - result.append("adb -s %s remount" % serial) - result.append("adb -s %s shell setenforce 0" % serial) - result.append("adb -s %s shell mkdir /bt_firmware" % serial) - result.append("adb -s %s shell chown system:system /bt_firmware" % serial) - result.append("adb -s %s shell chmod 650 /bt_firmware" % serial) - result.append("adb -s %s shell setenforce 1" % serial) - if repacked_imageset: - result.extend([ - "adb -s %s push {tools[%s/%s]} /system/lib64" % - (serial, common._ADDITIONAL_FILES_DIR, lib_file) - for lib_file in common.SDM845_LIB_LIST - ]) - else: - result.extend([ - "adb -s %s push {tmp_dir}/%s/%s /system/lib64" % (serial, serial, - lib_file) - for lib_file in common.SDM845_LIB_LIST - ]) - result.extend( - [("adb -s %s push ../testcases/DATA/xml/media_profiles_vendor.xml " - "/vendor/etc/media_profiles_vendor.xml") % serial]) - - result.append("shell -- rm {tmp_dir}/%s -rf" % serial) - result.append("adb -s %s reboot bootloader" % serial) - result.append("sleep 5") - # TODO(vtslab-dev): remove after b/112171990 is resolved - result.append("fastboot -s %s erase userdata" % serial) - # removed -w from below command - result.append("fastboot -s %s -- reboot" % serial) - if not repacked_imageset: - result.append("sleep 300") # wait for boot_complete (success) - - return result - - -def GenerateMt6739GsiFlashingCommands(serial, - gsi=False, - repacked_imageset=False): - """Returns a sequence of console commands to flash device imgs and GSI. - - Args: - serial: string, the target device serial number. - gsi: bool, whether to flash GSI over vendor images or not. - repacked_imageset: bool, True if this func is called directly from - the console, adjusts the resulting commands for - atomic flashing process. - - Returns: - a list of strings, each string is a console command. - """ - flash_img_cmd = ("fastboot -s %s flash %s " - "{device-image[full-zipfile-dir]}/%s") - flash_gsi_cmd = ("fastboot --timeout=900 -s %s flash system " - "{device-image[system.img]}") - result = [ - flash_img_cmd % (serial, partition, image) - for partition, image in ( - ("preloader", "preloader_SBOOT_DIS.img"), - ("loader_ext1", "loader_ext.img"), - ("loader_ext2", "loader_ext.img"), - ("tee1", "tee.img"), - ("tee2", "tee.img"), - ("lk", "lk.img"), - ("lk2", "lk.img"), - ) - ] - result.append("fastboot -s %s -- reboot bootloader" % serial) - # gpt is the partition table and must be flashed first. - # The bootloader reloads partition table automatically after flashing gpt. - result += [ - flash_img_cmd % (serial, partition, image) - for partition, image in ( - ("gpt", "PGPT"), - ("md1img", "md1img.img"), - ("md1dsp", "md1dsp.img"), - ("recovery", "recovery.img"), - ("spmfw", "spmfw.img"), - ("mcupmfw", "mcupmfw.img"), - ("boot", "boot.img"), - ("dtbo", "dtbo.img"), - ("vendor", "vendor.img"), - ("cache", "cache.img"), - ("userdata", "userdata.img"), - ) - ] - - if gsi: - result.append(flash_gsi_cmd % serial) - result.append("fastboot -s %s -- -w" % serial) - else: - flash_img_cmd += " --timeout=900" - result.append(flash_img_cmd % (serial, "system", "system.img")) - - result.append("fastboot -s %s reboot" % serial) - if not repacked_imageset: - result.append("sleep 300") # wait for boot_complete (success) - - return result - - -def GenerateUniversal9810GsiFlashingCommands(serial, - gsi=False, - repacked_imageset=False): - """Returns a sequence of console commands to flash device imgs and GSI. - - Args: - serial: string, the target device serial number. - gsi: bool, whether to flash GSI over vendor images or not. - repacked_imageset: bool, True if this func is called directly from - the console, adjusts the resulting commands for - atomic flashing process. - - Returns: - a list of strings, each string is a console command. - """ - result = [ - ("fastboot -s %s flash el3_mon " - "{device-image[full-zipfile-dir]}/el3_mon.img" % serial), - ("fastboot -s %s flash epbl " - "{device-image[full-zipfile-dir]}/epbl.img" % serial), - ("fastboot -s %s flash bootloader " - "{device-image[full-zipfile-dir]}/u-boot.img" % serial), - ("fastboot -s %s flash dtb " - "{device-image[full-zipfile-dir]}/dtb.img" % serial), - ("fastboot -s %s flash dtbo " - "{device-image[full-zipfile-dir]}/dtbo.img" % serial), - ("fastboot -s %s flash kernel " - "{device-image[full-zipfile-dir]}/kernel.img" % serial), - ("fastboot -s %s flash ramdisk " - "{device-image[full-zipfile-dir]}/ramdisk.img" % serial), - ("fastboot -s %s flash vendor " - "{device-image[full-zipfile-dir]}/vendor.img -- -S 300M" % serial), - ] - if gsi: - result.append(("fastboot --timeout=900 -s %s flash system " - "{device-image[system.img]} -- -S 512M" % serial)) - else: - result.append(( - "fastboot --timeout=900 -s %s flash system " - "{device-image[full-zipfile-dir]}/system.img -- -S 512M" % serial)) - result.append("fastboot -s %s reboot -- -w" % serial) - if not repacked_imageset: - result.append("sleep 300") # wait for boot_complete (success) - - return result - - -FLASH_COMMAND_EMITTER = { - common.K39TV1_BSP: GenerateMt6739GsiFlashingCommands, - common.K39TV1_BSP_1G: GenerateMt6739GsiFlashingCommands, - common.SDM845: GenerateSdm845GsiFlashingCommands, - common.UNIVERSAL9810: GenerateUniversal9810GsiFlashingCommands, -} diff --git a/harnesses/host_controller/campaigns/campaign_test.py b/harnesses/host_controller/campaigns/campaign_test.py deleted file mode 100644 index ddb0c54..0000000 --- a/harnesses/host_controller/campaigns/campaign_test.py +++ /dev/null @@ -1,75 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest - -from host_controller.campaigns import cts -from host_controller.campaigns import gts -from host_controller.campaigns import sts -from host_controller.campaigns import vts - -from host_controller.campaigns.testdata import default_testcase - - -class CampaignTest(unittest.TestCase): - """Unit tests for the campaign generators.""" - - def setUp(self): - self.maxDiff = None - - def testVtsBaseline(self): - """Tests the default device's vts scenario.""" - test_name = "vts/vts" - results = vts.EmitConsoleCommands( - **default_testcase.GenerateInputData(test_name)) - self.assertEqual( - default_testcase.GenerateOutputData(test_name), results) - - def testCtsOnGsiBaseline(self): - """Tests the default device's vts scenario.""" - test_name = "vts/cts-on-gsi" - results = vts.EmitConsoleCommands( - **default_testcase.GenerateInputData(test_name)) - self.assertEqual( - default_testcase.GenerateOutputData(test_name), results) - - def testCtsBaseline(self): - """Tests the default device's vts scenario.""" - test_name = "cts/cts" - results = cts.EmitConsoleCommands( - **default_testcase.GenerateInputData(test_name)) - self.assertEqual( - default_testcase.GenerateOutputData(test_name), results) - - def testGtsBaseline(self): - """Tests the default device's vts scenario.""" - test_name = "gts/gts" - results = gts.EmitConsoleCommands( - **default_testcase.GenerateInputData(test_name)) - self.assertEqual( - default_testcase.GenerateOutputData(test_name), results) - - def testStsBaseline(self): - """Tests the default device's vts scenario.""" - test_name = "sts/sts" - results = sts.EmitConsoleCommands( - **default_testcase.GenerateInputData(test_name)) - self.assertEqual( - default_testcase.GenerateOutputData(test_name), results) - - -if __name__ == '__main__': - unittest.main() diff --git a/harnesses/host_controller/campaigns/cts.py b/harnesses/host_controller/campaigns/cts.py deleted file mode 100644 index 28aa22d..0000000 --- a/harnesses/host_controller/campaigns/cts.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from host_controller.campaigns import campaign_common - - -def EmitConsoleCommands(**kwargs): - """Runs a common CTS test. - - This uses a given device branch information and automatically - selects a GSI branch and a test branch. - """ - return campaign_common.EmitCommonConsoleCommands(**kwargs) diff --git a/harnesses/host_controller/campaigns/gts.py b/harnesses/host_controller/campaigns/gts.py deleted file mode 100644 index fec0eba..0000000 --- a/harnesses/host_controller/campaigns/gts.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from host_controller.campaigns import campaign_common - - -def EmitConsoleCommands(**kwargs): - """Runs a common GTS test. - - This uses a given device branch information and automatically - selects a GSI branch and a test branch. - """ - return campaign_common.EmitCommonConsoleCommands(**kwargs) diff --git a/harnesses/host_controller/campaigns/sts.py b/harnesses/host_controller/campaigns/sts.py deleted file mode 100644 index 4850368..0000000 --- a/harnesses/host_controller/campaigns/sts.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from host_controller.campaigns import campaign_common - - -def EmitConsoleCommands(**kwargs): - """Runs a common STS test. - - This uses a given device branch information and automatically - selects a GSI branch and a test branch. - """ - return campaign_common.EmitCommonConsoleCommands(**kwargs) diff --git a/harnesses/host_controller/campaigns/testdata/__init__.py b/harnesses/host_controller/campaigns/testdata/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/campaigns/testdata/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/campaigns/testdata/default_testcase.py b/harnesses/host_controller/campaigns/testdata/default_testcase.py deleted file mode 100644 index 9aeb91e..0000000 --- a/harnesses/host_controller/campaigns/testdata/default_testcase.py +++ /dev/null @@ -1,141 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy - -# Based on JobModel defined in -# test/vti/test_serving/gae/webapp/src/proto/model.py -input_data = { - "test_type": 1, - "hostname": "my_hostname", - "priority": "low", - "test_name": "vts/vts", - "require_signed_device_build": True, - "has_bootloader_img": True, - "has_radio_img": True, - "device": "my_device", - "serial": ["my_serial1", "my_serial2", "my_serial3"], - - # device image information - "build_storage_type": 1, - "manifest_branch": "my_branch", - "build_target": "my_build_target", - "build_id": "my_build_id", - "pab_account_id": "my_pab_account_id", - "shards": 3, - "param": "", - "status": 1, - "period": 24 * 60, # 1 day - - # GSI information - "gsi_storage_type": 1, - "gsi_branch": "my_gsi_branch", - "gsi_build_target": "my_gsi_build_target", - "gsi_build_id": "my_gsi_build_id", - "gsi_pab_account_id": "my_gsi_pab_account_id", - # gsi_vendor_version: "8.1.0" - - # test suite information - "test_storage_type": 1, - "test_branch": "my_test_branch", - "test_build_target": "my_test_build_target", - "test_build_id": "my_test_build_id", - "test_pab_account_id": "my_test_pab_account_id", - - #timestamp = ndb.DateTimeProperty(auto_now=False) - #heartbeat_stamp = ndb.DateTimeProperty(auto_now=False) - "retry_count": 3, - "infra_log_url": "infra_log_url", - - #parent_schedule = ndb.KeyProperty(kind="ScheduleModel") - "image_package_repo_base": "image_package_repo_base", - "report_bucket": ["report_bucket"], - "report_spreadsheet_id": ["report_spreadsheet_id"], -} - -expected_output = [ - 'device --set_serial=my_serial1,my_serial2,my_serial3 --from_job_pool --interval=300', - 'fetch --type=pab --branch=my_branch --target=my_build_target --artifact_name=my_build_target-img-my_build_id.zip --build_id=my_build_id --account_id=my_pab_account_id --fetch_signed_build=True', - 'fetch --type=pab --branch=my_branch --target=my_build_target --artifact_name=bootloader.img --build_id=my_build_id --account_id=my_pab_account_id', - 'fetch --type=pab --branch=my_branch --target=my_build_target --artifact_name=radio.img --build_id=my_build_id --account_id=my_pab_account_id', - 'fetch --type=pab --branch=my_gsi_branch --target=my_gsi_build_target --gsi=True --artifact_name=my_gsi_build_target-img-{build_id}.zip --build_id=my_gsi_build_id --account_id=my_gsi_pab_account_id', - 'fetch --type=pab --branch=my_test_branch --target=my_test_build_target --artifact_name=android-{{test_suite}}.zip --build_id=my_test_build_id --account_id=my_test_pab_account_id', - 'info', 'gsispl --version_from_path=boot.img', 'info', - [[ - 'flash --current --serial my_serial1 --skip-vbmeta=True ', - 'adb -s my_serial1 root', - 'dut --operation=wifi_on --serial=my_serial1 --ap=GoogleGuest', - 'dut --operation=volume_mute --serial=my_serial1 --version=9.0' - ], [ - 'flash --current --serial my_serial2 --skip-vbmeta=True ', - 'adb -s my_serial2 root', - 'dut --operation=wifi_on --serial=my_serial2 --ap=GoogleGuest', - 'dut --operation=volume_mute --serial=my_serial2 --version=9.0' - ], [ - 'flash --current --serial my_serial3 --skip-vbmeta=True ', - 'adb -s my_serial3 root', - 'dut --operation=wifi_on --serial=my_serial3 --ap=GoogleGuest', - 'dut --operation=volume_mute --serial=my_serial3 --version=9.0' - ]], - 'test --suite {{test_suite}} --keep-result -- {{test_plan}} --shards 3 --serial my_serial1 --serial my_serial2 --serial my_serial3', - 'retry --suite {{test_suite}} --count 3 {{retry_plan}} --shards 3 --serial my_serial1 --serial my_serial2 --serial my_serial3{{cleanup_device}}', - 'upload --src={result_full} --dest=report_bucket/{suite_plan}/{{test_plan}}/{branch}/{target}/my_build_target_{build_id}_{timestamp}/ --report_path=report_bucket/suite_result/{timestamp_year}/{timestamp_month}/{timestamp_day}', - 'sheet --src {result_zip} --dest report_spreadsheet_id --extra_rows logs,report_bucket/{suite_plan}/{{test_plan}}/{branch}/{target}/my_build_target_{build_id}_{timestamp}/ --primary_abi_only --client_secrets DATA/vtslab-gcs.json', - 'device --update=stop', -] - - -def GenerateInputData(test_name): - """Returns an input data dict for a given `test_name`.""" - new_data = copy.copy(input_data) - new_data["test_name"] = test_name - return new_data - - -def GenerateOutputData(test_name): - """Returns an output data list for a given `test_name`.""" - test_suite, test_plan = test_name.split("/") - - def ReplaceChars(line): - line = line.replace('{{test_suite}}', test_suite) - line = line.replace('{{test_plan}}', test_plan) - if test_plan != "cts-on-gsi": - line = line.replace(' --primary_abi_only', '') - if (test_suite == "cts" or test_suite == "gts" or test_suite == "sts" - or test_plan.startswith("cts-")): - line = line.replace('--shards', "--shard-count") - if test_suite == "vts": - line = line.replace('{{retry_plan}}', - '--retry_plan=%s-retry' % test_plan) - else: - line = line.replace('{{retry_plan}}', '--retry_plan=retry') - line = line.replace('{{cleanup_device}}', - ' --cleanup_devices=True') - else: - line = line.replace('{{retry_plan}}', '') - line = line.replace('{{cleanup_device}}', '') - return line - - def RecursivelyApply(input_list, func): - for number, item in enumerate(input_list): - if type(item) is list: - input_list[number] = RecursivelyApply(input_list[number], func) - elif type(item) is str: - input_list[number] = func(item) - else: - return None - return input_list - - return RecursivelyApply(copy.copy(expected_output), ReplaceChars) diff --git a/harnesses/host_controller/campaigns/vts.py b/harnesses/host_controller/campaigns/vts.py deleted file mode 100644 index 0287455..0000000 --- a/harnesses/host_controller/campaigns/vts.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from host_controller.campaigns import campaign_common - - -def EmitConsoleCommands(**kwargs): - """Runs a common VTS-on-GSI or CTS-on-GSI test. - - This uses a given device branch information and automatically - selects a GSI branch and a test branch. - """ - return campaign_common.EmitCommonConsoleCommands(**kwargs) diff --git a/harnesses/host_controller/command_processor/__init__.py b/harnesses/host_controller/command_processor/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/command_processor/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/command_processor/base_command_processor.py b/harnesses/host_controller/command_processor/base_command_processor.py deleted file mode 100644 index fa53f74..0000000 --- a/harnesses/host_controller/command_processor/base_command_processor.py +++ /dev/null @@ -1,135 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -import logging -import re - -from host_controller import console_argument_parser - -# tmp_dir variable name. -TMP_DIR_VAR ="{tmp_dir}" - - -class BaseCommandProcessor(object): - '''Base class for command processors. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - arg_buffer: dict, stores last parsed argument, value pairs. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - ''' - - command = 'base' - command_detail = 'Command processor template' - - def _SetUp(self, console): - '''Internal SetUp function that will call subclass' Setup function. - - Args: - console: Console object. - ''' - self.console = console - self.arg_buffer = {} - self.arg_parser = console_argument_parser.ConsoleArgumentParser( - self.command, self.command_detail) - self.SetUp() - - def SetUp(self): - '''SetUp method for subclass to override.''' - pass - - def _Run(self, arg_line): - '''Internal function that will call subclass' Run function. - - Args: - arg_line: string, line of command arguments - ''' - ret = self.Run(arg_line) - args = self.arg_parser.ParseLine(arg_line) - for arg_tuple in args._get_kwargs(): - key = arg_tuple[0] - value = arg_tuple[1] - self.arg_buffer[key] = value - - if ret is not None: - if ret == True: # exit command executed. - return True - elif ret == False: - return False - else: - logging.warning("{} coommand returned {}".format( - self.command, ret)) - - def Run(self, arg_line): - '''Run method to perform action when invoked from console. - - Args: - arg_line: string, line of command - ''' - pass - - def _Help(self): - '''Internal function that will call subclass' Help function.''' - self.Help() - - def Help(self): - '''Help method to print help informations.''' - if hasattr(self, 'arg_parser') and hasattr(self.console, '_out_file'): - self.arg_parser.print_help(self.console._out_file) - - def _TearDown(self): - '''Internal function that will call subclass' TearDown function.''' - self.TearDown() - - def TearDown(self): - '''TearDown tasks to be called when console is shutting down.''' - pass - - def ReplaceVars(self, message_list): - """Replaces vars in a 'messsag_list' to their values.""" - new_message_list = [] - for message in message_list: - new_message = message - - vars = re.findall(r"{device-image\[[^]]+\]}", message) - if vars: - for var in vars: - var_name = var[len("{device-image")+1:-2] - if var_name and var_name in self.console.device_image_info: - new_message = new_message.replace( - var, self.console.device_image_info[var_name]) - else: - new_message = new_message.replace(var, "{undefined}") - - vars = re.findall(r"{tools\[[^]]+\]}", message) - if vars: - for var in vars: - var_name = var[len("{tools")+1:-2] - if var_name and var_name in self.console.tools_info: - new_message = new_message.replace( - var, self.console.tools_info[var_name]) - else: - new_message = new_message.replace(var, "{undefined}") - - if TMP_DIR_VAR in new_message: - new_message = new_message.replace( - TMP_DIR_VAR, self.console.tmpdir_default) - - new_message_list.append(new_message) - return new_message_list diff --git a/harnesses/host_controller/command_processor/command_acloud.py b/harnesses/host_controller/command_processor/command_acloud.py deleted file mode 100644 index 42b377f..0000000 --- a/harnesses/host_controller/command_processor/command_acloud.py +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging - -from host_controller.acloud import acloud_client -from host_controller.command_processor import base_command_processor - - -class CommandAcloud(base_command_processor.BaseCommandProcessor): - '''Command processor for acloud command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - ''' - - command = 'acloud' - command_detail = 'Create acloud instances.' - - def Run(self, arg_line): - '''Creates an acloud instance and connects to it via adb. - - Args: - arg_line: string, line of command arguments - ''' - args = self.arg_parser.ParseLine(arg_line) - - if args.provider == "ab": - if args.build_id.lower() == "latest": - build_id = self.console._build_provider["ab"].GetLatestBuildId( - args.branch, - args.target) - else: - # TODO(yuexima): support more provider types. - logging.error("Provider %s not supported yet." % args.provider) - return - - ac = acloud_client.ACloudClient() - ac.PrepareConfig(args.config_path) - ac.CreateInstance(args.build_id) - ac.ConnectInstanceToAdb(ah.GetInstanceIP()) - - def SetUp(self): - """Initializes the parser for acloud command.""" - self.arg_parser.add_argument( - "--build_id", - help="Build ID to use.") - self.arg_parser.add_argument( - "--provider", - default="ab", - choices=("local_fs", "gcs", "pab", "ab"), - help="Build provider type") - self.arg_parser.add_argument( - "--branch", # not required for local_fs - help="Branch to grab the artifact from.") - self.arg_parser.add_argument( - "--target", # not required for local_fs - help="Target product to grab the artifact from.") - self.arg_parser.add_argument( - "--config_path", - required=True, - help="Acloud config path.") diff --git a/harnesses/host_controller/command_processor/command_adb.py b/harnesses/host_controller/command_processor/command_adb.py deleted file mode 100644 index 1738820..0000000 --- a/harnesses/host_controller/command_processor/command_adb.py +++ /dev/null @@ -1,86 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging - -from host_controller import common -from host_controller.command_processor import base_command_processor -from host_controller.utils.usb import usb_utils - -from vts.utils.python.common import cmd_utils - - -class CommandAdb(base_command_processor.BaseCommandProcessor): - """Command processor for adb command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "adb" - command_detail = "Runs an ADB command." - - # @Override - def SetUp(self): - """Initializes the parser for device command.""" - self.arg_parser.add_argument( - "--serial", - "-s", - default=None, - help="The target device serial to run the command.") - self.arg_parser.add_argument( - "--timeout", - type=float, - default=common.DEFAULT_DEVICE_TIMEOUT_SECS, - help="The maximum timeout value of this command in seconds. " - "Set to 0 to disable the timeout functionality.") - self.arg_parser.add_argument( - "command", - metavar="COMMAND", - nargs="+", - help="The command to be executed. If the command contains " - "arguments starting with \"-\", place the command at end of line " - "after \"--\".") - - # @Override - def Run(self, arg_line): - """Runs an adb command.""" - args = self.arg_parser.ParseLine(arg_line) - cmd_list = ["adb"] - if args.serial: - if "," in args.serial: - logging.error("Only one serial can be specified") - return False - cmd_list.append("-s %s" % args.serial) - cmd_list.extend(self.ReplaceVars(args.command)) - if args.timeout == 0: - stdout, stderr, retcode = cmd_utils.ExecuteOneShellCommand( - " ".join(cmd_list)) - else: - stdout, stderr, retcode = cmd_utils.ExecuteOneShellCommand( - " ".join(cmd_list), args.timeout, - usb_utils.ResetUsbDeviceOfSerial_Callback, args.serial) - if stdout: - logging.info(stdout) - if stderr: - logging.error(stderr) - if self.console.job_pool and args.serial: - self.console.device_status[ - args.serial] = common._DEVICE_STATUS_DICT["error"] - if retcode != 0: - return False diff --git a/harnesses/host_controller/command_processor/command_build.py b/harnesses/host_controller/command_processor/command_build.py deleted file mode 100644 index b1479d8..0000000 --- a/harnesses/host_controller/command_processor/command_build.py +++ /dev/null @@ -1,249 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import httplib2 -import logging -import socket -import threading -import time - -from googleapiclient import errors - -from host_controller import common -from host_controller.command_processor import base_command_processor -from host_controller.console_argument_parser import ConsoleArgumentError -from host_controller.tradefed import remote_operation - - -class CommandBuild(base_command_processor.BaseCommandProcessor): - """Command processor for build command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - build_thread: dict containing threading.Thread instances(s) that - update build info regularly. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "build" - command_detail = "Specifies branches and targets to monitor." - - def UpdateBuild(self, account_id, branch, targets, artifact_type, method, - userinfo_file, noauth_local_webserver, verify_signed): - """Updates the build state. - - Args: - account_id: string, Partner Android Build account_id to use. - branch: string, branch to grab the artifact from. - targets: string, a comma-separate list of build target product(s). - artifact_type: string, artifact type (`device`, 'gsi' or `test'). - method: string, method for getting build information. - userinfo_file: string, the path of a file containing email and - password (if method == POST). - noauth_local_webserver: boolean, True to not use a local websever. - verify_signed: A Boolean indicating whether to verify signed build. - """ - builds = [] - - self.console._build_provider["pab"].Authenticate( - userinfo_file=userinfo_file, - noauth_local_webserver=noauth_local_webserver) - for target in targets.split(","): - try: - listed_builds = self.console._build_provider["pab"].GetBuildList( - account_id=account_id, - branch=branch, - target=target, - page_token="", - max_results=100, - method=method, - verify_signed=verify_signed) - except ValueError as e: - logging.exception(e) - continue - - for listed_build in listed_builds: - if method == "GET": - if "successful" in listed_build: - if listed_build["successful"]: - build = {} - build["manifest_branch"] = branch - build["build_id"] = listed_build["build_id"] - if "-" in target: - build["build_target"], build[ - "build_type"] = target.split("-") - else: - build["build_target"] = target - build["build_type"] = "" - build["artifact_type"] = artifact_type - build["artifacts"] = [] - if "signed" in listed_build: - build["signed"] = listed_build["signed"] - else: - build["signed"] = False - builds.append(build) - else: - logging.error("Error: listed_build %s", listed_build) - else: # POST - build = {} - build["manifest_branch"] = branch - build["build_id"] = listed_build[u"1"] - if "-" in target: - (build["build_target"], - build["build_type"]) = target.split("-") - else: - build["build_target"] = target - build["build_type"] = "" - build["artifact_type"] = artifact_type - build["artifacts"] = [] - build["signed"] = False - builds.append(build) - self.console._vti_endpoint_client.UploadBuildInfo(builds) - - def UpdateBuildLoop(self, account_id, branch, target, artifact_type, - method, userinfo_file, noauth_local_webserver, - update_interval, verify_signed): - """Regularly updates the build information. - - Args: - account_id: string, Partner Android Build account_id to use. - branch: string, branch to grab the artifact from. - targets: string, a comma-separate list of build target product(s). - artifact_type: string, artifcat type (`device`, 'gsi' or `test). - method: string, method for getting build information. - userinfo_file: string, the path of a file containing email and - password (if method == POST). - noauth_local_webserver: boolean, True to not use a local websever. - update_interval: int, number of seconds before repeating - """ - thread = threading.currentThread() - while getattr(thread, 'keep_running', True): - try: - self.UpdateBuild(account_id, branch, target, artifact_type, - method, userinfo_file, noauth_local_webserver, - verify_signed) - except (socket.error, remote_operation.RemoteOperationException, - httplib2.HttpLib2Error, errors.HttpError) as e: - logging.exception(e) - time.sleep(update_interval) - - # @Override - def SetUp(self): - """Initializes the parser for build command.""" - self.build_thread = {} - self.arg_parser.add_argument( - "--update", - choices=("single", "start", "stop", "list"), - default="start", - help="Update build info") - self.arg_parser.add_argument( - "--id", - default=None, - help="session ID only required for 'stop' update command") - self.arg_parser.add_argument( - "--interval", - type=int, - default=30, - help="Interval (seconds) to repeat build update.") - self.arg_parser.add_argument( - "--artifact-type", - choices=("device", "gsi", "test"), - default="device", - help="The type of an artifact to update") - self.arg_parser.add_argument( - "--branch", - required=True, - help="Branch to grab the artifact from.") - self.arg_parser.add_argument( - "--target", - required=True, - help="a comma-separate list of build target product(s).") - self.arg_parser.add_argument( - "--account_id", - default=common._DEFAULT_ACCOUNT_ID, - help="Partner Android Build account_id to use.") - self.arg_parser.add_argument( - "--method", - default="GET", - choices=("GET", "POST"), - help="Method for getting build information") - self.arg_parser.add_argument( - "--userinfo-file", - help= - "Location of file containing email and password, if using POST.") - self.arg_parser.add_argument( - "--noauth_local_webserver", - default=False, - type=bool, - help="True to not use a local webserver for authentication.") - self.arg_parser.add_argument( - "--verify-signed-build", - default=False, - type=bool, - help="True to verify whether the build is signed.") - # @Override - def Run(self, arg_line): - """Updates build info.""" - args = self.arg_parser.ParseLine(arg_line) - if args.update == "single": - self.UpdateBuild(args.account_id, args.branch, args.target, - args.artifact_type, args.method, - args.userinfo_file, args.noauth_local_webserver, - args.verify_signed_build) - elif args.update == "list": - logging.info("Running build update sessions:") - for id in self.build_thread: - logging.info(" ID %d", id) - elif args.update == "start": - if args.interval <= 0: - raise ConsoleArgumentError("update interval must be positive") - # do not allow user to create new - # thread if one is currently running - if args.id is None: - if not self.build_thread: - args.id = 1 - else: - args.id = max(self.build_thread) + 1 - else: - args.id = int(args.id) - if args.id in self.build_thread and not hasattr( - self.build_thread[args.id], 'keep_running'): - logging.warning( - 'build update (session ID: %s) already running. ' - 'run build --update stop first.', args.id) - return - self.build_thread[args.id] = threading.Thread( - target=self.UpdateBuildLoop, - args=( - args.account_id, - args.branch, - args.target, - args.artifact_type, - args.method, - args.userinfo_file, - args.noauth_local_webserver, - args.interval, - args.verify_signed_build, - )) - self.build_thread[args.id].daemon = True - self.build_thread[args.id].start() - elif args.update == "stop": - if args.id is None: - logging.error("--id must be set for stop") - else: - self.build_thread[int(args.id)].keep_running = False diff --git a/harnesses/host_controller/command_processor/command_config.py b/harnesses/host_controller/command_processor/command_config.py deleted file mode 100644 index a8b1a24..0000000 --- a/harnesses/host_controller/command_processor/command_config.py +++ /dev/null @@ -1,440 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import httplib2 -import itertools -import logging -import os -import socket -import threading -import time - -from googleapiclient import errors -from google.protobuf import text_format - -from host_controller import common -from host_controller.command_processor import base_command_processor -from host_controller.console_argument_parser import ConsoleArgumentError -from host_controller.tradefed import remote_operation - -from vti.test_serving.proto import TestLabConfigMessage_pb2 as LabCfgMsg -from vti.test_serving.proto import TestScheduleConfigMessage_pb2 as SchedCfgMsg - - -class CommandConfig(base_command_processor.BaseCommandProcessor): - """Command processor for config command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - schedule_thread: dict containing threading.Thread instances(s) that - update schedule info regularly. - """ - - command = "config" - command_detail = "Specifies a global config type to monitor." - - def UpdateConfig(self, account_id, branch, targets, config_type, method, - update_build, clear_schedule, clear_labinfo): - """Updates the global configuration data. - - Args: - account_id: string, Partner Android Build account_id to use. - branch: string, branch to grab the artifact from. - targets: string, a comma-separate list of build target product(s). - config_type: string, config type (`prod` or `test'). - method: string, HTTP method for fetching. - update_build: boolean, indicating whether to upload build info. - clear_schedule: bool, True to clear all schedule data exist on the - scheduler - clear_labinfo: bool, True to clear all lab data exist on the - scheduler - """ - for target in targets.split(","): - fetch_path = self.FetchConfig( - account_id=account_id, - branch=branch, - target=target, - config_type=config_type, - method=method) - if fetch_path: - self.UploadConfig( - path=fetch_path, - update_build=update_build, - clear_schedule=clear_schedule, - clear_labinfo=clear_labinfo) - - def FetchConfig(self, account_id, branch, target, config_type, method): - """Fetches config files from the PAB build provider. - - Args: - account_id: string, Partner Android Build account_id to use. - branch: string, branch to grab the artifact from. - target: string, build target. - config_type: string, config type (`prod` or `test'). - method: string, HTTP method for fetching. - - Returns: - string, a path to the temp directory where config files are stored. - """ - path = "" - self.console._build_provider["pab"].Authenticate() - try: - listed_builds = self.console._build_provider["pab"].GetBuildList( - account_id=account_id, - branch=branch, - target=target, - page_token="", - max_results=1, - method="GET") - except ValueError as e: - logging.exception(e) - return path - - if listed_builds and len(listed_builds) > 0: - listed_build = listed_builds[0] - if listed_build["successful"]: - device_images, test_suites, artifacts, configs = ( - self.console._build_provider["pab"].GetArtifact( - account_id=account_id, - branch=branch, - target=target, - artifact_name=( - "vti-global-config-%s.zip" % config_type), - build_id=listed_build["build_id"], - method=method)) - path = os.path.dirname(configs[config_type]) - - return path - - def UploadConfig(self, path, update_build, clear_schedule, clear_labinfo): - """Uploads configs to VTI server. - - Args: - path: string, a path where config files are stored. - update_build: boolean, indicating whether to upload build info. - clear_schedule: bool, True to clear all schedule data exist on the - scheduler - clear_labinfo: bool, True to clear all lab data exist on the - scheduler - """ - schedules_pbs = [] - lab_pbs = [] - for root, dirs, files in os.walk(path): - for config_file in files: - full_path = os.path.join(root, config_file) - try: - if config_file.endswith(".schedule_config"): - with open(full_path, "r") as fd: - context = fd.read() - sched_cfg_msg = SchedCfgMsg.ScheduleConfigMessage() - text_format.Merge(context, sched_cfg_msg) - schedules_pbs.append(sched_cfg_msg) - logging.info(sched_cfg_msg.manifest_branch) - elif config_file.endswith(".lab_config"): - with open(full_path, "r") as fd: - context = fd.read() - lab_cfg_msg = LabCfgMsg.LabConfigMessage() - text_format.Merge(context, lab_cfg_msg) - lab_pbs.append(lab_cfg_msg) - except text_format.ParseError as e: - logging.error("ERROR: Config parsing error %s", e) - if update_build: - commands = self.GetBuildCommands(schedules_pbs) - if commands: - for command in commands: - ret = self.console.onecmd(command) - if ret == False: - break - self.console._vti_endpoint_client.UploadScheduleInfo( - schedules_pbs, clear_schedule) - self.console._vti_endpoint_client.UploadLabInfo(lab_pbs, clear_labinfo) - - def UpdateConfigLoop(self, account_id, branch, target, config_type, method, - update_build, update_interval, clear_schedule, - clear_labinfo): - """Regularly updates the global configuration. - - Args: - account_id: string, Partner Android Build account_id to use. - branch: string, branch to grab the artifact from. - targets: string, a comma-separate list of build target product(s). - config_type: string, config type (`prod` or `test'). - method: string, HTTP method for fetching. - update_build: boolean, indicating whether to upload build info. - update_interval: int, number of seconds before repeating - clear_schedule: bool, True to clear all schedule data exist on the - scheduler - clear_labinfo: bool, True to clear all lab data exist on the - scheduler - """ - thread = threading.currentThread() - while getattr(thread, 'keep_running', True): - try: - self.UpdateConfig(account_id, branch, target, config_type, - method, update_build, clear_schedule, - clear_labinfo) - except (socket.error, remote_operation.RemoteOperationException, - httplib2.HttpLib2Error, errors.HttpError) as e: - logging.exception(e) - time.sleep(update_interval) - - def GetBuildCommands(self, schedule_pbs): - """Generates a list of build commands with given schedules. - - Args: - schedule_pbs: a list of TestScheduleConfig protobuf messages. - - Returns: - a list of build command strings - """ - attrs = {} - attrs["device"] = [ - "build_storage_type", "manifest_branch", "pab_account_id", - "require_signed_device_build", "name" - ] - attrs["gsi"] = [ - "gsi_storage_type", "gsi_branch", "gsi_pab_account_id", - "gsi_build_target" - ] - attrs["test"] = [ - "test_storage_type", "test_branch", "test_pab_account_id", - "test_build_target" - ] - - class BuildInfo(object): - """A build information class.""" - - def __init__(self, _build_type): - if _build_type in attrs: - for attribute in attrs[_build_type]: - setattr(self, attribute, "") - - def __eq__(self, compare): - return self.__dict__ == compare.__dict__ - - build_commands = [] - if not schedule_pbs: - return build_commands - - # parses the given protobuf and stores as BuildInfo object. - builds = {"device": [], "gsi": [], "test": []} - for pb in schedule_pbs: - for build_target in pb.build_target: - build_type = "device" - device = BuildInfo(build_type) - for attr in attrs[build_type]: - if hasattr(pb, attr): - setattr(device, attr, getattr(pb, attr, None)) - elif hasattr(build_target, attr): - setattr(device, attr, getattr(build_target, attr, - None)) - if not [x for x in builds[build_type] if x == device]: - builds[build_type].append(device) - for test_schedule in build_target.test_schedule: - build_type = "gsi" - gsi = BuildInfo(build_type) - for attr in attrs[build_type]: - if hasattr(test_schedule, attr): - setattr(gsi, attr, - getattr(test_schedule, attr, None)) - if not [x for x in builds[build_type] if x == gsi]: - builds[build_type].append(gsi) - - build_type = "test" - test = BuildInfo(build_type) - for attr in attrs[build_type]: - if hasattr(test_schedule, attr): - setattr(test, attr, - getattr(test_schedule, attr, None)) - if not [x for x in builds[build_type] if x == test]: - builds[build_type].append(test) - - # groups by artifact, branch, and account id, and builds a command. - for artifact in attrs: - load_attrs = attrs[artifact] - if artifact == "device": - storage_type_text = "build_storage_type" - else: - storage_type_text = "" + artifact + "_storage_type" - pab_builds = [ - x for x in builds[artifact] - if getattr(x, storage_type_text) == - SchedCfgMsg.BUILD_STORAGE_TYPE_PAB - ] - pab_builds.sort(key=lambda x: tuple([getattr(x, attribute) - for attribute in load_attrs])) - groups = [list(g) for k, g in itertools.groupby( - pab_builds, lambda x: tuple([getattr(x, attribute) - for attribute - in load_attrs[1:-1]]))] - for group in groups: - command = ("build --artifact-type={} --method=GET " - "--noauth_local_webserver=True --update=single". - format(artifact)) - if artifact == "device": - if group[0].manifest_branch: - command += " --branch={}".format( - group[0].manifest_branch) - else: - logging.debug( - "Device manifest branch is a mandatory field.") - continue - if group[0].pab_account_id: - command += " --account_id={}".format( - group[0].pab_account_id) - if group[0].require_signed_device_build: - command += " --verify-signed-build=True" - targets = ",".join([x.name for x in group if x.name]) - if targets: - command += " --target={}".format(targets) - build_commands.append(command) - else: - if getattr(group[0], "" + artifact + "_branch"): - command += " --branch={}".format( - getattr(group[0], "" + artifact + "_branch")) - else: - logging.debug( - "{} branch is a mandatory field.".format(artifact)) - continue - if getattr(group[0], "" + artifact + "_pab_account_id"): - command += " --account_id={}".format( - getattr(group[0], - "" + artifact + "_pab_account_id")) - targets = ",".join([ - getattr(x, "" + artifact + "_build_target") - for x in group - if getattr(x, "" + artifact + "_build_target") - ]) - if targets: - command += " --target={}".format(targets) - build_commands.append(command) - - return build_commands - - # @Override - def SetUp(self): - """Initializes the parser for config command.""" - self.schedule_thread = {} - self.arg_parser.add_argument( - "--update", - choices=("single", "start", "stop", "list"), - default="start", - help="Update build info") - self.arg_parser.add_argument( - "--id", - default=None, - help="session ID only required for 'stop' update command") - self.arg_parser.add_argument( - "--interval", - type=int, - default=60, - help="Interval (seconds) to repeat build update.") - self.arg_parser.add_argument( - "--config-type", - choices=("prod", "test"), - default="prod", - help="Whether it's for prod") - self.arg_parser.add_argument( - "--branch", - required=True, - help="Branch to grab the artifact from.") - self.arg_parser.add_argument( - "--target", - required=True, - help="a comma-separate list of build target product(s).") - self.arg_parser.add_argument( - "--account_id", - default=common._DEFAULT_ACCOUNT_ID, - help="Partner Android Build account_id to use.") - self.arg_parser.add_argument( - '--method', - default='GET', - choices=('GET', 'POST'), - help='Method for fetching') - self.arg_parser.add_argument( - '--update_build', - dest='update_build', - action='store_true', - help='A boolean value indicating whether to upload build info.') - self.arg_parser.add_argument( - "--clear_schedule", - default=False, - help="True to clear all schedule data on the scheduler cloud") - self.arg_parser.add_argument( - "--clear_labinfo", - default=False, - help="True to clear all lab info data on the scheduler cloud") - - # @Override - def Run(self, arg_line): - """Updates global config.""" - args = self.arg_parser.ParseLine(arg_line) - if args.update == "single": - self.UpdateConfig(args.account_id, args.branch, args.target, - args.config_type, args.method, args.update_build, - args.clear_schedule, args.clear_labinfo) - elif args.update == "list": - logging.info("Running config update sessions:") - for id in self.schedule_thread: - logging.info(" ID %d", id) - elif args.update == "start": - if args.interval <= 0: - raise ConsoleArgumentError("update interval must be positive") - # do not allow user to create new - # thread if one is currently running - if args.id is None: - if not self.schedule_thread: - args.id = 1 - else: - args.id = max(self.schedule_thread) + 1 - else: - args.id = int(args.id) - if args.id in self.schedule_thread and not hasattr( - self.schedule_thread[args.id], 'keep_running'): - logging.warning('config update already running. ' - 'run config --update=stop --id=%s first.', - args.id) - return - self.schedule_thread[args.id] = threading.Thread( - target=self.UpdateConfigLoop, - args=( - args.account_id, - args.branch, - args.target, - args.config_type, - args.method, - args.update_build, - args.interval, - args.clear_schedule, - args.clear_labinfo, - )) - self.schedule_thread[args.id].daemon = True - self.schedule_thread[args.id].start() - elif args.update == "stop": - if args.id is None: - logging.error("--id must be set for stop") - else: - self.schedule_thread[int(args.id)].keep_running = False - - def Help(self): - base_command_processor.BaseCommandProcessor.Help(self) - logging.info("Sample: config --branch=<branch name> " - "--target=<build target> " - "--account_id=<account id> --config-type=[prod|test] " - "--update=single") diff --git a/harnesses/host_controller/command_processor/command_config_local.py b/harnesses/host_controller/command_processor/command_config_local.py deleted file mode 100644 index 6c6e5e0..0000000 --- a/harnesses/host_controller/command_processor/command_config_local.py +++ /dev/null @@ -1,166 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import httplib2 -import logging -import socket -import threading -import time - -from googleapiclient import errors - -from host_controller.command_processor import base_command_processor -from host_controller.command_processor import command_config -from host_controller.console_argument_parser import ConsoleArgumentError -from host_controller.tradefed import remote_operation - - -class CommandConfigLocal(command_config.CommandConfig): - """Command processor for config-local command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - schedule_thread: dict containing threading.Thread instances(s) that - update schedule info regularly. - """ - - command = "config_local" - command_detail = "Uploads configs from local path." - - # @Override - def UpdateConfig(self, path, update_build, clear_schedule, clear_labinfo): - """Updates the global configuration data. - - Args: - path: string, a path where config files are stored. - update_build: boolean, indicating whether to upload build info. - """ - if path: - self.UploadConfig( - path=path, - update_build=update_build, - clear_schedule=clear_schedule, - clear_labinfo=clear_labinfo) - - # @Override - def UpdateConfigLoop(self, path, update_build, update_interval, - clear_schedule, clear_labinfo): - """Regularly updates the global configuration. - - Args: - path: string, a path where config files are stored. - update_build: boolean, indicating whether to upload build info. - update_interval: int, number of seconds before repeating - """ - thread = threading.currentThread() - while getattr(thread, 'keep_running', True): - try: - self.UpdateConfig(path, update_build, clear_schedule, - clear_labinfo) - except (socket.error, remote_operation.RemoteOperationException, - httplib2.HttpLib2Error, errors.HttpError) as e: - logging.exception(e) - time.sleep(update_interval) - - # @Override - def SetUp(self): - """Initializes the parser for config command.""" - self.schedule_thread = {} - self.arg_parser.add_argument( - "--update", - choices=("single", "start", "stop", "list"), - default="single", - help="Update build info") - self.arg_parser.add_argument( - "--id", - default=None, - help="session ID only required for 'stop' update command") - self.arg_parser.add_argument( - "--interval", - type=int, - default=60, - help="Interval (seconds) to repeat build update.") - self.arg_parser.add_argument( - "--path", - required=True, - help="A path where config files are stored.") - self.arg_parser.add_argument( - '--update_build', - dest='update_build', - action='store_true', - help='A boolean value indicating whether to upload build info.') - self.arg_parser.add_argument( - "--clear_schedule", - default=False, - help="True to clear all schedule data on the scheduler cloud") - self.arg_parser.add_argument( - "--clear_labinfo", - default=False, - help="True to clear all lab info data on the scheduler cloud") - - # @Override - def Run(self, arg_line): - """Updates global config.""" - args = self.arg_parser.ParseLine(arg_line) - if args.update == "single": - self.UpdateConfig(args.path, args.update_build, - args.clear_schedule, args.clear_labinfo) - elif args.update == "list": - logging.info("Running config update sessions:") - for id in self.schedule_thread: - logging.info(" ID %d", id) - elif args.update == "start": - if args.interval <= 0: - raise ConsoleArgumentError("update interval must be positive") - # do not allow user to create new - # thread if one is currently running - if args.id is None: - if not self.schedule_thread: - args.id = 1 - else: - args.id = max(self.schedule_thread) + 1 - else: - args.id = int(args.id) - if args.id in self.schedule_thread and not hasattr( - self.schedule_thread[args.id], 'keep_running'): - logging.warning( - 'config update already running. ' - 'run config-local --update=stop --id=%s first.', args.id) - return - self.schedule_thread[args.id] = threading.Thread( - target=self.UpdateConfigLoop, - args=( - args.path, - args.update_build, - args.interval, - args.clear_schedule, - args.clear_labinfo, - )) - self.schedule_thread[args.id].daemon = True - self.schedule_thread[args.id].start() - elif args.update == "stop": - if args.id is None: - logging.error("--id must be set for stop") - else: - self.schedule_thread[int(args.id)].keep_running = False - - def Help(self): - base_command_processor.BaseCommandProcessor.Help(self) - logging.info("Sample: config-local --path=/usr/local/home/config/prod " - "--update=single --update_build") diff --git a/harnesses/host_controller/command_processor/command_copy.py b/harnesses/host_controller/command_processor/command_copy.py deleted file mode 100644 index 880e55e..0000000 --- a/harnesses/host_controller/command_processor/command_copy.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os -import shutil - -from host_controller.command_processor import base_command_processor - - -class CommandCopy(base_command_processor.BaseCommandProcessor): - """Command processor for copy command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "copy" - command_detail = "Copy a file." - - # @Override - def Run(self, arg_line): - """Copy a file from source to destination path.""" - src, dst = arg_line.split() - if dst == "{vts_tf_home}": - dst = os.path.dirname(self.console.test_suite_info["vts"]) - elif "{" in dst: - logging.error("unknown dst %s", dst) - return - shutil.copy(src, dst)
\ No newline at end of file diff --git a/harnesses/host_controller/command_processor/command_device.py b/harnesses/host_controller/command_processor/command_device.py deleted file mode 100644 index 32d51dd..0000000 --- a/harnesses/host_controller/command_processor/command_device.py +++ /dev/null @@ -1,354 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import httplib2 -import logging -import socket -import threading -import time - -from googleapiclient import errors - -from host_controller import common -from host_controller.command_processor import base_command_processor -from host_controller.console_argument_parser import ConsoleArgumentError -from host_controller.tradefed import remote_operation -from host_controller.utils.usb import usb_utils - -from vts.utils.python.common import cmd_utils - - -class CommandDevice(base_command_processor.BaseCommandProcessor): - """Command processor for Device command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - update_thread: threading.Thread that updates device state regularly. - """ - - command = "device" - command_detail = "Selects device(s) under test." - - def UpdateDevice(self, - server_type, - host, - lease, - suppress_lock_warning=True, - from_job_pool=False): - """Updates the device state of all devices on a given host. - - Args: - server_type: string, the type of a test secheduling server. - host: HostController object - lease: boolean, True to lease and execute jobs. - suppress_lock_warning: bool, True to suppress the warning msg from - file_lock. - from_job_pool: bool, True if the 'device' command is executed from - one of the job pool processes. Checks only - the availability of the devices when set. - """ - if server_type == "vti": - devices = [] - - if from_job_pool: - devices_dict = {} - for serial in self.console.GetSerials(): - device = {} - device["serial"] = serial - device["status"] = common._DEVICE_STATUS_DICT[ - "no-response"] - device["product"] = "error" - devices_dict[serial] = device - - stdout, stderr, returncode = cmd_utils.ExecuteOneShellCommand( - "adb devices") - lines_adb = stdout.split("\n") - stdout, stderr, returncode = cmd_utils.ExecuteOneShellCommand( - "fastboot devices") - lines_fastboot = stdout.split("\n") - - for line in lines_adb: - if (len(line.strip()) and not (line.startswith("* ") - or line.startswith("List "))): - device = {} - device["serial"] = line.split()[0] - serial = device["serial"] - - if from_job_pool: - if (serial in devices_dict - and line.split()[1] == "device"): - devices_dict[serial][ - "status"] = common._DEVICE_STATUS_DICT[ - "online"] - product = (self.console._vti_endpoint_client. - GetJobDeviceProductName()) - if product: - devices_dict[serial]["product"] = product - continue - - if self.console.file_lock.LockDevice( - serial, suppress_lock_warning) == False: - self.console.device_status[ - serial] = common._DEVICE_STATUS_DICT["use"] - if not suppress_lock_warning: - logging.info("Device %s already locked." % serial) - continue - - stdout, _, retcode = cmd_utils.ExecuteOneShellCommand( - "adb -s %s reboot bootloader" % device["serial"], - common.DEFAULT_DEVICE_TIMEOUT_SECS, - usb_utils.ResetUsbDeviceOfSerial_Callback, - device["serial"]) - if retcode == 0: - lines_fastboot.append(line) - - self.console.file_lock.UnlockDevice(serial) - - for line in lines_fastboot: - if len(line.strip()): - device = {} - device["serial"] = line.split()[0] - serial = device["serial"] - - if from_job_pool: - if serial in devices_dict: - devices_dict[serial][ - "status"] = common._DEVICE_STATUS_DICT[ - "fastboot"] - product = (self.console._vti_endpoint_client. - GetJobDeviceProductName()) - if product: - devices_dict[serial]["product"] = product - continue - - if self.console.file_lock.LockDevice( - serial, suppress_lock_warning) == False: - self.console.device_status[ - serial] = common._DEVICE_STATUS_DICT["use"] - if not suppress_lock_warning: - logging.info("Device %s already locked." % serial) - continue - - _, stderr, retcode = cmd_utils.ExecuteOneShellCommand( - "fastboot -s %s getvar product" % device["serial"], - common.DEFAULT_DEVICE_TIMEOUT_SECS, - usb_utils.ResetUsbDeviceOfSerial_Callback, - device["serial"]) - if retcode == 0: - res = stderr.splitlines()[0].rstrip() - if ":" in res: - device["product"] = res.split(":")[1].strip() - elif "waiting for %s" % serial in res: - res = stderr.splitlines()[1].rstrip() - device["product"] = res.split(":")[1].strip() - else: - device["product"] = "error" - self.console.device_status[ - serial] = common._DEVICE_STATUS_DICT["fastboot"] - else: - device["product"] = "error" - self.console.device_status[ - serial] = common._DEVICE_STATUS_DICT["no-response"] - - device["status"] = self.console.device_status[serial] - devices.append(device) - - self.console.file_lock.UnlockDevice(serial) - - if from_job_pool: - devices = devices_dict.values() - if devices: - self.console._vti_endpoint_client.UploadDeviceInfo( - host.hostname, devices) - return - - self.console._vti_endpoint_client.UploadDeviceInfo( - host.hostname, devices) - - if lease: - self.console._job_in_queue.put("lease") - - if self.console.vtslab_version: - self.console._vti_endpoint_client.UploadHostVersion( - host.hostname, self.console.vtslab_version) - elif server_type == "tfc": - devices = host.ListDevices() - for device in devices: - device.Extend(['sim_state', 'sim_operator', 'mac_address']) - snapshots = self.console._tfc_client.CreateDeviceSnapshot( - host._cluster_ids[0], host.hostname, devices) - self.console._tfc_client.SubmitHostEvents([snapshots]) - else: - logging.error("Error: unknown server_type %s for UpdateDevice", - server_type) - - def UpdateDeviceRepeat(self, - server_type, - host, - lease, - update_interval, - suppress_lock_warning=True, - from_job_pool=False): - """Regularly updates the device state of devices on a given host. - - Args: - server_type: string, the type of a test secheduling server. - host: HostController object - lease: boolean, True to lease and execute jobs. - update_interval: int, number of seconds before repeating - suppress_lock_warning: bool, True to suppress the warning msg from - file_lock. - from_job_pool: bool, True if the 'device' command is executed form - one of the job pool processes. - """ - thread = threading.currentThread() - while getattr(thread, 'keep_running', True): - try: - self.UpdateDevice(server_type, host, lease, - suppress_lock_warning, from_job_pool) - except (socket.error, remote_operation.RemoteOperationException, - httplib2.HttpLib2Error, errors.HttpError) as e: - logging.exception(e) - time.sleep(update_interval) - - def RunUSBResetTimer(self, serial, interval): - """Sets up a timer to run the target function after 'interval' secs. - - Args: - serial: string, serial number of the device whose USB device file - will reset when the timeout happens. - interval: int, sets up the timer for the target function to be - executed after 'interval' seconds, if not canceled. - - Returns: - threading.Timer, set to reset USB port corresponding the device - with the given serial number. - """ - usb_reset_timer = threading.Timer(interval, self.USBResetCallback, - (serial, )) - usb_reset_timer.daemon = True - usb_reset_timer.start() - - return usb_reset_timer - - def USBResetCallback(self, serial): - """Resets USB device file corresponding to the given device serial. - - Args: - serial: string, serial number of the device whose USB device file - will reset. - """ - device_file_path = usb_utils.GetDevicesUSBFilePath() - if serial in device_file_path: - logging.error( - "Device %s not responding. Resetting device file %s.", serial, - device_file_path[serial]) - usb_utils.ResetDeviceUsb(device_file_path[serial]) - - # @Override - def SetUp(self): - """Initializes the parser for device command.""" - self.update_thread = None - self.arg_parser.add_argument( - "--set_serial", - default="", - help="Serial number for device. Can be a comma-separated list.") - self.arg_parser.add_argument( - "--update", - choices=("single", "start", "stop"), - default="start", - help="Update device info on cloud scheduler") - self.arg_parser.add_argument( - "--interval", - type=int, - default=30, - help="Interval (seconds) to repeat device update.") - self.arg_parser.add_argument( - "--host", type=int, help="The index of the host.") - self.arg_parser.add_argument( - "--server_type", - choices=("vti", "tfc"), - default="vti", - help="The type of a cloud-based test scheduler server.") - self.arg_parser.add_argument( - "--lease", - default=False, - type=bool, - help="Whether to lease jobs and execute them.") - self.arg_parser.add_argument( - "--suppress_lock_warning", - default=True, - help="Whether to suppress device lock warning messages.") - self.arg_parser.add_argument( - "--from_job_pool", - action="store_true", - help="Whether the command is executed from the job pool. " - "Check only the availability of the devices when set.") - - # @Override - def Run(self, arg_line): - """Sets device info such as serial number.""" - args = self.arg_parser.ParseLine(arg_line) - if args.set_serial: - self.console.SetSerials(args.set_serial.split(",")) - logging.info("serials: %s", self.console._serials) - if args.update: - if args.host is None: - if len(self.console._hosts) > 1: - raise ConsoleArgumentError("More than one host.") - args.host = 0 - host = self.console._hosts[args.host] - - if args.suppress_lock_warning: - if (type(args.suppress_lock_warning) != str - or args.suppress_lock_warning.lower() == "true"): - suppress_lock_warning = True - else: - suppress_lock_warning = False - - if args.update == "single": - self.UpdateDevice(args.server_type, host, args.lease, - suppress_lock_warning, args.from_job_pool) - elif args.update == "start": - if args.interval <= 0: - raise ConsoleArgumentError( - "update interval must be positive") - # do not allow user to create new - # thread if one is currently running - if self.update_thread is not None and not hasattr( - self.update_thread, 'keep_running'): - logging.warning('device update already running. ' - 'run device --update stop first.') - return - self.update_thread = threading.Thread( - target=self.UpdateDeviceRepeat, - args=( - args.server_type, - host, - args.lease, - args.interval, - suppress_lock_warning, - args.from_job_pool, - )) - self.update_thread.daemon = True - self.update_thread.start() - elif args.update == "stop": - self.update_thread.keep_running = False - if self.console.GetSerials(): - self.console.ResetSerials() diff --git a/harnesses/host_controller/command_processor/command_device_test.py b/harnesses/host_controller/command_processor/command_device_test.py deleted file mode 100644 index f951c56..0000000 --- a/harnesses/host_controller/command_processor/command_device_test.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import re -import unittest - -try: - from unittest import mock -except ImportError: - import mock - -from host_controller import common -from host_controller.command_processor import command_device - - -def cmd_util_side_effect(value, timeout=0, callback_on_timeout=None, *args): - ret = ("", "", 0) - if value == "adb devices": - ret = ("List of devices attached\ndevice1\tdevice\ndevice2\tdevice", - "", 0) - elif value == "fastboot devices": - ret = ("device3\tfastboot\n", "", 0) - elif re.match("fastboot -s .* getvar product", value): - ret = ("", "product: somefish", 0) - return ret - - -class CommandDeviceTest(unittest.TestCase): - """Tests for device command processor""" - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_device.cmd_utils") - def testUpdateDevice(self, mock_cmd_utils, mock_console): - command = command_device.CommandDevice() - command._SetUp(mock_console) - mock_host = mock.Mock() - mock_host.hostname = "vtslab-001" - mock_cmd_utils.ExecuteOneShellCommand.side_effect = cmd_util_side_effect - command.UpdateDevice("vti", mock_host, False) - mock_console._vti_endpoint_client.UploadDeviceInfo.assert_called_with( - "vtslab-001", [{ - "status": mock.ANY, - "serial": "device3", - "product": "somefish" - }, { - "status": mock.ANY, - "serial": "device1", - "product": "somefish" - }, { - "status": mock.ANY, - "serial": "device2", - "product": "somefish" - }]) - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_device.cmd_utils") - def testUpdateDeviceLeaseJob(self, mock_cmd_utils, mock_console): - command = command_device.CommandDevice() - command._SetUp(mock_console) - mock_host = mock.Mock() - mock_host.hostname = "vtslab-001" - mock_cmd_utils.ExecuteOneShellCommand.side_effect = cmd_util_side_effect - command.UpdateDevice("vti", mock_host, True) - mock_console._job_in_queue.put.assert_called_with("lease") - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_device.cmd_utils") - def testUpdateDeviceFromJobPool(self, mock_cmd_utils, mock_console): - mock_console.GetSerials.return_value = ["device1", "device4"] - command = command_device.CommandDevice() - command._SetUp(mock_console) - mock_host = mock.Mock() - mock_host.hostname = "vtslab-001" - mock_cmd_utils.ExecuteOneShellCommand.side_effect = cmd_util_side_effect - command.UpdateDevice("vti", mock_host, False, from_job_pool=True) - mock_console._vti_endpoint_client.UploadDeviceInfo.assert_called_with( - "vtslab-001", - [{ - 'status': common._DEVICE_STATUS_DICT["no-response"], - 'serial': 'device4', - 'product': 'error' - }, { - 'status': common._DEVICE_STATUS_DICT["online"], - 'serial': 'device1', - 'product': mock.ANY - }]) - - @mock.patch("host_controller.console.Console") - def testCommandDeviceUpdateSingle(self, mock_console): - command = command_device.CommandDevice() - command.UpdateDevice = mock.Mock() - command._SetUp(mock_console) - ret = command._Run("--update=single") - self.assertIsNone(ret) - command.UpdateDevice.assert_called_with("vti", mock.ANY, False, True, - False) - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_device.threading") - def testCommandDeviceUpdateSetSerial(self, mock_threading, mock_console): - mock_thread = mock.Mock() - mock_threading.Thread.return_value = mock_thread - command = command_device.CommandDevice() - command.UpdateDevice = mock.Mock() - command._SetUp(mock_console) - ret = command._Run("--set_serial=device1,device2,device3") - self.assertIsNone(ret) - mock_console.SetSerials.assert_called_with( - ["device1", "device2", "device3"]) - mock_threading.Thread.assert_called_with( - args=("vti", mock.ANY, False, 30, True, False), - target=command.UpdateDeviceRepeat) - mock_thread.start.assert_called_with() - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_device.threading") - def testCommandDeviceUpdateStop(self, mock_threading, mock_console): - mock_thread = mock.Mock() - mock_threading.Thread.return_value = mock_thread - command = command_device.CommandDevice() - command.UpdateDevice = mock.Mock() - command._SetUp(mock_console) - ret = command._Run("") - self.assertIsNone(ret) - mock_threading.Thread.assert_called_with( - args=("vti", mock.ANY, False, 30, True, False), - target=command.UpdateDeviceRepeat) - mock_thread.start.assert_called_with() - ret = command._Run("--update=stop") - self.assertIsNone(ret) - self.assertFalse(mock_thread.keep_running) - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/command_processor/command_dut.py b/harnesses/host_controller/command_processor/command_dut.py deleted file mode 100644 index a722b41..0000000 --- a/harnesses/host_controller/command_processor/command_dut.py +++ /dev/null @@ -1,141 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging - -from host_controller import common -from host_controller.command_processor import base_command_processor - -from vts.utils.python.common import cmd_utils -from vts.utils.python.controllers import adb -from vts.utils.python.controllers import android_device - -# Default index of setStreamVolume() from IAudioService.aidl (1 based) -SETSTREAMVOLUME_INDEX_DEFAULT = 3 - -# Indices of each stream type (can be checked with "adb shell dumpsys audio" on the command line) -STREAM_TYPE_CALL = 0 -STREAM_TYPE_RINGTONE = 2 -STREAM_TYPE_MEDIA = 3 -STREAM_TYPE_ALARM = 4 - -STREAM_TYPE_LIST = [ - STREAM_TYPE_CALL, - STREAM_TYPE_RINGTONE, - STREAM_TYPE_MEDIA, - STREAM_TYPE_ALARM, -] - - -class CommandDUT(base_command_processor.BaseCommandProcessor): - """Command processor for DUT command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "dut" - command_detail = "Performs certain operations on DUT (Device Under Test)." - - # @Override - def SetUp(self): - """Initializes the parser for dut command.""" - self.arg_parser.add_argument( - "--operation", - choices=("wifi_on", "wifi_off", 'volume_mute', 'volume_max'), - default="", - required=True, - help="Operation to perform.") - self.arg_parser.add_argument( - "--serial", default="", required=True, help="The device serial.") - self.arg_parser.add_argument( - "--ap", - default="", # Required only for wifi_on - help="Access point (AP) name for 'wifi_on' operation.") - self.arg_parser.add_argument( - "--volume_levels", - type=int, - default=30, # Required only for volume_mute and volume_max - help="The number of volume control levels.") - self.arg_parser.add_argument( - "--version", - type=float, - default=8.0, - help="System version information of the device on which " - "the test will run.") - - # @Override - def Run(self, arg_line): - """Performs the requested operation on the selected DUT.""" - args = self.arg_parser.ParseLine(arg_line) - device = android_device.AndroidDevice( - args.serial, device_callback_port=-1) - boot_complete = device.waitForBootCompletion() - if not boot_complete: - logging.error("Device %s failed to bootup.", args.serial) - self.console.device_status[ - args.serial] = common._DEVICE_STATUS_DICT["error"] - self.console.vti_endpoint_client.SetJobStatusFromLeasedTo( - "bootup-err") - return False - - adb_proxy = adb.AdbProxy(serial=args.serial) - adb_proxy.root() - try: - if args.operation == "wifi_on": - adb_proxy.shell("svc wifi enable") - if args.ap: - adb_proxy.install( - "../testcases/DATA/app/WifiUtil/WifiUtil.apk") - adb_proxy.shell( - "am instrument -e method \"connectToNetwork\" " - "-e ssid %s " - "-w com.android.tradefed.utils.wifi/.WifiUtil" % - args.ap) - elif args.operation == "wifi_off": - adb_proxy.shell("svc wifi disable") - elif args.operation == "volume_mute": - for _ in range(args.volume_levels): - adb_proxy.shell("input keyevent 25") - self.SetOtherVolumes(adb_proxy, 0, args.version) - elif args.operation == "volume_max": - for _ in range(args.volume_levels): - adb_proxy.shell("input keyevent 24") - self.SetOtherVolumes(adb_proxy, args.volume_levels, - args.version) - except adb.AdbError as e: - logging.exception(e) - return False - - def SetOtherVolumes(self, adb_proxy, volume_level, version=None): - """Sets device's call/media/alarm volumes a certain level. - - Args: - adb_proxy: AdbProxy, used for interacting with the device via adb. - volume_level: int, volume level value. - version: float, Android system version value. The index of - setStreamVolume() depends on the Android version. - """ - setStreamVolume_index = SETSTREAMVOLUME_INDEX_DEFAULT - if version and version >= 9.0: - setStreamVolume_index = 7 - for stream_type in STREAM_TYPE_LIST: - adb_volume_command = "service call audio %s i32 %s i32 %s i32 1" % ( - setStreamVolume_index, stream_type, volume_level) - adb_proxy.shell(adb_volume_command) diff --git a/harnesses/host_controller/command_processor/command_dut_test.py b/harnesses/host_controller/command_processor/command_dut_test.py deleted file mode 100644 index fff5c4d..0000000 --- a/harnesses/host_controller/command_processor/command_dut_test.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest - -try: - from unittest import mock -except ImportError: - import mock - -from host_controller import common -from host_controller.command_processor import command_dut - - -class CommandDUTTest(unittest.TestCase): - """Tests for DUT command processor""" - - def setUp(self): - """Creates CommandSheet.""" - self._command = command_dut.CommandDUT() - mock_console = mock.Mock() - mock_console.device_status = {} - self._command._SetUp(mock_console) - - def testSetOtherVolumesWithoutVersionInfo(self): - mock_adb_proxy = mock.Mock() - self._command.SetOtherVolumes(mock_adb_proxy, 5) - self.assertEqual(mock_adb_proxy.shell.mock_calls, [ - mock.call("service call audio 3 i32 0 i32 5 i32 1"), - mock.call("service call audio 3 i32 2 i32 5 i32 1"), - mock.call("service call audio 3 i32 3 i32 5 i32 1"), - mock.call("service call audio 3 i32 4 i32 5 i32 1"), - ]) - - def testSetOtherVolumesWithVersionInfo(self): - mock_adb_proxy = mock.Mock() - self._command.SetOtherVolumes(mock_adb_proxy, 0, 8.1) - self.assertEqual(mock_adb_proxy.shell.mock_calls, [ - mock.call("service call audio 3 i32 0 i32 0 i32 1"), - mock.call("service call audio 3 i32 2 i32 0 i32 1"), - mock.call("service call audio 3 i32 3 i32 0 i32 1"), - mock.call("service call audio 3 i32 4 i32 0 i32 1"), - ]) - mock_adb_proxy.shell.mock_calls = [] - self._command.SetOtherVolumes(mock_adb_proxy, 10, 9.0) - self.assertEqual(mock_adb_proxy.shell.mock_calls, [ - mock.call("service call audio 7 i32 0 i32 10 i32 1"), - mock.call("service call audio 7 i32 2 i32 10 i32 1"), - mock.call("service call audio 7 i32 3 i32 10 i32 1"), - mock.call("service call audio 7 i32 4 i32 10 i32 1"), - ]) - - @mock.patch("host_controller.command_processor.command_dut.android_device") - @mock.patch("host_controller.command_processor.command_dut.logging") - def testCommandDUTBootupFail(self, mock_logging, mock_android_device): - mock_device = mock.Mock() - mock_device.waitForBootCompletion.return_value = False - mock_android_device.AndroidDevice.return_value = mock_device - ret = self._command._Run("--serial device1 --operation wifi_on") - self.assertFalse(ret) - mock_logging.error.assert_called_with("Device %s failed to bootup.", - "device1") - - @mock.patch("host_controller.command_processor.command_dut.android_device") - @mock.patch("host_controller.command_processor.command_dut.adb") - def testCommandDUTWifiOnWithoutAP(self, mock_adb, mock_android_device): - mock_adb_proxy = mock.Mock() - mock_adb.AdbProxy.return_value = mock_adb_proxy - mock_device = mock.Mock() - mock_device.waitForBootCompletion.return_value = True - mock_android_device.AndroidDevice.return_value = mock_device - ret = self._command._Run("--serial device1 --operation wifi_on") - self.assertIsNone(ret) - mock_adb_proxy.root.assert_called_once() - mock_adb_proxy.shell.assert_called_with("svc wifi enable") - mock_adb_proxy.install.assert_not_called() - - @mock.patch("host_controller.command_processor.command_dut.android_device") - @mock.patch("host_controller.command_processor.command_dut.adb") - def testCommandDUTWifiOn(self, mock_adb, mock_android_device): - mock_adb_proxy = mock.Mock() - mock_adb.AdbProxy.return_value = mock_adb_proxy - mock_device = mock.Mock() - mock_device.waitForBootCompletion.return_value = True - mock_android_device.AndroidDevice.return_value = mock_device - ret = self._command._Run("--serial device1 --operation wifi_on --ap %s" - % common._DEFAULT_WIFI_AP) - self.assertIsNone(ret) - mock_adb_proxy.root.assert_called_once() - mock_adb_proxy.shell.assert_any_call("svc wifi enable") - mock_adb_proxy.install.assert_called_with( - "../testcases/DATA/app/WifiUtil/WifiUtil.apk") - mock_adb_proxy.shell.assert_called_with( - "am instrument -e method \"connectToNetwork\" -e ssid GoogleGuest " - "-w com.android.tradefed.utils.wifi/.WifiUtil") - - @mock.patch("host_controller.command_processor.command_dut.android_device") - @mock.patch("host_controller.command_processor.command_dut.adb") - def testCommandDUTWifiOff(self, mock_adb, mock_android_device): - mock_adb_proxy = mock.Mock() - mock_adb.AdbProxy.return_value = mock_adb_proxy - mock_device = mock.Mock() - mock_device.waitForBootCompletion.return_value = True - mock_android_device.AndroidDevice.return_value = mock_device - ret = self._command._Run("--serial device1 --operation wifi_off") - self.assertIsNone(ret) - mock_adb_proxy.root.assert_called_once() - mock_adb_proxy.shell.assert_any_call("svc wifi disable") - - @mock.patch("host_controller.command_processor.command_dut.android_device") - @mock.patch("host_controller.command_processor.command_dut.adb") - def testCommandDUTVolumeMute(self, mock_adb, mock_android_device): - self._command.SetOtherVolumes = mock.Mock() - mock_adb_proxy = mock.Mock() - mock_adb.AdbProxy.return_value = mock_adb_proxy - mock_device = mock.Mock() - mock_device.waitForBootCompletion.return_value = True - mock_android_device.AndroidDevice.return_value = mock_device - ret = self._command._Run("--serial device1 --operation volume_mute") - self.assertIsNone(ret) - mock_adb_proxy.root.assert_called_once() - self.assertEqual(mock_adb_proxy.shell.mock_calls, - [mock.call("input keyevent 25")] * 30) - self._command.SetOtherVolumes.assert_called_with(mock.ANY, 0, 8.0) - - @mock.patch("host_controller.command_processor.command_dut.android_device") - @mock.patch("host_controller.command_processor.command_dut.adb") - def testCommandDUTVolumeMax(self, mock_adb, mock_android_device): - self._command.SetOtherVolumes = mock.Mock() - mock_adb_proxy = mock.Mock() - mock_adb.AdbProxy.return_value = mock_adb_proxy - mock_device = mock.Mock() - mock_device.waitForBootCompletion.return_value = True - mock_android_device.AndroidDevice.return_value = mock_device - ret = self._command._Run( - "--serial device1 --operation volume_max --version 9.0") - self.assertIsNone(ret) - mock_adb_proxy.root.assert_called_once() - self.assertEqual(mock_adb_proxy.shell.mock_calls, - [mock.call("input keyevent 24")] * 30) - self._command.SetOtherVolumes.assert_called_with(mock.ANY, 30, 9.0) - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/command_processor/command_exit.py b/harnesses/host_controller/command_processor/command_exit.py deleted file mode 100644 index 638e6ad..0000000 --- a/harnesses/host_controller/command_processor/command_exit.py +++ /dev/null @@ -1,63 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging - -from host_controller.command_processor import base_command_processor - - -class CommandExit(base_command_processor.BaseCommandProcessor): - """Command processor for exit command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "exit" - command_detail = "" - - # @Override - def SetUp(self): - """Initializes the parser for request command.""" - self.arg_parser.add_argument( - "--wait_for_jobs", - default=True, - help="True to wait for the running jobs to complete before exiting." - ) - - # @Override - def Run(self, arg_line): - """Terminates the console. - - Returns: - True, which stops the cmdloop. - """ - args = self.arg_parser.ParseLine(arg_line) - - self.console.onecmd("device --update stop") - - if args.wait_for_jobs: - if (type(args.wait_for_jobs) != str or - args.wait_for_jobs.lower() == "true"): - logging.info("waiting for running jobs to complete...") - self.console.WaitForJobsToExit() - - self.console.StopJobThreadAndProcessPool() - self.console.__exit__() - return True diff --git a/harnesses/host_controller/command_processor/command_fastboot.py b/harnesses/host_controller/command_processor/command_fastboot.py deleted file mode 100644 index 459a079..0000000 --- a/harnesses/host_controller/command_processor/command_fastboot.py +++ /dev/null @@ -1,100 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging - -from host_controller import common -from host_controller.command_processor import base_command_processor -from host_controller.utils.ipc import file_lock_semaphore -from host_controller.utils.usb import usb_utils - -from vts.utils.python.common import cmd_utils - - -class CommandFastboot(base_command_processor.BaseCommandProcessor): - """Command processor for fastboot command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "fastboot" - command_detail = "Runs a fastboot command." - - # @Override - def SetUp(self): - """Initializes the parser for device command.""" - self.sem_fastboot = file_lock_semaphore.FileLockSemaphore("fastboot") - self.arg_parser.add_argument( - "--serial", - "-s", - required=True, - default=None, - help="The target device serial to run the command.") - self.arg_parser.add_argument( - "--retry", - type=int, - default=2, - help="The number of times to retry if a command fails.") - self.arg_parser.add_argument( - "--timeout", - type=float, - default=common.DEFAULT_DEVICE_TIMEOUT_SECS, - help="The maximum timeout value of this command in seconds. " - "Set to 0 to disable the timeout functionality.") - self.arg_parser.add_argument( - "command", - metavar="COMMAND", - nargs="+", - help="The command to be executed. If the command contains " - "arguments starting with \"-\", place the command at end of line " - "after \"--\".") - - # @Override - def Run(self, arg_line): - """Runs a fastboot command.""" - args = self.arg_parser.ParseLine(arg_line) - cmd_list = ["fastboot"] - if args.serial: - if "," in args.serial: - logging.error("Only one serial can be specified") - return False - cmd_list.append("-s %s" % args.serial) - cmd_list.extend(self.ReplaceVars(args.command)) - cmd = " ".join(cmd_list) - for _ in range(args.retry + 1): - self.sem_fastboot.Acquire() - if args.timeout == 0: - stdout, stderr, retcode = cmd_utils.ExecuteOneShellCommand(cmd) - else: - stdout, stderr, retcode = cmd_utils.ExecuteOneShellCommand( - cmd, args.timeout, - usb_utils.ResetUsbDeviceOfSerial_Callback, args.serial) - self.sem_fastboot.Release() - if stdout: - logging.info(stdout) - if stderr: - logging.error(stderr) - if retcode == 0: - return - logging.warn("Retrying... (%s)", cmd) - - if self.console.job_pool and args.serial: - self.console.device_status[ - args.serial] = common._DEVICE_STATUS_DICT["error"] - return False diff --git a/harnesses/host_controller/command_processor/command_fetch.py b/harnesses/host_controller/command_processor/command_fetch.py deleted file mode 100644 index f43731d..0000000 --- a/harnesses/host_controller/command_processor/command_fetch.py +++ /dev/null @@ -1,204 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os - -from host_controller import common -from host_controller.command_processor import base_command_processor - - -class CommandFetch(base_command_processor.BaseCommandProcessor): - """Command processor for fetch command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "fetch" - command_detail = "Fetch a build artifact." - - # @Override - def SetUp(self): - """Initializes the parser for fetch command.""" - self.arg_parser.add_argument( - '--type', - default='pab', - choices=('local_fs', 'gcs', 'pab', 'ab'), - help='Build provider type') - self.arg_parser.add_argument( - '--method', - default='GET', - choices=('GET', 'POST'), - help='Method for fetching') - self.arg_parser.add_argument( - "--path", # required for local_fs - help="The path of a local directory which keeps the artifacts.") - self.arg_parser.add_argument( - "--branch", # required for pab - help="Branch to grab the artifact from.") - self.arg_parser.add_argument( - "--target", # required for pab - help="Target product to grab the artifact from.") - # TODO(lejonathan): find a way to not specify this? - self.arg_parser.add_argument( - "--account_id", - default=common._DEFAULT_ACCOUNT_ID, - help="Partner Android Build account_id to use.") - self.arg_parser.add_argument( - '--build_id', - default='latest', - help='Build ID to use default latest.') - self.arg_parser.add_argument( - "--artifact_name", # required for pab - help= - "Name of the artifact to be fetched. {id} replaced with build id.") - self.arg_parser.add_argument( - "--userinfo-file", - help= - "Location of file containing email and password, if using POST.") - self.arg_parser.add_argument( - "--noauth_local_webserver", - default=False, - type=bool, - help="True to not use a local webserver for authentication.") - self.arg_parser.add_argument( - "--fetch_signed_build", - default=False, - type=bool, - help="True to fetch only signed build images.") - self.arg_parser.add_argument( - "--full_device_images", - default=False, - type=bool, - help="True to skip checking whether the fetched artifacts are " - "fully packaged device images.") - self.arg_parser.add_argument( - "--gsi", - default=False, - type=bool, - help="True if a target is GSI. Only system.img and " - "vbmeta.img are taken.") - self.arg_parser.add_argument( - "--set_suite_as", - default="", - choices=("", "vts", "cts", "gts", "sts"), - help="To specify the type of a test suite that is being fetched." - "Used when the artifact's file name does not follow the " - "standard naming convention.") - - # @Override - def Run(self, arg_line): - """Makes the host download a build artifact from PAB.""" - args = self.arg_parser.ParseLine(arg_line) - - if args.type not in self.console._build_provider: - logging.error("ERROR: uninitialized fetch type %s", args.type) - return False - - provider = self.console._build_provider[args.type] - if args.type == "pab": - # do we want this somewhere else? No harm in doing multiple times - provider.Authenticate(args.userinfo_file, - args.noauth_local_webserver) - if not args.fetch_signed_build: - (device_images, test_suites, fetch_environment, - _) = provider.GetArtifact( - account_id=args.account_id, - branch=args.branch, - target=args.target, - artifact_name=args.artifact_name, - build_id=args.build_id, - method=args.method, - full_device_images=args.full_device_images) - self.console.fetch_info["fetch_signed_build"] = False - else: - (device_images, test_suites, fetch_environment, - _) = provider.GetSignedBuildArtifact( - account_id=args.account_id, - branch=args.branch, - target=args.target, - artifact_name=args.artifact_name, - build_id=args.build_id, - method=args.method, - full_device_images=args.full_device_images) - self.console.fetch_info["fetch_signed_build"] = True - - self.console.fetch_info["build_id"] = fetch_environment["build_id"] - elif args.type == "local_fs": - device_images, test_suites = provider.Fetch( - args.path, args.full_device_images) - self.console.fetch_info["build_id"] = None - elif args.type == "gcs": - device_images, test_suites, tools = provider.Fetch( - args.path, args.full_device_images, args.set_suite_as) - self.console.fetch_info["build_id"] = None - elif args.type == "ab": - device_images, test_suites, fetch_environment = provider.Fetch( - branch=args.branch, - target=args.target, - artifact_name=args.artifact_name, - build_id=args.build_id, - full_device_images=args.full_device_images) - self.console.fetch_info["build_id"] = fetch_environment["build_id"] - else: - logging.error("ERROR: unknown fetch type %s", args.type) - return False - - if args.gsi: - filtered_images = {} - image_names = device_images.keys() - for image_name in image_names: - if image_name.endswith(".img") and image_name not in [ - "system.img", "vbmeta.img" - ]: - provider.RemoveDeviceImage(image_name) - continue - filtered_images[image_name] = device_images[image_name] - device_images = filtered_images - - if args.type == "gcs": - gcs_path, filename = os.path.split(args.path) - self.console.fetch_info["branch"] = gcs_path - self.console.fetch_info["target"] = filename - self.console.fetch_info["build_id"] = "latest" - self.console.fetch_info["account_id"] = "" - else: - self.console.fetch_info["branch"] = args.branch - self.console.fetch_info["target"] = args.target - self.console.fetch_info["account_id"] = args.account_id - - self.console.UpdateFetchInfo(provider.GetFetchedArtifactType()) - - self.console.device_image_info.update(device_images) - self.console.test_suite_info.update(test_suites) - self.console.tools_info.update(provider.GetAdditionalFile()) - - if self.console.device_image_info: - logging.info("device images:\n%s", "\n".join( - image + ": " + path - for image, path in self.console.device_image_info.iteritems())) - if self.console.test_suite_info: - logging.info("test suites:\n%s", "\n".join( - suite + ": " + path - for suite, path in self.console.test_suite_info.iteritems())) - if self.console.tools_info: - logging.info("additional files:\n%s", "\n".join( - rel_path + ": " + full_path for rel_path, full_path in - self.console.tools_info.iteritems())) diff --git a/harnesses/host_controller/command_processor/command_flash.py b/harnesses/host_controller/command_processor/command_flash.py deleted file mode 100644 index b8a2d7e..0000000 --- a/harnesses/host_controller/command_processor/command_flash.py +++ /dev/null @@ -1,210 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import importlib -import os -import stat - -from host_controller import common -from host_controller.build import build_flasher -from host_controller.command_processor import base_command_processor - - -class CommandFlash(base_command_processor.BaseCommandProcessor): - """Command processor for flash command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "flash" - command_detail = "Flash images to a device." - - # @Override - def SetUp(self): - """Initializes the parser for flash command.""" - self.arg_parser.add_argument( - "--image", - help=("The file name of an image to flash." - " Used to flash a single image.")) - self.arg_parser.add_argument( - "--current", - metavar="PARTITION_IMAGE", - nargs="*", - type=lambda x: x.split("="), - help="The partitions and images to be flashed. The format is " - "<partition>=<image>. If PARTITION_IMAGE list is empty, " - "currently fetched " + ", ".join(common._DEFAULT_FLASH_IMAGES) + - " will be flashed.") - self.arg_parser.add_argument( - "--serial", default="", help="Serial number for device.") - self.arg_parser.add_argument( - "--build_dir", - help="Directory containing build images to be flashed.") - self.arg_parser.add_argument( - "--gsi", help="Path to generic system image") - self.arg_parser.add_argument("--vbmeta", help="Path to vbmeta image") - self.arg_parser.add_argument( - "--flasher_type", - default="fastboot", - help="Flasher type. Valid arguments are \"fastboot\", \"custom\", " - "and full module name followed by class name. The class must " - "inherit build_flasher.BuildFlasher, and implement " - "__init__(serial, flasher_path) and " - "Flash(device_images, additional_files, *flasher_args).") - self.arg_parser.add_argument( - "--flasher_path", default=None, help="Path to a flasher binary") - self.arg_parser.add_argument( - "flasher_args", - metavar="ARGUMENTS", - nargs="*", - help="The arguments passed to the flasher binary. If any argument " - "starts with \"-\", place all of them after \"--\" at end of " - "line.") - self.arg_parser.add_argument( - "--reboot_mode", - default="bootloader", - choices=("bootloader", "download"), - help="Reboot device to bootloader/download mode") - self.arg_parser.add_argument( - "--repackage", - default="tar.md5", - choices=("tar.md5"), - help="Repackage artifacts into given format before flashing.") - self.arg_parser.add_argument( - "--wait-for-boot", - default="true", - help="false to not wait for device booting.") - self.arg_parser.add_argument( - "--reboot", default="false", help="true to reboot the device(s).") - self.arg_parser.add_argument( - "--skip-vbmeta", - default=False, - type=bool, - help="true to skip flashing vbmeta.img if the device does not have " - "the vbmeta slot .") - - # @Override - def Run(self, arg_line): - """Flash GSI or build images to a device connected with ADB.""" - args = self.arg_parser.ParseLine(arg_line) - - # path - if (self.console.tools_info is not None - and args.flasher_path in self.console.tools_info): - flasher_path = self.console.tools_info[args.flasher_path] - elif args.flasher_path: - flasher_path = args.flasher_path - else: - flasher_path = "" - if os.path.exists(flasher_path): - flasher_mode = os.stat(flasher_path).st_mode - os.chmod(flasher_path, - flasher_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - - # serial numbers - if args.serial: - flasher_serials = [args.serial] - elif self.console._serials: - flasher_serials = self.console._serials - else: - flasher_serials = [""] - - # images - if args.image: - partition_image = {} - partition_image[args.image] = self.console.device_image_info[ - args.image] - else: - if args.current: - partition_image = dict((partition, - self.console.device_image_info[image]) - for partition, image in args.current) - else: - partition_image = dict( - (image.rsplit(".img", 1)[0], - self.console.device_image_info[image]) - for image in common._DEFAULT_FLASH_IMAGES - if image in self.console.device_image_info) - - # type - if args.flasher_type in ("fastboot", "custom"): - flasher_class = build_flasher.BuildFlasher - else: - class_path = args.flasher_type.rsplit(".", 1) - flasher_module = importlib.import_module(class_path[0]) - flasher_class = getattr(flasher_module, class_path[1]) - if not issubclass(flasher_class, build_flasher.BuildFlasher): - raise TypeError( - "%s is not a subclass of BuildFlasher." % class_path[1]) - - flashers = [flasher_class(s, flasher_path) for s in flasher_serials] - - # Can be parallelized as long as that's proven reliable. - for flasher in flashers: - ret_flash = True - if args.flasher_type == "fastboot": - if args.image is not None: - ret_flash = flasher.FlashImage(partition_image, True - if args.reboot == "true" - else False) - elif args.current is not None: - ret_flash = flasher.Flash(partition_image, - args.skip_vbmeta) - else: - if args.gsi is None and args.build_dir is None: - self.arg_parser.error("Nothing requested: " - "specify --gsi or --build_dir") - return False - if args.build_dir is not None: - ret_flash = flasher.Flashall(args.build_dir) - if args.gsi is not None: - ret_flash = flasher.FlashGSI( - args.gsi, - args.vbmeta, - skip_vbmeta=args.skip_vbmeta) - elif args.flasher_type == "custom": - if flasher_path is not None: - if args.repackage is not None: - flasher.RepackageArtifacts( - self.console.device_image_info, args.repackage) - ret_flash = flasher.FlashUsingCustomBinary( - self.console.device_image_info, args.reboot_mode, - args.flasher_args, 300) - else: - self.arg_parser.error( - "Please specify the path to custom flash tool.") - return False - else: - ret_flash = flasher.Flash(partition_image, - self.console.tools_info, - *args.flasher_args) - if ret_flash == False: - return False - - if args.wait_for_boot == "true": - for flasher in flashers: - ret_wait = flasher.WaitForDevice() - if ret_wait == False: - self.console.device_status[ - flasher.device.serial] = common._DEVICE_STATUS_DICT[ - "error"] - self.console.vti_endpoint_client.SetJobStatusFromLeasedTo( - "bootup-err") - return False diff --git a/harnesses/host_controller/command_processor/command_gsispl.py b/harnesses/host_controller/command_processor/command_gsispl.py deleted file mode 100644 index 9ebcd68..0000000 --- a/harnesses/host_controller/command_processor/command_gsispl.py +++ /dev/null @@ -1,153 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import datetime -import logging -import os -import shutil -import tempfile -import zipfile - -from host_controller import common -from host_controller.command_processor import base_command_processor -from host_controller.utils.gsi import img_utils - -from vts.utils.python.common import cmd_utils - - -class CommandGsispl(base_command_processor.BaseCommandProcessor): - """Command processor for gsispl command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "gsispl" - command_detail = "Changes security patch level on a selected GSI file." - - # @Override - def SetUp(self): - """Initializes the parser for device command.""" - self.arg_parser.add_argument( - "--gsi", - help="Path to GSI image to change security patch level. " - "If path is not given, the most recently fetched system.img " - "kept in device_image_info dictionary is used and then " - "device_image_info will be updated with the new GSI file.") - self.arg_parser.add_argument( - "--version", help="New version ID. It should be YYYY-mm-dd format") - self.arg_parser.add_argument( - "--version_from_path", - help="Path to vendor provided image file to retrieve SPL version. " - "If just a file name is given, the most recently fetched .img " - "file will be used.") - self.arg_parser.add_argument( - "--vendor_version", - help="The version of vendor.img that will be used (e.g., 8.1.0).") - - # @Override - def Run(self, arg_line): - """Changes security patch level on a selected GSI file.""" - args = self.arg_parser.ParseLine(arg_line) - if args.gsi: - if os.path.isfile(args.gsi): - gsi_path = args.gsi - else: - logging.error("Cannot find system image in given path") - return - elif "system.img" in self.console.device_image_info: - gsi_path = self.console.device_image_info["system.img"] - else: - logging.error("Cannot find system image.") - return False - - if args.version: - try: - version_date = datetime.datetime.strptime( - args.version, "%Y-%m-%d") - version = "{:04d}-{:02d}-{:02d}".format( - version_date.year, version_date.month, version_date.day) - except ValueError as e: - logging.error("version ID should be YYYY-mm-dd format.") - return - elif args.version_from_path: - dest_path = None - if os.path.isabs(args.version_from_path) and os.path.exists( - args.version_from_path): - img_path = args.version_from_path - elif args.version_from_path in self.console.device_image_info: - img_path = self.console.device_image_info[ - args.version_from_path] - elif (args.version_from_path == "boot.img" - and "full-zipfile" in self.console.device_image_info): - tempdir_base = os.path.join(os.getcwd(), "tmp") - if not os.path.exists(tempdir_base): - os.mkdir(tempdir_base) - dest_path = tempfile.mkdtemp(dir=tempdir_base) - - with zipfile.ZipFile( - self.console.device_image_info["full-zipfile"], - 'r') as zip_ref: - zip_ref.extractall(dest_path) - img_path = os.path.join(dest_path, "boot.img") - if not os.path.exists(img_path): - logging.error("No %s file in device img .zip.", - args.version_from_path) - shutil.rmtree(dest_path) - return - else: - logging.error("Cannot find %s file.", args.version_from_path) - return False - - version_dict = img_utils.GetSPLVersionFromBootImg(img_path) - if dest_path: - shutil.rmtree(dest_path) - if "year" in version_dict and "month" in version_dict: - version = "{:04d}-{:02d}-{:02d}".format( - version_dict["year"], version_dict["month"], - common._SPL_DEFAULT_DAY) - else: - logging.error("Failed to fetch SPL version from %s file.", - img_path) - return False - else: - logging.error("version ID or path of .img file must be given.") - return False - - output_path = os.path.join( - os.path.dirname(os.path.abspath(gsi_path)), - "system-{}.img".format(version)) - command = "{} {} {} {}".format( - os.path.join(os.getcwd(), "..", "bin", - "change_security_patch_ver.sh"), gsi_path, - output_path, version) - if args.vendor_version: - command = command + " -v " + args.vendor_version - if self.console.password.value: - command = "echo {} | sudo -S {}".format( - self.console.password.value, command) - stdout, stderr, err_code = cmd_utils.ExecuteOneShellCommand(command) - if err_code is 0: - if not args.gsi: - logging.info( - "system.img path is updated to : {}".format(output_path)) - self.console.device_image_info["system.img"] = output_path - else: - logging.error("gsispl error: {} {}".format(stdout, stderr)) - return False diff --git a/harnesses/host_controller/command_processor/command_info.py b/harnesses/host_controller/command_processor/command_info.py deleted file mode 100644 index 715f913..0000000 --- a/harnesses/host_controller/command_processor/command_info.py +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging - -from host_controller.command_processor import base_command_processor - - -class CommandInfo(base_command_processor.BaseCommandProcessor): - '''Command processor for info command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - ''' - - command = 'info' - command_detail = 'Show status.' - - def Run(self, arg_line): - '''Shows the console's session status information. - - Args: - arg_line: string, line of command arguments - ''' - logging.info('device image: %s', self.console.device_image_info) - logging.info('test suite: %s', self.console.test_suite_info) - logging.info('test result: %s', self.console.test_results) - logging.info('fetch info: %s', self.console.fetch_info) - logging.info('detailed fetch info: %s', self.console.detailed_fetch_info) diff --git a/harnesses/host_controller/command_processor/command_lease.py b/harnesses/host_controller/command_processor/command_lease.py deleted file mode 100644 index cc21306..0000000 --- a/harnesses/host_controller/command_processor/command_lease.py +++ /dev/null @@ -1,59 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from host_controller.command_processor import base_command_processor -from host_controller.console_argument_parser import ConsoleArgumentError - - -class CommandLease(base_command_processor.BaseCommandProcessor): - """Command processor for lease command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "lease" - command_detail = "Make a host lease command tasks from TFC." - - def _PrintTasks(self, tasks): - """Shows a list of command tasks. - - Args: - devices: A list of DeviceInfo objects. - """ - attr_names = ("request_id", "command_id", "task_id", "device_serials", - "command_line") - self.console._PrintObjects(tasks, attr_names) - - # @Override - def SetUp(self): - """Initializes the parser for lease command.""" - self.arg_parser.add_argument( - "--host", type=int, help="The index of the host.") - - # @Override - def Run(self, arg_line): - """Makes a host lease command tasks from TFC.""" - args = self.arg_parser.ParseLine(arg_line) - if args.host is None: - if len(self.console._hosts) > 1: - raise ConsoleArgumentError("More than one hosts.") - args.host = 0 - tasks = self.console._hosts[args.host].LeaseCommandTasks() - self._PrintTasks(tasks)
\ No newline at end of file diff --git a/harnesses/host_controller/command_processor/command_list.py b/harnesses/host_controller/command_processor/command_list.py deleted file mode 100644 index e196a1a..0000000 --- a/harnesses/host_controller/command_processor/command_list.py +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging - -from host_controller.command_processor import base_command_processor - - -class CommandList(base_command_processor.BaseCommandProcessor): - """Command processor for list command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "list" - command_detail = "Show information about the hosts." - - def _PrintHosts(self, hosts): - """Shows a list of host controllers. - - Args: - hosts: A list of HostController objects. - """ - self.console._Print("index name") - for ind, host in enumerate(hosts): - self.console._Print("[%3d] %s" % (ind, host.hostname)) - - def _PrintDevices(self, devices): - """Shows a list of devices. - - Args: - devices: A list of DeviceInfo objects. - """ - attr_names = ("device_serial", "state", "run_target", "build_id", - "sdk_version", "stub") - self.console._PrintObjects(devices, attr_names) - - # @Override - def SetUp(self): - """Initializes the parser for list command.""" - self.arg_parser.add_argument( - "--host", type=int, help="The index of the host.") - self.arg_parser.add_argument( - "type", - choices=("hosts", "devices"), - help="The type of the shown objects.") - - # @Override - def Run(self, arg_line): - """Shows information about the hosts.""" - args = self.arg_parser.ParseLine(arg_line) - if args.host is None: - hosts = enumerate(self.console._hosts) - else: - hosts = [(args.host, self.console._hosts[args.host])] - if args.type == "hosts": - self._PrintHosts(self.console._hosts) - elif args.type == "devices": - for ind, host in hosts: - devices = host.ListDevices() - self.console._Print("[%3d] %s" % (ind, host.hostname)) - self._PrintDevices(devices) - - def Help(self): - base_command_processor.BaseCommandProcessor.Help(self) - logging.info("Sample: build --target=aosp_sailfish-userdebug " - "--branch=<branch name> --artifact-type=device") diff --git a/harnesses/host_controller/command_processor/command_password.py b/harnesses/host_controller/command_processor/command_password.py deleted file mode 100644 index c767f22..0000000 --- a/harnesses/host_controller/command_processor/command_password.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import getpass -import logging - -from host_controller.command_processor import base_command_processor - - -class CommandPassword(base_command_processor.BaseCommandProcessor): - """Command processor for password command. - - Attributes: - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "password" - command_detail = "Sets password." - - # @Override - def Run(self, arg_line): - """Gets a new password.""" - new_password = getpass.getpass() - if new_password: - self.console.password.value = new_password - logging.info("new password set.") - else: - logging.warn("password is not updated.") - - def Help(self): - base_command_processor.BaseCommandProcessor.Help(self) diff --git a/harnesses/host_controller/command_processor/command_release.py b/harnesses/host_controller/command_processor/command_release.py deleted file mode 100644 index 71eccbf..0000000 --- a/harnesses/host_controller/command_processor/command_release.py +++ /dev/null @@ -1,267 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import datetime -import logging -import stat -import threading -import zipfile - -from host_controller import common -from host_controller.command_processor import base_command_processor - -_REPACKAGE_ADDITIONAL_FILE_LIST = [ - "android-vtslab/testcases/DATA/app/WifiUtil/WifiUtil.apk", - "android-vtslab/testcases/DATA/vtslab-gcs.json", - "android-vtslab/testcases/DATA/xml/media_profiles_vendor.xml", - "android-vtslab/testcases/host_controller/build/client_secrets.json", - "android-vtslab/testcases/host_controller/build/credentials", -] - -_REPACKAGE_ADDITIONAL_BIN_LIST = [ - "android-vtslab/bin/adb", -] - -# Path to the version.txt file in the fetched vtslab package zip. -_VERSION_INFO_FILE_PATH = "android-vtslab/testcases/version.txt" - -# List of strings for supported ak versions. -AK_VERSIONS = ["8.0.0", "8.0.1", "8.1.0", "9", "O", "OMR1", "P", "Q"] - -for version in AK_VERSIONS: - file_path = "android-vtslab/testcases/DATA/ak/.%s.ak" % version - _REPACKAGE_ADDITIONAL_FILE_LIST.append(file_path) - file_path += ".pub" - _REPACKAGE_ADDITIONAL_FILE_LIST.append(file_path) - - -class CommandRelease(base_command_processor.BaseCommandProcessor): - """Command processor for update command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - _timers: dict, instances of scheduled threading.Timer. - Uses timestamp("%H:%M") string as a key. - _vtslab_package_version: string, version information of the fetched - vtslab package. - (<git commit timestamp>:<git commit hash value>) - """ - - command = "release" - command_detail = "Release HC. Used for fetching HC package from PAB and uploading to GCS." - - # @Override - def SetUp(self): - """Initializes the parser for update command.""" - self._timers = {} - self._vtslab_package_version = "" - self.arg_parser.add_argument( - "--schedule-for", - default="17:00", - help="Schedule to update HC package at the given time every day. " - "Example: --schedule-for=%%H:%%M") - self.arg_parser.add_argument( - "--account_id", - default=common._DEFAULT_ACCOUNT_ID, - help="Partner Android Build account_id to use.") - self.arg_parser.add_argument( - "--branch", help="Branch to grab the artifact from.") - self.arg_parser.add_argument( - "--target", - help="a comma-separate list of build target product(s).") - self.arg_parser.add_argument( - "--dest", - help="Google Cloud Storage URL to which the file is uploaded.") - self.arg_parser.add_argument( - "--cancel", help="Cancel all scheduled release if given.") - self.arg_parser.add_argument( - "--print-all", help="Print all scheduled timers.") - self.arg_parser.add_argument( - "--additional_files_bucket", - default="gs://vtslab-release", - help="GCS bucket URL from where to fetch the additional files " - "required for HC to run properly.") - - # @Override - def Run(self, arg_line): - """Schedule a host_constroller package release at a certain time.""" - args = self.arg_parser.ParseLine(arg_line) - - if args.print_all: - logging.info(self._timers) - return - - if not args.cancel: - if args.schedule_for == "now": - self.ReleaseCallback(args.schedule_for, args.account_id, - args.branch, args.target, args.dest, - args.additional_files_bucket) - return - - elif len(args.schedule_for.split(":")) != 2: - logging.error("The format of --schedule-for flag is %H:%M") - return False - - if (int(args.schedule_for.split(":")[0]) not in range(24) - or int(args.schedule_for.split(":")[-1]) not in range(60)): - logging.error("The value of --schedule-for flag must be in " - "\"00:00\"..\"23:59\" inclusive") - return False - - if not args.schedule_for in self._timers: - delta_time = datetime.datetime.now().replace( - hour=int(args.schedule_for.split(":")[0]), - minute=int(args.schedule_for.split(":")[-1]), - second=0, - microsecond=0) - datetime.datetime.now() - - if delta_time <= datetime.timedelta(0): - delta_time += datetime.timedelta(days=1) - - self._timers[args.schedule_for] = threading.Timer( - delta_time.total_seconds(), self.ReleaseCallback, - (args.schedule_for, args.account_id, args.branch, - args.target, args.dest, args.additional_files_bucket)) - self._timers[args.schedule_for].daemon = True - self._timers[args.schedule_for].start() - logging.info("Release job scheduled for {}".format( - datetime.datetime.now() + delta_time)) - else: - self.CancelAllEvents() - - def FetchVtslab(self, account_id, branch, target, bucket): - """Fetchs android-vtslab.zip and return the fetched file path. - - Args: - account_id: string, Partner Android Build account_id to use. - branch: string, branch to grab the artifact from. - targets: string, a comma-separate list of build target product(s). - bucket: string, GCS bucket URL from where to fetch the additional - files. - - Returns: - path to the fetched android-vtslab.zip file. None if the fetching - has failed. - """ - self.console.build_provider["pab"].Authenticate() - fetched_path = self.console.build_provider[ - "pab"].FetchLatestBuiltHCPackage(account_id, branch, target) - - with zipfile.ZipFile(fetched_path, mode="a") as vtslab_package: - if _VERSION_INFO_FILE_PATH in vtslab_package.namelist(): - self._vtslab_package_version = vtslab_package.open( - _VERSION_INFO_FILE_PATH).readline().strip() - else: - self._vtslab_package_version = "" - - for path in _REPACKAGE_ADDITIONAL_FILE_LIST: - additional_file = os.path.join(bucket, path) - self.console.build_provider["gcs"].Fetch(additional_file) - try: - logging.info("Adding file %s into %s" % - (os.path.basename(path), - os.path.basename(fetched_path))) - additional_file_path = self.console.build_provider[ - "gcs"].GetAdditionalFile(os.path.basename(path)) - vtslab_package.write(additional_file_path, path) - except KeyError as e: - logging.exception(e) - - for bin in _REPACKAGE_ADDITIONAL_BIN_LIST: - additional_bin = os.path.join(bucket, bin) - self.console.build_provider["gcs"].Fetch(additional_bin) - try: - logging.info("Adding executable %s into %s" % - (os.path.basename(bin), - os.path.basename(fetched_path))) - additional_bin_path = self.console.build_provider[ - "gcs"].GetAdditionalFile(os.path.basename(bin)) - bin_mode = os.stat(additional_bin_path).st_mode - os.chmod( - additional_bin_path, - bin_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) - vtslab_package.write(additional_bin_path, bin) - except KeyError as e: - logging.exception(e) - - return fetched_path - - def UploadVtslab(self, package_file_path, dest_path): - """upload repackaged vtslab package to GCS. - - Args: - package_file_path: string, path to the vtslab package file. - dest_path: string, URL to GCS. - """ - if dest_path and dest_path.endswith("/"): - split_list = os.path.basename(package_file_path).split(".") - if self._vtslab_package_version: - try: - timestamp, hash = self._vtslab_package_version.split(":") - split_list[0] += "-%s-%s" % (timestamp, hash) - except ValueError as e: - logging.exception(e) - split_list[0] += "-{timestamp_date}" - else: - split_list[0] += "-{timestamp_date}" - dest_path += ".".join(split_list) - - upload_command = "upload --src %s --dest %s" % (package_file_path, - dest_path) - self.console.onecmd(upload_command) - - def ReleaseCallback(self, schedule_for, account_id, branch, target, dest, - bucket): - """Target function for the scheduled Timer. - - Args: - schedule_for: string, scheduled time for this Timer. - Format: "%H:%M" (from "00:00" to "23:59" inclusive) - account_id: string, Partner Android Build account_id to use. - branch: string, branch to grab the artifact from. - targets: string, a comma-separate list of build target product(s). - dest: string, URL to GCS. - bucket: string, GCS bucket URL from where to fetch the additional - files. - """ - fetched_path = self.FetchVtslab(account_id, branch, target, bucket) - if fetched_path: - self.UploadVtslab(fetched_path, dest) - - if schedule_for != "now": - delta_time = datetime.datetime.now().replace( - hour=int(schedule_for.split(":")[0]), - minute=int(schedule_for.split(":")[-1]), - second=0, - microsecond=0) - datetime.datetime.now() + datetime.timedelta( - days=1) - self._timers[schedule_for] = threading.Timer( - delta_time.total_seconds(), self.ReleaseCallback, - (schedule_for, account_id, branch, target, dest, bucket)) - self._timers[schedule_for].daemon = True - self._timers[schedule_for].start() - logging.info("Release job scheduled for {}".format( - datetime.datetime.now() + delta_time)) - - def CancelAllEvents(self): - """Cancel all scheduled Timer.""" - for scheduled_time in self._timers: - self._timers[scheduled_time].cancel() - self._timers = {} diff --git a/harnesses/host_controller/command_processor/command_repack.py b/harnesses/host_controller/command_processor/command_repack.py deleted file mode 100644 index 98a9168..0000000 --- a/harnesses/host_controller/command_processor/command_repack.py +++ /dev/null @@ -1,147 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os -import re -import shutil -import tempfile -import zipfile - -from host_controller import common -from host_controller.command_processor import base_command_processor - -# Name of android-info.txt file which contains prerequisite data for the img.zip -_ANDROID_INFO_TXT_FILENAME = "android-info.txt" - - -class CommandRepack(base_command_processor.BaseCommandProcessor): - """Command processor for repack command.""" - - command = "repack" - command_detail = ("Repackage the whole device image files, including GSI " - "images if exist.") - - # @Override - def SetUp(self): - """Initializes the parser for repack command.""" - self.arg_parser.add_argument( - "--dest", - default="gs://vts-release/img_package", - help="Google Cloud Storage base URL to which the file is uploaded." - ) - self.arg_parser.add_argument( - "--additional_files", - nargs="*", - default=[], - help="Additional files that need to be added to the zip file.") - - # @Override - def Run(self, arg_line): - """Runs an repack command.""" - args = self.arg_parser.ParseLine(arg_line) - try: - device_zipfile_path = self.console.device_image_info[ - common.FULL_ZIPFILE] - except KeyError as e: - logging.exception(e) - logging.error( - "please execute this command after fetching at least one " - "device img set.") - return False - - with zipfile.ZipFile(device_zipfile_path, 'r') as zip_ref: - zip_ref.extract(_ANDROID_INFO_TXT_FILENAME, - common.FULL_ZIPFILE_DIR) - self.console.tools_info[_ANDROID_INFO_TXT_FILENAME] = os.path.join( - common.FULL_ZIPFILE_DIR, _ANDROID_INFO_TXT_FILENAME) - - tempdir_base = os.path.join(os.getcwd(), "tmp") - tmpdir_rezip = tempfile.mkdtemp(dir=tempdir_base) - - dest_url_base, new_zipfile_name = os.path.split( - self.GetDestURL(args.dest)) - - new_zipfile_path = os.path.join(tmpdir_rezip, new_zipfile_name) - with zipfile.ZipFile( - new_zipfile_path, "w", allowZip64=True) as zip_ref: - for img_path in self.console.device_image_info: - if img_path not in (common.FULL_ZIPFILE, - common.FULL_ZIPFILE_DIR, - common.GSI_ZIPFILE, - common.GSI_ZIPFILE_DIR): - logging.info("Adding %s into the zip archive.", img_path) - zip_ref.write( - self.console.device_image_info[img_path], - img_path, - compress_type=zipfile.ZIP_DEFLATED) - if args.additional_files: - additional_file_list = self.ReplaceVars(args.additional_files) - for file_path in additional_file_list: - file_name = os.path.basename(file_path) - logging.info( - "Adding additional file %s into the zip archive.", - file_name) - zip_ref.write( - file_path, - os.path.join(common._ADDITIONAL_FILES_DIR, file_name), - compress_type=zipfile.ZIP_DEFLATED) - zip_ref.write( - self.console.tools_info[_ANDROID_INFO_TXT_FILENAME], - _ANDROID_INFO_TXT_FILENAME, - compress_type=zipfile.ZIP_DEFLATED) - - logging.info("Repackaged image set: %s", new_zipfile_path) - logging.info("Uploading %s to %s.", new_zipfile_name, dest_url_base) - - self.console.onecmd("upload --src=%s --dest=%s/%s" % - (new_zipfile_path, dest_url_base, - new_zipfile_name)) - - shutil.rmtree(tmpdir_rezip, ignore_errors=True) - - def GetDestURL(self, dest_base_url): - """Generates the destination URL to GCS bucket based on dest_base_url. - - Args: - dest_base_url: string, URL to a GCS bucket. - - Returns: - A string, device/gsi img sets branch/target info and the final - .zip file name appended to the dest_base_url. - """ - device_branch = re.sub( - "git_", "", self.console.detailed_fetch_info["device"]["branch"]) - device_target = self.console.detailed_fetch_info["device"]["target"] - device_build_id = self.console.detailed_fetch_info["device"][ - "build_id"] - new_zipfile_name = ("%s_%s_%s.zip" % (device_branch, device_target, - device_build_id)) - dest_url_base = os.path.join(dest_base_url, device_branch, - device_target) - - if common.GSI_ZIPFILE in self.console.device_image_info: - gsi_branch = re.sub( - "git_", "", self.console.detailed_fetch_info["gsi"]["branch"]) - gsi_target = self.console.detailed_fetch_info["gsi"]["target"] - gsi_build_id = self.console.detailed_fetch_info["gsi"]["build_id"] - new_zipfile_name = new_zipfile_name[:-4] + ("_%s_%s_%s.zip" % ( - gsi_branch, gsi_target, gsi_build_id)) - dest_url_base = os.path.join(dest_url_base, gsi_branch, gsi_target) - - ret = os.path.join(dest_url_base, new_zipfile_name) - self.console.repack_dest_path = ret - return ret diff --git a/harnesses/host_controller/command_processor/command_repack_test.py b/harnesses/host_controller/command_processor/command_repack_test.py deleted file mode 100644 index 6eca960..0000000 --- a/harnesses/host_controller/command_processor/command_repack_test.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import unittest - -try: - from unittest import mock -except ImportError: - import mock - -from host_controller import common -from host_controller.command_processor import command_repack - -_SAMPLE_COMMAND_REPACK = "repack --dest" - - -class CommandRepackTest(unittest.TestCase): - """Tests for repack command processor""" - - @mock.patch("host_controller.console.Console") - def testGetDestURLWithoutGSI(self, mock_console): - mock_console.detailed_fetch_info = { - "device": { - "branch": "git_device_branch", - "target": "device_userdebug", - "build_id": "1234567" - } - } - command = command_repack.CommandRepack() - command._SetUp(mock_console) - ret = command.GetDestURL("gs://bucket/base/url") - - self.assertEqual(ret, - "gs://bucket/base/url/device_branch/device_userdebug" - "/device_branch_device_userdebug_1234567.zip") - - @mock.patch("host_controller.console.Console") - def testGetDestURLWithGSI(self, mock_console): - mock_console.device_image_info = {common.GSI_ZIPFILE: ""} - mock_console.detailed_fetch_info = { - "device": { - "branch": "git_device_branch", - "target": "device-userdebug", - "build_id": "1234567" - }, - "gsi": { - "branch": "git_aosp_gsi_branch", - "target": "gsi-userdebug", - "build_id": "2345678" - } - } - - command = command_repack.CommandRepack() - command._SetUp(mock_console) - ret = command.GetDestURL("gs://bucket/base/url") - - self.assertEqual( - ret, "gs://bucket/base/url/device_branch/device-userdebug/" - "aosp_gsi_branch/gsi-userdebug/device_branch_device-userdebug" - "_1234567_aosp_gsi_branch_gsi-userdebug_2345678.zip") - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_repack.logging") - def testRepackFullDeviceImageAbsent(self, mock_logger, mock_console): - mock_console.device_image_info = {} - command = command_repack.CommandRepack() - command._SetUp(mock_console) - ret = command._Run("--dest=gs://bucket/path/") - - self.assertFalse(ret) - mock_logger.error.assert_called_with( - "please execute this command after fetching at least" - " one device img set.") - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_repack.zipfile") - @mock.patch("host_controller.command_processor.command_repack.os") - @mock.patch("host_controller.command_processor.command_repack.tempfile") - @mock.patch("host_controller.command_processor.command_repack.shutil") - def testRepackWithFullDeviceImage(self, mock_shutil, mock_tempfile, - mock_os, mock_zipfile, mock_console): - mock_zip_ref = mock.Mock() - mock_zip_ref.__enter__ = mock.Mock(return_value=mock_zip_ref) - mock_zip_ref.__exit__ = mock.Mock(return_value=None) - mock_zipfile.ZipFile.return_value = mock_zip_ref - mock_console.device_image_info = { - common.FULL_ZIPFILE: "/path/to/full-zipfile.zip", - "system.img": "/path/to/full-zipfile.zip.dir/system.img", - "odm.img": "/path/to/full-zipfile.zip.dir/odm.img" - } - mock_os.getcwd.return_value = "/abs/path/to/current/dir" - mock_os.path.split = os.path.split - command = command_repack.CommandRepack() - command.GetDestURL = mock.Mock( - return_value="gs://dest/gs/url/file.zip") - command._SetUp(mock_console) - ret = command._Run("--dest=gs://bucket/path/") - - self.assertIsNone(ret) - mock_zip_ref.extract.assert_called_with("android-info.txt", - common.FULL_ZIPFILE_DIR) - mock_zip_ref.write.assert_any_call( - "/path/to/full-zipfile.zip.dir/system.img", - "system.img", - compress_type=mock.ANY) - mock_zip_ref.write.assert_any_call( - "/path/to/full-zipfile.zip.dir/odm.img", - "odm.img", - compress_type=mock.ANY) - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_repack.zipfile") - @mock.patch("host_controller.command_processor.command_repack.os") - @mock.patch("host_controller.command_processor.command_repack.tempfile") - @mock.patch("host_controller.command_processor.command_repack.shutil") - def testRepackWithAdditionalFiles(self, mock_shutil, mock_tempfile, - mock_os, mock_zipfile, mock_console): - mock_zip_ref = mock.Mock() - mock_zip_ref.__enter__ = mock.Mock(return_value=mock_zip_ref) - mock_zip_ref.__exit__ = mock.Mock(return_value=None) - mock_zipfile.ZipFile.return_value = mock_zip_ref - mock_console.device_image_info = { - common.FULL_ZIPFILE: "/path/to/full-zipfile.zip", - "system.img": "/path/to/full-zipfile.zip.dir/system.img", - "odm.img": "/path/to/full-zipfile.zip.dir/odm.img" - } - mock_os.getcwd.return_value = "/abs/path/to/current/dir" - mock_os.path.split = os.path.split - mock_os.path.join = os.path.join - mock_os.path.basename = os.path.basename - command = command_repack.CommandRepack() - command.GetDestURL = mock.Mock( - return_value="gs://dest/gs/url/file.zip") - command._SetUp(mock_console) - ret = command._Run( - "--dest=gs://bucket/path/ --additional_files /path/to/tmp/af1" - " /path/to/tmp/af2") - - self.assertIsNone(ret) - mock_zip_ref.extract.assert_called_with("android-info.txt", - common.FULL_ZIPFILE_DIR) - mock_zip_ref.write.assert_any_call( - "/path/to/full-zipfile.zip.dir/system.img", - "system.img", - compress_type=mock.ANY) - mock_zip_ref.write.assert_any_call( - "/path/to/full-zipfile.zip.dir/odm.img", - "odm.img", - compress_type=mock.ANY) - mock_zip_ref.write.assert_any_call( - "/path/to/tmp/af1", "additional_file/af1", compress_type=mock.ANY) - mock_zip_ref.write.assert_any_call( - "/path/to/tmp/af2", "additional_file/af2", compress_type=mock.ANY) - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/command_processor/command_reproduce.py b/harnesses/host_controller/command_processor/command_reproduce.py deleted file mode 100644 index 7d4f68f..0000000 --- a/harnesses/host_controller/command_processor/command_reproduce.py +++ /dev/null @@ -1,301 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import imp # Python v2 compatibility -import logging -import os -import re -import subprocess -import zipfile - -from host_controller import common -from host_controller.command_processor import base_command_processor -from host_controller.utils.gcp import gcs_utils - -from vti.dashboard.proto import TestSuiteResultMessage_pb2 as SuiteResMsg -from vti.test_serving.proto import TestScheduleConfigMessage_pb2 as SchedCfgMsg - - -class CommandReproduce(base_command_processor.BaseCommandProcessor): - """Command processor for reproduce command. - - Attributes: - campaign_common: campaign module. Dynamically imported since - the campaign might need to be separated from the - host controller itself. - """ - - command = "reproduce" - command_detail = ("Reproduce the test environment for a pre-run test and " - "execute the tradefed command prompt of the fetched " - "test suite. Setup the test env " - "(fetching, flashing devices, etc) and retrieve " - "formerly run test result to retry on, if the path " - "to the report protobuf file is given.") - - # @Override - def SetUp(self): - """Initializes the parser for reproduce command.""" - self.campaign_common = None - self.arg_parser.add_argument( - "--suite", - default="vts", - choices=("vts", "cts", "gts", "sts"), - help="To specify the type of a test suite to be run.") - self.arg_parser.add_argument( - "--report_path", - required=True, - help="Google Cloud Storage URL, the path of a report protobuf file." - ) - self.arg_parser.add_argument( - "--serial", - default=None, - help="The serial numbers for flashing and testing. " - "Multiple serial numbers are separated by commas.") - self.arg_parser.add_argument( - "--automated_retry", - action="store_true", - help="Retries automatically until all test cases are passed " - "or the number or the failed test cases is the same as " - "the previous one.") - - # @Override - def Run(self, arg_line): - """Reproduces the test env of the pre-run test.""" - args = self.arg_parser.ParseLine(arg_line) - - if args.report_path: - gsutil_path = gcs_utils.GetGsutilPath() - if not gsutil_path: - logging.error( - "Please check whether gsutil is installed and on your PATH" - ) - return False - - if (not args.report_path.startswith("gs://") - or not gcs_utils.IsGcsFile(gsutil_path, args.report_path)): - logging.error("%s is not a valid GCS path.", args.report_path) - return False - - dest_path = os.path.join("".join(self.ReplaceVars(["{tmp_dir}"])), - os.path.basename(args.report_path)) - gcs_utils.Copy(gsutil_path, args.report_path, dest_path) - report_msg = SuiteResMsg.TestSuiteResultMessage() - try: - with open(dest_path, "r") as report_fd: - report_msg.ParseFromString(report_fd.read()) - except IOError as e: - logging.exception(e) - return False - serial = [] - if args.serial: - serial = args.serial.split(",") - setup_command_list = self.GenerateSetupCommands(report_msg, serial) - if not setup_command_list: - suite_fetch_command = self.GenerateTestSuiteFetchCommand( - report_msg) - if suite_fetch_command: - setup_command_list.append(suite_fetch_command) - for command in setup_command_list: - self.console.onecmd(command) - - if not self.GetResultFromGCS(gsutil_path, report_msg, args.suite): - return False - else: - logging.error("Path to a report protobuf file is required.") - return False - - if args.suite not in self.console.test_suite_info: - logging.error("test_suite_info doesn't have '%s': %s", args.suite, - self.console.test_suite_info) - return False - - if args.automated_retry: - if self.campaign_common is None: - self.campaign_common = imp.load_source( - 'campaign_common', - os.path.join(os.getcwd(), "host_controller", "campaigns", - "campaign_common.py")) - retry_command = self.campaign_common.GenerateRetryCommand( - report_msg.schedule_config.build_target[0].name, - report_msg.branch, report_msg.suite_name.lower(), - report_msg.suite_plan, serial) - self.console.onecmd(retry_command) - else: - subprocess.call(self.console.test_suite_info[args.suite]) - - def GenerateSetupCommands(self, report_msg, serial): - """Generates fetch, flash commands using fetch info from report_msg. - - Args: - report_msg: pb2, contains fetch info of the test suite. - serial: list of string, serial number(s) of the device(s) - to be flashed. - - Returns: - list of string, console commands to fetch device/gsi images - and flash the device(s). - """ - ret = [] - schedule_config = report_msg.schedule_config - if not schedule_config.manifest_branch: - logging.error("Report contains no fetch information. " - "Aborting pre-test setups on the device(s).") - elif not serial: - logging.error("Device serial number(s) not given. " - "Aborting pre-test setups on the device(s).") - else: - try: - build_target_msg = schedule_config.build_target[0] - test_schedule_msg = build_target_msg.test_schedule[0] - except IndexError as e: - logging.exception(e) - return ret - kwargs = {} - # common fetch info - kwargs["shards"] = str(len(serial)) - kwargs["test_name"] = "%s/%s" % (report_msg.suite_name.lower(), - report_msg.suite_plan) - kwargs["serial"] = serial - - # fetch info for device images - kwargs["manifest_branch"] = schedule_config.manifest_branch - kwargs["build_target"] = build_target_msg.name - kwargs["build_id"] = report_msg.vendor_build_id - kwargs["pab_account_id"] = schedule_config.pab_account_id - if kwargs["manifest_branch"].startswith("gs://"): - kwargs[ - "build_storage_type"] = SchedCfgMsg.BUILD_STORAGE_TYPE_GCS - else: - kwargs[ - "build_storage_type"] = SchedCfgMsg.BUILD_STORAGE_TYPE_PAB - kwargs["require_signed_device_build"] = ( - build_target_msg.require_signed_device_build) - kwargs["has_bootloader_img"] = build_target_msg.has_bootloader_img - kwargs["has_radio_img"] = build_target_msg.has_radio_img - - # fetch info for gsi images and gsispl command - kwargs["gsi_branch"] = test_schedule_msg.gsi_branch - kwargs["gsi_build_target"] = test_schedule_msg.gsi_build_target - kwargs["gsi_build_id"] = report_msg.gsi_build_id - kwargs["gsi_pab_account_id"] = test_schedule_msg.gsi_pab_account_id - if kwargs["gsi_branch"].startswith("gs://"): - kwargs["gsi_storage_type"] = SchedCfgMsg.BUILD_STORAGE_TYPE_GCS - else: - kwargs["gsi_storage_type"] = SchedCfgMsg.BUILD_STORAGE_TYPE_PAB - kwargs["gsi_vendor_version"] = test_schedule_msg.gsi_vendor_version - - # fetch info for test suite - kwargs["test_build_id"] = report_msg.build_id - kwargs["test_branch"] = report_msg.branch - kwargs["test_build_target"] = report_msg.target - if kwargs["test_build_target"].endswith(".zip"): - kwargs["test_build_target"] = kwargs["test_build_target"][:-4] - kwargs[ - "test_pab_account_id"] = test_schedule_msg.test_pab_account_id - if kwargs["test_branch"].startswith("gs://"): - kwargs[ - "test_storage_type"] = SchedCfgMsg.BUILD_STORAGE_TYPE_GCS - else: - kwargs["gsi_storage_type"] = SchedCfgMsg.BUILD_STORAGE_TYPE_PAB - - self.campaign_common = imp.load_source( - "campaign_common", - os.path.join(os.getcwd(), "host_controller", "campaigns", - "campaign_common.py")) - fetch_commands_result, gsi = self.campaign_common.EmitFetchCommands( - **kwargs) - ret.extend(fetch_commands_result) - flash_commands_result = self.campaign_common.EmitFlashCommands( - gsi, **kwargs) - ret.extend(flash_commands_result) - - return ret - - def GenerateTestSuiteFetchCommand(self, report_msg): - """Generates a fetch command line using fetch info from report_msg. - - Args: - report_msg: pb2, contains fetch info of the test suite. - - Returns: - string, console command to fetch a test suite artifact. - """ - ret = "fetch" - - if report_msg.branch.startswith("gs://"): - ret += " --type=gcs --path=%s/%s --set_suite_as=%s" % ( - report_msg.branch, report_msg.target, - report_msg.suite_name.lower()) - else: - ret += (" --type=pab --branch=%s --target=%s --build_id=%s" - " --artifact_name=android-%s.zip") % ( - report_msg.branch, report_msg.target, - report_msg.build_id, report_msg.suite_name.lower()) - try: - build_target_msg = report_msg.schedule_config.build_target[0] - test_schedule_msg = build_target_msg.test_schedule[0] - except IndexError as e: - logging.exception(e) - test_schedule_msg = SchedCfgMsg.TestScheduleConfigMessage() - if test_schedule_msg.test_pab_account_id: - ret += " --account_id=%s" % test_schedule_msg.test_pab_account_id - else: - ret += " --account_id=%s" % common._DEFAULT_ACCOUNT_ID_INTERNAL - - return ret - - def GetResultFromGCS(self, gsutil_path, report_msg, suite): - """Downloads results.zip from GCS and unzip it to the results directory. - - Args: - gsutil_path: string, path to the gsutil binary. - report_msg: pb2, contains fetch info of the test suite. - suite: string, specifies the type of test suite fetched. - - Returns: - True if successful. False otherwise - """ - result_base_path = report_msg.result_path - try: - tools_path = os.path.dirname(self.console.test_suite_info[suite]) - except KeyError as e: - logging.exception(e) - return False - local_results_path = os.path.join(tools_path, - common._RESULTS_BASE_PATH) - - if not os.path.exists(local_results_path): - os.mkdir(local_results_path) - - result_path_list = gcs_utils.List(gsutil_path, result_base_path) - result_zip_url = "" - for result_path in result_path_list: - if re.match(".*results_.*\.zip", result_path): - result_zip_url = result_path - break - - if (not result_zip_url or not gcs_utils.Copy( - gsutil_path, result_zip_url, local_results_path)): - logging.error("Fail to copy from %s.", result_base_path) - return False - - result_zip_local_path = os.path.join(local_results_path, - os.path.basename(result_zip_url)) - with zipfile.ZipFile(result_zip_local_path, mode="r") as zip_ref: - zip_ref.extractall(local_results_path) - - return True diff --git a/harnesses/host_controller/command_processor/command_reproduce_test.py b/harnesses/host_controller/command_processor/command_reproduce_test.py deleted file mode 100644 index a5ab4f6..0000000 --- a/harnesses/host_controller/command_processor/command_reproduce_test.py +++ /dev/null @@ -1,353 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import unittest - -try: - from unittest import mock -except ImportError: - import mock - -from host_controller import common -from host_controller.command_processor import command_reproduce - - -def emit_fetch_commands(**kwargs): - gsi = "gsi_branch" in kwargs - return [], gsi - - -def emit_flash_commands(gsi, **kwargs): - return [] - - -class CommandReproduceTest(unittest.TestCase): - """Tests for reproduce command processor""" - - def setUp(self): - mock_console = mock.Mock() - self._command = command_reproduce.CommandReproduce() - self._command._SetUp(mock_console) - - @mock.patch("host_controller.command_processor.command_reproduce.logging") - def testGenerateSetupCommandsNoFetchInfo(self, mock_logging): - mock_msg = mock.Mock() - mock_msg.schedule_config.manifest_branch = "" - ret = self._command.GenerateSetupCommands(mock_msg, - ["serial1", "serial2"]) - self.assertEqual(ret, []) - mock_logging.error.assert_called_with( - "Report contains no fetch information. " - "Aborting pre-test setups on the device(s).") - - @mock.patch("host_controller.command_processor.command_reproduce.logging") - def testGenerateSetupCommandsNoSerial(self, mock_logging): - mock_msg = mock.Mock() - mock_msg.schedule_config.manifest_branch = "some_branch" - ret = self._command.GenerateSetupCommands(mock_msg, []) - self.assertEqual(ret, []) - mock_logging.error.assert_called_with( - "Device serial number(s) not given. " - "Aborting pre-test setups on the device(s).") - - @mock.patch("host_controller.command_processor.command_reproduce.imp") - def testGenerateSetupCommands(self, mock_imp): - mock_campiagn = mock.Mock() - mock_campiagn.EmitFetchCommands.side_effect = emit_fetch_commands - mock_campiagn.EmitFlashCommands.side_effect = emit_flash_commands - mock_imp.load_source.return_value = mock_campiagn - mock_msg = mock.Mock() - mock_msg.suite_name = "vts" - mock_msg.suite_plan = "vts" - mock_msg.vendor_build_id = "1234567" - mock_msg.gsi_build_id = "2345678" - mock_msg.build_id = "3456789" - mock_msg.branch = "git_whatever-release" - mock_msg.target = "test_suites_bitness" - mock_msg.schedule_config.manifest_branch = "git_whatever-release" - mock_msg.schedule_config.pab_account_id = common._DEFAULT_ACCOUNT_ID_INTERNAL - mock_msg.schedule_config.build_target = [] - mock_build_target_msg = mock.Mock() - mock_build_target_msg.name = "somefish-userdebug" - mock_build_target_msg.test_schedule = [] - mock_build_target_msg.require_signed_device_build = False - mock_build_target_msg.has_bootloader_img = True - mock_build_target_msg.has_radio_img = True - mock_test_schedule_msg = mock.Mock() - mock_test_schedule_msg.gsi_branch = "git_whatever-gsi" - mock_test_schedule_msg.gsi_build_target = "aosp_bitness-userdebug" - mock_test_schedule_msg.gsi_pab_account_id = common._DEFAULT_ACCOUNT_ID - mock_test_schedule_msg.test_pab_account_id = common._DEFAULT_ACCOUNT_ID - mock_build_target_msg.test_schedule.append(mock_test_schedule_msg) - mock_msg.schedule_config.build_target.append(mock_build_target_msg) - self._command.GenerateSetupCommands(mock_msg, ["serial1", "serial2"]) - mock_campiagn.EmitFetchCommands.assert_called_with( - build_id="1234567", - build_storage_type=1, - build_target="somefish-userdebug", - gsi_branch="git_whatever-gsi", - gsi_build_id="2345678", - gsi_build_target="aosp_bitness-userdebug", - gsi_pab_account_id="543365459", - gsi_storage_type=1, - gsi_vendor_version=mock.ANY, - has_bootloader_img=True, - has_radio_img=True, - manifest_branch="git_whatever-release", - pab_account_id="541462473", - require_signed_device_build=False, - serial=["serial1", "serial2"], - shards="2", - test_branch="git_whatever-release", - test_build_id="3456789", - test_build_target="test_suites_bitness", - test_name="vts/vts", - test_pab_account_id="543365459") - mock_campiagn.EmitFlashCommands.assert_called_with( - True, - build_id="1234567", - build_storage_type=1, - build_target="somefish-userdebug", - gsi_branch="git_whatever-gsi", - gsi_build_id="2345678", - gsi_build_target="aosp_bitness-userdebug", - gsi_pab_account_id="543365459", - gsi_storage_type=1, - gsi_vendor_version=mock.ANY, - has_bootloader_img=True, - has_radio_img=True, - manifest_branch="git_whatever-release", - pab_account_id="541462473", - require_signed_device_build=False, - serial=["serial1", "serial2"], - shards="2", - test_branch="git_whatever-release", - test_build_id="3456789", - test_build_target="test_suites_bitness", - test_name="vts/vts", - test_pab_account_id="543365459") - - def testGenerateTestSuiteFetchCommandGCS(self): - report_msg = mock.Mock() - report_msg.branch = "gs://bucket/path/to/vts/release" - report_msg.target = "android-vts.zip" - report_msg.suite_name = "VTS" - ret = self._command.GenerateTestSuiteFetchCommand(report_msg) - self.assertEqual( - ret, "fetch --type=gcs " - "--path=gs://bucket/path/to/vts/release/android-vts.zip " - "--set_suite_as=vts") - - @mock.patch( - "host_controller.command_processor.command_reproduce.SchedCfgMsg") - @mock.patch("host_controller.command_processor.command_reproduce.logging") - def testGenerateTestSuiteFetchCommandIndexError(self, mock_logging, - mock_sched_cfg_msg): - report_msg = mock.Mock() - report_msg.branch = "git_whatever-release" - report_msg.target = "test_suites_bitness" - report_msg.build_id = "1234567" - report_msg.suite_name = "VTS" - report_msg.schedule_config.build_target = [] - mock_test_schedule_msg = mock.Mock() - mock_test_schedule_msg.test_pab_account_id = "1234567898765" - mock_sched_cfg_msg.TestScheduleConfigMessage.return_value = ( - mock_test_schedule_msg) - ret = self._command.GenerateTestSuiteFetchCommand(report_msg) - mock_logging.exception.assert_called() - self.assertEqual(ret, "fetch --type=pab --branch=git_whatever-release " - "--target=test_suites_bitness --build_id=1234567 " - "--artifact_name=android-vts.zip " - "--account_id=1234567898765") - - @mock.patch( - "host_controller.command_processor.command_reproduce.SchedCfgMsg") - def testGenerateTestSuiteFetchCommandPAB(self, mock_sched_cfg_msg): - report_msg = mock.Mock() - report_msg.branch = "git_whatever-release" - report_msg.target = "test_suites_bitness" - report_msg.build_id = "1234567" - report_msg.suite_name = "VTS" - mock_build_target_msg = mock.Mock() - mock_test_schedule_msg = mock.Mock() - mock_test_schedule_msg.test_pab_account_id = "987654321" - mock_build_target_msg.test_schedule = [] - mock_build_target_msg.test_schedule.append(mock_test_schedule_msg) - report_msg.schedule_config.build_target = [] - report_msg.schedule_config.build_target.append(mock_build_target_msg) - mock_sched_cfg_msg.TestScheduleConfigMessage.return_value = ( - mock_test_schedule_msg) - - ret = self._command.GenerateTestSuiteFetchCommand(report_msg) - self.assertEqual(ret, "fetch --type=pab --branch=git_whatever-release " - "--target=test_suites_bitness --build_id=1234567 " - "--artifact_name=android-vts.zip " - "--account_id=987654321") - - @mock.patch("host_controller.command_processor.command_reproduce.logging") - def testGetResultFromGCSNoTestSuite(self, mock_logging): - report_msg = mock.Mock() - report_msg.result_path = "gs://bucket/path/to/log/files" - self._command.console.test_suite_info = {} - ret = self._command.GetResultFromGCS("/mock_bin/gsutil", report_msg, - "vts") - self.assertFalse(ret) - mock_logging.exception.assert_called() - - @mock.patch("host_controller.command_processor.command_reproduce.os") - def testGetResultFromGCSMkdirResults(self, mock_os): - report_msg = mock.Mock() - report_msg.result_path = "gs://bucket/path/to/log/files" - self._command.console.test_suite_info = { - "vts": "tmp/android-vts/tools/vts-tradefed" - } - mock_os.path.exists.return_value = False - mock_os.path.join = os.path.join - mock_os.path.dirname = os.path.dirname - self._command.GetResultFromGCS("/mock_bin/gsutil", report_msg, "vts") - mock_os.mkdir.assert_called_with("tmp/android-vts/tools/../results") - - @mock.patch( - "host_controller.command_processor.command_reproduce.gcs_utils") - @mock.patch("host_controller.command_processor.command_reproduce.os") - @mock.patch("host_controller.command_processor.command_reproduce.logging") - def testGetResultFromGCSNoResult(self, mock_logging, mock_os, - mock_gcs_util): - report_msg = mock.Mock() - report_msg.result_path = "gs://bucket/path/to/log/files" - self._command.console.test_suite_info = { - "vts": "tmp/android-vts/tools/vts-tradefed" - } - mock_gcs_util.List.return_value = [ - "some_log1.zip", "some_log2.zip", "not_a_result.zip" - ] - ret = self._command.GetResultFromGCS("/mock_bin/gsutil", report_msg, - "vts") - self.assertFalse(ret) - - @mock.patch( - "host_controller.command_processor.command_reproduce.gcs_utils") - @mock.patch("host_controller.command_processor.command_reproduce.os") - @mock.patch("host_controller.command_processor.command_reproduce.zipfile") - def testGetResultFromGCS(self, mock_zipfile, mock_os, mock_gcs_util): - report_msg = mock.Mock() - report_msg.result_path = "gs://bucket/path/to/log/files" - self._command.console.test_suite_info = { - "vts": "tmp/android-vts/tools/vts-tradefed" - } - mock_zip_ref = mock.Mock() - mock_zip_ref.__enter__ = mock.Mock(return_value=mock_zip_ref) - mock_zip_ref.__exit__ = mock.Mock(return_value=None) - mock_zipfile.ZipFile.return_value = mock_zip_ref - mock_gcs_util.List.return_value = [ - "some_log1.zip", "some_log2.zip", "results_some_hash.zip" - ] - mock_gcs_util.Copy.return_value = True - mock_os.path.join = os.path.join - mock_os.path.exists.return_value = False - mock_os.path.dirname = os.path.dirname - ret = self._command.GetResultFromGCS("/mock_bin/gsutil", report_msg, - "vts") - self.assertTrue(ret) - mock_zip_ref.extractall.assert_called_with( - "tmp/android-vts/tools/../results") - - @mock.patch( - "host_controller.command_processor.command_reproduce.gcs_utils") - @mock.patch("host_controller.command_processor.command_reproduce.logging") - def testCommandReproduceGsutilAbsent(self, mock_logging, mock_gcs_util): - mock_gcs_util.GetGsutilPath.return_value = "" - ret = self._command._Run( - "--report_path=gs://bucket/path/to/report/file") - self.assertFalse(ret) - mock_logging.error.assert_called_with( - "Please check whether gsutil is installed and on your PATH") - - @mock.patch( - "host_controller.command_processor.command_reproduce.gcs_utils") - @mock.patch("host_controller.command_processor.command_reproduce.logging") - def testCommandReproduceInvalidURL(self, mock_logging, mock_gcs_util): - mock_gcs_util.GetGsutilPath.return_value = "/mock_bin/gsutil" - ret = self._command._Run("--report_path=/some/path/to/report/file") - self.assertFalse(ret) - mock_logging.error.assert_called_with("%s is not a valid GCS path.", - "/some/path/to/report/file") - - @mock.patch( - "host_controller.command_processor.command_reproduce.gcs_utils") - @mock.patch( - "host_controller.command_processor.command_reproduce.open", - new_callable=mock.mock_open) - @mock.patch("host_controller.command_processor.command_reproduce.imp") - def testCommandReproduceAutomatedRetry(self, mock_imp, mock_open, - mock_gcs_util): - mock_gcs_util.GetGsutilPath.return_value = "/mock_bin/gsutil" - mock_gcs_util.IsGcsFile.return_value = True - mock_gcs_util.Copy = mock.Mock(return_value=True) - command_reproduce.SuiteResMsg.ParseFromString = mock.Mock( - return_value={}) - self._command.console.test_suite_info = { - "vts": "tmp/android-vts/tools/vts-tradefed" - } - mock_campiagn = mock.Mock() - mock_imp.load_source.return_value = mock_campiagn - self._command.ReplaceVars = mock.Mock(return_value="tmp") - self._command.GetResultFromGCS = mock.Mock(return_value=True) - self._command.GenerateSetupCommands = mock.Mock(return_value=[]) - self._command.GenerateTestSuiteFetchCommand = mock.Mock( - return_value=[]) - ret = self._command._Run("--report_path=gs://some/path/to/report/file" - " --serial=device1 --automated_retry") - self.assertIsNone(ret) - self._command.console.onecmd.assert_called_with( - "retry --suite vts --serial device1") - - @mock.patch( - "host_controller.command_processor.command_reproduce.gcs_utils") - @mock.patch( - "host_controller.command_processor.command_reproduce.open", - new_callable=mock.mock_open) - @mock.patch("host_controller.command_processor.command_reproduce.imp") - def testCommandReproduceAutomatedRetryShardOption( - self, mock_imp, mock_open, mock_gcs_util): - mock_gcs_util.GetGsutilPath.return_value = "/mock_bin/gsutil" - mock_gcs_util.IsGcsFile.return_value = True - mock_gcs_util.Copy = mock.Mock(return_value=True) - command_reproduce.SuiteResMsg.ParseFromString = mock.Mock() - self._command.console.test_suite_info = { - "vts": "tmp/android-vts/tools/vts-tradefed" - } - mock_campiagn = mock.Mock() - mock_campiagn.GetVersion.return_value = 8.0 - mock_imp.load_source.return_value = mock_campiagn - self._command.ReplaceVars = mock.Mock(return_value="tmp") - self._command.GetResultFromGCS = mock.Mock(return_value=True) - self._command.GenerateSetupCommands = mock.Mock(return_value=[]) - self._command.GenerateTestSuiteFetchCommand = mock.Mock( - return_value=[]) - ret = self._command._Run( - "--suite=vts --report_path=gs://some/path/to/report/file" - " --serial=device1,device2 --automated_retry") - self.assertIsNone(ret) - self._command.console.onecmd.assert_called_with( - "retry --suite vts --shards 2 " - "--serial device1 --serial device2") - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/command_processor/command_request.py b/harnesses/host_controller/command_processor/command_request.py deleted file mode 100644 index 3851823..0000000 --- a/harnesses/host_controller/command_processor/command_request.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from host_controller.command_processor import base_command_processor -from host_controller.tfc import request - - -class CommandRequest(base_command_processor.BaseCommandProcessor): - """Command processor for request command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "request" - command_detail = "Send TFC a request to execute a command." - - # @Override - def SetUp(self): - """Initializes the parser for request command.""" - self.arg_parser.add_argument( - "--cluster", - required=True, - help="The cluster to which the request is submitted.") - self.arg_parser.add_argument( - "--run-target", - required=True, - help="The target device to run the command.") - self.arg_parser.add_argument( - "--user", - required=True, - help="The name of the user submitting the request.") - self.arg_parser.add_argument( - "command", - metavar="COMMAND", - nargs="+", - help='The command to be executed. If the command contains ' - 'arguments starting with "-", place the command after ' - '"--" at end of line.') - - # @Override - def Run(self, arg_line): - """Sends TFC a request to execute a command.""" - args = self.arg_parser.ParseLine(arg_line) - req = request.Request( - cluster=args.cluster, - command_line=" ".join(args.command), - run_target=args.run_target, - user=args.user) - self.console._tfc_client.NewRequest(req) diff --git a/harnesses/host_controller/command_processor/command_retry.py b/harnesses/host_controller/command_processor/command_retry.py deleted file mode 100644 index 3108b4b..0000000 --- a/harnesses/host_controller/command_processor/command_retry.py +++ /dev/null @@ -1,301 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import itertools -import logging -import os -import zipfile - -from host_controller import common -from host_controller.command_processor import base_command_processor -from host_controller.utils.gcp import gcs_utils -from host_controller.utils.parser import xml_utils - -from vts.utils.python.common import cmd_utils - -# The command list for cleaning up each devices listed for the retry command. -_DEVICE_CLEANUP_COMMAND_LIST = [ - "adb -s {serial} reboot bootloader", - "fastboot -s {serial} erase metadata -- -w", - "fastboot -s {serial} reboot", - "adb -s {serial} wait-for-device", - "dut --operation=wifi_on --serial={serial} --ap=" + - common._DEFAULT_WIFI_AP, -] - - -class CommandRetry(base_command_processor.BaseCommandProcessor): - """Command processor for retry command.""" - - command = "retry" - command_detail = "Retry last run test plan for certain times." - - def IsResultZipFile(self, zip_ref): - """Determines whether the given zip_ref is the right result archive. - - Need to check the number of contents of the zip file since - the "log-result_<>.zip" file only contains "test_result.xml", - but cannot parsed by vts-tf properly when trying to do the retry. - - Args: - zip_ref: ZipFile, reference to the downloaded results_<>.zip file - - Returns: - True if the downloaded zip file is usable from vts-fs, - False otherwise. - """ - if len(zip_ref.namelist()) > 1: - for name in zip_ref.namelist(): - if common._TEST_RESULT_XML in name: - return True - return False - - def GetResultFromGCS(self, gcs_result_path, local_results_dir): - """Downloads a vts-tf result zip archive from GCS. - - And unzip the file to "android-vts/results/" path so - the vts-tf will parse the result correctly. - - Args: - gcs_result_path: string, path to GCS file. - local_results_dir: string, abs path to the result directory of - currently running vts-tf. - Returns: - A string which is the name of unzipped result directory. - None if the download has failed or the downloaded zip file - is not a correct result archive. - """ - gsutil_path = gcs_utils.GetGsutilPath() - if not gsutil_path: - logging.error("Please check gsutil is installed and on your PATH") - return None - - if (not gcs_result_path.startswith("gs://") - or not gcs_utils.IsGcsFile(gsutil_path, gcs_result_path)): - logging.error("%s is not correct GCS url.", gcs_result_path) - return None - if not gcs_result_path.endswith(".zip"): - logging.error("%s is not a correct result archive file.", - gcs_result_path) - return None - - if not os.path.exists(local_results_dir): - os.mkdir(local_results_dir) - if not gcs_utils.Copy(gsutil_path, gcs_result_path, local_results_dir): - logging.error("Fail to copy from %s.", gcs_result_path) - return None - result_zip = os.path.join(local_results_dir, - gcs_result_path.split("/")[-1]) - with zipfile.ZipFile(result_zip, mode="r") as zip_ref: - if self.IsResultZipFile(zip_ref): - unzipped_result_dir = zip_ref.namelist()[0].rstrip("/") - zip_ref.extractall(local_results_dir) - return unzipped_result_dir - else: - logging.error("Not a correct vts-tf result archive file.") - return None - - # @Override - def SetUp(self): - """Initializes the parser for retry command.""" - self.arg_parser.add_argument( - "--suite", - default="vts", - choices=("vts", "cts", "gts", "sts"), - help="To specify the type of a test suite to be run.") - self.arg_parser.add_argument( - "--count", - type=int, - default=common.DEFAULT_RETRY_COUNT, - help="Retry count. Default retry count is %s." % - common.DEFAULT_RETRY_COUNT) - self.arg_parser.add_argument( - "--force-count", - type=int, - default=3, - help="Forced retry count. Retry certain test plan for the given " - "times whether all testcases has passed or not.") - self.arg_parser.add_argument( - "--result-from-gcs", - help="Google Cloud Storage URL from which the result is downloaded. " - "Will retry based on the fetched result data") - self.arg_parser.add_argument( - "--serial", - action="append", - default=[], - help="Serial number for device. Can pass this flag multiple times." - ) - self.arg_parser.add_argument( - "--shards", type=int, help="Test plan's shard count.") - self.arg_parser.add_argument( - "--shard-count", - type=int, - help= - "Test plan's shard count. Same as the \"--shards\" flag but the " - "value will be passed to the tradefed with \"--shard-count\" flag." - ) - self.arg_parser.add_argument( - "--cleanup_devices", - default=False, - type=bool, - help="True to erase metadata and userdata (equivalent to " - "factory reset) between retries.") - self.arg_parser.add_argument( - "--retry_plan", help="The name of a retry plan to use.") - - # @Override - def Run(self, arg_line): - """Retry last run plan for certain times.""" - args = self.arg_parser.ParseLine(arg_line) - retry_count = args.count - force_retry_count = args.force_count - - if args.suite not in self.console.test_suite_info: - logging.error("test_suite_info doesn't have '%s': %s", args.suite, - self.console.test_suite_info) - return False - - tools_path = os.path.dirname(self.console.test_suite_info[args.suite]) - results_path = os.path.join(tools_path, common._RESULTS_BASE_PATH) - - unzipped_result_dir = "" - unzipped_result_session_id = -1 - if args.result_from_gcs: - unzipped_result_dir = self.GetResultFromGCS( - args.result_from_gcs, results_path) - if not unzipped_result_dir: - return False - - former_results = [ - result for result in os.listdir(results_path) - if os.path.isdir(os.path.join(results_path, result)) - and not os.path.islink(os.path.join(results_path, result)) - ] - former_result_count = len(former_results) - if former_result_count < 1: - logging.error( - "No test plan has been run yet, former results count is %d", - former_result_count) - return False - - if unzipped_result_dir: - former_results.sort() - unzipped_result_session_id = former_results.index( - unzipped_result_dir) - - for result_index in range(retry_count): - if unzipped_result_session_id >= 0: - session_id = unzipped_result_session_id - unzipped_result_session_id = -1 - latest_result_xml_path = os.path.join( - results_path, unzipped_result_dir, common._TEST_RESULT_XML) - else: - session_id = former_result_count - 1 + result_index - latest_result_xml_path = os.path.join(results_path, "latest", - common._TEST_RESULT_XML) - if not os.path.exists(latest_result_xml_path): - latest_result_xml_path = os.path.join( - results_path, former_results[-1], - common._TEST_RESULT_XML) - - result_attrs = xml_utils.GetAttributes( - latest_result_xml_path, common._RESULT_TAG, - [common._SUITE_PLAN_ATTR_KEY]) - - summary_attrs = xml_utils.GetAttributes( - latest_result_xml_path, common._SUMMARY_TAG, [ - common._FAILED_ATTR_KEY, common._MODULES_TOTAL_ATTR_KEY, - common._MODULES_DONE_ATTR_KEY - ]) - - result_fail_count = int(summary_attrs[common._FAILED_ATTR_KEY]) - result_skip_count = int( - summary_attrs[common._MODULES_TOTAL_ATTR_KEY]) - int( - summary_attrs[common._MODULES_DONE_ATTR_KEY]) - - if (result_index >= force_retry_count and result_skip_count == 0 - and result_fail_count == 0): - logging.info("All modules have run and passed. " - "Skipping remaining %d retry runs.", - (retry_count - result_index)) - break - - shard_flag_literal = "" - if args.shards: - shard_flag_literal = "--shards" - shard_num = args.shards - if args.shard_count: - shard_flag_literal = "--shard-count" - shard_num = args.shard_count - - if args.retry_plan: - retry_plan = args.retry_plan - else: - retry_plan = result_attrs[common._SUITE_PLAN_ATTR_KEY] - if shard_flag_literal: - retry_test_command = ( - "test --suite=%s --keep-result -- %s --retry %d %s %d" % - (args.suite, retry_plan, session_id, shard_flag_literal, - shard_num)) - else: - retry_test_command = ( - "test --suite=%s --keep-result -- %s --retry %d" % - (args.suite, retry_plan, session_id)) - if args.serial: - for serial in args.serial: - retry_test_command += " --serial %s" % serial - - if args.cleanup_devices: - for (command, serial) in itertools.product( - _DEVICE_CLEANUP_COMMAND_LIST, args.serial): - if not self.console.onecmd(command.format(serial=serial)): - logging.error( - "Factory reset failed on the devices %s. " - "Skipping retry run(s)", serial) - self.console.device_status[ - serial] = common._DEVICE_STATUS_DICT["use"] - return - - self.console.onecmd(retry_test_command) - - for result in os.listdir(results_path): - new_result = os.path.join(results_path, result) - if (os.path.isdir(new_result) - and not os.path.islink(new_result) - and result not in former_results): - former_results.append(result) - break - - summary_after_retry = xml_utils.GetAttributes( - os.path.join(results_path, former_results[-1], - common._TEST_RESULT_XML), common._SUMMARY_TAG, - [ - common._FAILED_ATTR_KEY, common._MODULES_TOTAL_ATTR_KEY, - common._MODULES_DONE_ATTR_KEY - ]) - fail_count_after_retry = int( - summary_after_retry[common._FAILED_ATTR_KEY]) - skip_count_after_retry = int( - summary_after_retry[common._MODULES_TOTAL_ATTR_KEY]) - int( - summary_after_retry[common._MODULES_DONE_ATTR_KEY]) - if (result_index >= force_retry_count - and fail_count_after_retry == result_fail_count - and skip_count_after_retry == result_skip_count): - logging.warning( - "Same result as the former test run from the retry run. " - "Skipping remaining %d retry runs.", - (retry_count - result_index)) - break diff --git a/harnesses/host_controller/command_processor/command_sheet.py b/harnesses/host_controller/command_processor/command_sheet.py deleted file mode 100644 index 090f707..0000000 --- a/harnesses/host_controller/command_processor/command_sheet.py +++ /dev/null @@ -1,421 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import csv -import os -import logging -import re -import shutil -import tempfile -import zipfile - -try: - # TODO: Remove when we stop supporting Python 2 - import StringIO as string_io_module -except ImportError: - import io as string_io_module - -import gspread - -from oauth2client.service_account import ServiceAccountCredentials - -from host_controller import common -from host_controller.command_processor import base_command_processor -from host_controller.utils.gcp import gcs_utils -from host_controller.utils.parser import result_utils -from host_controller.utils.parser import xml_utils - -# Attributes shown on spreadsheet -_RESULT_ATTR_KEYS = [ - common._SUITE_NAME_ATTR_KEY, common._SUITE_PLAN_ATTR_KEY, - common._SUITE_VERSION_ATTR_KEY, common._SUITE_BUILD_NUM_ATTR_KEY, - common._START_DISPLAY_TIME_ATTR_KEY, - common._END_DISPLAY_TIME_ATTR_KEY -] - -_BUILD_ATTR_KEYS = [ - common._FINGERPRINT_ATTR_KEY, - common._SYSTEM_FINGERPRINT_ATTR_KEY, - common._VENDOR_FINGERPRINT_ATTR_KEY -] - -_SUMMARY_ATTR_KEYS = [ - common._PASSED_ATTR_KEY, common._FAILED_ATTR_KEY, - common._MODULES_TOTAL_ATTR_KEY, common._MODULES_DONE_ATTR_KEY -] - -# Texts on spreadsheet -_TABLE_HEADER = ("BITNESS", "TEST_MODULE", "TEST_CLASS", "TEST_CASE", "RESULT") - -_CMP_TABLE_HEADER = _TABLE_HEADER + ("REFERENCE_RESULT",) - -_TOO_MANY_DATA = "too many to be displayed" - - -class CommandSheet(base_command_processor.BaseCommandProcessor): - """Command processor for sheet command. - - Attributes: - _SCOPE: The scope needed to access Google Sheets. - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - _SCOPE = "https://www.googleapis.com/auth/drive" - command = "sheet" - command_detail = "Convert and upload a file to Google Sheets." - - # @Override - def SetUp(self): - """Initializes the parser for sheet command.""" - self.arg_parser.add_argument( - "--src", - required=True, - help="The local file or GCS URL to be uploaded to Google Sheets. " - "This command supports the results produced by TradeFed in XML " - "and ZIP formats. Variables enclosed in {} are replaced with the " - "the values stored in the console.") - self.arg_parser.add_argument( - "--dest", - required=True, - help="The ID of the spreadsheet to which the file is uploaded.") - self.arg_parser.add_argument( - "--ref", - default=None, - help="The reference file to be compared with src. If a test in " - "src fails, its result in ref is also written to the spreadsheet.") - self.arg_parser.add_argument( - "--extra_rows", - nargs="*", - default=[], - help="The extra rows written to the spreadsheet. Each argument " - "is a row. Cells in a row are separated by commas. Each cell can " - "contain variables enclosed in {}.") - self.arg_parser.add_argument( - "--max", - default=30000, - type=int, - help="Maximum number of results written to the spreadsheet. " - "If there are too many results, only failing ones are written.") - self.arg_parser.add_argument( - "--primary_abi_only", - action="store_true", - help="Whether to upload only the test results for primary ABI. If " - "ref is also specified, this command loads the primary ABI " - "results from ref and compares regardless of bitness.") - self.arg_parser.add_argument( - "--client_secrets", - default=None, - help="The path to the client secrets file in JSON format for " - "authentication. If this argument is not specified, this command " - "uses PAB client secrets.") - - # @Override - def Run(self, arg_line): - """Uploads args.src file to args.dest on Google Sheets.""" - args = self.arg_parser.ParseLine(arg_line) - - try: - src_path = self.console.FormatString(args.src) - ref_path = (None if args.ref is None else - self.console.FormatString(args.ref)) - extra_rows = [] - for row in args.extra_rows: - extra_rows.append([self.console.FormatString(cell) - for cell in row.split(",")]) - except KeyError as e: - logging.error( - "Unknown or uninitialized variable in arguments: %s", e) - return False - - if args.client_secrets is not None: - credentials = ServiceAccountCredentials.from_json_keyfile_name( - args.client_secrets, scopes=self._SCOPE) - else: - credentials = self.console.build_provider["pab"].Authenticate( - scopes=self._SCOPE) - client = gspread.authorize(credentials) - - # Load result_attrs, build_attrs, summary_attrs, - # src_dict, ref_dict, and exceed_max - temp_dir = tempfile.mkdtemp() - try: - src_path = _GetResultAsXml(src_path, os.path.join(temp_dir, "src")) - if not src_path: - return False - - with open(src_path, "r") as src_file: - (result_attrs, - build_attrs, - summary_attrs) = result_utils.LoadTestSummary(src_file) - src_file.seek(0) - if args.primary_abi_only: - abis = build_attrs.get( - common._ABIS_ATTR_KEY, "").split(",") - src_bitness = str(result_utils.GetAbiBitness(abis[0])) - src_dict, exceed_max = _LoadSrcResults(src_file, args.max, - src_bitness) - else: - src_dict, exceed_max = _LoadSrcResults(src_file, args.max) - - if ref_path: - ref_path = _GetResultAsXml( - ref_path, os.path.join(temp_dir, "ref")) - if not ref_path: - return False - with open(ref_path, "r") as ref_file: - if args.primary_abi_only: - ref_build_attrs = xml_utils.GetAttributes( - ref_file, common._BUILD_TAG, - (common._ABIS_ATTR_KEY, )) - ref_file.seek(0) - abis = ref_build_attrs[ - common._ABIS_ATTR_KEY].split(",") - ref_bitness = str(result_utils.GetAbiBitness(abis[0])) - ref_dict = _LoadRefResults(ref_file, src_dict, - ref_bitness, src_bitness) - else: - ref_dict = _LoadRefResults(ref_file, src_dict) - finally: - shutil.rmtree(temp_dir) - - # Output - csv_file = string_io_module.StringIO() - try: - writer = csv.writer(csv_file, lineterminator="\n") - - writer.writerows(extra_rows) - - for keys, attrs in ( - (_RESULT_ATTR_KEYS, result_attrs), - (_BUILD_ATTR_KEYS, build_attrs), - (_SUMMARY_ATTR_KEYS, summary_attrs)): - writer.writerows((k, attrs.get(k, "")) for k in keys) - - src_list = sorted(src_dict.items()) - if ref_path: - _WriteComparisonToCsv(src_list, ref_dict, writer) - else: - _WriteResultsToCsv(src_list, writer) - - if exceed_max: - writer.writerow((_TOO_MANY_DATA,)) - - client.import_csv(args.dest, csv_file.getvalue()) - finally: - csv_file.close() - - -def _DownloadResultZipFromGcs(gcs_url, local_dir): - """Downloads a result ZIP from GCS. - - If the GCS URL is a directory, this function searches the directory for the - file whose name matches the pattern of result ZIP. - - Args: - gcs_url: The URL of the ZIP file or the directory. - local_dir: The local directory where the ZIP is downloaded. - - Returns: - The path to the downloaded ZIP file. - None if fail to download. - """ - gsutil_path = gcs_utils.GetGsutilPath() - if not gsutil_path: - return False - - if gcs_utils.IsGcsFile(gsutil_path, gcs_url): - gcs_urls = [gcs_url] - else: - ls_urls = gcs_utils.List(gsutil_path, gcs_url) - gcs_urls = [x for x in ls_urls if - re.match(".+/results_\\d*\\.zip$", x)] - if not gcs_urls: - gcs_urls = [x for x in ls_urls if - re.match(".+/log-result_\\d*\\.zip$", x)] - - if not gcs_urls: - logging.error("No results on %s", gcs_url) - return None - if len(gcs_urls) > 1: - logging.warning("More than one result. Select %s", gcs_urls[0]) - - if not os.path.exists(local_dir): - os.makedirs(local_dir) - if not gcs_utils.Copy(gsutil_path, gcs_urls[0], local_dir): - logging.error("Fail to copy from %s", gcs_urls[0]) - return None - - return os.path.join(local_dir, gcs_urls[0].rpartition("/")[2]) - - -def _GetResultAsXml(src, temp_dir): - """Downloads and extracts an XML result. - - If src is a GCS URL, it is downloaded to temp_dir. - If the file is a ZIP, it is extracted to temp_dir. - - Args: - src: The location of the file, can be a directory on GCS, - a ZIP file on GCS, a local ZIP file, or a local XML file. - temp_dir: The directory where the ZIP is downloaded and extracted. - - Returns: - The path to the XML file. - None if fails to the find the XML. - """ - original_src = src - if src.startswith("gs://"): - src = _DownloadResultZipFromGcs(src, os.path.join(temp_dir, "zipped")) - if not src: - return None - - if zipfile.is_zipfile(src): - with zipfile.ZipFile(src, mode="r") as zip_file: - src = result_utils.ExtractResultZip( - zip_file, os.path.join(temp_dir, "unzipped")) - if not src: - logging.error("Cannot find XML result in %s", original_src) - return None - - return src - - -def _FilterTestResults(xml_file, max_return, filter_func): - """Loads test results from XML to dictionary with a filter. - - Args: - xml_file: The input file object in XML format. - max_return: Maximum number of output results. - filter_func: A function taking the test name and result as parameters, - and returning whether it should be included. - - Returns: - A dict of {name: result} where name is a tuple of strings and result - is a string. - """ - result_dict = dict() - for module, testcase, test in result_utils.IterateTestResults(xml_file): - if len(result_dict) >= max_return: - break - test_name = result_utils.GetTestName(module, testcase, test) - result = test.attrib.get(common._RESULT_ATTR_KEY, "") - if filter_func(test_name, result): - result_dict[test_name] = result - - return result_dict - - -def _LoadSrcResults(src_xml, max_return, bitness=""): - """Loads test results from XML to dictionary. - - If number of results exceeds max_return, only failures are returned. - If number of failures exceeds max_return, the results are truncated. - - Args - src_xml: The file object in XML format. - max_return: Maximum number of returned results. - bitness: A string, the bitness of the returned results. - - Returns: - A dict of {name: result} and a boolean which represents whether the - results are truncated. - """ - def FilterBitness(name): - return not bitness or bitness == name[0] - - results = _FilterTestResults( - src_xml, max_return + 1, lambda name, result: FilterBitness(name)) - - if len(results) > max_return: - src_xml.seek(0) - results = _FilterTestResults( - src_xml, max_return + 1, - lambda name, result: result == "fail" and FilterBitness(name)) - - exceed_max = len(results) > max_return - if results and exceed_max: - del results[max(results)] - - return results, exceed_max - - -def _LoadRefResults(ref_xml, base_results, ref_bitness="", base_bitness=""): - """Loads reference results from XML to dictionary. - - A test result in ref_xml is returned if the test fails in base_results. - - Args: - ref_xml: The file object in XML format. - base_results: A dict of {name: result} containing the test names to be - loaded from ref_xml. - ref_bitness: A string, the bitness of the results to be loaded from - ref_xml. - base_bitness: A string, the bitness of the returned results. If this - argument is specified, the function ignores bitness when - comparing test names. - - Returns: - A dict of {name: result}, the test name in base_results and the result - in ref_xml. - """ - ref_results = dict() - for module, testcase, test in result_utils.IterateTestResults(ref_xml): - if len(ref_results) >= len(base_results): - break - result = test.attrib.get(common._RESULT_ATTR_KEY, "") - name = result_utils.GetTestName(module, testcase, test) - - if ref_bitness and name[0] != ref_bitness: - continue - if base_bitness: - name_in_base = (base_bitness, ) + name[1:] - else: - name_in_base = name - - if base_results.get(name_in_base, "") == "fail": - ref_results[name_in_base] = result - - return ref_results - - -def _WriteResultsToCsv(result_list, writer): - """Writes a list of test names and results to a CSV file. - - Args: - result_list: The list of (name, result). - writer: The object of CSV writer. - """ - writer.writerow(_TABLE_HEADER) - writer.writerows(name + (result,) for name, result in result_list) - - -def _WriteComparisonToCsv(result_list, reference_dict, writer): - """Writes test names, results, and reference results to a CSV file. - - Args: - result_list: The list of (name, result). - reference_dict: The dict of {name: reference_result}. - writer: The object of CSV writer. - """ - writer.writerow(_CMP_TABLE_HEADER) - for name, result in result_list: - if result == "fail": - reference = reference_dict.get(name, "no_data") - else: - reference = "" - writer.writerow(name + (result, reference)) diff --git a/harnesses/host_controller/command_processor/command_sheet_test.py b/harnesses/host_controller/command_processor/command_sheet_test.py deleted file mode 100644 index 1c61349..0000000 --- a/harnesses/host_controller/command_processor/command_sheet_test.py +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import tempfile -import unittest -import zipfile - -try: - from unittest import mock -except ImportError: - # TODO: Remove when we stop supporting Python 2 - import mock - -from host_controller import common -from host_controller.command_processor import command_sheet - -# Input -_EXTRA_ROWS = ("logs,gs://a/b", "logs,gs://c/{suite_plan}") - -_XML_HEAD = """\ -<?xml version='1.0' encoding='UTF-8' standalone='no' ?> -<?xml-stylesheet type="text/xsl" href="compatibility_result.xsl"?> -<Result start="1528285394762" end="1528286467560" start_display="Wed Jun 06 04:43:14 PDT 2018" end_display="Wed Jun 06 05:01:07 PDT 2018" suite_name="VTS" suite_version="9.0_R1" suite_plan="vts" suite_build_number="4000000" report_version="5.0" command_line_args="vts" devices="TEST123456" host_name="unit.test" os_name="Linux" os_version="4.13.0-41-generic" os_arch="amd64" java_vendor="Oracle Corporation" java_version="1.8.0_171"> - <Build build_abis_64="arm64-v8a" build_manufacturer="Google" build_reference_fingerprint2="" build_board="unit_test" build_serial="TEST123456" build_system_fingerprint="system_fp" build_reference_fingerprint="null" build_fingerprint="vendor_fp" build_abis="arm64-v8a,armeabi-v7a,armeabi" build_device="unit_test" build_abi="arm64-v8a" build_abi2="null" build_version_release="9" build_version_base_os="" build_type="userdebug" build_vendor_model="Unit Test" build_abis_32="armeabi-v7a,armeabi" build_product="unit_test" build_brand="google" build_abi22="" build_version_security_patch="2018-06-05" build_vendor_manufacturer="Google" build_version_sdk="28" build_id="test_build" build_model="Unit Test" build_vendor_fingerprint="vendor_fp" build_version_incremental="4000000" build_fingerprint2="system_fp" build_tags="test-keys" /> -""" - -_XML_1 = _XML_HEAD + """\ - <Summary pass="1" failed="3" modules_done="2" modules_total="2" /> - <Module name="module1" abi="armeabi-v7a" runtime="1" done="true" pass="0"> - <TestCase name="testcase1"> - <Test result="fail" name="test1" /> - </TestCase> - </Module> - <Module name="module2" abi="arm64-v8a" runtime="1" done="true" pass="1"> - <TestCase name="testcase2"> - <Test result="pass" name="test1" /> - <Test result="fail" name="test2" /> - <Test result="fail" name="test3" /> - </TestCase> - </Module> -</Result> -""" - -_XML_2 = _XML_HEAD + """\ - <Summary pass="3" failed="1" modules_done="2" modules_total="2" /> - <Module name="module1" abi="armeabi-v7a" runtime="1" done="true" pass="1"> - <TestCase name="testcase1"> - <Test result="pass" name="test1,test2" /> - </TestCase> - </Module> - <Module name="module2" abi="arm64-v8a" runtime="1" done="true" pass="2"> - <TestCase name="testcase2"> - <Test result="pass" name="test1" /> - <Test result="pass" name="test2" /> - <Test result="fail" name="test3" /> - </TestCase> - </Module> -</Result> -""" - -# Expected output -_CSV_HEAD = """\ -logs,gs://a/b -logs,gs://c/vts -suite_name,VTS -suite_plan,vts -suite_version,9.0_R1 -suite_build_number,4000000 -start_display,Wed Jun 06 04:43:14 PDT 2018 -end_display,Wed Jun 06 05:01:07 PDT 2018 -build_fingerprint,vendor_fp -build_system_fingerprint,system_fp -build_vendor_fingerprint,vendor_fp -""" - -_ALL_RESULTS_1 = _CSV_HEAD + """\ -pass,1 -failed,3 -modules_total,2 -modules_done,2 -BITNESS,TEST_MODULE,TEST_CLASS,TEST_CASE,RESULT -32,module1,testcase1,test1,fail -64,module2,testcase2,test1,pass -64,module2,testcase2,test2,fail -64,module2,testcase2,test3,fail -""" - -_FAILING_RESULTS_1 = _CSV_HEAD + """\ -pass,1 -failed,3 -modules_total,2 -modules_done,2 -BITNESS,TEST_MODULE,TEST_CLASS,TEST_CASE,RESULT -32,module1,testcase1,test1,fail -64,module2,testcase2,test2,fail -64,module2,testcase2,test3,fail -""" - -_TRUNCATED_RESULTS_1 = _CSV_HEAD + """\ -pass,1 -failed,3 -modules_total,2 -modules_done,2 -BITNESS,TEST_MODULE,TEST_CLASS,TEST_CASE,RESULT -32,module1,testcase1,test1,fail -too many to be displayed -""" - -_PRIMARY_ABI_RESULTS_1 = _CSV_HEAD + """\ -pass,1 -failed,3 -modules_total,2 -modules_done,2 -BITNESS,TEST_MODULE,TEST_CLASS,TEST_CASE,RESULT -64,module2,testcase2,test1,pass -64,module2,testcase2,test2,fail -64,module2,testcase2,test3,fail -""" - -_COMPARISON_1_2 = _CSV_HEAD + """\ -pass,1 -failed,3 -modules_total,2 -modules_done,2 -BITNESS,TEST_MODULE,TEST_CLASS,TEST_CASE,RESULT,REFERENCE_RESULT -32,module1,testcase1,test1,fail,no_data -64,module2,testcase2,test2,fail,pass -64,module2,testcase2,test3,fail,fail -""" - -_PRIMARY_ABI_COMPARISON_1_2 = _CSV_HEAD + """\ -pass,1 -failed,3 -modules_total,2 -modules_done,2 -BITNESS,TEST_MODULE,TEST_CLASS,TEST_CASE,RESULT,REFERENCE_RESULT -64,module2,testcase2,test1,pass, -64,module2,testcase2,test2,fail,pass -64,module2,testcase2,test3,fail,fail -""" - -_COMPARISON_2_1 = _CSV_HEAD + """\ -pass,3 -failed,1 -modules_total,2 -modules_done,2 -BITNESS,TEST_MODULE,TEST_CLASS,TEST_CASE,RESULT,REFERENCE_RESULT -32,module1,testcase1,"test1,test2",pass, -64,module2,testcase2,test1,pass, -64,module2,testcase2,test2,pass, -64,module2,testcase2,test3,fail,fail -""" - - -@mock.patch("host_controller.command_processor.command_sheet.gspread") -@mock.patch("host_controller.command_processor.command_sheet." - "ServiceAccountCredentials") -class CommandSheetTest(unittest.TestCase): - """Unit test for CommandSheet. - - Attributes: - _cmd: The CommandSheet being tested. - _temp_files: The paths to the temporary files. - """ - - def setUp(self): - """Creates CommandSheet.""" - self._cmd = command_sheet.CommandSheet() - mock_console = mock.Mock() - mock_console.FormatString = lambda x: x.replace("{suite_plan}", "vts") - self._cmd._SetUp(mock_console) - self._temp_files = [] - - def tearDown(self): - """Deletes CommandSheet and temoprary files.""" - self._cmd._TearDown() - for temp_file in self._temp_files: - os.remove(temp_file) - - def _CreateXml(self, xml_string): - """Creates a test result in XML format. - - Args: - xml_string: The file contents. - - Returns: - The path to the XML file. - """ - with tempfile.NamedTemporaryFile( - suffix=".xml", delete=False) as temp_file: - self._temp_files.append(temp_file.name) - temp_file.write(xml_string) - return temp_file.name - - def _CreateZip(self, xml_string): - """Creates a zipped test result. - - Args: - xml_string: The file contents. - - Returns: - The path to the ZIP file. - """ - with tempfile.NamedTemporaryFile( - suffix=".zip", delete=False) as temp_file: - self._temp_files.append(temp_file.name) - with zipfile.ZipFile(temp_file, "w") as zip_file: - zip_file.writestr(common._TEST_RESULT_XML, xml_string) - return temp_file.name - - def testUploadZip(self, mock_credentials, mock_gspread): - """Tests uploading zipped result.""" - mock_client = mock.Mock() - mock_gspread.authorize.return_value = mock_client - - self._cmd.Run("--src %s --dest 123 --client_secret /abc " - "--extra_rows %s" % - (self._CreateZip(_XML_1), " ".join(_EXTRA_ROWS))) - - mock_client.import_csv.assert_called_with("123", _ALL_RESULTS_1) - - def testShowFailureOnly(self, mock_credentials, mock_gspread): - """Tests showing only failing tests.""" - mock_client = mock.Mock() - mock_gspread.authorize.return_value = mock_client - - self._cmd.Run("--src %s --dest 123 --client_secret /abc " - "--extra_rows %s --max 3" % - (self._CreateXml(_XML_1), " ".join(_EXTRA_ROWS))) - - mock_client.import_csv.assert_called_with("123", _FAILING_RESULTS_1) - - def testTruncate(self, mock_credentials, mock_gspread): - """Tests truncating output.""" - mock_client = mock.Mock() - mock_gspread.authorize.return_value = mock_client - - self._cmd.Run("--src %s --dest 123 --client_secret /abc " - "--extra_rows %s --max 1" % - (self._CreateXml(_XML_1), " ".join(_EXTRA_ROWS))) - - mock_client.import_csv.assert_called_with("123", _TRUNCATED_RESULTS_1) - - def testPrimaryAbiOnly(self, mock_credentials, mock_gspread): - """Tests showing only results for primary ABI.""" - """Tests showing only failing tests.""" - mock_client = mock.Mock() - mock_gspread.authorize.return_value = mock_client - - self._cmd.Run("--src %s --dest 123 --client_secret /abc " - "--extra_rows %s --primary_abi_only" % - (self._CreateXml(_XML_1), " ".join(_EXTRA_ROWS))) - - mock_client.import_csv.assert_called_with("123", _PRIMARY_ABI_RESULTS_1) - - def testCompareLocal(self, mock_credentials, mock_gspread): - """Tests comparing two local XML files.""" - mock_client = mock.Mock() - mock_gspread.authorize.return_value = mock_client - - self._cmd.Run("--src %s --ref %s --dest 123 --client_secret /abc " - "--extra_rows %s --max 3" % - (self._CreateXml(_XML_1), self._CreateZip(_XML_2), - " ".join(_EXTRA_ROWS))) - - mock_client.import_csv.assert_called_with("123", _COMPARISON_1_2) - - def testComparePrimaryAbi(self, mock_credentials, mock_gspread): - """Tests comparing primary ABI only.""" - mock_client = mock.Mock() - mock_gspread.authorize.return_value = mock_client - - self._cmd.Run("--src %s --ref %s --dest 123 --client_secret /abc " - "--extra_rows %s --primary_abi_only" % - (self._CreateXml(_XML_1), self._CreateZip(_XML_2), - " ".join(_EXTRA_ROWS))) - - mock_client.import_csv.assert_called_with("123", - _PRIMARY_ABI_COMPARISON_1_2) - - @mock.patch("host_controller.command_processor.command_sheet.gcs_utils") - def testCompareGcs(self, mock_gcs_utils, mock_credentials, mock_gspread): - """Tests comparing a local XML with a ZIP on GCS.""" - - zip_name = "results_123.zip" - gcs_dir = "gs://unit/test" - zip_url = gcs_dir + "/" + zip_name - - def MockCopy(gsutil_path, gcs_url, local_dir): - self.assertEqual(zip_url, gcs_url) - with zipfile.ZipFile(os.path.join(local_dir, zip_name), - "w") as zip_file: - zip_file.writestr(common._TEST_RESULT_XML, _XML_1) - return True - - mock_gcs_utils.GetGsutilPath.return_value = "mock_gsutil" - mock_gcs_utils.IsGcsFile.return_value = False - mock_gcs_utils.List.return_value = [zip_url] - mock_gcs_utils.Copy = MockCopy - - mock_client = mock.Mock() - mock_gspread.authorize.return_value = mock_client - - self._cmd.Run("--src %s --ref %s --dest 123 --client_secret /abc " - "--extra_rows %s" % - (self._CreateXml(_XML_2), gcs_dir, - " ".join(_EXTRA_ROWS))) - - mock_client.import_csv.assert_called_with("123", _COMPARISON_2_1) - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/command_processor/command_shell.py b/harnesses/host_controller/command_processor/command_shell.py deleted file mode 100644 index abe3274..0000000 --- a/harnesses/host_controller/command_processor/command_shell.py +++ /dev/null @@ -1,59 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging - -from host_controller.command_processor import base_command_processor - -from vts.utils.python.common import cmd_utils - - -class CommandShell(base_command_processor.BaseCommandProcessor): - """Command processor for shell command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "shell" - command_detail = "Runs a shell command on the host OS." - - # @Override - def SetUp(self): - """Initializes the parser for device command.""" - self.arg_parser.add_argument( - "command", - metavar="COMMAND", - nargs="+", - help="The command to be executed. If the command contains " - "arguments starting with \"-\", place the command at end of line " - "after \"--\".") - - # @Override - def Run(self, arg_line): - """Runs a shell command.""" - args = self.arg_parser.ParseLine(arg_line) - cmd_list = self.ReplaceVars(args.command) - stdout, stderr, retcode = cmd_utils.ExecuteOneShellCommand( - " ".join(cmd_list)) - if stdout: - logging.info(stdout) - if stderr: - logging.error(stderr) - if retcode != 0: - return False diff --git a/harnesses/host_controller/command_processor/command_sleep.py b/harnesses/host_controller/command_processor/command_sleep.py deleted file mode 100644 index 9517496..0000000 --- a/harnesses/host_controller/command_processor/command_sleep.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import time - -from host_controller.command_processor import base_command_processor - - -class CommandSleep(base_command_processor.BaseCommandProcessor): - """Command processor for sleep command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "sleep" - command_detail = "Sleeps for N seconds." - - # @Override - def SetUp(self): - """Initializes the parser for device command.""" - self.arg_parser.add_argument( - "seconds", - metavar="COMMAND", - nargs=1, - help=("an integer indicating the amount of time to sleep " - "in seconds")) - - # @Override - def Run(self, arg_line): - """Blocks for a specified time interval.""" - args = self.arg_parser.ParseLine(arg_line) - try: - time.sleep(int(args.seconds[0])) - except ValueError as e: - logging.exception(e) - return False diff --git a/harnesses/host_controller/command_processor/command_test.py b/harnesses/host_controller/command_processor/command_test.py deleted file mode 100644 index 8e5adb7..0000000 --- a/harnesses/host_controller/command_processor/command_test.py +++ /dev/null @@ -1,226 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os -import shutil -import subprocess -import tempfile -import threading -import zipfile - -from host_controller.command_processor import base_command_processor -from host_controller.utils.parser import xml_utils -from vts.runners.host import utils - - -class CommandTest(base_command_processor.BaseCommandProcessor): - """Command processor for test command. - - Attributes: - _RESULT_ATTRIBUTES: The attributes of <Result> in the XML report. - After test execution, the attributes are loaded - from report to console's dictionary. - _result_dir: the path to the temporary result directory. - """ - - command = "test" - command_detail = "Executes a command on TF." - _RESULT_TAG = "Result" - _RESULT_ATTRIBUTES = ["suite_plan"] - - # @Override - def SetUp(self): - """Initializes the parser for test command.""" - self._result_dir = None - self.arg_parser.add_argument( - "--suite", - default="vts", - choices=("vts", "cts", "gts", "sts"), - help="To specify the type of a test suite to be run.") - self.arg_parser.add_argument( - "--serial", - "-s", - default=None, - help="The target device serial to run the command. " - "A comma-separate list.") - self.arg_parser.add_argument( - "--test-exec-mode", - default="subprocess", - help="The target exec model.") - self.arg_parser.add_argument( - "--keep-result", - action="store_true", - help="Keep the path to the result in the console instance.") - self.arg_parser.add_argument( - "command", - metavar="COMMAND", - nargs="+", - help="The command to be executed. If the command contains " - "arguments starting with \"-\", place the command after " - "\"--\" at end of line. format: plan -m module -t testcase") - - def _ClearResultDir(self): - """Deletes all files in the result directory.""" - if self._result_dir is None: - self._result_dir = tempfile.mkdtemp() - return - - for file_name in os.listdir(self._result_dir): - shutil.rmtree(os.path.join(self._result_dir, file_name)) - - @staticmethod - def _GenerateTestSuiteCommand(bin_path, command, serials, result_dir=None): - """Generates a *ts-tradefed command. - - Args: - bin_path: the path to *ts-tradefed. - command: a list of strings, the command arguments. - serials: a list of strings, the serial numbers of the devices. - result_dir: the path to the temporary directory where the result is - saved. - - Returns: - a list of strings, the *ts-tradefed command. - """ - cmd = [bin_path, "run", "commandAndExit"] - cmd.extend(str(c) for c in command) - - for serial in serials: - cmd.extend(["-s", str(serial)]) - - if result_dir: - cmd.extend(["--log-file-path", result_dir, "--use-log-saver"]) - - return cmd - - @staticmethod - def _ExecuteCommand(cmd): - """Executes a command and logs output in real time. - - Args: - cmd: a list of strings, the command to execute. - """ - - def LogOutputStream(log_level, stream): - try: - while True: - line = stream.readline() - if not line: - break - logging.log(log_level, line.rstrip()) - finally: - stream.close() - - proc = subprocess.Popen( - cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - out_thread = threading.Thread( - target=LogOutputStream, args=(logging.INFO, proc.stdout)) - err_thread = threading.Thread( - target=LogOutputStream, args=(logging.ERROR, proc.stderr)) - out_thread.daemon = True - err_thread.daemon = True - out_thread.start() - err_thread.start() - proc.wait() - logging.info("Return code: %d", proc.returncode) - proc.stdin.close() - out_thread.join() - err_thread.join() - - # @Override - def Run(self, arg_line): - """Executes a command using a *TS-TF instance. - - Args: - arg_line: string, line of command arguments. - """ - args = self.arg_parser.ParseLine(arg_line) - if args.serial: - serials = args.serial.split(",") - elif self.console.GetSerials(): - serials = self.console.GetSerials() - else: - serials = [] - - if args.test_exec_mode == "subprocess": - if args.suite not in self.console.test_suite_info: - logging.error("test_suite_info doesn't have '%s': %s", - args.suite, self.console.test_suite_info) - return - - if args.keep_result: - self._ClearResultDir() - result_dir = self._result_dir - else: - result_dir = None - - cmd = self._GenerateTestSuiteCommand( - self.console.test_suite_info[args.suite], args.command, - serials, result_dir) - - logging.info("Command: %s", cmd) - self._ExecuteCommand(cmd) - - if result_dir: - result_paths = [ - os.path.join(dir_name, file_name) - for dir_name, file_name in utils.iterate_files(result_dir) - if file_name.startswith("log-result") - and file_name.endswith(".zip") - ] - - if len(result_paths) != 1: - logging.warning("Unexpected number of results: %s", - result_paths) - - self.console.test_result.clear() - result = {} - if len(result_paths) > 0: - with zipfile.ZipFile( - result_paths[0], mode="r") as result_zip: - with result_zip.open( - "log-result.xml", mode="rU") as result_xml: - result = xml_utils.GetAttributes( - result_xml, self._RESULT_TAG, - self._RESULT_ATTRIBUTES) - if not result: - logging.warning("Nothing loaded from report.") - result["result_zip"] = result_paths[0] - - result_paths_full = [ - os.path.join(dir_name, file_name) - for dir_name, file_name in utils.iterate_files(result_dir) - if file_name.endswith(".zip") - ] - result["result_full"] = " ".join(result_paths_full) - result["suite_name"] = args.suite - - logging.debug(result) - self.console.test_result.update(result) - else: - logging.error("unsupported exec mode: %s", args.test_exec_mode) - return False - - # @Override - def TearDown(self): - """Deletes the result directory.""" - if self._result_dir: - shutil.rmtree(self._result_dir, ignore_errors=True) diff --git a/harnesses/host_controller/command_processor/command_upload.py b/harnesses/host_controller/command_processor/command_upload.py deleted file mode 100644 index db57089..0000000 --- a/harnesses/host_controller/command_processor/command_upload.py +++ /dev/null @@ -1,346 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os -import shutil -import socket -import time - -from host_controller import common -from host_controller.command_processor import base_command_processor -from host_controller.utils.gcp import gcs_utils -from host_controller.utils.parser import xml_utils - -from vts.utils.python.common import cmd_utils - -from vti.dashboard.proto import TestSuiteResultMessage_pb2 as SuiteResMsg -from vti.test_serving.proto import TestScheduleConfigMessage_pb2 as SchedCfgMsg - - -class CommandUpload(base_command_processor.BaseCommandProcessor): - """Command processor for upload command. - - Attributes: - arg_parser: ConsoleArgumentParser object, argument parser. - console: cmd.Cmd console object. - command: string, command name which this processor will handle. - command_detail: string, detailed explanation for the command. - """ - - command = "upload" - command_detail = "Upload <src> file to <dest> Google Cloud Storage. In <src> and <dest>, variables enclosed in {} are replaced with the values stored in the console." - - # @Override - def SetUp(self): - """Initializes the parser for upload command.""" - self.arg_parser.add_argument( - "--src", - required=True, - default="latest-system.img", - help="Path to a source file to upload. Only single file can be " - "uploaded per once. Use 'latest- prefix to upload the latest " - "fetch images. e.g. --src=latest-system.img If argument " - "value is not given, the recently fetched system.img will be " - "uploaded.") - self.arg_parser.add_argument( - "--dest", - required=True, - help="Google Cloud Storage URL to which the file is uploaded.") - self.arg_parser.add_argument( - "--report_path", - help="Google Cloud Storage URL, the dest path of a report file") - self.arg_parser.add_argument( - "--clear_dest", - action="store_true", - help="Delete dest recursively before the upload.") - self.arg_parser.add_argument( - "--clear_results", - default=False, - help="True to clear all the results after the upload.") - self.arg_parser.add_argument( - "--result_from_suite", - default="", - choices=("", "vts", "cts", "gts", "sts"), - help="To specify the type of a test suite report, since there can " - "be multiple numbers of result sets from different test " - "suites. If not specified, the HC will upload the report " - "from last run suite and plan.") - self.arg_parser.add_argument( - "--result_from_plan", - default="", - help="To specify the type of the plan name from which " - "the report is generated.") - - # @Override - def Run(self, arg_line): - """Upload args.src file to args.dest Google Cloud Storage.""" - args = self.arg_parser.ParseLine(arg_line) - - gsutil_path = gcs_utils.GetGsutilPath() - if not gsutil_path: - logging.error("Please check gsutil is installed and on your PATH") - return False - - if args.src.startswith("latest-"): - src_name = args.src[7:] - if src_name in self.console.device_image_info: - src_paths = self.console.device_image_info[src_name] - else: - logging.error( - "Unable to find {} in device_image_info".format(src_name)) - return False - else: - try: - src_paths = self.console.FormatString(args.src) - except KeyError as e: - logging.error("Unknown or uninitialized variable in src: %s", - e) - return False - - src_path_list_tmp = src_paths.split(" ") - src_path_list = [] - if src_path_list_tmp: - for src_path in src_path_list_tmp: - file_path = src_path.strip() - if os.path.isfile(file_path): - src_path_list.append(file_path) - else: - logging.error("Cannot find a file: {}".format(file_path)) - src_paths = " ".join(src_path_list) - - try: - dest_path = self.console.FormatString(args.dest) - except KeyError as e: - logging.error("Unknown or uninitialized variable in dest: %s", e) - return False - - if not dest_path.startswith("gs://"): - logging.error("{} is not correct GCS url.".format(dest_path)) - return False - """ TODO(jongmok) : Before upload, login status, authorization, - and dest check are required. """ - if args.clear_dest: - if not gcs_utils.Remove(gsutil_path, dest_path, recursive=True): - logging.error("Fail to remove %s", dest_path) - - if not gcs_utils.Copy(gsutil_path, src_paths, dest_path): - logging.error("Fail to copy %s to %s", src_paths, dest_path) - - if args.report_path or args.clear_results: - tools_path = "" - if args.result_from_suite: - tools_path = os.path.dirname( - self.console.test_suite_info[args.result_from_suite]) - else: - try: - tools_path = os.path.dirname(self.console.test_suite_info[ - self.console.FormatString("{suite_name}")]) - except KeyError: - if self.console.vti_endpoint_client.CheckBootUpStatus(): - logging.error( - "No test results found from any fetched test suite." - " Please fetch a test suite and run 'test' command," - " then try running 'upload' command again.") - return False - results_base_path = os.path.join(tools_path, - common._RESULTS_BASE_PATH) - - if args.report_path: - report_path = self.console.FormatString(args.report_path) - if not report_path.startswith("gs://"): - logging.error( - "{} is not correct GCS url.".format(report_path)) - else: - self.UploadReport( - gsutil_path, report_path, dest_path, results_base_path, - args.result_from_suite, args.result_from_plan) - - if args.clear_results: - shutil.rmtree(results_base_path, ignore_errors=True) - - def UploadReport(self, gsutil_path, report_path, log_path, results_path, - suite_name, plan_name): - """Uploads report summary file to the given path. - - Args: - gsutil_path: string, the path of a gsutil binary. - report_path: string, the dest GCS URL to which the summarized report - file will be uploaded. - log_path: string, GCS URL where the log files from the test run - have been uploaded. - results_path: string, the base path for the results. - """ - suite_res_msg = SuiteResMsg.TestSuiteResultMessage() - suite_res_msg.result_path = log_path - suite_res_msg.branch = self.console.FormatString("{branch}") - suite_res_msg.target = self.console.FormatString("{target}") - vti = self.console.vti_endpoint_client - suite_res_msg.boot_success = vti.CheckBootUpStatus() - suite_res_msg.test_type = vti.GetJobTestType() - - device_fetch_info = self.console.detailed_fetch_info[ - common._ARTIFACT_TYPE_DEVICE] - gsi_fetch_info = None - if common._ARTIFACT_TYPE_GSI in self.console.detailed_fetch_info: - gsi_fetch_info = self.console.detailed_fetch_info[ - common._ARTIFACT_TYPE_GSI] - - if vti.CheckBootUpStatus(): - former_results = [ - result for result in os.listdir(results_path) - if os.path.isdir(os.path.join(results_path, result)) - and not os.path.islink(os.path.join(results_path, result)) - ] - - if not former_results: - logging.error("No test result found.") - return False - - former_results.sort() - latest_result = former_results[-1] - latest_result_xml_path = os.path.join(results_path, latest_result, - common._TEST_RESULT_XML) - - result_attrs = xml_utils.GetAttributes( - latest_result_xml_path, common._RESULT_TAG, [ - common._SUITE_NAME_ATTR_KEY, common._SUITE_PLAN_ATTR_KEY, - common._SUITE_VERSION_ATTR_KEY, - common._SUITE_BUILD_NUM_ATTR_KEY, - common._START_TIME_ATTR_KEY, common._END_TIME_ATTR_KEY, - common._HOST_NAME_ATTR_KEY - ]) - build_attrs = xml_utils.GetAttributes( - latest_result_xml_path, common._BUILD_TAG, [ - common._FINGERPRINT_ATTR_KEY, - common._SYSTEM_FINGERPRINT_ATTR_KEY, - common._VENDOR_FINGERPRINT_ATTR_KEY - ]) - summary_attrs = xml_utils.GetAttributes( - latest_result_xml_path, common._SUMMARY_TAG, [ - common._PASSED_ATTR_KEY, common._FAILED_ATTR_KEY, - common._MODULES_TOTAL_ATTR_KEY, - common._MODULES_DONE_ATTR_KEY - ]) - - suite_res_msg.build_id = result_attrs[ - common._SUITE_BUILD_NUM_ATTR_KEY] - suite_res_msg.suite_name = result_attrs[ - common._SUITE_NAME_ATTR_KEY] - suite_res_msg.suite_plan = result_attrs[ - common._SUITE_PLAN_ATTR_KEY] - suite_res_msg.suite_version = result_attrs[ - common._SUITE_VERSION_ATTR_KEY] - suite_res_msg.suite_build_number = result_attrs[ - common._SUITE_BUILD_NUM_ATTR_KEY] - suite_res_msg.start_time = long( - result_attrs[common._START_TIME_ATTR_KEY]) - suite_res_msg.end_time = long( - result_attrs[common._END_TIME_ATTR_KEY]) - suite_res_msg.host_name = result_attrs[common._HOST_NAME_ATTR_KEY] - if common._SYSTEM_FINGERPRINT_ATTR_KEY in build_attrs: - suite_res_msg.build_system_fingerprint = build_attrs[ - common._SYSTEM_FINGERPRINT_ATTR_KEY] - else: - suite_res_msg.build_system_fingerprint = build_attrs[ - common._FINGERPRINT_ATTR_KEY] - if common._VENDOR_FINGERPRINT_ATTR_KEY in build_attrs: - suite_res_msg.build_vendor_fingerprint = build_attrs[ - common._VENDOR_FINGERPRINT_ATTR_KEY] - else: - suite_res_msg.build_vendor_fingerprint = build_attrs[ - common._FINGERPRINT_ATTR_KEY] - suite_res_msg.passed_test_case_count = int( - summary_attrs[common._PASSED_ATTR_KEY]) - suite_res_msg.failed_test_case_count = int( - summary_attrs[common._FAILED_ATTR_KEY]) - suite_res_msg.modules_done = int( - summary_attrs[common._MODULES_DONE_ATTR_KEY]) - suite_res_msg.modules_total = int( - summary_attrs[common._MODULES_TOTAL_ATTR_KEY]) - else: - suite_res_msg.build_id = self.console.fetch_info["build_id"] - suite_res_msg.suite_name = suite_name - suite_res_msg.suite_plan = plan_name - suite_res_msg.suite_version = "" - suite_res_msg.suite_build_number = suite_res_msg.build_id - suite_res_msg.start_time = long(time.time() * 1000) - suite_res_msg.end_time = suite_res_msg.start_time - suite_res_msg.host_name = socket.gethostname() - suite_res_msg.build_vendor_fingerprint = "%s/%s/%s" % ( - device_fetch_info["branch"], device_fetch_info["target"], - device_fetch_info["build_id"]) - if gsi_fetch_info: - suite_res_msg.build_system_fingerprint = "%s/%s/%s" % ( - gsi_fetch_info["branch"], gsi_fetch_info["target"], - gsi_fetch_info["build_id"]) - else: - suite_res_msg.build_system_fingerprint = suite_res_msg.build_vendor_fingerprint - suite_res_msg.passed_test_case_count = 0 - suite_res_msg.failed_test_case_count = 0 - suite_res_msg.modules_done = 0 - suite_res_msg.modules_total = 0 - - suite_res_msg.infra_log_path = self.console.FormatString( - "{hc_log_upload_path}") - repack_path_list = [] - repack_path_list.append(self.console.FormatString("{repack_path}")) - suite_res_msg.repacked_image_path.extend(repack_path_list) - - suite_res_msg.schedule_config.build_target.extend( - [SchedCfgMsg.BuildScheduleConfigMessage()]) - build_target_msg = suite_res_msg.schedule_config.build_target[0] - build_target_msg.test_schedule.extend( - [SchedCfgMsg.TestScheduleConfigMessage()]) - test_schedule_msg = build_target_msg.test_schedule[0] - - suite_res_msg.vendor_build_id = device_fetch_info["build_id"] - suite_res_msg.schedule_config.manifest_branch = str( - device_fetch_info["branch"]) - build_target_msg.name = str(device_fetch_info["target"]) - if device_fetch_info["account_id"]: - suite_res_msg.schedule_config.pab_account_id = str( - device_fetch_info["account_id"]) - if device_fetch_info["fetch_signed_build"]: - build_target_msg.require_signed_device_build = device_fetch_info[ - "fetch_signed_build"] - if gsi_fetch_info: - test_schedule_msg.gsi_branch = str(gsi_fetch_info["branch"]) - test_schedule_msg.gsi_build_target = str(gsi_fetch_info["target"]) - suite_res_msg.gsi_build_id = str(gsi_fetch_info["build_id"]) - if gsi_fetch_info["account_id"]: - test_schedule_msg.gsi_pab_account_id = str( - gsi_fetch_info["account_id"]) - test_schedule_msg.gsi_vendor_version = str( - self.console.FormatString("{gsispl.vendor_version}")) - test_schedule_msg.test_pab_account_id = str( - self.console.FormatString("{account_id}")) - build_target_msg.has_bootloader_img = "bootloader.img" in self.console.device_image_info - build_target_msg.has_radio_img = "radio.img" in self.console.device_image_info - - report_file_path = os.path.join( - self.console.tmp_logdir, - self.console.FormatString("{timestamp_time}.bin")) - with open(report_file_path, "w") as fd: - fd.write(suite_res_msg.SerializeToString()) - fd.close() - - copy_command = "{} cp {} {}".format( - gsutil_path, report_file_path, - os.path.join(report_path, os.path.basename(report_file_path))) - _, stderr, err_code = cmd_utils.ExecuteOneShellCommand(copy_command) - if err_code: - logging.error(stderr) diff --git a/harnesses/host_controller/command_processor/command_upload_test.py b/harnesses/host_controller/command_processor/command_upload_test.py deleted file mode 100644 index 67ccb69..0000000 --- a/harnesses/host_controller/command_processor/command_upload_test.py +++ /dev/null @@ -1,312 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import unittest - -try: - from unittest import mock -except ImportError: - import mock - -from host_controller import common -from host_controller.command_processor import command_upload - - -def side_effect(value): - return value - - -class CommandUploadTest(unittest.TestCase): - """Tests for upload command processor""" - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_upload.open") - @mock.patch("host_controller.command_processor.command_upload.cmd_utils") - @mock.patch("host_controller.command_processor.command_upload.SuiteResMsg") - @mock.patch("host_controller.command_processor.command_upload.SchedCfgMsg") - def testUploadReportBootupErr(self, mock_sched_config_msg, - mock_suite_res_msg, mock_cmd_util, mock_open, - mock_console): - mock_open.__enter__ = mock.Mock(return_value=mock_open) - mock_open.__exit__ = mock.Mock(return_value=None) - mock_cmd_util.ExecuteOneShellCommand = mock.Mock( - return_value=("", "", 0)) - mock_console.vti_endpoint_client.CheckBootUpStatus.return_value = False - mock_console.FormatString.side_effect = side_effect - mock_console.fetch_info = { - "branch": "git_device_branch", - "target": "device-userdebug", - "build_id": "1234567", - "account_id": common._DEFAULT_ACCOUNT_ID, - } - mock_console.detailed_fetch_info = { - "device": { - "branch": "git_device_branch", - "target": "device-userdebug", - "build_id": "1234567", - "account_id": common._DEFAULT_ACCOUNT_ID, - "fetch_signed_build": False, - }, - "gsi": { - "branch": "git_aosp_gsi_branch", - "target": "gsi-userdebug", - "build_id": "2345678", - "account_id": common._DEFAULT_ACCOUNT_ID_INTERNAL, - } - } - mock_console.tmp_logdir = "tmp/log" - mock_console.device_image_info = { - "bootloader.img": "path/to/bootloader.img" - } - mock_pb2 = mock.Mock() - mock_pb2.repacked_image_path = [] - mock_pb2.schedule_config.build_target = [] - mock_build_sched_config_pb2 = mock.Mock() - mock_build_sched_config_pb2.test_schedule = [] - mock_test_sched_config_pb2 = mock.Mock() - mock_suite_res_msg.TestSuiteResultMessage.return_value = mock_pb2 - mock_sched_config_msg.BuildScheduleConfigMessage.return_value = ( - mock_build_sched_config_pb2) - mock_sched_config_msg.TestScheduleConfigMessage.return_value = ( - mock_test_sched_config_pb2) - command = command_upload.CommandUpload() - command._SetUp(mock_console) - command.UploadReport("/path/to/bin/gsutil", "gs://report-bucket/", - "tmp/console.log", "tmp/result.log", "vts", - "some_plan") - self.assertEqual(mock_pb2.build_id, "1234567") - self.assertEqual(mock_pb2.suite_name, "vts") - self.assertEqual(mock_pb2.suite_plan, "some_plan") - self.assertEqual(mock_pb2.suite_build_number, mock_pb2.build_id) - self.assertEqual(mock_pb2.build_system_fingerprint, - "git_aosp_gsi_branch/gsi-userdebug/2345678") - self.assertEqual(mock_pb2.build_vendor_fingerprint, - "git_device_branch/device-userdebug/1234567") - self.assertEqual(mock_pb2.repacked_image_path, ["{repack_path}"]) - self.assertEqual(mock_pb2.schedule_config.manifest_branch, - "git_device_branch") - self.assertEqual(mock_pb2.schedule_config.pab_account_id, - common._DEFAULT_ACCOUNT_ID) - - self.assertTrue(mock_build_sched_config_pb2.has_bootloader_img) - self.assertFalse(mock_build_sched_config_pb2.has_radio_img) - - self.assertEqual(mock_test_sched_config_pb2.gsi_branch, - "git_aosp_gsi_branch") - self.assertEqual(mock_test_sched_config_pb2.gsi_build_target, - "gsi-userdebug") - self.assertEqual(mock_test_sched_config_pb2.gsi_pab_account_id, - common._DEFAULT_ACCOUNT_ID_INTERNAL) - mock_cmd_util.ExecuteOneShellCommand.assert_called_with( - "/path/to/bin/gsutil cp tmp/log/{timestamp_time}.bin " - "gs://report-bucket/{timestamp_time}.bin") - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_upload.open") - @mock.patch("host_controller.command_processor.command_upload.cmd_utils") - @mock.patch("host_controller.command_processor.command_upload.SuiteResMsg") - @mock.patch("host_controller.command_processor.command_upload.os") - @mock.patch("host_controller.command_processor.command_upload.xml_utils") - @mock.patch("host_controller.command_processor.command_upload.SchedCfgMsg") - def testUploadReportBootupOk(self, mock_sched_config_msg, mock_xml_util, - mock_os, mock_suite_res_msg, mock_cmd_util, - mock_open, mock_console): - mock_open.__enter__ = mock.Mock(return_value=mock_open) - mock_open.__exit__ = mock.Mock(return_value=None) - mock_cmd_util.ExecuteOneShellCommand = mock.Mock( - return_value=("", "", 0)) - mock_console.vti_endpoint_client.CheckBootUpStatus.return_value = True - mock_console.FormatString.side_effect = side_effect - mock_console.tmp_logdir = "tmp/log" - mock_pb2 = mock.Mock() - mock_pb2.repacked_image_path = [] - mock_pb2.schedule_config.build_target = [] - mock_build_sched_config_pb2 = mock.Mock() - mock_build_sched_config_pb2.test_schedule = [] - mock_test_sched_config_pb2 = mock.Mock() - mock_suite_res_msg.TestSuiteResultMessage.return_value = mock_pb2 - mock_sched_config_msg.BuildScheduleConfigMessage.return_value = ( - mock_build_sched_config_pb2) - mock_sched_config_msg.TestScheduleConfigMessage.return_value = ( - mock_test_sched_config_pb2) - mock_os.listdir.return_value = ["1", "2", "3"] - mock_os.path.isdir.return_value = True - mock_os.path.islink.return_value = False - mock_os.path.join = os.path.join - mock_os.path.basename = os.path.basename - mock_xml_util.GetAttributes.return_value = { - common._SUITE_NAME_ATTR_KEY: - "vts", - common._SUITE_PLAN_ATTR_KEY: - "some_plan", - common._SUITE_VERSION_ATTR_KEY: - "8.0_R1", - common._SUITE_BUILD_NUM_ATTR_KEY: - "1234567", - common._START_TIME_ATTR_KEY: - "0", - common._END_TIME_ATTR_KEY: - "1", - common._HOST_NAME_ATTR_KEY: - "this-host", - common._FINGERPRINT_ATTR_KEY: - "git_device_branch/device-userdebug/1234567", - common._SYSTEM_FINGERPRINT_ATTR_KEY: - "git_aosp_gsi_branch/gsi-userdebug/2345678", - common._VENDOR_FINGERPRINT_ATTR_KEY: - "git_device_branch/device-userdebug/1234567", - common._PASSED_ATTR_KEY: - "1265", - common._FAILED_ATTR_KEY: - "43", - common._MODULES_TOTAL_ATTR_KEY: - "100", - common._MODULES_DONE_ATTR_KEY: - "98", - } - command = command_upload.CommandUpload() - command._SetUp(mock_console) - command.UploadReport("/path/to/bin/gsutil", "gs://report-bucket/", - "tmp/console.log", "tmp/vts/results", "vts", - "some_plan") - self.assertEqual(mock_pb2.build_id, "1234567") - self.assertEqual(mock_pb2.suite_name, "vts") - self.assertEqual(mock_pb2.suite_plan, "some_plan") - self.assertEqual(mock_pb2.suite_version, "8.0_R1") - self.assertEqual(mock_pb2.suite_build_number, mock_pb2.build_id) - self.assertEqual(mock_pb2.start_time, 0) - self.assertEqual(mock_pb2.end_time, 1) - self.assertEqual(mock_pb2.host_name, "this-host") - self.assertEqual(mock_pb2.build_system_fingerprint, - "git_aosp_gsi_branch/gsi-userdebug/2345678") - self.assertEqual(mock_pb2.build_vendor_fingerprint, - "git_device_branch/device-userdebug/1234567") - self.assertEqual(mock_pb2.passed_test_case_count, 1265) - self.assertEqual(mock_pb2.failed_test_case_count, 43) - self.assertEqual(mock_pb2.modules_total, 100) - self.assertEqual(mock_pb2.modules_done, 98) - self.assertEqual(mock_pb2.repacked_image_path, ["{repack_path}"]) - mock_cmd_util.ExecuteOneShellCommand.assert_called_with( - "/path/to/bin/gsutil cp tmp/log/{timestamp_time}.bin " - "gs://report-bucket/{timestamp_time}.bin") - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_upload.os") - @mock.patch("host_controller.command_processor.command_upload.logging") - def testUploadReportResultDirAbsent(self, mock_logger, mock_os, - mock_console): - mock_console.vti_endpoint_client.CheckBootUpStatus.return_value = True - mock_console.FormatString.side_effect = side_effect - mock_console.tmp_logdir = "tmp/log" - mock_os.listdir.return_value = [] - command = command_upload.CommandUpload() - command._SetUp(mock_console) - ret = command.UploadReport("/path/to/bin/gsutil", - "gs://report-bucket/", "tmp/console.log", - "tmp/result.log", "vts", "some_plan") - self.assertFalse(ret) - mock_logger.error.assert_called_with("No test result found.") - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_upload.gcs_utils") - @mock.patch("host_controller.command_processor.command_upload.logging") - def testCommandUploadGsutilAbsent(self, mock_logger, mock_gcs_util, - mock_console): - mock_gcs_util.GetGsutilPath.return_value = "" - command = command_upload.CommandUpload() - command.UploadReport = mock.Mock() - command._SetUp(mock_console) - ret = command._Run("--src=tmp/result.log --dest=gs://report-bucket/") - self.assertFalse(ret) - mock_logger.error.assert_called_with( - "Please check gsutil is installed and on your PATH") - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_upload.gcs_utils") - @mock.patch("host_controller.command_processor.command_upload.logging") - def testCommandUploadLatestSrc(self, mock_logger, mock_gcs_util, - mock_console): - mock_gcs_util.GetGsutilPath.return_value = "/path/to/bin/gsutil" - command = command_upload.CommandUpload() - command.UploadReport = mock.Mock() - command._SetUp(mock_console) - ret = command._Run( - "--src=latest-something.img --dest=gs://report-bucket/") - self.assertFalse(ret) - mock_logger.error.assert_called_with( - "Unable to find something.img in device_image_info") - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_upload.gcs_utils") - @mock.patch("host_controller.command_processor.command_upload.os") - def testCommandUploadLatestLegitSrc(self, mock_os, mock_gcs_util, - mock_console): - mock_os.path.isfile.return_value = True - mock_console.device_image_info = {"system.img": "path/to/system.img"} - mock_console.FormatString.side_effect = side_effect - mock_gcs_util.GetGsutilPath.return_value = "/path/to/bin/gsutil" - command = command_upload.CommandUpload() - command.UploadReport = mock.Mock() - command._SetUp(mock_console) - ret = command._Run( - "--src=latest-system.img --dest=gs://report-bucket/dir " - "--clear_dest") - self.assertIsNone(ret) - mock_gcs_util.Remove.assert_called_with( - "/path/to/bin/gsutil", "gs://report-bucket/dir", recursive=True) - mock_gcs_util.Copy.assert_called_with('/path/to/bin/gsutil', - 'path/to/system.img', - 'gs://report-bucket/dir') - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_upload.gcs_utils") - @mock.patch("host_controller.command_processor.command_upload.os") - @mock.patch("host_controller.command_processor.command_upload.logging") - def testCommandUploadFalseDest(self, mock_logger, mock_os, mock_gcs_util, - mock_console): - mock_os.path.isfile.return_value = True - mock_console.device_image_info = {"system.img": "path/to/system.img"} - mock_console.FormatString.side_effect = side_effect - mock_gcs_util.GetGsutilPath.return_value = "/path/to/bin/gsutil" - command = command_upload.CommandUpload() - command.UploadReport = mock.Mock() - command._SetUp(mock_console) - ret = command._Run("--src=latest-system.img --dest=/report-bucket/") - self.assertFalse(ret) - mock_logger.error.assert_called_with( - "/report-bucket/ is not correct GCS url.") - - @mock.patch("host_controller.console.Console") - @mock.patch("host_controller.command_processor.command_upload.gcs_utils") - @mock.patch("host_controller.command_processor.command_upload.os") - def testCommandUploadMultipleFiles(self, mock_os, mock_gcs_util, - mock_console): - mock_os.path.isfile.return_value = True - mock_console.FormatString.side_effect = side_effect - mock_gcs_util.GetGsutilPath.return_value = "/path/to/bin/gsutil" - command = command_upload.CommandUpload() - command.UploadReport = mock.Mock() - command._SetUp(mock_console) - ret = command._Run("--src=result.zip --dest=gs://report-bucket/") - self.assertIsNone(ret) - mock_gcs_util.Copy.assert_called_with( - '/path/to/bin/gsutil', 'result.zip', 'gs://report-bucket/') - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/common.py b/harnesses/host_controller/common.py deleted file mode 100644 index 76bb697..0000000 --- a/harnesses/host_controller/common.py +++ /dev/null @@ -1,212 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# The default Partner Android Build (PAB) public account. -# To obtain access permission, please reach out to Android partner engineering -# department of Google LLC. - -from vti.test_serving.proto import TestScheduleConfigMessage_pb2 as pb - -_DEFAULT_ACCOUNT_ID = '543365459' - -# The default Partner Android Build (PAB) internal account. -_DEFAULT_ACCOUNT_ID_INTERNAL = '541462473' - -# The key value used for getting a fetched .zip android img file. -FULL_ZIPFILE = "full-zipfile" -# The key of an item that stores the unziped files of a full image zip file. -FULL_ZIPFILE_DIR = "full-zipfile-dir" - -# The key of an item that stores the fetch GSI image (.zip) file. -GSI_ZIPFILE = "gsi-zipfile" -# The key of an item that stores the unziped files of a GSI image zip file. -GSI_ZIPFILE_DIR = "gsi-zipfile-dir" - -# The default value for "flash --current". -_DEFAULT_FLASH_IMAGES = [ - FULL_ZIPFILE, - FULL_ZIPFILE_DIR, - "bootloader.img", - "boot.img", - "cache.img", - "radio.img", - "system.img", - "userdata.img", - "vbmeta.img", - "vendor.img", -] - -# The environment variable for default serial numbers. -_ANDROID_SERIAL = "ANDROID_SERIAL" - -_DEVICE_STATUS_DICT = { - "unknown": 0, - "fastboot": 1, - "online": 2, - "ready": 3, - "use": 4, - "error": 5, - "no-response": 6 -} - -_STORAGE_TYPE_DICT = { - pb.UNKNOWN_BUILD_STORAGE_TYPE: "unknown", - pb.BUILD_STORAGE_TYPE_PAB: "pab", - pb.BUILD_STORAGE_TYPE_GCS: "gcs", -} - -_STORAGE_TYPE_DICT_REVERSE = { - "unknown": pb.UNKNOWN_BUILD_STORAGE_TYPE, - "pab": pb.BUILD_STORAGE_TYPE_PAB, - "gcs": pb.BUILD_STORAGE_TYPE_GCS, -} - -# Default SPL date, used for gsispl command -_SPL_DEFAULT_DAY = 5 - -# Maximum number of leased jobs per host. -_MAX_LEASED_JOBS = 14 - -# Defualt access point for dut wifi_on command. -_DEFAULT_WIFI_AP = "GoogleGuest" - -# SoC name list. -K39TV1_BSP = "k39tv1_bsp" -K39TV1_BSP_1G = "k39tv1_bsp_1g" - -SDM845 = "sdm845" - -UNIVERSAL9810 = "universal9810" - -# Lib files from SDM845 vendor system image need to be re-pushed -# into GSI system image to boot the devices up properly. -SDM845_LIB_LIST = [ - "libdrm.so", - "vendor.display.color@1.0.so", - "vendor.display.config@1.0.so", - "vendor.display.config@1.1.so", - "vendor.display.postproc@1.0.so", - "vendor.qti.hardware.perf@1.0.so", -] - -# Dir name in which the addtional files need to be repacked. -_ADDITIONAL_FILES_DIR = "additional_file" - -# Relative path to the "results" directory from the tools directory. -_RESULTS_BASE_PATH = "../results" - -# Test result file contains invoked test plan results. -_TEST_RESULT_XML = "test_result.xml" - -_LOG_RESULT_XML = "log-result.xml" - -# XML tag name whose attributes represent a module. -_MODULE_TAG = "Module" - -# XML tag name whose attribute is test plan. -_RESULT_TAG = "Result" - -# XML tag name whose attributes represent a test case in a module. -_TESTCASE_TAG = "TestCase" - -# XML tag name whose attributes represent a test result in a test case. -_TEST_TAG = "Test" - -# XML tag name whose attributes are about the build info of the device. -_BUILD_TAG = "Build" - -# XML tag name whose attributes are pass/fail count, modules run/total count. -_SUMMARY_TAG = "Summary" - -# The key value for retrieving test plan and etc. from the result xml -_SUITE_PLAN_ATTR_KEY = "suite_plan" - -_SUITE_BUILD_NUM_ATTR_KEY = "suite_build_number" - -_SUITE_VERSION_ATTR_KEY = "suite_version" - -_HOST_NAME_ATTR_KEY = "host_name" - -_START_TIME_ATTR_KEY = "start" - -_START_DISPLAY_TIME_ATTR_KEY = "start_display" - -_END_TIME_ATTR_KEY = "end" - -_END_DISPLAY_TIME_ATTR_KEY = "end_display" - -_SUITE_NAME_ATTR_KEY = "suite_name" - -# The key value for retrieving build fingerprint values from the result xml. -_ABIS_ATTR_KEY = "build_abis" - -_FINGERPRINT_ATTR_KEY = "build_fingerprint" - -_SYSTEM_FINGERPRINT_ATTR_KEY = "build_system_fingerprint" - -_VENDOR_FINGERPRINT_ATTR_KEY = "build_vendor_fingerprint" - -# The key value for retrieving passed testcase count -_PASSED_ATTR_KEY = "pass" - -# The key value for retrieving failed testcase count -_FAILED_ATTR_KEY = "failed" - -# The key value for retrieving total module count -_MODULES_TOTAL_ATTR_KEY = "modules_total" - -# The key value for retrieving run module count -_MODULES_DONE_ATTR_KEY = "modules_done" - -# The key value for retrieving name of a test, testcase, or module. -_NAME_ATTR_KEY = "name" - -# The key value for retrieving ABI of a module. -_ABI_ATTR_KEY = "abi" - -# The key value for retrieving result of a test. -_RESULT_ATTR_KEY = "result" - -# VTSLAB package version file -_VTSLAB_VERSION_TXT = "version.txt" - -_VTSLAB_VERSION_DEFAULT_VALUE = "000000_000000:00000000" - -# String representations of the artifact types can be fetched. -_ARTIFACT_TYPE_DEVICE = "device" -_ARTIFACT_TYPE_GSI = "gsi" -_ARTIFACT_TYPE_TEST_SUITE = "test_suite" -# Artifact type that are usually for custom tools fetched from GCS. -_ARTIFACT_TYPE_INFRA = "infra" - -# List of artifact types. -_ARTIFACT_TYPE_LIST = [ - _ARTIFACT_TYPE_DEVICE, - _ARTIFACT_TYPE_GSI, - _ARTIFACT_TYPE_TEST_SUITE, - _ARTIFACT_TYPE_INFRA, -] -# Directory relative to the home directory, in which the devices' lock files will be. -_DEVLOCK_DIR = ".devlock" - -# Default timeout for "adb reboot/fastboot getvar" command in secs. -DEFAULT_DEVICE_TIMEOUT_SECS = 300 - -# Maximum number of concurrent adb/fastboot processes. -MAX_ADB_FASTBOOT_PROCESS = 2 - -# Default number of the actual retry runs for the "retry" command. -DEFAULT_RETRY_COUNT = 30
\ No newline at end of file diff --git a/harnesses/host_controller/console.py b/harnesses/host_controller/console.py deleted file mode 100644 index 6ab0722..0000000 --- a/harnesses/host_controller/console.py +++ /dev/null @@ -1,920 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import cmd -import ctypes -import datetime -import imp # Python v2 compatibility -import logging -import multiprocessing -import multiprocessing.pool -import os -import re -import shutil -import signal -import socket -import sys -import tempfile -import threading -import time -import urlparse - -from host_controller import common -from host_controller.command_processor import command_adb -from host_controller.command_processor import command_build -from host_controller.command_processor import command_config -from host_controller.command_processor import command_config_local -from host_controller.command_processor import command_copy -from host_controller.command_processor import command_device -from host_controller.command_processor import command_dut -from host_controller.command_processor import command_exit -from host_controller.command_processor import command_fastboot -from host_controller.command_processor import command_fetch -from host_controller.command_processor import command_flash -from host_controller.command_processor import command_gsispl -from host_controller.command_processor import command_info -from host_controller.command_processor import command_lease -from host_controller.command_processor import command_list -from host_controller.command_processor import command_password -from host_controller.command_processor import command_release -from host_controller.command_processor import command_retry -from host_controller.command_processor import command_request -from host_controller.command_processor import command_repack -from host_controller.command_processor import command_sheet -from host_controller.command_processor import command_shell -from host_controller.command_processor import command_sleep -from host_controller.command_processor import command_test -from host_controller.command_processor import command_reproduce -from host_controller.command_processor import command_upload -from host_controller.build import build_info -from host_controller.build import build_provider_ab -from host_controller.build import build_provider_gcs -from host_controller.build import build_provider_local_fs -from host_controller.build import build_provider_pab -from host_controller.utils.ipc import file_lock -from host_controller.utils.ipc import shared_dict -from host_controller.vti_interface import vti_endpoint_client -from vts.runners.host import logger -from vts.utils.python.common import cmd_utils - -COMMAND_PROCESSORS = [ - command_adb.CommandAdb, - command_build.CommandBuild, - command_config.CommandConfig, - command_config_local.CommandConfigLocal, - command_copy.CommandCopy, - command_device.CommandDevice, - command_dut.CommandDUT, - command_exit.CommandExit, - command_fastboot.CommandFastboot, - command_fetch.CommandFetch, - command_flash.CommandFlash, - command_gsispl.CommandGsispl, - command_info.CommandInfo, - command_lease.CommandLease, - command_list.CommandList, - command_password.CommandPassword, - command_release.CommandRelease, - command_retry.CommandRetry, - command_request.CommandRequest, - command_repack.CommandRepack, - command_sheet.CommandSheet, - command_shell.CommandShell, - command_sleep.CommandSleep, - command_test.CommandTest, - command_reproduce.CommandReproduce, - command_upload.CommandUpload, -] - - -class NonDaemonizedProcess(multiprocessing.Process): - """Process class which is not daemonized.""" - - def _get_daemon(self): - return False - - def _set_daemon(self, value): - pass - - daemon = property(_get_daemon, _set_daemon) - - -class NonDaemonizedPool(multiprocessing.pool.Pool): - """Pool class which is not daemonized.""" - - Process = NonDaemonizedProcess - - -def JobMain(vti_address, in_queue, out_queue, device_status, password, hosts): - """Main() for a child process that executes a leased job. - - Currently, lease jobs must use VTI (not TFC). - - Args: - vti_client: VtiEndpointClient needed to create Console. - in_queue: Queue to get new jobs. - out_queue: Queue to put execution results. - device_status: SharedDict, contains device status information. - shared between processes. - password: multiprocessing.managers.ValueProxy, a proxy instance of a - string(ctypes.c_char_p) represents the password which is - to be passed to the prompt when executing certain command - as root user. - hosts: A list of HostController objects. Needed for the device command. - """ - - def SigTermHandler(signum, frame): - """Signal handler for exiting pool process explicitly. - - Added to resolve orphaned pool process issue. - """ - sys.exit(0) - - signal.signal(signal.SIGTERM, SigTermHandler) - - vti_client = vti_endpoint_client.VtiEndpointClient(vti_address) - console = Console(vti_client, None, None, hosts, job_pool=True) - console.device_status = device_status - console.password = password - multiprocessing.util.Finalize(console, console.__exit__, exitpriority=0) - - while True: - command = in_queue.get() - if command == "exit": - break - elif command == "lease": - filepath, kwargs = vti_client.LeaseJob(socket.gethostname(), True) - logging.debug("Job %s -> %s" % (os.getpid(), kwargs)) - if filepath is not None: - # TODO: redirect console output and add - # console command to access them. - - console._build_provider[ - "pab"] = build_provider_pab.BuildProviderPAB() - console._build_provider[ - "gcs"] = build_provider_gcs.BuildProviderGCS() - - for serial in kwargs["serial"]: - console.ChangeDeviceState( - serial, common._DEVICE_STATUS_DICT["use"]) - print_to_console = True - if not print_to_console: - sys.stdout = out - sys.stderr = err - - ret, gcs_log_url = console.ProcessConfigurableScript( - os.path.join(os.getcwd(), "host_controller", "campaigns", - filepath), **kwargs) - if ret: - job_status = "complete" - else: - job_status = "infra-err" - - vti_client.StopHeartbeat(job_status, gcs_log_url) - logging.info("Job execution complete. " - "Setting job status to {}".format(job_status)) - - if not print_to_console: - sys.stdout = sys.__stdout__ - sys.stderr = sys.__stderr__ - - for serial in kwargs["serial"]: - console.ChangeDeviceState( - serial, common._DEVICE_STATUS_DICT["ready"]) - - del console._build_provider["pab"] - del console._build_provider["gcs"] - console.fetch_info = {} - console._detailed_fetch_info = {} - else: - logging.error("Unknown job command %s", command) - - out_queue.put("exit") - - -class Console(cmd.Cmd): - """The console for host controllers. - - Attributes: - command_processors: dict of string:BaseCommandProcessor, - map between command string and command processors. - device_image_info: dict containing info about device image files. - prompt: The prompt string at the beginning of each command line. - test_result: dict containing info about the last test result. - test_suite_info: dict containing info about test suite package files. - tools_info: dict containing info about custom tool files. - scheduler_thread: dict containing threading.Thread instances(s) that - update configs regularly. - _build_provider_pab: The BuildProviderPAB used to download artifacts. - _vti_address: string, VTI service URI. - _vti_client: VtiEndpoewrClient, used to upload data to a test - scheduling infrastructure. - _tfc_client: The TfcClient that the host controllers connect to. - _hosts: A list of HostController objects. - _in_file: The input file object. - _out_file: The output file object. - _serials: A list of string where each string is a device serial. - _device_status: SharedDict, shared with process pool. - contains status data on each devices. - _job_pool: bool, True if Console is created from job pool process - context. - _password: multiprocessing.managers.ValueProxy, a proxy instance of a - string(ctypes.c_char_p) represents the password which is - to be passed to the prompt when executing certain command - as root user. - _manager: SyncManager. an instance of a manager for shared objects and - values between processes. - _vtslab_version: string, contains version information of vtslab package. - (<git commit timestamp>:<git commit hash value>) - _detailed_fetch_info: A nested dict, holds the branch and target value - of the device, gsi, or test suite artifact. - _file_lock: FileLock, an instance used for synchronizing the devices' - use when the automated self-update happens. - """ - - def __init__(self, - vti_endpoint_client, - tfc, - pab, - host_controllers, - vti_address=None, - in_file=sys.stdin, - out_file=sys.stdout, - job_pool=False, - password=None): - """Initializes the attributes and the parsers.""" - # cmd.Cmd is old-style class. - cmd.Cmd.__init__(self, stdin=in_file, stdout=out_file) - self._build_provider = {} - self._job_pool = job_pool - if not self._job_pool: - self._build_provider["pab"] = pab - self._build_provider["gcs"] = build_provider_gcs.BuildProviderGCS() - self._build_provider[ - "local_fs"] = build_provider_local_fs.BuildProviderLocalFS() - self._build_provider["ab"] = build_provider_ab.BuildProviderAB() - self._manager = multiprocessing.Manager() - self._device_status = shared_dict.SharedDict(self._manager) - self._password = self._manager.Value(ctypes.c_char_p, password) - try: - with open(common._VTSLAB_VERSION_TXT, "r") as file: - self._vtslab_version = file.readline().strip() - file.close() - logging.info("VTSLAB version: %s" % self._vtslab_version) - except IOError as e: - logging.exception(e) - logging.error("Version info missing in vtslab package. " - "Setting version as %s", - common._VTSLAB_VERSION_DEFAULT_VALUE) - self._vtslab_version = common._VTSLAB_VERSION_DEFAULT_VALUE - self._logfile_upload_path = "" - - self._vti_endpoint_client = vti_endpoint_client - self._vti_address = vti_address - self._tfc_client = tfc - self._hosts = host_controllers - self._in_file = in_file - self._out_file = out_file - self.prompt = "> " - self.command_processors = {} - self.device_image_info = build_info.BuildInfo() - self.test_result = {} - self.test_suite_info = build_info.BuildInfo() - self.tools_info = build_info.BuildInfo() - self.fetch_info = {} - self._detailed_fetch_info = {} - self.test_results = {} - self._file_lock = file_lock.FileLock() - self.repack_dest_path = "" - - if common._ANDROID_SERIAL in os.environ: - self._serials = [os.environ[common._ANDROID_SERIAL]] - else: - self._serials = [] - - self.InitCommandModuleParsers() - self.SetUpCommandProcessors() - - tempdir_base = os.path.join(os.getcwd(), "tmp") - if not os.path.exists(tempdir_base): - os.mkdir(tempdir_base) - self._tmpdir_default = tempfile.mkdtemp(dir=tempdir_base) - self._tmp_logdir = tempfile.mkdtemp(dir=tempdir_base) - if not self._job_pool: - self._logfile_path = logger.setupTestLogger( - self._tmp_logdir, create_symlink=False) - - def __exit__(self): - """Finalizes the build provider attributes explicitly when exited.""" - for bp in self._build_provider: - self._build_provider[bp].__del__() - if os.path.exists(self._tmp_logdir): - shutil.rmtree(self._tmp_logdir) - - @property - def job_pool(self): - """getter for self._job_pool""" - return self._job_pool - - @property - def device_status(self): - """getter for self._device_status""" - return self._device_status - - @device_status.setter - def device_status(self, device_status): - """setter for self._device_status""" - self._device_status = device_status - - @property - def build_provider(self): - """getter for self._build_provider""" - return self._build_provider - - @property - def tmpdir_default(self): - """getter for self._password""" - return self._tmpdir_default - - @tmpdir_default.setter - def tmpdir_default(self, tmpdir): - """getter for self._password""" - self._tmpdir_default = tmpdir - - @property - def password(self): - """getter for self._password""" - return self._password - - @password.setter - def password(self, password): - """getter for self._password""" - self._password = password - - @property - def logfile_path(self): - """getter for self._logfile_path""" - return self._logfile_path - - @property - def tmp_logdir(self): - """getter for self._tmp_logdir""" - return self._tmp_logdir - - @property - def vti_endpoint_client(self): - """getter for self._vti_endpoint_client""" - return self._vti_endpoint_client - - @property - def vtslab_version(self): - """getter for self._vtslab_version""" - return self._vtslab_version - - @property - def detailed_fetch_info(self): - return self._detailed_fetch_info - - def UpdateFetchInfo(self, artifact_type): - if artifact_type in common._ARTIFACT_TYPE_LIST: - self._detailed_fetch_info[artifact_type] = {} - self._detailed_fetch_info[artifact_type].update(self.fetch_info) - else: - logging.error("Unrecognized artifact type: %s", artifact_type) - - @property - def file_lock(self): - """getter for self._file_lock""" - return self._file_lock - - def ChangeDeviceState(self, serial, state): - """Changes a device's state and (un)locks the file lock if necessary. - - Args: - serial: string, serial number of a device. - state: int, devices' status value pre-defined in - common._DEVICE_STATUS_DICT. - Returns: - True if the state change and locking/unlocking are successful. - False otherwise. - """ - if state == common._DEVICE_STATUS_DICT["use"]: - ret = self._file_lock.LockDevice(serial) - if ret == False: - return False - - current_status = self.device_status[serial] - self.device_status[serial] = state - - if (current_status in (common._DEVICE_STATUS_DICT["use"], - common._DEVICE_STATUS_DICT["error"]) - and current_status != state): - self._file_lock.UnlockDevice(serial) - - def InitCommandModuleParsers(self): - """Init all console command modules""" - for name in dir(self): - if name.startswith('_Init') and name.endswith('Parser'): - attr_func = getattr(self, name) - if hasattr(attr_func, '__call__'): - attr_func() - - def SetUpCommandProcessors(self): - """Sets up all command processors.""" - for command_processor in COMMAND_PROCESSORS: - cp = command_processor() - cp._SetUp(self) - do_text = "do_%s" % cp.command - help_text = "help_%s" % cp.command - setattr(self, do_text, cp._Run) - setattr(self, help_text, cp._Help) - self.command_processors[cp.command] = cp - - def TearDown(self): - """Removes all command processors.""" - for command_processor in self.command_processors.itervalues(): - command_processor._TearDown() - self.command_processors.clear() - self.__exit__() - - def FormatString(self, format_string): - """Replaces variables with the values in the console's dictionaries. - - Args: - format_string: The string containing variables enclosed in {}. - - Returns: - The formatted string. - - Raises: - KeyError if a variable is not found in the dictionaries or the - value is empty. - """ - - def ReplaceVariable(match): - """Replacement functioon for re.sub(). - - replaces string encased in braces with values in the console's dict. - - Args: - match: regex, used for extracting the variable name. - - Returns: - string value corresponding to the input variable name. - """ - name = match.group(1) - if name in ("build_id", "branch", "target", "account_id"): - value = self.fetch_info[name] - elif name in ("result_full", "result_zip", "suite_plan", - "suite_name"): - value = self.test_result[name] - elif "timestamp" in name: - current_datetime = datetime.datetime.now() - value_date = current_datetime.strftime("%Y%m%d") - value_time = current_datetime.strftime("%H%M%S") - if "_date" in name: - value = value_date - elif "_time" in name: - value = value_time - elif "_year" in name: - value = value_date[0:4] - elif "_month" in name: - value = value_date[4:6] - elif "_day" in name: - value = value_date[6:8] - else: - value = "%s-%s" % (value_date, value_time) - elif name in ("hc_log", "hc_log_file", "hc_log_upload_path"): - # hc_log: full abs path to the current process's infra log. - # hc_log_file: infra log file name, with no path information. - # hc_log_upload_path: path of the infra log file in GCS. - value = self._logfile_path - if name == "hc_log_file": - value = os.path.basename(value) - elif name == "hc_log_upload_path": - value = self._logfile_upload_path - elif name in ("repack_path"): - value = self.repack_dest_path - self.repack_dest_path = "" - elif name in ("hostname"): - value = socket.gethostname() - elif "." in name and name.split(".")[0] in self.command_processors: - command, arg = name.split(".") - try: - value = self.command_processors[command].arg_buffer[arg] - except KeyError as e: - logging.exception(e) - value = "" - if value is None: - value = "" - else: - value = None - - if value is None: - raise KeyError(name) - - return value - - return re.sub("{([^}]+)}", ReplaceVariable, format_string) - - def ProcessScript(self, script_file_path): - """Processes a .py script file. - - A script file implements a function which emits a list of console - commands to execute. That function emits an empty list or None if - no more command needs to be processed. - - Args: - script_file_path: string, the path of a script file (.py file). - - Returns: - True if successful; False otherwise - """ - if not script_file_path.endswith(".py"): - logging.error("Script file is not .py file: %s" % script_file_path) - return False - - script_module = imp.load_source('script_module', script_file_path) - - commands = script_module.EmitConsoleCommands() - if commands: - for command in commands: - ret = self.onecmd(command) - if ret == False: - return False - return True - - def ProcessConfigurableScript(self, script_file_path, **kwargs): - """Processes a .py script file. - - A script file implements a function which emits a list of console - commands to execute. That function emits an empty list or None if - no more command needs to be processed. - - Args: - script_file_path: string, the path of a script file (.py file). - kwargs: extra args for the interface function defined in - the script file. - - Returns: - True if successful; False otherwise - String which represents URL to the upload infra log file. - """ - if script_file_path and not script_file_path.endswith(".py"): - script_file_path += ".py" - - if not script_file_path.endswith(".py"): - logging.error("Script file is not .py file: %s", script_file_path) - return False - - ret = True - - self._logfile_path, file_handler = logger.addLogFile(self._tmp_logdir) - src = self.FormatString("{hc_log}") - dest = self.FormatString( - "gs://vts-report/infra_log/{hostname}/%s_{timestamp}/{hc_log_file}" - % kwargs["build_target"]) - self._logfile_upload_path = dest - - script_module = imp.load_source('script_module', script_file_path) - - commands = script_module.EmitConsoleCommands(**kwargs) - logging.info("Command list: %s", commands) - if commands: - logging.info("Console commands: %s", commands) - for command in commands: - ret = self.onecmd(command) - if ret == False: - break - else: - ret = False - - file_handler.flush() - infra_log_upload_command = "upload" - infra_log_upload_command += " --src=%s" % src - infra_log_upload_command += " --dest=%s" % dest - for serial in kwargs["serial"]: - if self.device_status[serial] == common._DEVICE_STATUS_DICT[ - "error"]: - self.vti_endpoint_client.SetJobStatusFromLeasedTo("bootup-err") - break - if not self.vti_endpoint_client.CheckBootUpStatus(): - infra_log_upload_command += (" --report_path=gs://vts-report/" - "suite_result/{timestamp_year}/" - "{timestamp_month}/{timestamp_day}") - suite_name, plan_name = kwargs["test_name"].split("/") - infra_log_upload_command += ( - " --result_from_suite=%s" % suite_name) - infra_log_upload_command += (" --result_from_plan=%s" % plan_name) - self.onecmd(infra_log_upload_command) - if self.GetSerials(): - self.onecmd("device --update=stop") - logging.getLogger().removeHandler(file_handler) - os.remove(self._logfile_path) - return (ret != False), dest - - def _Print(self, string): - """Prints a string and a new line character. - - Args: - string: The string to be printed. - """ - self._out_file.write(string + "\n") - - def _PrintObjects(self, objects, attr_names): - """Shows objects as a table. - - Args: - object: The objects to be shown, one object in a row. - attr_names: The attributes to be shown, one attribute in a column. - """ - width = [len(name) for name in attr_names] - rows = [attr_names] - for dev_info in objects: - attrs = [ - _ToPrintString(getattr(dev_info, name, "")) - for name in attr_names - ] - rows.append(attrs) - for index, attr in enumerate(attrs): - width[index] = max(width[index], len(attr)) - - for row in rows: - self._Print(" ".join( - attr.ljust(width[index]) for index, attr in enumerate(row))) - - def DownloadTestResources(self, request_id): - """Download all of the test resources for a TFC request id. - - Args: - request_id: int, TFC request id - """ - resources = self._tfc_client.TestResourceList(request_id) - for resource in resources: - self.DownloadTestResource(resource['url']) - - def DownloadTestResource(self, url): - """Download a test resource with build provider, given a url. - - Args: - url: a resource locator (not necessarily HTTP[s]) - with the scheme specifying the build provider. - """ - parsed = urlparse.urlparse(url) - path = (parsed.netloc + parsed.path).split('/') - if parsed.scheme == "pab": - if len(path) != 5: - logging.error("Invalid pab resource locator: %s", url) - return - account_id, branch, target, build_id, artifact_name = path - cmd = ("fetch" - " --type=pab" - " --account_id=%s" - " --branch=%s" - " --target=%s" - " --build_id=%s" - " --artifact_name=%s") % (account_id, branch, target, - build_id, artifact_name) - self.onecmd(cmd) - elif parsed.scheme == "ab": - if len(path) != 4: - logging.error("Invalid ab resource locator: %s", url) - return - branch, target, build_id, artifact_name = path - cmd = ("fetch" - "--type=ab" - " --branch=%s" - " --target=%s" - " --build_id=%s" - " --artifact_name=%s") % (branch, target, build_id, - artifact_name) - self.onecmd(cmd) - elif parsed.scheme == gcs: - cmd = "fetch --type=gcs --path=%s" % url - self.onecmd(cmd) - else: - logging.error("Invalid URL: %s", url) - - def SetSerials(self, serials): - """Sets the default serial numbers for flashing and testing. - - Args: - serials: A list of strings, the serial numbers. - """ - self._serials = serials - - def FlashImgPackage(self, package_path_gcs): - """Fetches a repackaged image set from GCS and flashes to the device(s). - - Args: - package_path_gcs: GCS URL to the packaged img zip file. May contain - the GSI imgs. - """ - self.onecmd("fetch --type=gcs --path=%s --full_device_images=True" % - package_path_gcs) - if common.FULL_ZIPFILE not in self.device_image_info: - logging.error("Failed to fetch the given file: %s", - package_path_gcs) - return False - - if not self._serials: - logging.error("Please specify the serial number(s) of target " - "device(s) for flashing.") - return False - - campaign_common = imp.load_source( - 'campaign_common', - os.path.join(os.getcwd(), "host_controller", "campaigns", - "campaign_common.py")) - flash_command_list = [] - - for serial in self._serials: - flash_commands = [] - cmd_utils.ExecuteOneShellCommand( - "adb -s %s reboot bootloader" % serial) - _, stderr, retcode = cmd_utils.ExecuteOneShellCommand( - "fastboot -s %s getvar product" % serial) - if retcode == 0: - res = stderr.splitlines()[0].rstrip() - if ":" in res: - product = res.split(":")[1].strip() - elif "waiting for %s" % serial in res: - res = stderr.splitlines()[1].rstrip() - product = res.split(":")[1].strip() - else: - product = "error" - else: - product = "error" - logging.info("Device %s product type: %s", serial, product) - if product in campaign_common.FLASH_COMMAND_EMITTER: - flash_commands.append( - campaign_common.FLASH_COMMAND_EMITTER[product]( - serial, repacked_imageset=True)) - elif product != "error": - flash_commands.append( - "flash --current --serial %s --skip-vbmeta=True" % serial) - else: - logging.error( - "Device %s does not exist. Omitting the flashing " - "to the device.", serial) - continue - flash_command_list.append(flash_commands) - - ret = self.onecmd(flash_command_list) - if ret == False: - logging.error("Flash failed on device %s.", self._serials) - else: - logging.info("Flash succeeded on device %s.", self._serials) - - return ret - - def GetSerials(self): - """Returns the serial numbers saved in the console. - - Returns: - A list of strings, the serial numbers. - """ - return self._serials - - def ResetSerials(self): - """Clears all the serial numbers set to this console obj.""" - self._serials = [] - - def JobThread(self): - """Job thread which monitors and uploads results.""" - thread = threading.currentThread() - while getattr(thread, "keep_running", True): - time.sleep(1) - - if self._job_pool: - self._job_pool.close() - self._job_pool.terminate() - self._job_pool.join() - - def StartJobThreadAndProcessPool(self): - """Starts a background thread to control leased jobs.""" - self._job_in_queue = multiprocessing.Queue() - self._job_out_queue = multiprocessing.Queue() - self._job_pool = NonDaemonizedPool( - common._MAX_LEASED_JOBS, JobMain, - (self._vti_address, self._job_in_queue, self._job_out_queue, - self._device_status, self._password, self._hosts)) - - self._job_thread = threading.Thread(target=self.JobThread) - self._job_thread.daemon = True - self._job_thread.start() - - def StopJobThreadAndProcessPool(self): - """Terminates the thread and processes that runs the leased job.""" - if hasattr(self, "_job_thread"): - self._job_thread.keep_running = False - self._job_thread.join() - - def WaitForJobsToExit(self): - """Wait for the running jobs to complete before exiting HC.""" - if self._job_pool: - pool_process_count = common._MAX_LEASED_JOBS - for _ in range(common._MAX_LEASED_JOBS): - self._job_in_queue.put("exit") - - while True: - response = self._job_out_queue.get() - if response == "exit": - pool_process_count -= 1 - if pool_process_count <= 0: - break - - # @Override - def onecmd(self, line, depth=1, ret_out_queue=None): - """Executes command(s) and prints any exception. - - Parallel execution only for 2nd-level list element. - - Args: - line: a list of string or string which keeps the command to run. - """ - if not line: - return - - if type(line) == list: - if depth == 1: # 1 to use multi-threading - jobs = [] - ret_queue = multiprocessing.Queue() - for sub_command in line: - p = multiprocessing.Process( - target=self.onecmd, - args=( - sub_command, - depth + 1, - ret_queue, - )) - jobs.append(p) - p.start() - for job in jobs: - job.join() - - ret_cmd_list = True - while not ret_queue.empty(): - ret_from_subprocess = ret_queue.get() - ret_cmd_list = ret_cmd_list and ret_from_subprocess - if ret_cmd_list == False: - return False - else: - for sub_command in line: - ret_cmd_list = self.onecmd(sub_command, depth + 1) - if ret_cmd_list == False and ret_out_queue: - ret_out_queue.put(False) - return False - return - - logging.info("Command: %s", line) - try: - ret_cmd = cmd.Cmd.onecmd(self, line) - if ret_cmd == False and ret_out_queue: - ret_out_queue.put(ret_cmd) - return ret_cmd - except Exception as e: - self._Print("%s: %s" % (type(e).__name__, e)) - if ret_out_queue: - ret_out_queue.put(False) - return False - - # @Override - def emptyline(self): - """Ignores empty lines.""" - pass - - # @Override - def default(self, line): - """Handles unrecognized commands. - - Returns: - True if receives EOF; otherwise delegates to default handler. - """ - if line == "EOF": - return self.do_exit(line) - return cmd.Cmd.default(self, line) - - -def _ToPrintString(obj): - """Converts an object to printable string on console. - - Args: - obj: The object to be printed. - """ - if isinstance(obj, (list, tuple, set)): - return ",".join(str(x) for x in obj) - return str(obj) diff --git a/harnesses/host_controller/console_argument_parser.py b/harnesses/host_controller/console_argument_parser.py deleted file mode 100644 index 71a756f..0000000 --- a/harnesses/host_controller/console_argument_parser.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -import logging - - -class ConsoleArgumentError(Exception): - """Raised when the console fails to parse commands.""" - pass - - -class ConsoleArgumentParser(argparse.ArgumentParser): - """The argument parser for a console command.""" - - def __init__(self, command_name, description): - """Initializes the ArgumentParser without --help option. - - Args: - command_name: A string, the first argument of the command. - description: The help message about the command. - """ - super(ConsoleArgumentParser, self).__init__( - prog=command_name, description=description, add_help=False) - - def ParseLine(self, line): - """Parses a command line. - - Args: - line: A string, the command line. - - Returns: - An argparse.Namespace object. - """ - return self.parse_args(line.split()) - - # @Override - def error(self, message): - """Raises an exception when failing to parse the command. - - Args: - message: The error message. - - Raises: - ConsoleArgumentError. - """ - raise ConsoleArgumentError(message)
\ No newline at end of file diff --git a/harnesses/host_controller/console_test.py b/harnesses/host_controller/console_test.py deleted file mode 100644 index bd4906a..0000000 --- a/harnesses/host_controller/console_test.py +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import unittest - -try: - from unittest import mock -except ImportError: - import mock - -try: - import StringIO as string_io_module -except ImportError: - import io as string_io_module - -from host_controller.build import build_flasher -from host_controller.tfc import command_task -from host_controller.tfc import device_info -from host_controller import common -from host_controller import console - - -class ConsoleTest(unittest.TestCase): - """A test for console.Console. - - Attribute: - _out_file: The console output buffer. - _host_controller: A mock tfc_host_controller.HostController. - _build_provider_pab: A mock build_provider_pab.BuildProviderPAB. - _tfc_client: A mock tfc_client.TfcClient. - _vti_client A mock vti_endpoint_client.VtiEndpointClient. - _console: The console being tested. - """ - _DEVICES = [ - device_info.DeviceInfo( - device_serial="ABC001", - run_target="sailfish", - state="Available", - build_id="111111", - sdk_version="27") - ] - _TASKS = [ - command_task.CommandTask( - request_id="1", - task_id="1-0", - command_id="2", - command_line="vts -m SampleShellTest", - device_serials=["ABC001"]) - ] - - def setUp(self): - """Creates the console.""" - self._out_file = string_io_module.StringIO() - self._host_controller = mock.Mock() - self._build_provider_pab = mock.Mock() - self._tfc_client = mock.Mock() - self._vti_client = mock.Mock() - self._console = console.Console( - self._vti_client, - self._tfc_client, - self._build_provider_pab, [self._host_controller], - None, - out_file=self._out_file) - self._console.device_image_info = {} - - def tearDown(self): - """Closes the output file.""" - self._out_file.close() - - def _IssueCommand(self, command_line): - """Issues a command in the console. - - Args: - command_line: A string, the input to the console. - - Returns: - A string, the output of the console. - """ - out_position = self._out_file.tell() - self._console.onecmd(command_line) - self._out_file.seek(out_position) - return self._out_file.read() - - def testLease(self): - """Tests the lease command.""" - self._host_controller.LeaseCommandTasks.return_value = self._TASKS - output = self._IssueCommand("lease") - expected = ( - "request_id command_id task_id device_serials command_line \n" - "1 2 1-0 ABC001 vts -m SampleShellTest\n" - ) - self.assertEqual(expected, output) - output = self._IssueCommand("lease --host 0") - self.assertEqual(expected, output) - - def testRequest(self): - """Tests the request command.""" - user = "user0" - cluster = "cluster0" - run_target = "sailfish" - command_line = "vts -m SampleShellTest" - self._IssueCommand("request --user %s --cluster %s --run-target %s " - "-- %s" % (user, cluster, run_target, command_line)) - req = self._tfc_client.NewRequest.call_args[0][0] - self.assertEqual(user, req.user) - self.assertEqual(cluster, req.cluster) - self.assertEqual(run_target, req.run_target) - self.assertEqual(command_line, req.command_line) - - def testListHosts(self): - """Tests the list command.""" - self._host_controller.hostname = "host0" - output = self._IssueCommand("list hosts") - self.assertEqual("index name\n" "[ 0] host0\n", output) - - def testListDevices(self): - """Tests the list command.""" - self._host_controller.ListDevices.return_value = self._DEVICES - self._host_controller.hostname = "host0" - output = self._IssueCommand("list devices") - expected = ( - "[ 0] host0\n" - "device_serial state run_target build_id sdk_version stub\n" - "ABC001 Available sailfish 111111 27 \n" - ) - self.assertEqual(expected, output) - output = self._IssueCommand("list devices --host 0") - self.assertEqual(expected, output) - - def testWrongHostIndex(self): - """Tests host index out of range.""" - output = self._IssueCommand("list devices --host 1") - expected = "IndexError: " - self.assertTrue(output.startswith(expected)) - output = self._IssueCommand("lease --host 1") - self.assertTrue(output.startswith(expected)) - - @mock.patch('host_controller.build.build_flasher.BuildFlasher') - def testFetchPOSTAndFlash(self, mock_class): - """Tests fetching from pab and flashing.""" - self._build_provider_pab.GetArtifact.return_value = ({ - "system.img": - "/mock/system.img", - "odm.img": - "/mock/odm.img" - }, {}, { - "build_id": - "build_id" - }, {}) - self._build_provider_pab.GetFetchedArtifactType.return_value = common._ARTIFACT_TYPE_DEVICE - self._IssueCommand( - "fetch --branch=aosp-master-ndk --target=darwin_mac " - "--account_id=100621237 " - "--artifact_name=foo-{build_id}.tar.bz2 --method=POST") - self._build_provider_pab.GetArtifact.assert_called_with( - account_id='100621237', - branch='aosp-master-ndk', - target='darwin_mac', - artifact_name='foo-{build_id}.tar.bz2', - build_id='latest', - method='POST', - full_device_images=False) - self.assertEqual(self._console.device_image_info, { - "system.img": "/mock/system.img", - "odm.img": "/mock/odm.img" - }) - - flasher = mock.Mock() - mock_class.return_value = flasher - self._IssueCommand("flash --current system=system.img odm=odm.img") - flasher.Flash.assert_called_with({ - "system": "/mock/system.img", - "odm": "/mock/odm.img" - }, False) - - def testFetchAndEnvironment(self): - """Tests fetching from pab and check stored os environment""" - build_id_return = "4328532" - target_return = "darwin_mac" - expected_fetch_info = {"build_id": build_id_return} - - self._build_provider_pab.GetArtifact.return_value = ({ - "system.img": - "/mock/system.img", - "odm.img": - "/mock/odm.img" - }, {}, expected_fetch_info, {}) - self._IssueCommand( - "fetch --branch=aosp-master-ndk --target=%s " - "--account_id=100621237 " - "--artifact_name=foo-{id}.tar.bz2 --method=POST" % target_return) - self._build_provider_pab.GetArtifact.assert_called_with( - account_id='100621237', - branch='aosp-master-ndk', - target='darwin_mac', - artifact_name='foo-{id}.tar.bz2', - build_id='latest', - full_device_images=False, - method='POST') - - expected = expected_fetch_info["build_id"] - self.assertEqual(build_id_return, expected) - - @mock.patch('host_controller.build.build_flasher.BuildFlasher') - def testFlashGSI(self, mock_class): - flasher = mock.Mock() - mock_class.return_value = flasher - self._IssueCommand("flash --gsi=system.img") - flasher.FlashGSI.assert_called_with( - 'system.img', None, skip_vbmeta=False) - - @mock.patch('host_controller.build.build_flasher.BuildFlasher') - def testFlashGSIWithVbmeta(self, mock_class): - flasher = mock.Mock() - mock_class.return_value = flasher - self._IssueCommand("flash --gsi=system.img --vbmeta=vbmeta.img") - flasher.FlashGSI.assert_called_with( - 'system.img', 'vbmeta.img', skip_vbmeta=False) - - @mock.patch('host_controller.build.build_flasher.BuildFlasher') - def testFlashall(self, mock_class): - flasher = mock.Mock() - mock_class.return_value = flasher - self._IssueCommand("flash --build_dir=path/to/dir/") - flasher.Flashall.assert_called_with('path/to/dir/') - - @mock.patch('host_controller.command_processor.command_flash.importlib') - @mock.patch('host_controller.command_processor.command_flash.issubclass') - def testImportFlasher(self, mock_issubclass, mock_importlib): - mock_issubclass.return_value = True - flasher_module = mock.Mock() - flasher = mock.Mock() - mock_importlib.import_module.return_value = flasher_module - flasher_module.Flasher.return_value = flasher - self._IssueCommand("flash --serial ABC001 " - "--flasher_type test.flasher.Flasher " - "--flasher_path /test/flasher " - "-- --unit test") - mock_issubclass.assert_called_once_with(flasher_module.Flasher, - build_flasher.BuildFlasher) - mock_importlib.import_module.assert_called_with("test.flasher") - flasher_module.Flasher.assert_called_with("ABC001", "/test/flasher") - flasher.Flash.assert_called_with({}, {}, "--unit", "test") - flasher.WaitForDevice.assert_called_with() - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/gsi/change_security_patch_ver.sh b/harnesses/host_controller/gsi/change_security_patch_ver.sh deleted file mode 100755 index 1a09178..0000000 --- a/harnesses/host_controller/gsi/change_security_patch_ver.sh +++ /dev/null @@ -1,288 +0,0 @@ -#!/bin/bash - -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script modifies the original GSI to match the vendor version and -# the security patch level. -# -# Usage: change_security_patch_ver.sh <system.img> [<output_system.img> \ -# [<new_security_patch_level> [<input_file_contexts.bin>]]] \ -# [-v <vendor_version>] -# -# Examples: -# change_security_patch_ver.sh system.img -# - Shows current version information. -# change_security_patch_ver.sh system.img new_system.img 2018-04-05 -# - Make new_system.img that has replaced SPL with 2018-04-05. -# change_security_patch_ver.sh system.img new_system.img -v 8.1.0 -# - Make new_system.img that includes the patches for vendor version 8.1.0. -# change_security_patch_ver.sh system.img new_system.img 2018-04-05 -v 8.1.0 -# - Make new_system.img that has both new SPL and vendor version. - -function unmount() { - echo "Unmounting..." - sudo umount "${MOUNT_POINT}/" -} - -SCRIPT_NAME=$(basename $0) - -declare -a SUPPORTED_VENDOR_VERSIONS=( - 8.1.0 - 9 -) -SUPPORTED_VENDOR_VERSIONS="${SUPPORTED_VENDOR_VERSIONS[@]}" - -param_count=0 -while [[ $# -gt 0 ]] -do -case $1 in --v|--vendor) # set vendor version - VENDOR_VERSION=$2 - shift - shift - ;; -*) # set the ordered parameters - ((param_count++)) - case $param_count in - 1) # The input file name for original GSI - SYSTEM_IMG=$1 - shift - ;; - 2) # The output file name for modified GSI - OUTPUT_SYSTEM_IMG=$1 - shift - ;; - 3) # New Security Patch Level to be written. It must be YYYY-MM-DD format. - NEW_SPL=$1 - shift - ;; - 4) # Selinux file context - FILE_CONTEXTS_BIN=$1 - shift - ;; - *) - ERROR=true - break - ;; - esac - ;; -esac -done - -if ((param_count == 0)) || [ "$ERROR" == "true" ]; then - echo "Usage: $SCRIPT_NAME <system.img> [<output_system.img> [<new_security_patch_level> [<input_file_contexts.bin>]]] [-v <vendor_version>]" - exit 1 -fi - -# SPL must have YYYY-MM-DD format -if ((param_count >= 3)) && [[ ! ${NEW_SPL} =~ ^[0-9]{4}-(0[0-9]|1[012])-([012][0-9]|3[01])$ ]]; then - echo "<new_security_patch_level> must have YYYY-MM-DD format" - exit 1 -fi - -if [ "$VENDOR_VERSION" != "" ] && [[ ! ${VENDOR_VERSION} =~ ^(${SUPPORTED_VENDOR_VERSIONS// /\|})$ ]]; then - echo "Available vendor_version: $SUPPORTED_VENDOR_VERSIONS" - exit 1 -fi - -if [ "$VENDOR_VERSION" != "" ] && [ "$OUTPUT_SYSTEM_IMG" == "" ]; then - echo "<output_system.img> must be provided to set vendor version" - exit 1 -fi - -REQUIRED_BINARIES_LIST=( - "img2simg" - "simg2img" -) -if [ ! -z "${FILE_CONTEXTS_BIN}" ]; then - REQUIRED_BINARIES_LIST+=("mkuserimg_mke2fs") -fi - -# number of binaries to find. -BIN_COUNT=${#REQUIRED_BINARIES_LIST[@]} - -# use an associative array to store binary path -declare -A BIN_PATH -for bin in ${REQUIRED_BINARIES_LIST[@]}; do - BIN_PATH[${bin}]="" -done - -# check current PATH environment first -for bin in ${REQUIRED_BINARIES_LIST[@]}; do - if command -v ${bin} >/dev/null 2>&1; then - echo "found ${bin} in PATH." - BIN_PATH[${bin}]=${bin} - ((BIN_COUNT--)) - fi -done - -if [ ${BIN_COUNT} -gt 0 ]; then - # listed in the recommended order. - PATH_LIST=("${PWD}") - if [ "${PWD##*/}" == "testcases" ] && [ -d "${PWD}/../bin" ]; then - PATH_LIST+=("${PWD}/../bin") - fi - if [ -d "${ANDROID_HOST_OUT}" ]; then - PATH_LIST+=("${ANDROID_HOST_OUT}/bin") - fi - - for dir in ${PATH_LIST[@]}; do - for bin in ${REQUIRED_BINARIES_LIST[@]}; do - if [ -z "${BIN_PATH[${bin}]}" ] && [ -f "${dir}/${bin}" ]; then - echo "found ${bin} in ${dir}." - BIN_PATH[${bin}]=${dir}/${bin} - ((BIN_COUNT--)) - if [ ${BIN_COUNT} -eq 0 ]; then break; fi - fi - done - done -fi - -if [ ${BIN_COUNT} -gt 0 ]; then - echo "Cannot find the required binaries. Need lunch; or run in a correct path." - exit 1 -fi -echo "Found all binaries." - -MOUNT_POINT="${PWD}/temp_mnt" -SPL_PROPERTY_NAME="ro.build.version.security_patch" -RELEASE_VERSION_PROPERTY_NAME="ro.build.version.release" -VNDK_VERSION_PROPERTY="ro.vndk.version" -VNDK_VERSION_PROPERTY_OMR1="${VNDK_VERSION_PROPERTY}=27" - -UNSPARSED_SYSTEM_IMG="${SYSTEM_IMG}.raw" -SYSTEM_IMG_MAGIC="$(xxd -g 4 -l 4 "$SYSTEM_IMG" | head -n1 | awk '{print $2}')" -if [ "$SYSTEM_IMG_MAGIC" = "3aff26ed" ]; then - echo "Unsparsing ${SYSTEM_IMG}..." - ${BIN_PATH["simg2img"]} "$SYSTEM_IMG" "$UNSPARSED_SYSTEM_IMG" -else - echo "Copying unsparse input system image ${SYSTEM_IMG}..." - cp "$SYSTEM_IMG" "$UNSPARSED_SYSTEM_IMG" -fi - -IMG_SIZE=$(stat -c%s "$UNSPARSED_SYSTEM_IMG") - -echo "Mounting..." -mkdir -p "$MOUNT_POINT" -sudo mount -t ext4 -o loop "$UNSPARSED_SYSTEM_IMG" "${MOUNT_POINT}/" - -# check the property file path -BUILD_PROP_PATH_LIST=( - "/system/build.prop" # layout of A/B support - "/build.prop" # layout of non-A/B support -) -BUILD_PROP_MOUNT_PATH="" -BUILD_PROP_PATH="" - -echo "Finding build.prop..." -for path in ${BUILD_PROP_PATH_LIST[@]}; do - if [ -f "${MOUNT_POINT}${path}" ]; then - BUILD_PROP_MOUNT_PATH="${MOUNT_POINT}${path}" - BUILD_PROP_PATH=${path} - echo " ${path}" - break - fi -done - -PROP_DEFAULT_PATH_LIST=( - "/system/etc/prop.default" # layout of A/B support - "/etc/prop.default" # layout of non-A/B support -) - -if [ "$BUILD_PROP_MOUNT_PATH" != "" ]; then - if [ "$OUTPUT_SYSTEM_IMG" != "" ]; then - echo "Replacing..." - fi - CURRENT_SPL=`sudo sed -n -r "s/^${SPL_PROPERTY_NAME}=(.*)$/\1/p" ${BUILD_PROP_MOUNT_PATH}` - CURRENT_VERSION=`sudo sed -n -r "s/^${RELEASE_VERSION_PROPERTY_NAME}=(.*)$/\1/p" ${BUILD_PROP_MOUNT_PATH}` - echo " Current security patch level: ${CURRENT_SPL}" - echo " Current release version: ${CURRENT_VERSION}" - - # Update SPL to <new_security_patch_level> - if [[ "$OUTPUT_SYSTEM_IMG" != "" && "$NEW_SPL" != "" ]]; then - if [[ "$CURRENT_SPL" == "" ]]; then - echo "ERROR: Cannot find ${SPL_PROPERTY_NAME} in ${BUILD_PROP_PATH}" - else - echo " New security patch level: ${NEW_SPL}" - seek=$(sudo grep --byte-offset "${SPL_PROPERTY_NAME}=" "${BUILD_PROP_MOUNT_PATH}" | cut -d':' -f 1) - seek=$(($seek + ${#SPL_PROPERTY_NAME} + 1)) # 1 is for '=' - echo "${NEW_SPL}" | sudo dd of="${BUILD_PROP_MOUNT_PATH}" seek="$seek" bs=1 count=10 conv=notrunc - fi - fi - - # Update release version to <vendor_version> - if [[ "$OUTPUT_SYSTEM_IMG" != "" && "$VENDOR_VERSION" != "" ]]; then - if [[ "$CURRENT_VERSION" == "" ]]; then - echo "ERROR: Cannot find ${RELEASE_VERSION_PROPERTY_NAME} in ${BUILD_PROP_PATH}" - else - echo " New release version for vendor.img: ${VENDOR_VERSION}" - sudo sed -i -e "s/^${RELEASE_VERSION_PROPERTY_NAME}=.*$/${RELEASE_VERSION_PROPERTY_NAME}=${VENDOR_VERSION}/" ${BUILD_PROP_MOUNT_PATH} - fi - - if [[ "$VENDOR_VERSION" == "8.1.0" ]]; then - # add ro.vndk.version for O-MR1 - echo "Finding prop.default..." - for path in ${PROP_DEFAULT_PATH_LIST[@]}; do - if [ -f "${MOUNT_POINT}${path}" ]; then - PROP_DEFAULT_PATH=${path} - echo " ${path}" - break - fi - done - - if [[ "$PROP_DEFAULT_PATH" != "" ]]; then - CURRENT_VNDK_VERSION=`sudo sed -n -r "s/^${VNDK_VERSION_PROPERTY}=(.*)$/\1/p" ${MOUNT_POINT}${PROP_DEFAULT_PATH}` - if [[ "$CURRENT_VNDK_VERSION" != "" ]]; then - echo "WARNING: ${VNDK_VERSION_PROPERTY} is already set to ${CURRENT_VNDK_VERSION} in ${PROP_DEFAULT_PATH}" - else - echo " Add \"${VNDK_VERSION_PROPERTY_OMR1}\" to ${PROP_DEFAULT_PATH} for O-MR1 vendor image." - sudo sed -i -e "\$a\#\n\# FOR O-MR1 DEVICES\n\#\n${VNDK_VERSION_PROPERTY_OMR1}" ${MOUNT_POINT}${PROP_DEFAULT_PATH} - fi - else - echo "ERROR: Cannot find prop.default." - fi - fi - fi -else - echo "ERROR: Cannot find build.prop." -fi - -if [ "$OUTPUT_SYSTEM_IMG" != "" ]; then - if [ "$FILE_CONTEXTS_BIN" != "" ]; then - echo "Writing ${OUTPUT_SYSTEM_IMG}..." - - (cd $ANDROID_BUILD_TOP - if [[ "$(whereis mkuserimg_mke2fs | wc -w)" < 2 ]]; then - make mkuserimg_mke2fs -j - fi - NON_AB=$(expr "$BUILD_PROP_PATH" == "/build.prop") - if [ $NON_AB -eq 1 ]; then - sudo /bin/bash -c "PATH=out/host/linux-x86/bin/:\$PATH mkuserimg_mke2fs -s ${MOUNT_POINT} $OUTPUT_SYSTEM_IMG ext4 system $IMG_SIZE -D ${MOUNT_POINT} -L system $FILE_CONTEXTS_BIN" - else - sudo /bin/bash -c "PATH=out/host/linux-x86/bin/:\$PATH mkuserimg_mke2fs -s ${MOUNT_POINT} $OUTPUT_SYSTEM_IMG ext4 / $IMG_SIZE -D ${MOUNT_POINT}/system -L / $FILE_CONTEXTS_BIN" - fi) - - unmount - else - unmount - - echo "Writing ${OUTPUT_SYSTEM_IMG}..." - ${BIN_PATH["img2simg"]} "$UNSPARSED_SYSTEM_IMG" "$OUTPUT_SYSTEM_IMG" - fi -else - unmount -fi - -echo "Done." diff --git a/harnesses/host_controller/invocation_thread.py b/harnesses/host_controller/invocation_thread.py deleted file mode 100644 index b8760f3..0000000 --- a/harnesses/host_controller/invocation_thread.py +++ /dev/null @@ -1,169 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import socket -import threading - -import httplib2 -from googleapiclient import errors - -from host_controller.tfc import command_attempt -from host_controller.tradefed import remote_operation - - -class InvocationThread(threading.Thread): - """The thread that remotely executes a command task. - - Attributes: - _remote_client: The RemoteClient which executes the command. - _tfc_client: The TfcClient to which the command events are sent. - _attempt: The CommandAttempt whose events are sent to TFC. - _command: A list of strings, the command and arguments. - device_serials: A list of strings, the serial numbers of the devices - which need to be allocated to the task. - _allocated_serials: A list of strings, the serial numbers of the devices - which are successfully allocated. - _tfc_heartbeat_interval: The interval of TestRunInProgress events in - seconds. - """ - - def __init__(self, - remote_client, - tfc_client, - attempt, - command, - device_serials, - tfc_heartbeat_interval=5 * 60): - """Initializes the attributes.""" - super(InvocationThread, self).__init__() - self._remote_client = remote_client - self._tfc_client = tfc_client - self._attempt = attempt - self._command = command - self.device_serials = device_serials - self._allocated_serials = None - # The value in Java implementation is 5 minutes. - self._tfc_heartbeat_interval = tfc_heartbeat_interval - - def _AllocateDevices(self): - """Allocates all of device_serial.""" - for serial in self.device_serials: - self._remote_client.SendOperation( - remote_operation.AllocateDevice(serial)) - self._allocated_serials.append(serial) - - def _StartInvocation(self): - """Starts executing command and sends the event to TFC.""" - self._remote_client.SendOperation( - remote_operation.ExecuteCommand(self.device_serials[0], - *self._command)) - event = self._attempt.CreateCommandEvent( - command_attempt.EventType.INVOCATION_STARTED) - self._tfc_client.SubmitCommandEvents([event]) - - def _WaitForCommandResult(self): - """Waits for command result and keeps sending heartbeat to TFC - - Returns: - A JSON object returned from TradeFed remote manager. - """ - while True: - result = self._remote_client.WaitForCommandResult( - self.device_serials[0], self._tfc_heartbeat_interval) - if result: - return result - event = self._attempt.CreateCommandEvent( - command_attempt.EventType.TEST_RUN_IN_PROGRESS) - self._tfc_client.SubmitCommandEvents([event]) - - def _CompleteInvocation(self, result): - """Sends InvocationCompleted event according to the result. - - Args: - result: A JSON object returned from TradeFed remote manager. - """ - if result["status"] == "INVOCATION_SUCCESS": - event = self._attempt.CreateInvocationCompletedEvent( - str(result), 1, 0) - else: - event = self._attempt.CreateInvocationCompletedEvent( - str(result), 1, 1, error=str(result)) - self._tfc_client.SubmitCommandEvents([event]) - - def _FreeAllocatedDevices(self): - """Frees allocated devices and tolerates RemoteOperationException.""" - for serial in self._allocated_serials: - try: - self._remote_client.SendOperation( - remote_operation.FreeDevice(serial)) - except remote_operation.RemoteOperationException as e: - logging.exception(e) - except socket.error as e: - logging.exception(e) - break - self._allocated_serials = [] - - def _SubmitErrorEvent(self, event_type, error_msg): - """Submits an error event and tolerates http exceptions. - - Args: - event_type: A string, the type of the command event. - error_msg: A string, the error message. - """ - try: - self._tfc_client.SubmitCommandEvents( - [self._attempt.CreateCommandEvent(event_type, error_msg)]) - except (httplib2.HttpLib2Error, errors.HttpError) as e: - logging.exception(e) - - # @Override - def run(self): - """Executes a command task with exception handling.""" - self._allocated_serials = [] - last_error = None - error_event = command_attempt.EventType.ALLOCATION_FAILED - try: - self._AllocateDevices() - error_event = command_attempt.EventType.EXECUTE_FAILED - self._StartInvocation() - result = self._WaitForCommandResult() - self._CompleteInvocation(result) - error_event = None - except errors.HttpError as e: - logging.exception(e) - last_error = e - except remote_operation.RemoteOperationException as e: - logging.exception(e) - last_error = e - # ConfigurationException on TradeFed remote manager. - if str(e).startswith("Config error: "): - error_event = command_attempt.EventType.CONFIGURATION_ERROR - except httplib2.HttpLib2Error as e: - logging.exception("Cannot communicate with TradeFed cluster: %s\n" - "Skip submitting event %s.", e, error_event) - last_error = e - error_event = None - except socket.error as e: - logging.exception("Cannot communicate with TradeFed remote " - "manager: %s\nSkip freeing devices %s.", - e, self._allocated_serials) - last_error = e - self._allocated_serials = [] - finally: - if error_event: - self._SubmitErrorEvent(error_event, str(last_error)) - self._FreeAllocatedDevices() diff --git a/harnesses/host_controller/invocation_thread_test.py b/harnesses/host_controller/invocation_thread_test.py deleted file mode 100644 index 07847b9..0000000 --- a/harnesses/host_controller/invocation_thread_test.py +++ /dev/null @@ -1,148 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest - -try: - from unittest import mock -except ImportError: - import mock - -from host_controller import invocation_thread -from host_controller.tfc import command_attempt -from host_controller.tradefed import remote_operation - - -class InvocationThreadTest(unittest.TestCase): - """A test for invocation_thread.InvocationThread. - - Attributes: - _remote_client: A mock remote_client.RemoteClient. - _tfc_client: A mock tfc_client.TfcClient. - _inv_thread: The InvocationThread being tested. - """ - - def setUp(self): - """Creates the InvocationThread.""" - self._remote_client = mock.Mock() - self._tfc_client = mock.Mock() - attempt = command_attempt.CommandAttempt( - task_id="321-0", - attempt_id="abcd-1234", - hostname="host0", - device_serial="ABCDEF") - command = ["vts", "-m", "SampleShellTest"] - serials = ["serial123", "serial456"] - self._inv_thread = invocation_thread.InvocationThread( - self._remote_client, self._tfc_client, - attempt, command, serials) - - def _GetSubmittedEventTypes(self): - """Gets the types of the events submitted by the mock TfcClient. - - Returns: - A list of strings, the event types. - """ - event_types = [] - for args, kwargs in self._tfc_client.SubmitCommandEvents.call_args_list: - event_types.extend(event["type"] for event in args[0]) - return event_types - - def _GetSentOperationTypes(self): - """Gets the types of the operations sent by the mock RemoteClient. - - Returns: - A list of strings, the operation types. - """ - operation_types = [args[0].type for args, kwargs in - self._remote_client.SendOperation.call_args_list] - return operation_types - - def testAllocationFailed(self): - """Tests AllocationFailed event.""" - self._remote_client.SendOperation.side_effect = ( - lambda op: _RaiseExceptionForOperation(op, "ALLOCATE_DEVICE")) - self._inv_thread.run() - self.assertEqual([command_attempt.EventType.ALLOCATION_FAILED], - self._GetSubmittedEventTypes()) - self.assertEqual(["ALLOCATE_DEVICE"], - self._GetSentOperationTypes()) - - def testExecuteFailed(self): - """Tests ExecuteFailed event.""" - self._remote_client.SendOperation.side_effect = ( - lambda op: _RaiseExceptionForOperation(op, "EXEC_COMMAND")) - self._inv_thread.run() - self.assertEqual([command_attempt.EventType.EXECUTE_FAILED], - self._GetSubmittedEventTypes()) - self.assertEqual(["ALLOCATE_DEVICE", - "ALLOCATE_DEVICE", - "EXEC_COMMAND", - "FREE_DEVICE", - "FREE_DEVICE"], - self._GetSentOperationTypes()) - - def testConfigurationError(self): - """Tests ConfigurationError event.""" - self._remote_client.SendOperation.side_effect = ( - lambda op: _RaiseExceptionForOperation(op, "EXEC_COMMAND", - "Config error: test")) - self._inv_thread.run() - self.assertEqual([command_attempt.EventType.CONFIGURATION_ERROR], - self._GetSubmittedEventTypes()) - self.assertEqual(["ALLOCATE_DEVICE", - "ALLOCATE_DEVICE", - "EXEC_COMMAND", - "FREE_DEVICE", - "FREE_DEVICE"], - self._GetSentOperationTypes()) - - def testInvocationCompleted(self): - """Tests InvocationCompleted event.""" - self._remote_client.WaitForCommandResult.side_effect = ( - None, {"status": "INVOCATION_SUCCESS"}) - self._inv_thread.run() - self.assertEqual([command_attempt.EventType.INVOCATION_STARTED, - command_attempt.EventType.TEST_RUN_IN_PROGRESS, - command_attempt.EventType.INVOCATION_COMPLETED], - self._GetSubmittedEventTypes()) - # GET_LAST_COMMAND_RESULT isn't called in mock WaitForCommandResult. - self.assertEqual(["ALLOCATE_DEVICE", - "ALLOCATE_DEVICE", - "EXEC_COMMAND", - "FREE_DEVICE", - "FREE_DEVICE"], - self._GetSentOperationTypes()) - - -def _RaiseExceptionForOperation(operation, op_type, error_msg="unit test"): - """Raises exception for specific operation type. - - Args: - operation: A remote_operation.RemoteOperation object. - op_type: A string, the expected type. - error_msg: The message in the exception. - - Raises: - RemoteOperationException if the operation's type matches op_type. - """ - if operation.type == op_type: - raise remote_operation.RemoteOperationException(error_msg) - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/main.py b/harnesses/host_controller/main.py deleted file mode 100644 index 348134e..0000000 --- a/harnesses/host_controller/main.py +++ /dev/null @@ -1,227 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -import json -import logging -import socket -import time -import threading -import sys - -from host_controller import console -from host_controller import tfc_host_controller -from host_controller.build import build_provider_pab -from host_controller.tfc import tfc_client -from host_controller.vti_interface import vti_endpoint_client -from host_controller.tradefed import remote_client -from vts.utils.python.os import env_utils - -_ANDROID_BUILD_TOP = "ANDROID_BUILD_TOP" -_SECONDS_PER_UNIT = { - "m": 60, - "h": 60 * 60, - "d": 60 * 60 * 24 -} - - -def _ParseInterval(interval_str): - """Parses string to time interval. - - Args: - interval_str: string, a floating-point number followed by time unit. - - Returns: - float, the interval in seconds. - - Raises: - ValueError if the argument format is wrong. - """ - if not interval_str: - raise ValueError("Argument is empty.") - - unit = interval_str[-1] - if unit not in _SECONDS_PER_UNIT: - raise ValueError("Unknown unit: %s" % unit) - - interval = float(interval_str[:-1]) - if interval < 0: - raise ValueError("Invalid time interval: %s" % interval) - - return interval * _SECONDS_PER_UNIT[unit] - - -def _ScriptLoop(hc_console, script_path, loop_interval): - """Runs a console script repeatedly. - - Args: - hc_console: the host controller console. - script_path: string, the path to the script. - loop_interval: float or integer, the interval in seconds. - """ - next_start_time = time.time() - while hc_console.ProcessScript(script_path): - if loop_interval == 0: - continue - current_time = time.time() - skip_cnt = (current_time - next_start_time) // loop_interval - if skip_cnt >= 1: - logging.warning("Script execution time is longer than loop " - "interval. Skip %d iteration(s).", skip_cnt) - next_start_time += (skip_cnt + 1) * loop_interval - if next_start_time - current_time >= 0: - time.sleep(next_start_time - current_time) - else: - logging.error("Unexpected timestamps: current=%f, next=%f", - current_time, next_start_time) - - -def main(): - """Parses arguments and starts console.""" - parser = argparse.ArgumentParser() - parser.add_argument("--config-file", - default=None, - type=argparse.FileType('r'), - help="The configuration file in JSON format") - parser.add_argument("--poll", action="store_true", - help="Disable console and start host controller " - "threads polling TFC.") - parser.add_argument("--use-tfc", action="store_true", - help="Enable TFC (TradeFed Cluster).") - parser.add_argument("--vti", - default=None, - help="The base address of VTI endpoint APIs") - parser.add_argument("--script", - default=None, - help="The path to a script file in .py format") - parser.add_argument("--serial", - default=None, - help="The default serial numbers for flashing and " - "testing in the console. Multiple serial numbers " - "are separated by comma.") - parser.add_argument("--loop", - default=None, - metavar="INTERVAL", - type=_ParseInterval, - help="The interval of repeating the script. " - "The format is a float followed by unit which is " - "one of 'm' (minute), 'h' (hour), and 'd' (day). " - "If this option is unspecified, the script will " - "be processed once.") - parser.add_argument("--console", action="store_true", - help="Whether to start a console after processing " - "a script.") - parser.add_argument("--password", - default=None, - help="Password string to pass to the prompt " - "when running certain command as root previlege.") - parser.add_argument("--flash", - default=None, - help="GCS URL to an img package. Fetches and flashes " - "the device(s) given as the '--serial' flag.") - args = parser.parse_args() - if args.config_file: - config_json = json.load(args.config_file) - else: - config_json = {} - config_json["log_level"] = "DEBUG" - config_json["hosts"] = [] - host_config = {} - host_config["cluster_ids"] = ["local-cluster-1", - "local-cluster-2"] - host_config["lease_interval_sec"] = 30 - config_json["hosts"].append(host_config) - - env_vars = env_utils.SaveAndClearEnvVars([_ANDROID_BUILD_TOP]) - - root_logger = logging.getLogger() - root_logger.setLevel(getattr(logging, config_json["log_level"])) - - if args.vti: - vti_endpoint = vti_endpoint_client.VtiEndpointClient(args.vti) - else: - vti_endpoint = None - - tfc = None - if args.use_tfc: - if args.config_file: - tfc = tfc_client.CreateTfcClient( - config_json["tfc_api_root"], - config_json["service_key_json_path"], - api_name=config_json["tfc_api_name"], - api_version=config_json["tfc_api_version"], - scopes=config_json["tfc_scopes"]) - else: - logging.warning("WARN: If --use_tfc is set, --config_file argument " - "value must be provided. Starting without TFC.") - - pab = build_provider_pab.BuildProviderPAB() - - hosts = [] - for host_config in config_json["hosts"]: - cluster_ids = host_config["cluster_ids"] - # If host name is not specified, use local host. - hostname = host_config.get("hostname", socket.gethostname()) - port = host_config.get("port", remote_client.DEFAULT_PORT) - cluster_ids = host_config["cluster_ids"] - remote = remote_client.RemoteClient(hostname, port) - host = tfc_host_controller.HostController(remote, tfc, hostname, - cluster_ids) - hosts.append(host) - if args.poll: - lease_interval_sec = host_config["lease_interval_sec"] - host_thread = threading.Thread(target=host.Run, - args=(lease_interval_sec,)) - host_thread.daemon = True - host_thread.start() - - if args.poll: - while True: - sys.stdin.readline() - else: - main_console = console.Console(vti_endpoint, tfc, pab, hosts, - vti_address=args.vti, - password=args.password) - if args.vti: - main_console.StartJobThreadAndProcessPool() - else: - logging.warning("vti address is not set. example : " - "$ run --vti=<url>") - - try: - if args.serial: - main_console.SetSerials(args.serial.split(",")) - if args.script: - if args.loop is None: - main_console.ProcessScript(args.script) - else: - _ScriptLoop(main_console, args.script, args.loop) - - if args.console: - main_console.cmdloop() - elif args.flash: - main_console.FlashImgPackage(args.flash) - else: # if not script, the default is console mode. - main_console.cmdloop() - finally: - main_console.TearDown() - - env_utils.RestoreEnvVars(env_vars) - - -if __name__ == "__main__": - main() diff --git a/harnesses/host_controller/script/pip_requirements.txt b/harnesses/host_controller/script/pip_requirements.txt deleted file mode 100644 index 92a83cb..0000000 --- a/harnesses/host_controller/script/pip_requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Required pip packages for host controller. -enum -future -futures -google-api-python-client -google-cloud-pubsub -gspread -httplib2 -multiprocessing -protobuf -requests -selenium -setuptools diff --git a/harnesses/host_controller/script/release.sh b/harnesses/host_controller/script/release.sh deleted file mode 100755 index 969cb6f..0000000 --- a/harnesses/host_controller/script/release.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# VTS lab release script -# Usage: <this script> prod|test -# Caution: only used by a VTS lab release engineer -rm out/host/linux-x86/vtslab/android-vtslab/testcases/tmp/ -rf -. build/envsetup.sh -lunch aosp_arm64 -make WifiUtil -j -mkdir -p out/host/linux-x86/vtslab/android-vtslab/testcases/DATA/app -cp out/target/product/generic_arm64/testcases/WifiUtil/ out/host/linux-x86/vtslab/android-vtslab/testcases/DATA/app/ -rf -make vtslab -j - -if [ $# -eq 1 ]; then - ls -al out/host/linux-x86/vtslab/android-vtslab.zip - echo "To upload, please run:" - echo gsutil cp out/host/linux-x86/vtslab/android-vtslab.zip gs://vtslab-release/$1/android-vtslab-$(date +%F).zip -fi diff --git a/harnesses/host_controller/tfc/__init__.py b/harnesses/host_controller/tfc/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/tfc/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/tfc/api_message.py b/harnesses/host_controller/tfc/api_message.py deleted file mode 100644 index d1b618e..0000000 --- a/harnesses/host_controller/tfc/api_message.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -class ApiMessage(object): - """The class for the messages defined by TFC API.""" - - def __init__(self, all_keys, **kwargs): - """Initializes the attributes. - - Args: - all_keys: A collection of attribute names. - **kwargs: The names and values of the attributes. The names must be - in all_keys. - - Raises: - KeyError if any key in kwargs is not in all_keys. - """ - for key, value in kwargs.iteritems(): - if key not in all_keys: - raise KeyError(key) - setattr(self, key, value) - - def ToJson(self, keys): - """Creates a JSON object containing the specified keys. - - Args: - keys: The attributes to be included in the object. - - Returns: - A JSON object. - """ - return dict((x, getattr(self, x)) for x in keys if hasattr(self, x)) diff --git a/harnesses/host_controller/tfc/command_attempt.py b/harnesses/host_controller/tfc/command_attempt.py deleted file mode 100644 index 930ec71..0000000 --- a/harnesses/host_controller/tfc/command_attempt.py +++ /dev/null @@ -1,137 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time - -from host_controller.tfc import api_message - - -class EventType(object): - """The types of command events.""" - ALLOCATION_FAILED = "AllocationFailed" - CONFIGURATION_ERROR = "ConfigurationError" - EXECUTE_FAILED = "ExecuteFailed" - FETCH_FAILED = "FetchFailed" - INVOCATION_COMPLETED = "InvocationCompleted" - INVOCATION_STARTED = "InvocationStarted" - TEST_RUN_IN_PROGRESS = "TestRunInProgress" - - -class CommandAttempt(api_message.ApiMessage): - """The command attempt defined by TFC API. - - Attributes: - _COMMAND_EVENT: The parameters of command_events.submit. - _COMMAND_EVENT_DATA: The fields in "data" parameter of command_events. - _LIST_ATTEMPT: The fields returned by commandAttempts.list. - """ - _COMMAND_EVENT = { - "attempt_id", - "data", - "device_serial", - "hostname", - "task_id", - "time", - "type"} - _COMMAND_EVENT_DATA = { - "error", - "failed_test_count", - "summary", - "test_run_name", - "total_test_count"} - _LIST_ATTEMPT = { - "attempt_id", - "command_id", - "create_time", - "end_time", - "error", - "device_serial", - "failed_test_count", - "hostname", - "request_id", - "start_time", - "state", - "status", - "summary", - "task_id", - "total_test_count", - "update_time"} - - def __init__(self, task_id, attempt_id, hostname, device_serial, **kwargs): - """Initializes the attributes. - - Args: - task_id: A string, the task id assigned by the server. - attempt_id: A string or UUID, the attempt id generated by the host. - hostname: The name of the TradeFed host. - device_serial: The serial number of the device. - **kwargs: The optional attributes. - """ - super(CommandAttempt, self).__init__(self._LIST_ATTEMPT, - task_id=task_id, - attempt_id=str(attempt_id), - hostname=hostname, - device_serial=device_serial, - **kwargs) - - def CreateCommandEvent(self, event_type, error=None, event_time=None): - """Creates an event defined by command_events.submit. - - Args: - event_type: A string in EventType. - error: A string, the error message for *Failed, *Error, and - *Completed events. - event_time: A float, Unix timestamp of the event in seconds. - - Returns: - A JSON object. - """ - obj = self.ToJson(self._COMMAND_EVENT) - obj["type"] = event_type - obj["time"] = int(event_time if event_time is not None else time.time()) - data_obj = self.ToJson(self._COMMAND_EVENT_DATA) - if error is not None: - data_obj["error"] = error - if data_obj: - obj["data"] = data_obj - return obj - - def CreateInvocationCompletedEvent(self, - summary, - total_test_count, - failed_test_count, - error=None, - event_time=None): - """Creates an InvocationCompleted event. - - Args: - summary: A string, the result of the command. - total_test_count: Number of test cases. - failed_test_count: Number of failed test cases. - error: A string, the error message. - event_time: A float, Unix timestamp of the event in seconds. - - Returns: - A JSON object. - """ - obj = self.CreateCommandEvent(EventType.INVOCATION_COMPLETED, - error, event_time) - if "data" not in obj: - obj["data"] = dict() - obj["data"].update({"summary": summary, - "total_test_count": total_test_count, - "failed_test_count": failed_test_count}) - return obj diff --git a/harnesses/host_controller/tfc/command_task.py b/harnesses/host_controller/tfc/command_task.py deleted file mode 100644 index d4aa53e..0000000 --- a/harnesses/host_controller/tfc/command_task.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from host_controller.tfc import api_message - - -class CommandTask(api_message.ApiMessage): - """The task of executing a command defined by TFC API. - - Attributes: - _LEASE_HOST_TASK: The fields returned by commandAttempts.list. - """ - _LEASE_HOST_TASK = { - "request_id", - "command_id", - "task_id", - "command_line", - "request_type", - "device_serials"} - - def __init__(self, task_id, command_line, device_serials, **kwargs): - super(CommandTask, self).__init__(self._LEASE_HOST_TASK, - task_id=task_id, - command_line=command_line, - device_serials=device_serials, - **kwargs) diff --git a/harnesses/host_controller/tfc/device_info.py b/harnesses/host_controller/tfc/device_info.py deleted file mode 100644 index 6a1b35d..0000000 --- a/harnesses/host_controller/tfc/device_info.py +++ /dev/null @@ -1,105 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from host_controller.tfc import api_message -from vts.utils.python.controllers import android_device - - -class DeviceInfo(api_message.ApiMessage): - """The device information defined by TFC API. - - Attributes: - _DEVICE_SNAPSHOT: The parameters of host_events. - _LEASE_HOST_TASKS: The parameters of leasehosttasks. - _OTHER: The data retrieved from TradeFed host but not used by the API. - _ALL_KEYS: Union of above. - """ - _DEVICE_SNAPSHOT = { - "battery_level", - "build_id", - "device_serial", - "group_name", - "mac_address", - "product", - "product_variant", - "run_target", - "sim_operator", - "sim_state", - "sdk_version", - "state"} - _LEASE_HOST_TASKS = { - "device_serial", - "group_name", - "run_target", - "state"} - _OTHER = {"stub"} - _ALL_KEYS = (_DEVICE_SNAPSHOT | _LEASE_HOST_TASKS | _OTHER) - - def __init__(self, device_serial, **kwargs): - """Initializes the attributes. - - Args: - device_serial: The serial number of the device. - **kwargs: The optional attributes. - """ - super(DeviceInfo, self).__init__(self._ALL_KEYS, - device_serial=device_serial, **kwargs) - - def IsAvailable(self): - """Returns whether the device is available for running commands. - - Returns: - A boolean. - """ - return getattr(self, "state", None) == "Available" - - def IsStub(self): - """Returns whether the device is a stub. - - Returns: - A boolean. - """ - return getattr(self, "stub", False) - - def ToDeviceSnapshotJson(self): - """Converts to the parameter of host_events. - - Returns: - A JSON object. - """ - return self.ToJson(self._DEVICE_SNAPSHOT) - - def ToLeaseHostTasksJson(self): - """Converts to the parameter of leasehosttasks. - - Returns: - A JSON object. - """ - return self.ToJson(self._LEASE_HOST_TASKS) - - def Extend(self, properties): - """Adds properties to a DeviceInfo object, using AndroidDevice - - Args: - properties: list of strings, list of properties to update - """ - serial = getattr(self, "device_serial", None) - ad = android_device.AndroidDevice(serial) - for prop in properties: - val = getattr(ad, prop, None) - if val is None: - continue - setattr(self, prop, val) diff --git a/harnesses/host_controller/tfc/request.py b/harnesses/host_controller/tfc/request.py deleted file mode 100644 index a086bf2..0000000 --- a/harnesses/host_controller/tfc/request.py +++ /dev/null @@ -1,74 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from host_controller.tfc import api_message - - -class Request(api_message.ApiMessage): - """The requests defined by TFC API. - - Attributes: - _BODY: The requests.new parameters that are put into http message body. - _PARAMETERS: The requests.new parameters put into http parameters. - _ALL_KEYS: Union of above. - """ - _BODY = { - "command_line", - "user"} - _PARAMETERS = { - "branch", - "build_flavor", - "build_id", - "build_os", - "cluster", - "no_build_args", - "run_target", - "shard_count" - "run_count"} - _ALL_KEYS = (_BODY | _PARAMETERS) - - def __init__(self, cluster, command_line, run_target, user, **kwargs): - """Initializes the attributes. - - Args: - cluster: The ID of the cluster to send this request to. - command_line: The command to execute on a host. - run_target: The target device to run the command. - user: The name of the user sending this request. - **kwargs: The optional attributes. - """ - super(Request, self).__init__(self._ALL_KEYS, - cluster=cluster, - command_line=command_line, - run_target=run_target, - user=user, - **kwargs) - - def GetBody(self): - """Returns the http message body. - - Returns: - A JSON object. - """ - return self.ToJson(self._BODY) - - def GetParameters(self): - """Returns the http parameters. - - Returns: - A dict of strings. - """ - return self.ToJson(self._PARAMETERS) diff --git a/harnesses/host_controller/tfc/tfc_client.py b/harnesses/host_controller/tfc/tfc_client.py deleted file mode 100644 index 8380573..0000000 --- a/harnesses/host_controller/tfc/tfc_client.py +++ /dev/null @@ -1,176 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import httplib2 -import logging -import threading -import time - -from googleapiclient import discovery -from googleapiclient import http -from oauth2client.service_account import ServiceAccountCredentials - -from host_controller.tfc import command_task - -API_NAME = "tradefed_cluster" -API_VERSION = "v1" -SCOPES = ['https://www.googleapis.com/auth/userinfo.email'] - - -class TfcClient(object): - """The class for accessing TFC API. - - Attributes: - _service: The TFC service. - """ - - def __init__(self, service): - self._service = service - - def LeaseHostTasks(self, cluster_id, next_cluster_ids, hostname, device_infos): - """Calls leasehosttasks. - - Args: - cluster_id: A string, the primary cluster to lease tasks from. - next_cluster_ids: A list of Strings, the secondary clusters to lease - tasks from. - hostname: A string, the name of the TradeFed host. - device_infos: A list of DeviceInfo, the information about the - devices connected to the host. - - Returns: - A list of command_task.CommandTask, the leased tasks. - """ - lease = {"hostname": hostname, - "cluster": cluster_id, - "next_cluster_ids": next_cluster_ids, - "device_infos": [x.ToLeaseHostTasksJson() - for x in device_infos]} - logging.info("tasks.leasehosttasks body=%s", lease) - tasks = self._service.tasks().leasehosttasks(body=lease).execute() - logging.info("tasks.leasehosttasks response=%s", tasks) - if "tasks" not in tasks: - return [] - return [command_task.CommandTask(**task) for task in tasks["tasks"]] - - def TestResourceList(self, request_id): - """Calls testResource.list. - - Args: - request_id: int, id of request to grab resources for - - Returns: - A list of TestResources - """ - logging.info("request.testResource.list request_id=%s", request_id) - test_resources = self._service.requests().testResource().list(request_id=request_id).execute() - logging.info("request.testResource.list response=%s", test_resources) - if 'test_resources' not in test_resources: - return {} - return test_resources['test_resources'] - - @staticmethod - def CreateDeviceSnapshot(cluster_id, hostname, dev_infos): - """Creates a DeviceSnapshot which can be uploaded as host event. - - Args: - cluster_id: A string, the cluster to upload snapshot to. - hostname: A string, the name of the TradeFed host. - dev_infos: A list of DeviceInfo. - - Returns: - A JSON object. - """ - obj = {"time": int(time.time()), - "data": {}, - "cluster": cluster_id, - "hostname": hostname, - "tf_version": "(unknown)", - "type": "DeviceSnapshot", - "device_infos": [x.ToDeviceSnapshotJson() for x in dev_infos]} - return obj - - def SubmitHostEvents(self, host_events): - """Calls host_events.submit. - - Args: - host_events: A list of JSON objects. Currently DeviceSnapshot is - the only type of host events. - """ - json_obj = {"host_events": host_events} - logging.info("host_events.submit body=%s", json_obj) - self._service.host_events().submit(body=json_obj).execute() - - def SubmitCommandEvents(self, command_events): - """Calls command_events.submit. - - Args: - command_events: A list of JSON objects converted from CommandAttempt. - """ - json_obj = {"command_events": command_events} - logging.info("command_events.submit body=%s", json_obj) - self._service.command_events().submit(body=json_obj).execute() - - def NewRequest(self, request): - """Calls requests.new. - - Args: - request: An instance of Request. - - Returns: - A JSON object, the new request queued in the cluster. - - Sample - {'state': 'UNKNOWN', - 'command_line': 'vts-codelab --run-target sailfish', - 'id': '2', - 'user': 'testuser'} - """ - body = request.GetBody() - params = request.GetParameters() - logging.info("requests.new parameters=%s body=%s", params, body) - return self._service.requests().new(body=body, **params).execute() - - -def CreateTfcClient(api_root, oauth2_service_json, - api_name=API_NAME, api_version=API_VERSION, scopes=SCOPES): - """Builds an object of TFC service from a given URL. - - Args: - api_root: The URL to the service. - oauth2_service_json: The path to service account key file. - - Returns: - A TfcClient object. - """ - discovery_url = "%s/discovery/v1/apis/%s/%s/rest" % ( - api_root, api_name, api_version) - logging.info("Build service from: %s", discovery_url) - credentials = ServiceAccountCredentials.from_json_keyfile_name( - oauth2_service_json, scopes=scopes) - # httplib2.Http is not thread-safe. Use thread local object. - thread_local = threading.local() - thread_local.http = credentials.authorize(httplib2.Http()) - def BuildHttpRequest(unused_http, *args, **kwargs): - if not hasattr(thread_local, "http"): - thread_local.http = credentials.authorize(httplib2.Http()) - return http.HttpRequest(thread_local.http, *args, **kwargs) - - service = discovery.build( - api_name, api_version, http=thread_local.http, - discoveryServiceUrl=discovery_url, - requestBuilder=BuildHttpRequest) - return TfcClient(service) diff --git a/harnesses/host_controller/tfc/tfc_client_test.py b/harnesses/host_controller/tfc/tfc_client_test.py deleted file mode 100644 index 10db850..0000000 --- a/harnesses/host_controller/tfc/tfc_client_test.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest - -try: - from unittest import mock -except ImportError: - import mock - -from host_controller.tfc import tfc_client -from host_controller.tfc import command_attempt -from host_controller.tfc import device_info -from host_controller.tfc import request - - -class TfcClientTest(unittest.TestCase): - """A test for tfc_client.TfcClient. - - Attributes: - _client: The tfc_client.TfcClient being tested. - _service: The mock service that _client connects to. - """ - _DEVICE_INFOS = [device_info.DeviceInfo( - device_serial="ABCDEF", group_name="group0", - run_target="sailfish", state="Available")] - - def setUp(self): - """Creates a TFC client connecting to a mock service.""" - self._service = mock.Mock() - self._client = tfc_client.TfcClient(self._service) - - def testNewRequest(self): - """Tests requests.new.""" - req = request.Request(cluster="cluster0", - command_line="vts-codelab", - run_target="sailfish", - user="user0") - self._client.NewRequest(req) - self._service.assert_has_calls([ - mock.call.requests().new().execute()]) - - def testLeaseHostTasks(self): - """Tests tasks.leasehosttasks.""" - tasks = {"tasks": [{"request_id": "2", - "command_line": "vts-codelab --serial ABCDEF", - "task_id": "1-0", - "device_serials": ["ABCDEF"], - "command_id": "1"}]} - self._service.tasks().leasehosttasks().execute.return_value = tasks - self._client.LeaseHostTasks("cluster0", ["cluster1", "cluster2"], - "host0", self._DEVICE_INFOS) - self._service.assert_has_calls([ - mock.call.tasks().leasehosttasks().execute()]) - - def testHostEvents(self): - """Tests host_events.submit.""" - device_snapshots = [ - self._client.CreateDeviceSnapshot("vts-staging", "host0", - self._DEVICE_INFOS), - self._client.CreateDeviceSnapshot("vts-presubmit", "host0", - self._DEVICE_INFOS)] - self._client.SubmitHostEvents(device_snapshots) - self._service.assert_has_calls([ - mock.call.host_events().submit().execute()]) - - def testCommandEvents(self): - """Tests command_events.submit.""" - cmd = command_attempt.CommandAttempt( - task_id="321-0", - attempt_id="abcd-1234", - hostname="host0", - device_serial="ABCDEF") - expected_event = { - "task_id": "321-0", - "attempt_id": "abcd-1234", - "hostname": "host0", - "device_serial": "ABCDEF", - "time": 1} - - normal_event = cmd.CreateCommandEvent( - command_attempt.EventType.INVOCATION_STARTED, - event_time=1) - expected_event["type"] = command_attempt.EventType.INVOCATION_STARTED - self.assertDictEqual(expected_event, normal_event) - - error_event = cmd.CreateCommandEvent( - command_attempt.EventType.EXECUTE_FAILED, - error="unit test", event_time=1.1) - expected_event["type"] = command_attempt.EventType.EXECUTE_FAILED - expected_event["data"] = {"error":"unit test"} - self.assertDictEqual(expected_event, error_event) - - complete_event = cmd.CreateInvocationCompletedEvent( - summary="complete", total_test_count=2, failed_test_count=1, - error="unit test") - expected_event["type"] = command_attempt.EventType.INVOCATION_COMPLETED - expected_event["data"] = {"summary": "complete", "error": "unit test", - "total_test_count": 2, "failed_test_count": 1} - del expected_event["time"] - self.assertDictContainsSubset(expected_event, complete_event) - self.assertIn("time", complete_event) - - self._client.SubmitCommandEvents([ - normal_event, error_event, complete_event]) - self._service.assert_has_calls([ - mock.call.command_events().submit().execute()]) - - def testWrongParameter(self): - """Tests raising exception for wrong parameter name.""" - self.assertRaisesRegexp(KeyError, "sdk", device_info.DeviceInfo, - device_serial="123", sdk="25") - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/tfc_host_controller.py b/harnesses/host_controller/tfc_host_controller.py deleted file mode 100644 index 6e8fb23..0000000 --- a/harnesses/host_controller/tfc_host_controller.py +++ /dev/null @@ -1,140 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import socket -import time -import uuid - -import httplib2 -from googleapiclient import errors - -from host_controller import invocation_thread -from host_controller.tradefed import remote_operation -from host_controller.tfc import command_attempt - - -class HostController(object): - """The class that relays commands between a TradeFed host and clusters. - - Attributes: - _remote_client: The RemoteClient which runs commands. - _tfc_client: The TfcClient from which the command tasks are leased. - _hostname: A string, the name of the TradeFed host. - _cluster_ids: A list of strings, the cluster IDs for leasing tasks. - _invocation_threads: The list of running InvocationThread. - """ - - def __init__(self, remote_client, tfc_client, hostname, cluster_ids): - """Initializes the attributes.""" - self._remote_client = remote_client - self._tfc_client = tfc_client - self._hostname = hostname - self._cluster_ids = cluster_ids - self._invocation_threads = [] - - @property - def hostname(self): - """Returns the name of the host.""" - return self._hostname - - def _JoinInvocationThreads(self): - """Removes terminated threads from _invocation_threads.""" - alive_threads = [] - for inv_thread in self._invocation_threads: - inv_thread.join(0) - if inv_thread.is_alive(): - alive_threads.append(inv_thread) - self._invocation_threads = alive_threads - - def _CreateInvocationThread(self, task): - """Creates an invocation thread from a command task. - - Args: - task: The CommandTask object. - - Returns: - An InvocationThread. - """ - attempt_id = uuid.uuid4() - attempt = command_attempt.CommandAttempt( - task.task_id, attempt_id, - self._hostname, task.device_serials[0]) - inv_thread = invocation_thread.InvocationThread( - self._remote_client, self._tfc_client, attempt, - task.command_line.split(), task.device_serials) - return inv_thread - - def ListDevices(self): - """Lists present devices on the host. - - Returns: - A list of DeviceInfo. - """ - devices = self._remote_client.ListDevices() - return [dev for dev in devices if not dev.IsStub()] - - def ListAvailableDevices(self): - """Lists available devices for command tasks. - - Returns: - A list of DeviceInfo. - """ - self._JoinInvocationThreads() - allocated_serials = set() - for inv_thread in self._invocation_threads: - allocated_serials.update(inv_thread.device_serials) - - present_devices = self.ListDevices() - return [dev for dev in present_devices if - dev.IsAvailable() and - dev.device_serial not in allocated_serials] - - def LeaseCommandTasks(self): - """Leases command tasks and creates threads to execute them. - - Returns: - A list of CommandTask. The leased command tasks. - """ - available_devices = self.ListAvailableDevices() - if not available_devices: - return [] - - tasks = self._tfc_client.LeaseHostTasks( - self._cluster_ids[0], self._cluster_ids[1:], - self._hostname, available_devices) - for task in tasks: - inv_thread = self._CreateInvocationThread(task) - inv_thread.daemon = True - inv_thread.start() - self._invocation_threads.append(inv_thread) - return tasks - - def Run(self, poll_interval): - """Starts polling TFC for tasks. - - Args: - poll_interval: The poll interval in seconds. - """ - while True: - try: - self.LeaseCommandTasks() - except (socket.error, - remote_operation.RemoteOperationException, - httplib2.HttpLib2Error, - errors.HttpError) as e: - logging.exception(e) - time.sleep(poll_interval) diff --git a/harnesses/host_controller/tfc_host_controller_test.py b/harnesses/host_controller/tfc_host_controller_test.py deleted file mode 100644 index 0d4b8b5..0000000 --- a/harnesses/host_controller/tfc_host_controller_test.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import threading -import unittest -import time - -try: - from unittest import mock -except ImportError: - import mock - -from host_controller import tfc_host_controller -from host_controller.tfc import command_task -from host_controller.tfc import device_info - - -class HostControllerTest(unittest.TestCase): - """A test for tfc_host_controller.HostController. - - Args: - _remote_client: A mock remote_client.RemoteClient. - _tfc_client: A mock tfc_client.TfcClient. - _host_controller: The HostController being tested. - """ - _AVAILABLE_DEVICE = device_info.DeviceInfo( - device_serial="ABC001", - run_target="sailfish", - state="Available") - _ALLOCATED_DEVICE = device_info.DeviceInfo( - device_serial="ABC002", - run_target="sailfish", - state="Allocated") - _STUB_DEVICE = device_info.DeviceInfo( - device_serial="emulator-5554", - run_target="unknown", - state="Available", - stub=True) - _DEVICES = [_AVAILABLE_DEVICE, _ALLOCATED_DEVICE, _STUB_DEVICE] - _TASKS = [command_task.CommandTask(task_id="1-0", - command_line="vts -m SampleShellTest", - device_serials=["ABC001"])] - - def setUp(self): - """Creates the HostController.""" - self._remote_client = mock.Mock() - self._tfc_client = mock.Mock() - self._host_controller = tfc_host_controller.HostController( - self._remote_client, self._tfc_client, "host1", ["cluster1"]) - - @mock.patch("host_controller.invocation_thread." - "InvocationThread.run") - def testDeviceStateDuringInvocation(self, mock_run): - """Tests LeaseHostTasks and ListAvailableDevices.""" - self._remote_client.ListDevices.return_value = self._DEVICES - self._tfc_client.LeaseHostTasks.return_value = self._TASKS - run_event = threading.Event() - mock_run.side_effect = lambda: run_event.wait() - - self._host_controller.LeaseCommandTasks() - devices = self._host_controller.ListAvailableDevices() - self.assertEqual([], devices) - run_event.set() - # Wait for thread termination - time.sleep(0.2) - devices = self._host_controller.ListAvailableDevices() - self.assertEqual([self._AVAILABLE_DEVICE], devices) - - def testListDevices(self): - """Tests ListDevices.""" - self._remote_client.ListDevices.return_value = self._DEVICES - devices = self._host_controller.ListDevices() - self.assertEqual([self._AVAILABLE_DEVICE, self._ALLOCATED_DEVICE], - devices) - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/tradefed/__init__.py b/harnesses/host_controller/tradefed/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/tradefed/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/tradefed/remote_client.py b/harnesses/host_controller/tradefed/remote_client.py deleted file mode 100644 index e369ba3..0000000 --- a/harnesses/host_controller/tradefed/remote_client.py +++ /dev/null @@ -1,135 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import socket -import time - -from host_controller.tradefed import remote_operation - -LOCALHOST = "localhost" -DEFAULT_PORT = 30103 - - -class RemoteClient(object): - """The class for sending remote operations to TradeFed.""" - - def __init__(self, host=LOCALHOST, port=DEFAULT_PORT, timeout=None): - """Initializes the client of TradeFed remote manager. - - Args: - _host: The host name of the remote manager. - _port: The port of the remote manager. - _timeout: The connect and receive timeout in seconds - """ - self._host = host - self._port = port - self._timeout = timeout if timeout else socket.getdefaulttimeout() - - def SendOperations(self, *operations): - """Sends a list of operations and waits for each response. - - Args: - *operations: A list of remote_operation.RemoteOperation objects. - - Returns: - A list of JSON objects. - - Raises: - socket.error if fails to communicate with remote manager. - remote_operation.RemoteOperationException if any operation fails or - has no response. - """ - op_socket = socket.create_connection((self._host, self._port), - self._timeout) - logging.info("Connect to %s:%d", self._host, self._port) - try: - if self._timeout is not None: - op_socket.settimeout(self._timeout) - ops_str = "\n".join(str(op) for op in operations) - logging.info("Send: %s", ops_str) - op_socket.send(ops_str) - op_socket.shutdown(socket.SHUT_WR) - resp_str = "" - while True: - buf = op_socket.recv(4096) - if not buf: - break - resp_str += buf - finally: - op_socket.close() - logging.info("Receive: %s", resp_str) - resp_lines = [x for x in resp_str.split("\n") if x] - if len(resp_lines) != len(operations): - raise remote_operation.RemoteOperationException( - "Unexpected number of responses: %d" % len(resp_lines)) - return [operations[i].ParseResponse(resp_lines[i]) - for i in range(len(resp_lines))] - - def SendOperation(self, operation): - """Sends one operation and waits for its response. - - Args: - operation: A remote_operation.RemoteOperation object. - - Returns: - A JSON object. - - Raises: - socket.error if fails to communicate with remote manager. - remote_operation.RemoteOperationException if the operation fails or - has no response. - """ - return self.SendOperations(operation)[0] - - def ListDevices(self): - """Sends ListDevices operation. - - Returns: - A list of device_info.DeviceInfo which are the devices connected to - the host. - """ - json_obj = self.SendOperation(remote_operation.ListDevices()) - return remote_operation.ParseListDevicesResponse(json_obj) - - def WaitForCommandResult(self, serial, timeout, poll_interval=5): - """Sends a series of operations to wait until a command finishes. - - Args: - serial: The serial number of the device. - timeout: A float, the timeout in seconds. - poll_interval: A float, the interval of each GetLastCommandResult - operation in seconds. - - Returns: - A JSON object which is the result of the command. - None if timeout. - - Sample - {'status': 'INVOCATION_SUCCESS', - 'free_device_state': 'AVAILABLE'} - """ - deadline = time.time() + timeout - get_result_op = remote_operation.GetLastCommandResult(serial) - while True: - result = self.SendOperation(get_result_op) - if result["status"] != "EXECUTING": - return result - if time.time() > deadline: - return None - time.sleep(poll_interval) - if time.time() > deadline: - return None diff --git a/harnesses/host_controller/tradefed/remote_client_test.py b/harnesses/host_controller/tradefed/remote_client_test.py deleted file mode 100644 index bc9cbb8..0000000 --- a/harnesses/host_controller/tradefed/remote_client_test.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import queue -import socket -import threading -import unittest - -from host_controller.tradefed import remote_client -from host_controller.tradefed import remote_operation - - -class MockRemoteManagerThread(threading.Thread): - """A thread which mocks remote manager. - - Attributes: - HOST: Local host name. - PORT: The port that the remote manager listens to. - _remote_mgr_socket: The remote manager socket. - _response_queue: A queue.Queue object containing the response strings. - _timeout: Socket timeout in seconds. - last_error: The exception which caused this thread to terminate. - """ - HOST = remote_client.LOCALHOST - PORT = 32123 - - def __init__(self, timeout): - """Creates and listens to remote manager socket.""" - super(MockRemoteManagerThread, self).__init__() - self._response_queue = queue.Queue() - self._timeout = timeout - self.last_error = None - self._remote_mgr_socket = socket.socket() - try: - self._remote_mgr_socket.settimeout(self._timeout) - self._remote_mgr_socket.bind((self.HOST, self.PORT)) - self._remote_mgr_socket.listen(1) - except socket.error: - self._remote_mgr_socket.close() - - def _Respond(self, response_str): - """Accepts a client connection and responds. - - Args: - response_str: The response string. - """ - (server_socket, client_address) = self._remote_mgr_socket.accept() - try: - server_socket.settimeout(self._timeout) - # Receive until connection is closed - while not server_socket.recv(4096): - pass - server_socket.send(response_str) - finally: - server_socket.close() - - def AddResponse(self, response_str): - """Add a response string to the queue. - - Args: - response_str: The response string. - """ - self._response_queue.put_nowait(response_str) - - def CloseSocket(self): - """Closes the remote manager socket.""" - if self._remote_mgr_socket: - self._remote_mgr_socket.close() - self._remote_mgr_socket = None - - # @Override - def run(self): - """Sends the queued responses to the clients.""" - try: - while True: - response_str = self._response_queue.get() - self._response_queue.task_done() - if response_str is None: - break - self._Respond(response_str) - except socket.error as e: - self.last_error = e - finally: - self.CloseSocket() - - -class RemoteClientTest(unittest.TestCase): - """A test for remote_client.RemoteClient. - - Attributes: - _remote_mgr_thread: An instance of MockRemoteManagerThread. - _client: The remote_client.RemoteClient being tested. - """ - - def setUp(self): - """Creates remote manager thread.""" - self._remote_mgr_thread = MockRemoteManagerThread(5) - self._remote_mgr_thread.daemon = True - self._remote_mgr_thread.start() - self._client = remote_client.RemoteClient(self._remote_mgr_thread.HOST, - self._remote_mgr_thread.PORT, - 5) - - def tearDown(self): - """Terminates remote manager thread.""" - self._remote_mgr_thread.AddResponse(None) - self._remote_mgr_thread.join(15) - self._remote_mgr_thread.CloseSocket() - self.assertFalse(self._remote_mgr_thread.is_alive(), - "Cannot stop remote manager thread.") - if self._remote_mgr_thread.last_error: - raise self._remote_mgr_thread.last_error - - def testListDevice(self): - """Tests ListDevices operation.""" - self._remote_mgr_thread.AddResponse('{"serials": []}') - self._client.ListDevices() - - def testAddCommand(self): - """Tests AddCommand operation.""" - self._remote_mgr_thread.AddResponse('{}') - self._client.SendOperation(remote_operation.AddCommand(0, "COMMAND")) - - def testMultipleOperations(self): - """Tests sending multiple operations via one connection.""" - self._remote_mgr_thread.AddResponse('{}\n{}') - self._client.SendOperations(remote_operation.ListDevices(), - remote_operation.ListDevices()) - - def testExecuteCommand(self): - """Tests executing a command and waiting for result.""" - self._remote_mgr_thread.AddResponse('{}') - self._client.SendOperation(remote_operation.AllocateDevice("serial123")) - self._remote_mgr_thread.AddResponse('{}') - self._client.SendOperation(remote_operation.ExecuteCommand( - "serial123", "vts", "-m", "SampleShellTest")) - - self._remote_mgr_thread.AddResponse('{"status": "EXECUTING"}') - result = self._client.WaitForCommandResult("serial123", - timeout=0.5, poll_interval=1) - self.assertIsNone(result, "Client returns result before command finishes.") - - self._remote_mgr_thread.AddResponse('{"status": "EXECUTING"}') - self._remote_mgr_thread.AddResponse('{"status": "INVOCATION_SUCCESS"}') - result = self._client.WaitForCommandResult("serial123", - timeout=5, poll_interval=1) - self._remote_mgr_thread.AddResponse('{}') - self._client.SendOperation(remote_operation.FreeDevice("serial123")) - self.assertIsNotNone(result, "Client doesn't return command result.") - - def testSocketError(self): - """Tests raising exception when socket error occurs.""" - self.assertRaises(socket.timeout, self._client.ListDevices) - self._remote_mgr_thread.AddResponse(None) - self.assertRaises(socket.error, self._client.ListDevices) - - def testRemoteOperationException(self): - """Tests raising exception when response is an error.""" - self._remote_mgr_thread.AddResponse('{"error": "unit test"}') - self.assertRaises(remote_operation.RemoteOperationException, - self._client.ListDevices) - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/tradefed/remote_operation.py b/harnesses/host_controller/tradefed/remote_operation.py deleted file mode 100644 index 00839d4..0000000 --- a/harnesses/host_controller/tradefed/remote_operation.py +++ /dev/null @@ -1,172 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json - -from host_controller.tfc import device_info - - -class RemoteOperationException(Exception): - """Raised when remote operation fails.""" - pass - - -class RemoteOperation(object): - """The operation sent to TradeFed remote manager. - - Args: - _obj: A JSON object with at least 2 entries, "type" and "version". - """ - CURRENT_PROTOCOL_VERSION = 8 - - def __init__(self, type, **kwargs): - """Initializes a remote operation. - - Args: - type: A string, the type of the operation. - **kwargs: The arguments which are specific to the operation type. - """ - self._obj = kwargs - self._obj["type"] = type - if "version" not in self._obj: - self._obj["version"] = self.CURRENT_PROTOCOL_VERSION - - def ParseResponse(self, response_str): - """Parses the response to the operation. - - Args: - response_str: A JSON string. - - Returns: - A JSON object. - - Raises: - RemoteOperationException if the response is an error. - """ - response = json.loads(response_str) - if "error" in response: - raise RemoteOperationException(response["error"]) - return response - - @property - def type(self): - """Returns the type of this operation.""" - return self._obj["type"] - - def __str__(self): - """Converts the JSON object to string.""" - return json.dumps(self._obj) - - -def ListDevices(): - """Creates an operation of listing devices.""" - return RemoteOperation("LIST_DEVICES") - - -def ParseListDevicesResponse(json_obj): - """Parses ListDevices response to a list of DeviceInfo. - - Sample response: - {"serials": [ - {"product": "unknown", "battery": "0", "variant": "unknown", - "stub": True, "state": "Available", "build": "unknown", - "serial": "emulator-5554", "sdk": "unknown"}, - ]} - - Args: - json_obj: A JSON object, the response to ListDevices. - - Returns: - A list of DeviceInfo object. - """ - dev_infos = [] - for dev_obj in json_obj["serials"]: - if dev_obj["product"] == dev_obj["variant"]: - run_target = dev_obj["product"] - else: - run_target = dev_obj["product"] + ":" + dev_obj["variant"] - dev_info = device_info.DeviceInfo( - battery_level=dev_obj["battery"], - build_id=dev_obj["build"], - device_serial=dev_obj["serial"], - product=dev_obj["product"], - product_variant=dev_obj["variant"], - run_target=run_target, - sdk_version=dev_obj["sdk"], - state=dev_obj["state"], - stub=dev_obj["stub"]) - dev_infos.append(dev_info) - return dev_infos - - -def AllocateDevice(serial): - """Creates an operation of allocating a device. - - Args: - serial: The serial number of the device. - """ - return RemoteOperation("ALLOCATE_DEVICE", serial=serial) - - -def FreeDevice(serial): - """Creates an operation of freeing a device. - - Args: - serial: The serial number of the device. - """ - return RemoteOperation("FREE_DEVICE", serial=serial) - - -def Close(): - """Creates an operation of stopping the remote manager.""" - return RemoteOperation("CLOSE") - - -def AddCommand(time, *command_args): - """Creates an operation of adding a command to the queue. - - Args: - time: The time in ms that the command has been executing for. The value - is non-zero in handover situation. - command_args: The command to execute. - """ - return RemoteOperation("ADD_COMMAND", time=time, commandArgs=command_args) - - -def ExecuteCommand(serial, *command_args): - """Creates an operation of executing a command on a device. - - Args: - serial: The serial number of the device. - command_args: The command to execute. - """ - return RemoteOperation( - "EXEC_COMMAND", serial=serial, commandArgs=command_args) - - -def GetLastCommandResult(serial): - """Creates an operation of getting last EXEC_COMMAND result on a device. - - Sample response: - {"status": "INVOCATION_ERROR", - "invocation_error": "java.lang.NullPointerException", - "free_device_state": "AVAILABLE" - } - - Args: - serial: The serial number of the device. - """ - return RemoteOperation("GET_LAST_COMMAND_RESULT", serial=serial) diff --git a/harnesses/host_controller/utils/__init__.py b/harnesses/host_controller/utils/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/utils/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/utils/gcp/__init__.py b/harnesses/host_controller/utils/gcp/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/utils/gcp/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/utils/gcp/gcs_utils.py b/harnesses/host_controller/utils/gcp/gcs_utils.py deleted file mode 100644 index ffe0913..0000000 --- a/harnesses/host_controller/utils/gcp/gcs_utils.py +++ /dev/null @@ -1,104 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging - -from vts.utils.python.common import cmd_utils - - -def GetGsutilPath(): - """Finds gsutil in PATH. - - Instead of a Python library, gsutil binary is used to avoid packaging GCS - PIP package as part of VTS HC (Host Controller). - - Returns: - The gsutil file path if found; None otherwise. - """ - sh_stdout, sh_stderr, ret_code = cmd_utils.ExecuteOneShellCommand( - "which gsutil") - if ret_code == 0: - return sh_stdout.strip() - else: - logging.fatal("`gsutil` doesn't exist on the host; " - "please install Google Cloud SDK before retrying.") - return None - - -def IsGcsFile(gsutil_path, url): - """Checks whether a given path is for a GCS file. - - Args: - gsutil_path: string, the path of a gsutil binary. - url: string, the GCS URL. e.g., gs://<bucket>/<file>. - - Returns: - True if url is a file, False otherwise. - """ - check_command = "%s stat %s" % (gsutil_path, url) - _, _, ret_code = cmd_utils.ExecuteOneShellCommand(check_command) - return ret_code == 0 - - -def Copy(gsutil_path, src_urls, dst_url): - """Copies files between local file system and GCS. - - Args: - gsutil_path: string, the path of a gsutil binary. - src_urls: string, the source paths or GCS URLs separated by spaces. - dst_url: string, the destination path or GCS URL. - - Returns: - True if the command succeeded, False otherwise. - """ - copy_command = "%s cp %s %s" % (gsutil_path, src_urls, dst_url) - _, _, ret_code = cmd_utils.ExecuteOneShellCommand(copy_command) - return ret_code == 0 - - -def List(gsutil_path, url): - """Lists a directory or file on GCS. - - Args: - gsutil_path: string, the path of a gsutil binary. - url: string, the GCS URL of the directory or file. - - Returns: - list of strings, the GCS URLs of the listed files. - """ - ls_command = "%s ls %s" % (gsutil_path, url) - stdout, _, ret_code = cmd_utils.ExecuteOneShellCommand(ls_command) - return stdout.strip("\n").split("\n") if ret_code == 0 else [] - - -def Remove(gsutil_path, url, recursive=False): - """Removes a directory or file on GCS. - - Args: - gsutil_path: string, the path of a gsutil binary. - url: string, the GCS URL of the directory or file. - recursive: boolean, whether to remove the directory recursively. - - Returns: - True if the command succeeded, False otherwise. - """ - if "/" not in url.lstrip("gs://").rstrip("/"): - logging.error("Cannot remove bucket %s", url) - return False - rm_command = "%s rm -f%s %s" % ( - gsutil_path, ("r" if recursive else ""), url) - ret_code = cmd_utils.ExecuteOneShellCommand(rm_command) - return ret_code == 0 diff --git a/harnesses/host_controller/utils/gsi/__init__.py b/harnesses/host_controller/utils/gsi/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/utils/gsi/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/utils/gsi/img_utils.py b/harnesses/host_controller/utils/gsi/img_utils.py deleted file mode 100644 index ae197ab..0000000 --- a/harnesses/host_controller/utils/gsi/img_utils.py +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright 2018 - The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import struct - -OS_VERSOIN_OFFSET_BOOTIMG = 44 -SPL_YEAR_BASE_BOOTIMG = 2000 -SPL_YEAR_MASK_BOOTIMG = 127 << 4 -SPL_MONTH_MASK_BOOTIMG = 15 - - -def GetSPLVersionFromBootImg(img_file_path): - """Gets SPL version from given boot.img file path. - - Args: - img_file_path : string, path to the boot.img file. - Returns: - A dict contains SPL version's year and date value. - """ - try: - fo = open(img_file_path, "rb") - fo.seek(OS_VERSOIN_OFFSET_BOOTIMG, 0) - - os_version = fo.read(4) - except IOError as e: - logging.error(e.strerror) - return {} - - os_version_int = struct.unpack("i", os_version)[0] - - year = (os_version_int & SPL_YEAR_MASK_BOOTIMG) >> 4 - month = os_version_int & SPL_MONTH_MASK_BOOTIMG - - version_date = {} - version_date["year"] = SPL_YEAR_BASE_BOOTIMG + year - version_date["month"] = month - - return version_date
\ No newline at end of file diff --git a/harnesses/host_controller/utils/ipc/__init__.py b/harnesses/host_controller/utils/ipc/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/utils/ipc/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/utils/ipc/file_lock.py b/harnesses/host_controller/utils/ipc/file_lock.py deleted file mode 100644 index 67b6bf7..0000000 --- a/harnesses/host_controller/utils/ipc/file_lock.py +++ /dev/null @@ -1,119 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import fcntl -import logging -import os - -from host_controller import common - - -class FileLock(object): - """Class for using files as a locking mechanism for devices. - - This is for checking whether a certain device is running a job or not when - the automated self-update happens. - - Attributes: - _devlock_dir: string, represent the home directory of the user. - _lock_fd: dict, maps serial number of the devices and file descriptor. - """ - - def __init__(self, file_name=None, mode=None): - """Initializes the file lock managing class instance. - - Args: - file_name: string, name of the file to be opened. Existing lock - files will be opened if given as None - mode: string, the mode argument to open() function. - """ - self._lock_fd = {} - self._devlock_dir = os.path.join( - os.path.expanduser("~"), common._DEVLOCK_DIR) - if not os.path.exists(self._devlock_dir): - os.mkdir(self._devlock_dir) - - if file_name: - file_list = [file_name] - else: - file_list = [file_name for file_name in os.listdir(self._devlock_dir)] - logging.debug("file_list for file lock: %s", file_list) - - for file_name in file_list: - if os.path.isfile(os.path.join(self._devlock_dir, file_name)): - self._OpenFile(file_name, mode) - - def _OpenFile(self, serial, mode=None): - """Opens the given lock file and store the file descriptor to _lock_fd. - - Args: - serial: string, serial number of a device. - mode: string, the mode argument to open() function. - Set to "w+" if given as None. - """ - if serial in self._lock_fd and self._lock_fd[serial]: - logging.info("Lock for the device %s already exists." % serial) - return - - try: - if not mode: - mode = "w+" - self._lock_fd[serial] = open( - os.path.join(self._devlock_dir, serial), mode, 0) - except IOError as e: - logging.exception(e) - return False - - def LockDevice(self, serial, suppress_lock_warning=False, block=False): - """Tries to lock the file corresponding to "serial". - - Args: - serial: string, serial number of a device. - suppress_lock_warning: bool, True to suppress warning log output. - block: bool, True to block at the fcntl.lockf(), waiting for it - to be unlocked. - - Returns: - True if successfully acquired the lock. False otherwise. - """ - if serial not in self._lock_fd: - ret = self._OpenFile(serial) - if ret == False: - return ret - - try: - operation = fcntl.LOCK_EX - if not block: - operation |= fcntl.LOCK_NB - fcntl.lockf(self._lock_fd[serial], operation) - except IOError as e: - if not suppress_lock_warning: - logging.exception(e) - return False - - return True - - def UnlockDevice(self, serial): - """Releases the lock file corresponding to "serial". - - Args: - serial: string, serial number of a device. - """ - if serial not in self._lock_fd: - logging.error("Lock for the device %s does not exist." % serial) - return False - - fcntl.lockf(self._lock_fd[serial], fcntl.LOCK_UN) diff --git a/harnesses/host_controller/utils/ipc/file_lock_semaphore.py b/harnesses/host_controller/utils/ipc/file_lock_semaphore.py deleted file mode 100644 index 724c7b0..0000000 --- a/harnesses/host_controller/utils/ipc/file_lock_semaphore.py +++ /dev/null @@ -1,124 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the 'License'); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import time - -from host_controller import common -from host_controller.utils.ipc import file_lock - -MIN_SLEEP_TIME_IN_SECS = 1 -MAX_SLEEP_TIME_IN_SECS = 60 - - -class FileLockSemaphore(file_lock.FileLock): - """Class for system-wide semaphores for inter process-group sync. - - Inherited FileLock to make the decrement and increment operation atomic, - thus any read and write operations to the file must be leaded by Lock() - and be followed by Unlock() operations, defined in FileLock - - Attributes: - _name: string, name of the file used as semaphore. - """ - - def __init__(self, name): - super(FileLockSemaphore, self).__init__(name, "r+") - self._name = name - self._InitSemaphore() - - def Acquire(self): - """P() operation for the FileLockSemaphore. - - To minimize the starvation, the sleep time will decrease - for the processes that have waited longer than the others. - """ - sleep_time = MAX_SLEEP_TIME_IN_SECS - while self.LockDevice(self._name, True, True): - if self.sem_value > 0: - break - self.UnlockDevice(self._name) - time.sleep(sleep_time) - sleep_time = max(sleep_time / 2, MIN_SLEEP_TIME_IN_SECS) - - self._Decrement() - self.UnlockDevice(self._name) - - def Release(self): - """V() operation for the FileLockSemaphore.""" - self.LockDevice(self._name, True, True) - self._Increment() - self.UnlockDevice(self._name) - - def _InitSemaphore(self): - """Initializes the file content for semaphore operations. - - The value of the counter must remain in the range - [0, common.MAX_ADB_FASTBOOT_PROCESS] inclusive. - """ - self.LockDevice(self._name, True, True) - - try: - diff = common.MAX_ADB_FASTBOOT_PROCESS - self.sem_max_value - value_to_init = min( - max(0, self.sem_value + diff), common.MAX_ADB_FASTBOOT_PROCESS) - except IndexError: - value_to_init = common.MAX_ADB_FASTBOOT_PROCESS - - self._lock_fd[self._name].seek(0) - self._lock_fd[self._name].truncate(0) - self._lock_fd[self._name].write( - "%s\n%s" % (common.MAX_ADB_FASTBOOT_PROCESS, value_to_init)) - - self.UnlockDevice(self._name) - - @property - def sem_value(self): - """Reads the current counter value of the semaphore. - - Returns: - int, the value of the current counter. - """ - self._lock_fd[self._name].seek(0) - return int(self._lock_fd[self._name].read().split()[1]) - - @property - def sem_max_value(self): - """Reads the current maximum counter value of the semaphore. - - Since the maximum counter value may vary at the deployment time, - the existing HC process group needs to look for the maximum value - every time it tries to access the semaphore - - Returns: - int, the value of the maximum counter. - """ - self._lock_fd[self._name].seek(0) - return int(self._lock_fd[self._name].read().split()[0]) - - def _Decrement(self): - """Decrements the internal counter of the semaphore.""" - current_value = self.sem_value - current_max = self.sem_max_value - self._lock_fd[self._name].seek(len(str(current_max)) + 1) - self._lock_fd[self._name].write("%s" % max(0, (current_value - 1))) - - def _Increment(self): - """Increments the internal counter of the semaphore.""" - current_value = self.sem_value - current_max = self.sem_max_value - self._lock_fd[self._name].seek(len(str(current_max)) + 1) - self._lock_fd[self._name].write("%s" % min(current_max, - (current_value + 1))) diff --git a/harnesses/host_controller/utils/ipc/file_lock_semaphore_test.py b/harnesses/host_controller/utils/ipc/file_lock_semaphore_test.py deleted file mode 100644 index e1bae8b..0000000 --- a/harnesses/host_controller/utils/ipc/file_lock_semaphore_test.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import os -import threading -import time -import unittest - -try: - from unittest import mock -except ImportError: - import mock - -from host_controller import common -from host_controller.utils.ipc import file_lock_semaphore - - -class FileLockSemaphoreTest(unittest.TestCase): - """Tests for FileLockSemaphore""" - - def setUp(self): - self.sem = file_lock_semaphore.FileLockSemaphore("fls_test") - - def tearDown(self): - os.remove(os.path.join(os.path.expanduser("~"), common._DEVLOCK_DIR, "fls_test")) - - def testFileLockSemaphoreAcquire(self): - current_sem_value = self.sem.sem_value - self.sem.Acquire() - self.assertEqual(current_sem_value - 1, self.sem.sem_value) - self.sem.Release() - - def testFileLockSemaphoreRelease(self): - self.sem.Acquire() - current_sem_value = self.sem.sem_value - self.sem.Release() - self.assertEqual(current_sem_value + 1, self.sem.sem_value) - - def testFileLockSemaphoreOverflow(self): - current_sem_value = self.sem.sem_value - self.sem.Release() - self.assertEqual(current_sem_value, self.sem.sem_value) - - def _AcquireRelease(self): - self.sem.Acquire() - self.assertEqual(self.sem.sem_value, 0) - self.sem.Release() - - def testFileLockSemaphoreUnderflow(self): - for _ in range(common.MAX_ADB_FASTBOOT_PROCESS): - self.sem.Acquire() - - file_lock_semaphore.MAX_SLEEP_TIME_IN_SECS = 2 - self._thread = threading.Thread(target=self._AcquireRelease) - self._thread.daemon = True - self._thread.start() - - time.sleep(1) - self.sem.Release() - self._thread.join() - - -if __name__ == "__main__": - unittest.main() diff --git a/harnesses/host_controller/utils/ipc/shared_dict.py b/harnesses/host_controller/utils/ipc/shared_dict.py deleted file mode 100644 index a0e707f..0000000 --- a/harnesses/host_controller/utils/ipc/shared_dict.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import multiprocessing - -from host_controller import common - - -class SharedDict(object): - """Class for the state of devices connected to the host. - - shared between the main process and the process pool. - - Attributes: - _manager: SyncManager. an instance of a manager for shared objects and - values between processes. - _dict: multiprocessing.SyncManager.dict. A dictionary - shared among processes. - """ - - def __init__(self, manager=None): - if manager is not None: - self._manager = manager - else: - self._manager = multiprocessing.Manager() - self._dict = self._manager.dict() - - def __getitem__(self, key): - """getitem for self._dict. - - Args: - key: string, serial number of a device. - - Returns: - integer value defined in _DEVICE_STATUS_DICT. - """ - if key not in self._dict: - self._dict[key] = common._DEVICE_STATUS_DICT["unknown"] - return self._dict[key] - - def __setitem__(self, key, value): - """setitem for self._dict. - - if the value is not a defined one, device status is set to "unknown". - - Args: - key: string, serial number of a device. - value: integer, status value defined in _DEVICE_STATUS_DICT. - """ - if value not in range(len(common._DEVICE_STATUS_DICT)): - self._dict[key] = common._DEVICE_STATUS_DICT["unknown"] - else: - self._dict[key] = value
\ No newline at end of file diff --git a/harnesses/host_controller/utils/parser/__init__.py b/harnesses/host_controller/utils/parser/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/utils/parser/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/utils/parser/pb2_utils.py b/harnesses/host_controller/utils/parser/pb2_utils.py deleted file mode 100644 index af96588..0000000 --- a/harnesses/host_controller/utils/parser/pb2_utils.py +++ /dev/null @@ -1,81 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import logging -import requests - -# timeout seconds for requests -REQUESTS_TIMEOUT_SECONDS = 60 - - -def FillDictAndPost(msg, - dict_to_fill, - url, - headers, - filters={}, - caller_name=""): - """Fills up a dict using contents of the pb2 message and uploads it. - - Args: - msg: pb2 msg, containing info about all task schedules. - dict_to_fill: dict, to be converted into a json object before - upload. - url: string, URL for the endpoit API to be called when - the dict_to_fill fills up. - - Returns: - True if successful, False otherwise. - """ - terminal = True - ret = True - sub_msg_list = [] - for field in msg.DESCRIPTOR.fields: - if field.type == field.TYPE_MESSAGE: - terminal = False - for sub_msg in msg.__getattribute__(field.name): - # make all the messages to be processed after other attrs, - # otherwise dict_to_fill would be incomplete. - sub_msg_list.append(sub_msg) - else: - if filters and field.name in filters: - filtered_key = filters[field.name] - else: - filtered_key = field.name - - if field.label == field.LABEL_REPEATED: - dict_to_fill[filtered_key] = list( - msg.__getattribute__(field.name)) - else: - dict_to_fill[filtered_key] = msg.__getattribute__(field.name) - - for sub_msg in sub_msg_list: - ret = ret and FillDictAndPost(sub_msg, dict_to_fill, url, headers, - filters, caller_name) - - if terminal: - try: - response = requests.post(url, data=json.dumps(dict_to_fill), - headers=headers, - timeout=REQUESTS_TIMEOUT_SECONDS) - except requests.exceptions.Timeout as e: - logging.exception(e) - return False - if response.status_code != requests.codes.ok: - logging.error("%s error: %s", caller_name, response) - ret = ret and False - - return ret diff --git a/harnesses/host_controller/utils/parser/result_utils.py b/harnesses/host_controller/utils/parser/result_utils.py deleted file mode 100644 index 9b9b51e..0000000 --- a/harnesses/host_controller/utils/parser/result_utils.py +++ /dev/null @@ -1,129 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -import os - -from xml.etree import ElementTree - -from host_controller import common - - -def ExtractResultZip(zip_file, output_dir, xml_only=True): - """Extracts XML result from a zip file. - - Args: - zip_file: An object of zipfile.ZipFile. - output_dir: The directory where the zip is extracted. - xml_only: Whether to extract log-result.xml or test_result.xml only. - - Returns: - The path to the XML in the output directory. - None if the zip does not contain the XML result. - """ - try: - xml_name = next(x for x in zip_file.namelist() if - x.endswith(common._TEST_RESULT_XML) or - x.endswith(common._LOG_RESULT_XML)) - except StopIteration: - return None - - if not os.path.exists(output_dir): - os.makedirs(output_dir) - if xml_only: - return zip_file.extract(xml_name, path=output_dir) - else: - zip_file.extractall(path=output_dir) - return os.path.join(output_dir, xml_name) - - -def LoadTestSummary(result_xml): - """Gets attributes of <Result>, <Build>, and <Summary>. - - Args: - result_xml: A file object of the TradeFed report in XML format. - - Returns: - 3 dictionaries, the attributes of <Result>, <Build>, and <Summary>. - """ - result_attrib = {} - build_attrib = {} - summary_attrib = {} - for event, elem in ElementTree.iterparse(result_xml, events=("start", )): - if all((result_attrib, build_attrib, summary_attrib)): - break - if elem.tag == common._RESULT_TAG: - result_attrib = dict(elem.attrib) - elif elem.tag == common._BUILD_TAG: - build_attrib = dict(elem.attrib) - elif elem.tag == common._SUMMARY_TAG: - summary_attrib = dict(elem.attrib) - return result_attrib, build_attrib, summary_attrib - - -def IterateTestResults(result_xml): - """Yields test records in test_result.xml. - - Args: - result_xml: A file object of the TradeFed report in XML format. - - Yields: - A tuple of ElementTree.Element, (Module, TestCase, Test) for each - </Test>. - """ - module_elem = None - testcase_elem = None - for event, elem in ElementTree.iterparse( - result_xml, events=("start", "end")): - if event == "start": - if elem.tag == common._MODULE_TAG: - module_elem = elem - elif elem.tag == common._TESTCASE_TAG: - testcase_elem = elem - - if event == "end" and elem.tag == common._TEST_TAG: - yield module_elem, testcase_elem, elem - - -def GetAbiBitness(abi): - """Gets bitness of an ABI. - - Args: - abi: A string, the ABI name. - - Returns: - 32 or 64, the ABI bitness. - """ - return 64 if "arm64" in abi or "x86_64" in abi else 32 - - -def GetTestName(module, testcase, test): - """Gets the bitness and the full test name. - - Args: - module: The Element for <Module>. - testcase: The Element for <TestCase>. - test: The Element for <Test>. - - Returns: - A tuple of (bitness, module_name, testcase_name, test_name). - """ - abi = module.attrib.get(common._ABI_ATTR_KEY, "") - bitness = str(GetAbiBitness(abi)) - module_name = module.attrib.get(common._NAME_ATTR_KEY, "") - testcase_name = testcase.attrib.get(common._NAME_ATTR_KEY, "") - test_name = test.attrib.get(common._NAME_ATTR_KEY, "") - return bitness, module_name, testcase_name, test_name diff --git a/harnesses/host_controller/utils/parser/xml_utils.py b/harnesses/host_controller/utils/parser/xml_utils.py deleted file mode 100644 index 3717d5a..0000000 --- a/harnesses/host_controller/utils/parser/xml_utils.py +++ /dev/null @@ -1,43 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging - -from xml.etree import ElementTree - - -def GetAttributes(xml_file, tag, attrs): - """Parses attributes in the given tag from xml file. - - Args: - xml_file: The xml file object. - tag: a string, tag to parse from tag xml_file. - attrs: a list of attributes to look up inside the tag. - - Returns: - A dict containing values of the attributes in tag from the xml file. - """ - result = {} - for _, elem in ElementTree.iterparse(xml_file, ("start", )): - if elem.tag == tag: - for attr in attrs: - if attr in elem.attrib: - result[attr] = elem.attrib[attr] - else: - logging.warning( - "Cannot find an attribute {} in <{}>".format( - attr, tag)) - return result diff --git a/harnesses/host_controller/utils/usb/__init__.py b/harnesses/host_controller/utils/usb/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/utils/usb/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/utils/usb/usb_utils.py b/harnesses/host_controller/utils/usb/usb_utils.py deleted file mode 100644 index 5c42c7b..0000000 --- a/harnesses/host_controller/utils/usb/usb_utils.py +++ /dev/null @@ -1,112 +0,0 @@ -# -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import fcntl -import logging -import os - -from vts.utils.python.common import cmd_utils - -# _IO('U', 20) -USBDEVFS_RESET = ord("U") << 8 | 20 - - -def GetDevicesUSBFilePath(): - """Sweeps through connected USB device info and maps serial number to them. - - Returns: - A dict, serial numbers of the devices as the key, device file path - corresponding to the serial number as the value. - """ - ret = {} - - sh_stdout, sh_stderr, ret_code = cmd_utils.ExecuteOneShellCommand("which udevadm") - if ret_code != 0: - logging.error("`udevadm` doesn't exist on the host; " - "please install 'udev' pkg before retrying.") - return ret - - sh_stdout, _, _ = cmd_utils.ExecuteOneShellCommand( - "find /sys/bus/usb/devices/usb*/ -name dev") - lines = sh_stdout.split() - for line in lines: - syspath = os.path.dirname(line) - devname, _, _ = cmd_utils.ExecuteOneShellCommand( - "udevadm info -q name -p %s" % syspath) - if devname.startswith("bus/"): - serial, _, _ = cmd_utils.ExecuteOneShellCommand( - "udevadm info -q property --export -p %s | grep ID_SERIAL_SHORT" - % syspath) - if serial: - device_serial = serial.split("=")[1].strip().strip("'") - ret[device_serial] = os.path.join("/dev", devname.strip()) - - return ret - - -def ResetDeviceUsb(dev_file_path): - """Invokes ioctl that resets the USB device on the given file path. - - Args: - dev_file_path: string, abs path to the device file - - Returns: - True if successful, False otherwise. - """ - try: - with open(dev_file_path, "wb") as usb_fd: - fcntl.ioctl(usb_fd, USBDEVFS_RESET, 0) - except IOError as e: - logging.exception(e) - return False - - return True - - -def ResetUsbDeviceOfSerial(serial): - """Resets a USB device file corresponding to the given serial. - - Args: - serial: string, serial number of the device whose USB device file - will reset. - - Returns: - True if successful, False otherwise. - """ - device_file_path = GetDevicesUSBFilePath() - if serial in device_file_path: - logging.error( - "Device %s not responding. Resetting device file %s.", serial, - device_file_path[serial]) - return ResetDeviceUsb(device_file_path[serial]) - return False - - -def ResetUsbDeviceOfSerial_Callback(*args): - """Wrapper of the ResetUsbDeviceOfSerial(), for handling the *args. - - Args: - args: tuple of arguments, expected to have the serial number of - the devices as the first element. - - Returns: - True if successful, False otherwise. - """ - try: - return ResetUsbDeviceOfSerial(args[0]) - except IndexError as e: - logging.exception(e) - return False
\ No newline at end of file diff --git a/harnesses/host_controller/vti_interface/__init__.py b/harnesses/host_controller/vti_interface/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/harnesses/host_controller/vti_interface/__init__.py +++ /dev/null diff --git a/harnesses/host_controller/vti_interface/vti_endpoint_client.py b/harnesses/host_controller/vti_interface/vti_endpoint_client.py deleted file mode 100644 index 17fdc9d..0000000 --- a/harnesses/host_controller/vti_interface/vti_endpoint_client.py +++ /dev/null @@ -1,452 +0,0 @@ -# -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import logging -import requests -import threading -import time - -from host_controller.utils.parser import pb2_utils - -# Job status dict -JOB_STATUS_DICT = { - # scheduled but not leased yet - "ready": 0, - # scheduled and in running - "leased": 1, - # completed job - "complete": 2, - # unexpected error during running - "infra-err": 3, - # never leased within schedule period - "expired": 4, - # device boot error after flashing the given img sets - "bootup-err": 5 -} - -SCHEDULE_INFO_PB2_ATTR_FILTERS = { - "pab_account_id": "device_pab_account_id", - "name": "build_target", -} - -# timeout seconds for requests -REQUESTS_TIMEOUT_SECONDS = 60 - - -class VtiEndpointClient(object): - """VTI (Vendor Test Infrastructure) endpoint client. - - Attributes: - _headers: A dictionary, containing HTTP request header information. - _url: string, the base URL of an endpoint API. - _job: dict, currently leased job info. - """ - - def __init__(self, url): - if url == "localhost": - url = "http://localhost:8080/_ah/api/" - else: - if not url.startswith(("https://")) and not url.startswith("http://"): - url = "https://" + url - if url.endswith("appspot.com"): - url += "/_ah/api/" - self._headers = {"content-type": "application/json", - "Accept-Charset": "UTF-8"} - self._url = url - self._job = {} - self._heartbeat_thread = None - - def UploadBuildInfo(self, builds): - """Uploads the given build information to VTI. - - Args: - builds: a list of dictionaries, containing info about all new - builds found. - - Returns: - True if successful, False otherwise. - """ - url = self._url + "build/v1/set" - fail = False - for build in builds: - try: - response = requests.post(url, data=json.dumps(build), - headers=self._headers, - timeout=REQUESTS_TIMEOUT_SECONDS) - if response.status_code != requests.codes.ok: - logging.error("UploadBuildInfo error: %s", response) - fail = True - except requests.exceptions.Timeout as e: - logging.exception(e) - fail = True - if fail: - return False - return True - - def UploadDeviceInfo(self, hostname, devices): - """Uploads the given device information to VTI. - - Args: - hostname: string, the hostname of a target host. - devices: a list of dicts, containing info about all detected - devices that are attached to the host. - - Returns: - True if successful, False otherwise. - """ - url = self._url + "host/v1/set" - payload = {} - payload["hostname"] = hostname - payload["devices"] = [] - for device in devices: - new_device = { - "serial": device["serial"], - "product": device["product"], - "status": device["status"]} - payload["devices"].append(new_device) - - try: - response = requests.post(url, data=json.dumps(payload), - headers=self._headers, - timeout=REQUESTS_TIMEOUT_SECONDS) - except (requests.exceptions.ConnectionError, - requests.exceptions.Timeout) as e: - logging.exception(e) - return False - if response.status_code != requests.codes.ok: - logging.error("UploadDeviceInfo error: %s", response) - return False - return True - - def UploadScheduleInfo(self, pbs, clear_schedule): - """Uploads the given schedule information to VTI. - - Args: - pbs: a list of dicts, containing info about all task schedules. - clear_schedule: bool, True to clear all schedule data exist on the - scheduler - - Returns: - True if successful, False otherwise. - """ - if pbs is None or len(pbs) == 0: - return False - - url = self._url + "schedule/v1/clear" - succ = True - if clear_schedule: - try: - response = requests.post( - url, data=json.dumps({"manifest_branch": "na"}), - headers=self._headers, timeout=REQUESTS_TIMEOUT_SECONDS) - except requests.exceptions.Timeout as e: - logging.exception(e) - return False - if response.status_code != requests.codes.ok: - logging.error("UploadScheduleInfo error: %s", response) - succ = False - - if not succ: - return False - - url = self._url + "schedule/v1/set" - for pb in pbs: - schedule = {} - succ = succ and pb2_utils.FillDictAndPost( - pb, schedule, url, self._headers, - SCHEDULE_INFO_PB2_ATTR_FILTERS, "UploadScheduleInfo") - - return succ - - def UploadLabInfo(self, pbs, clear_labinfo): - """Uploads the given lab information to VTI. - - Args: - pbs: a list of dicts, containing info about all known labs. - clear_labinfo: bool, True to clear all lab data exist on the - scheduler - - Returns: - True if successful, False otherwise. - """ - if pbs is None or len(pbs) == 0: - return - - url = self._url + "lab/v1/clear" - succ = True - if clear_labinfo: - try: - response = requests.post(url, data=json.dumps({"name": "na"}), - headers=self._headers, - timeout=REQUESTS_TIMEOUT_SECONDS) - except requests.exceptions.Timeout as e: - logging.exception(e) - return False - if response.status_code != requests.codes.ok: - logging.error("UploadLabInfo error: %s", response) - succ = False - - if not succ: - return False - - url = self._url + "lab/v1/set" - for pb in pbs: - lab = {} - lab["name"] = pb.name - lab["owner"] = pb.owner - lab["admin"] = [] - lab["admin"].extend(pb.admin) - lab["host"] = [] - for host in pb.host: - new_host = {} - new_host["hostname"] = host.hostname - new_host["ip"] = host.ip - new_host["script"] = host.script - if host.host_equipment: - new_host["host_equipment"] = [] - new_host["host_equipment"].extend(host.host_equipment) - new_host["device"] = [] - if host.device: - for device in host.device: - new_device = {} - new_device["serial"] = device.serial - new_device["product"] = device.product - if device.device_equipment: - new_device["device_equipment"] = [] - new_device["device_equipment"].extend( - device.device_equipment) - new_host["device"].append(new_device) - lab["host"].append(new_host) - try: - response = requests.post(url, data=json.dumps(lab), - headers=self._headers, - timeout=REQUESTS_TIMEOUT_SECONDS) - if response.status_code != requests.codes.ok: - logging.error("UploadLabInfo error: %s", response) - succ = False - except requests.exceptions.Timeout as e: - logging.exception(e) - succ = False - return succ - - def LeaseJob(self, hostname, execute=True): - """Leases a job for the given host, 'hostname'. - - Args: - hostname: string, the hostname of a target host. - execute: boolean, True to lease and execute StartHeartbeat, which is - the case that the leased job will be executed on this - process's context. - - Returns: - True if successful, False otherwise. - """ - if not hostname: - return None, {} - - url = self._url + "job/v1/lease" - try: - response = requests.post(url, data=json.dumps({"hostname": hostname}), - headers=self._headers, - timeout=REQUESTS_TIMEOUT_SECONDS) - except requests.exceptions.Timeout as e: - logging.exception(e) - return None, {} - - if response.status_code != requests.codes.ok: - logging.error("LeaseJob error: %s", response.status_code) - return None, {} - - response_json = json.loads(response.text) - if ("return_code" in response_json - and response_json["return_code"] != "SUCCESS"): - logging.debug("LeaseJob error: %s", response_json) - return None, {} - - if "jobs" not in response_json: - logging.error( - "LeaseJob jobs not found in response json %s", response.text) - return None, {} - - jobs = response_json["jobs"] - if jobs and len(jobs) > 0: - for job in jobs: - if execute == True: - self._job = job - self.StartHeartbeat("leased", 60) - return job["test_name"].split("/")[0], job - return None, {} - - def ExecuteJob(self, job): - """Executes leased job passed from parent process. - - Args: - job: dict, information the on leased job. - - Returns: - a string which is path to a script file for onecmd(). - a dict contains info on the leased job, will be passed to onecmd(). - """ - logging.info("Job info : {}".format(json.dumps(job))) - if job is not None: - self._job = job - self.StartHeartbeat("leased", 60) - return job["test_name"].split("/")[0], job - - return None, {} - - def UpdateLeasedJobStatus(self, status, update_interval): - """Updates the status of the leased job. - - Args: - status: string, status value. - update_interval: int, time between heartbeats in second. - """ - if self._job is None: - return - - url = self._url + "job/v1/heartbeat" - self._job["status"] = JOB_STATUS_DICT[status] - - thread = threading.currentThread() - while getattr(thread, 'keep_running', True): - try: - response = requests.post(url, data=json.dumps(self._job), - headers=self._headers, - timeout=REQUESTS_TIMEOUT_SECONDS) - if response.status_code != requests.codes.ok: - logging.error("UpdateLeasedJobStatus error: %s", response) - except requests.exceptions.Timeout as e: - logging.exception(e) - time.sleep(update_interval) - - def StartHeartbeat(self, status="leased", update_interval=60): - """Starts the hearbeat_thread. - - Args: - status: string, status value. - update_interval: int, time between heartbeats in second. - """ - if (self._heartbeat_thread is None - or hasattr(self._heartbeat_thread, 'keep_running')): - self._heartbeat_thread = threading.Thread( - target=self.UpdateLeasedJobStatus, - args=( - status, - update_interval, - )) - self._heartbeat_thread.daemon = True - self._heartbeat_thread.start() - - def StopHeartbeat(self, status="complete", infra_log_url=""): - """Stops the hearbeat_thread and sets current job's status. - - Args: - status: string, status value. - infra_log_url: string, URL to the uploaded infra log. - """ - self._heartbeat_thread.keep_running = False - - if self._job is None: - return - - url = self._url + "job/v1/heartbeat" - self.SetJobStatusFromLeasedTo(status) - self._job["infra_log_url"] = infra_log_url - - try: - response = requests.post(url, data=json.dumps(self._job), - headers=self._headers, - timeout=REQUESTS_TIMEOUT_SECONDS) - if response.status_code != requests.codes.ok: - logging.error("StopHeartbeat error: %s", response) - except requests.exceptions.Timeout as e: - logging.exception(e) - - self._job = None - - def SetJobStatusFromLeasedTo(self, status): - """Sets current job's status only when the job's status is 'leased'. - - Args: - status: string, status value. - """ - if (self._job is not None and - self._job["status"] == JOB_STATUS_DICT["leased"]): - self._job["status"] = JOB_STATUS_DICT[status] - - def UploadHostVersion(self, hostname, vtslab_version): - """Uploads vtslab version. - - Args: - hostname: string, the name of the host. - vtslab_version: string, current version of vtslab package. - """ - url = self._url + "lab/v1/set_version" - host = {} - host["hostname"] = hostname - host["vtslab_version"] = vtslab_version - - try: - response = requests.post(url, data=json.dumps(host), - headers=self._headers, - timeout=REQUESTS_TIMEOUT_SECONDS) - except (requests.exceptions.ConnectionError, - requests.exceptions.Timeout) as e: - logging.exception(e) - return - if response.status_code != requests.codes.ok: - logging.error("UploadHostVersion error: %s", response) - - def CheckBootUpStatus(self): - """Checks whether the device_img + gsi from the job fails to boot up. - - Returns: - True if the devices flashed with the given imgs from the leased job - succeed to boot up. False otherwise. - """ - if self._job: - return (self._job["status"] != JOB_STATUS_DICT["bootup-err"]) - return False - - def GetJobTestType(self): - """Returns the test type of the leased job. - - Returns: - int, test_type attr in the job message. 0 when there is no job - leased to this vti_endpoint_client. - """ - if self._job and "test_type" in self._job: - try: - return int(self._job["test_type"]) - except ValueError as e: - logging.exception(e) - return 0 - - def GetJobDeviceProductName(self): - """Returns the product name of the DUTs of the leased job. - - Returns: - string, product name. An empty string if there is no job leased or - "device" attr of the job obj is not well formatted. - """ - if self._job and "device" in self._job: - try: - return self._job["device"].split("/")[1] - except IndexError as e: - logging.exception(e) - return "" diff --git a/host_setup/fabfile.py b/host_setup/fabfile.py deleted file mode 100644 index d37bb58..0000000 --- a/host_setup/fabfile.py +++ /dev/null @@ -1,386 +0,0 @@ -# -# Copyright 2018 - The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import imp -import os -import sys -import time - -from fabric.api import env -from fabric.api import local -from fabric.api import run -from fabric.api import settings -from fabric.api import sudo -from fabric.contrib.files import contains -from fabric.contrib.files import exists -from fabric.context_managers import cd - -_PIP_REQUIREMENTS_PATHS = [ - "test/vts/script/pip_requirements.txt", - "test/framework/harnesses/host_controller/script/pip_requirements.txt" -] - -# Path to the file that contains the abs path to the deployed vtslab pakcage. -_VTSLAB_PACKAGE_PATH_FILENAME = ".vtslab_package_path" - -# Zone filter list for GCE instances. -_DEFAULT_GCE_ZONE_LIST = [ - "us-east1-b", - "asia-northeast1-a", -] - - -def SetPassword(password): - """Sets password for hosts to access through ssh and to run sudo commands - - usage: $ fab SetPassword:<password for hosts> - - Args: - password: string, password for hosts. - """ - env.password = password - - -def GetHosts(hosts_file_path, gce_instance_name=None, account=None): - """Configures env.hosts to a given list of hosts. - - usage: $ fab GetHosts:<path to a source file contains hosts info> - - Args: - hosts_file_path: string, path to a python file passed from command file - input. - gce_instance_name: string, GCE instance name. - account: string, account name used for the ssh connection. - """ - if hosts_file_path.endswith(".py"): - hosts_module = imp.load_source('hosts_module', hosts_file_path) - env.hosts = hosts_module.EmitHostList() - else: - if not gce_instance_name or not account: - print( - "Please specify gce_instance_name and account using -H option. " - "Example: -H <Google_Cloud_project>,<GCE_instance>,<account>") - sys.exit(0) - env.key_filename = "~/.ssh/google_compute_engine" - gce_list_out = local( - "gcloud compute instances list --project=%s --filter=\"zone:(%s)\"" - % (hosts_file_path, " ".join(_DEFAULT_GCE_ZONE_LIST)), - capture=True) - for line in gce_list_out.split("\n")[1:]: - if line.startswith(gce_instance_name): - env.hosts.append("%s@%s" % (account, line.strip().split()[-2])) - - -def SetupIptables(ip_address_file_path): - """Configures iptables setting for all hosts listed. - - usage: $ fab SetupIptables:<path to a source file contains ip addresses of - certified machines> - - Args: - ip_address_file_path: string, path to a python file passed from command - file input. - """ - ip_addresses_module = imp.load_source('ip_addresses_module', - ip_address_file_path) - ip_address_list = ip_addresses_module.EmitIPAddressList() - - sudo("apt-get install -y iptables-persistent") - sudo("iptables -P INPUT ACCEPT") - sudo("iptables -P FORWARD ACCEPT") - sudo("iptables -F") - - for ip_address in ip_address_list: - sudo( - "iptables -A INPUT -p tcp -s %s --dport 22 -j ACCEPT" % ip_address) - - sudo("iptables -P INPUT DROP") - sudo("iptables -P FORWARD DROP") - sudo("iptables -A INPUT -p icmp -j ACCEPT") - sudo("netfilter-persistent save") - sudo("netfilter-persistent reload") - - -def SetupSudoers(): - """Append sudo rules for vtslab user. - - usage: $ fab SetupSudoers - """ - if not contains("/etc/sudoers", "vtslab", use_sudo=True): - sudo("echo '' | sudo tee -a /etc/sudoers") - sudo("echo '# Let vtslab account have all authorization' | " - "sudo tee -a /etc/sudoers") - sudo("echo 'vtslab ALL=(ALL:ALL) ALL' | sudo tee -a /etc/sudoers") - - -def SetupUSBPermission(): - """Sets up the USB permission for adb and fastboot. - - usage: $ fab SetupUSBPermission - """ - sudo("curl --create-dirs -L -o /etc/udev/rules.d/51-android.rules -O -L " - "https://raw.githubusercontent.com/snowdream/51-android/master/" - "51-android.rules") - sudo("chmod a+r /etc/udev/rules.d/51-android.rules") - sudo("service udev restart") - - -def SetupADBVendorKeysEnvVar(): - """Appends scripts for ADB_VENDOR_KEYS path setup. - - In setup step, this function looks into .bashrc file for this script, and - if there is not then appends the below scripts to .bashrc. - Later when shell env created (through ssh or screen instance creation time), - .bashrc file will look for _VTSLAB_PACKAGE_PATH_FILENAME and use the - contents of the file to set ADB_VENDOR_KEYS. - - usage: $ fab SetupADBVendorKeysEnvVar - """ - if not contains("~/.bashrc", _VTSLAB_PACKAGE_PATH_FILENAME): - run("echo '' >> ~/.bashrc", ) - run("echo '# Set $ADB_VENDOR_KEYS as paths to adb private key files " - "within the vtslab-package' >> ~/.bashrc") - run("echo 'if [ -f ~/%s ]; then' >> ~/.bashrc" % - _VTSLAB_PACKAGE_PATH_FILENAME) - run("echo ' export ADB_VENDOR_KEYS=$(find $(cat ~/%s)/android-vtslab/" - "testcases/DATA/ak -name \".*.ak\" | tr \"\\n\" \":\")' " - ">> ~/.bashrc" % _VTSLAB_PACKAGE_PATH_FILENAME) - run("echo 'fi' >> ~/.bashrc") - - -def _CheckADBVendorKeysEnvVar(vtslab_package_file_name=""): - """Checks if there is a change in ADB_VENDOR_KEYS env variable. - - if there is, then the adbd needs to be restarted in the screen context - before running the HC. - - Args: - vtslab_package_file_name: string, the HC package file name that is about - to be deployed. - - Returns: - True if the list of the adb vendor key files have changed, - False otherwise. - """ - former_key_set = set() - current_key_set = set() - set_keyfile_set = lambda set, path_list: map(set.add, map(os.path.basename, - path_list)) - vtslab_package_path_filepath = "~/%s" % _VTSLAB_PACKAGE_PATH_FILENAME - - if exists(vtslab_package_path_filepath): - former_HC_package_path = run("cat %s" % vtslab_package_path_filepath) - former_HC_package_adbkey_path = os.path.join( - former_HC_package_path, "android-vtslab/testcases/DATA/ak") - if exists(former_HC_package_adbkey_path): - adb_vendor_keys_list = run("find %s -name \".*.ak\"" % - former_HC_package_adbkey_path).split() - set_keyfile_set(former_key_set, adb_vendor_keys_list) - - if exists("~/run/%s.dir/android-vtslab/testcases/DATA/ak" % - vtslab_package_file_name): - adb_vendor_keys_list = run( - "find ~/run/%s.dir/android-vtslab/testcases/DATA/ak -name \".*.ak\"" - % vtslab_package_file_name).split() - set_keyfile_set(current_key_set, adb_vendor_keys_list) - - return former_key_set != current_key_set - - -def SetupPackages(ip_address_file_path=None): - """Sets up the execution environment for vts `run` command. - - Need to temporarily open the ports for apt-get and pip commands. - - usage: $ fab SetupPackages - - Args: - ip_address_file_path: string, path to a python file passed from command - file input. Will be passed to SetupIptables(). - """ - sudo("iptables -P INPUT ACCEPT") - - # todo : replace "kr.ubuntu." to "ubuntu" in /etc/apt/sources.list - sudo("apt-get upgrade -y") - sudo("apt-get update -y") - sudo("apt-get install -y git-core gnupg flex bison gperf build-essential " - "zip curl zlib1g-dev gcc-multilib g++-multilib x11proto-core-dev " - "libx11-dev lib32z-dev ccache libgl1-mesa-dev libxml2-utils xsltproc " - "unzip liblz4-tool udev screen") - - sudo("apt-get install -y android-tools-adb") - sudo("usermod -aG plugdev $LOGNAME") - - SetupUSBPermission() - - sudo("apt-get update") - sudo("apt-get install -y python2.7") - sudo("apt-get install -y python-pip") - run("pip install --upgrade pip") - sudo("apt-get install -y python-virtualenv") - - sudo("apt-get install -y python-dev python-protobuf protobuf-compiler " - "python-setuptools") - - for req_path in _PIP_REQUIREMENTS_PATHS: - full_path = os.path.join(os.environ["ANDROID_BUILD_TOP"], req_path) - pip_requirement_list = [] - try: - requirements_fd = open(full_path, "r") - lines = requirements_fd.readlines() - for line in lines: - req = line.rstrip() - if req != "" and not req.startswith("#"): - pip_requirement_list.append(req) - except IOError as e: - print("%s: %s" % (e.strerror, full_path)) - return - sudo("pip install %s" % " ".join(pip_requirement_list)) - - sudo("pip install --upgrade protobuf") - - lsb_result = run("lsb_release -c -s") - sudo("echo \"deb http://packages.cloud.google.com/apt cloud-sdk-%s " - "main\" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list" % - lsb_result) - sudo("curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | " - "sudo apt-key add -") - sudo("apt-get update && sudo apt-get install -y google-cloud-sdk") - sudo("apt-get install -y google-cloud-sdk-app-engine-java " - "google-cloud-sdk-app-engine-python kubectl") - - sudo("apt-get install -y m4 bison") - - if ip_address_file_path is not None: - SetupIptables(ip_address_file_path) - - SetupADBVendorKeysEnvVar() - - -def DeployVtslab(vtslab_package_gcs_url=None): - """Deploys vtslab package. - - Fetches and deploy vtslab by going through the processes described below - 1. Send the "exit --wait_for_jobs=True" command to all detached screen. - And let the screen to terminate itself. - 2. Create a new screen instance that downloads and runs the new HC, - give password and device command to the HC without actually attaching it - - usage: $ fab DeployVtslab -p <password> -H hosts.py -f <gs://vtslab-release/...> - - Args: - vtslab_package_gcs_url: string, URL to a certain vtslab package file. - """ - if not vtslab_package_gcs_url: - print("Please specify vtslab package file URL using -f option.") - return - elif not vtslab_package_gcs_url.startswith("gs://vtslab-release/"): - print("Please spcify a valid URL for the vtslab package.") - return - else: - vti = "vtslab-schedule-" + vtslab_package_gcs_url[len( - "gs://vtslab-release/"):].split("/")[0] + ".appspot.com" - with settings(warn_only=True): - screen_list_result = run("screen -list") - lines = screen_list_result.split("\n") - for line in lines: - if "(Detached)" in line: - screen_name = line.split("\t")[1] - print(screen_name) - with settings(warn_only=True): - run("screen -S %s -X stuff \"exit --wait_for_jobs=True\"" % - screen_name) - run("screen -S %s -X stuff \"^M\"" % screen_name) - run("screen -S %s -X stuff \"exit\"" % screen_name) - run("screen -S %s -X stuff \"^M\"" % screen_name) - - vtslab_package_file_name = os.path.basename(vtslab_package_gcs_url) - run("mkdir -p ~/run/%s.dir/" % vtslab_package_file_name) - with cd("~/run/%s.dir" % vtslab_package_file_name): - run("gsutil cp %s ./" % vtslab_package_gcs_url) - run("unzip -o %s" % vtslab_package_file_name) - adb_vendor_keys_changed = _CheckADBVendorKeysEnvVar( - vtslab_package_file_name) - run("pwd > ~/%s" % _VTSLAB_PACKAGE_PATH_FILENAME) - - with cd("~/run/%s.dir/android-vtslab/tools" % vtslab_package_file_name): - new_screen_name = run("cat ../testcases/version.txt") - - with cd("~/run/%s.dir/android-vtslab/tools" % vtslab_package_file_name): - run("./make_screen %s ; sleep 1" % new_screen_name) - - if adb_vendor_keys_changed: - reset_adbd = "" - while reset_adbd.lower() not in ["y", "n"]: - if reset_adbd: - print("Please type 'y' or 'n'") - reset_adbd = raw_input( - "Reset adb server daemon on host %s (y/n)? " % env.host) - if reset_adbd.lower() == "y": - run("screen -S %s -X stuff \"adb kill-server^M\"" % - new_screen_name) - run("screen -S %s -X stuff \"adb start-server^M\"" % - new_screen_name) - run("screen -S %s -X stuff \"./run --vti=%s\"" % (new_screen_name, vti)) - run("screen -S %s -X stuff \"^M\"" % new_screen_name) - time.sleep(5) - run("screen -S %s -X stuff \"password\"" % new_screen_name) - run("screen -S %s -X stuff \"^M\"" % new_screen_name) - run("screen -S %s -X stuff \"%s\"" % (new_screen_name, env.password)) - run("screen -S %s -X stuff \"^M\"" % new_screen_name) - run("screen -S %s -X stuff \"device --lease=True\"" % new_screen_name) - run("screen -S %s -X stuff \"^M\"" % new_screen_name) - - with cd("~/run/%s.dir" % vtslab_package_file_name): - run("rm %s" % vtslab_package_file_name) - - -def DeployGCE(vtslab_package_gcs_url=None): - """Deploys a vtslab package to GCE nodes. - - Fetches and deploy vtslab on monitor-hc by doing; - 1. Download android-vtslab-<>.zip from GCS using the given URL and upzip it. - 2. Send the Ctrl-c key input to all detached screen, then cursor-up - key input and enter key input, making the screen to execute - the last run command. - - usage: $ fab DeployVtslab -p <password> -H <Google Cloud Platform project name> -f <gs://vtslab-release/...> - - Args: - vtslab_package_gcs_url: string, URL to a certain vtslab package file. - """ - if not vtslab_package_gcs_url: - print("Please specify vtslab package file URL using -f option.") - return - elif not vtslab_package_gcs_url.startswith("gs://"): - print("Please spcify a valid URL for the vtslab package.") - return - - vtslab_package_file_name = os.path.basename(vtslab_package_gcs_url) - with cd("~/run"): - run("gsutil cp %s ./" % vtslab_package_gcs_url) - run("unzip -o %s" % vtslab_package_file_name) - run("rm %s" % vtslab_package_file_name) - - with settings(warn_only=True): - screen_list_result = run("screen -list") - lines = screen_list_result.split("\n") - for line in lines: - if "(Detached)" in line: - screen_name = line.split("\t")[1] - run("screen -S %s -X stuff \"^C\"" % screen_name) - run("screen -S %s -X stuff \"\033[A\"" % screen_name) - run("screen -S %s -X stuff \"^M\"" % screen_name) diff --git a/host_setup/host_setup.sh b/host_setup/host_setup.sh deleted file mode 100755 index fa52755..0000000 --- a/host_setup/host_setup.sh +++ /dev/null @@ -1,96 +0,0 @@ -#/bin/bash -# -# Copyright 2018 - The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -SCRIPT_NAME=$(basename $0) -usage() -{ - echo "Usage: $SCRIPT_NAME <task_name> [options]" - echo "" - echo "Options:" - echo " -p PASSWORD password for hosts." - echo " -H PATH path to a .py file contains hosts information" - echo " -i PATH path to a .py file contains ip address information of certified machines" - echo " -f GCS_URL URL to the vtslab package file to be deployed." - exit 1 -} - -FABRIC_EXISTS=$(pip show fabric) -if [ -z "$FABRIC_EXISTS" ]; then - INSTALL_FABRIC=true -else - FABRIC_VERSION=$(fab -V | grep Fabric) - if [ "${FABRIC_VERSION:7:1}" -ne 1 ]; then - INSTALL_FABRIC=true - fi -fi -if [ "$INSTALL_FABRIC" == true ]; then - sudo pip install fabric==1.14.0 --force -fi - -TASK=$1 -if [[ ${TASK:0:1} == "-" ]]; then - usage -fi - -shift -PASSWORD="" -HOSTS_PATH="hosts.py" -IPADDRESSES_PATH="" -VTSLAB_PACKAGE_GCS_URL="" - -while getopts ":p:H:i:f:" opt; do - case $opt in - p) - PASSWORD=$OPTARG - ;; - H) - HOSTS_PATH=$OPTARG - ;; - i) - IPADDRESSES_PATH=$OPTARG - ;; - f) - VTSLAB_PACKAGE_GCS_URL=$OPTARG - ;; - \?) - echo "Invalid option: -$OPTARG" - usage - ;; - :) - echo "Option -$OPTARG requires an argument." - usage - ;; - esac -done - -if [ "$TASK" == "SetupIptables" ]; then - fab SetPassword:$PASSWORD GetHosts:$HOSTS_PATH $TASK:$IPADDRESSES_PATH -elif [ "$TASK" == "SetupPackages" ]; then - if [ -z "$IPADDRESSES_PATH" ]; then - fab SetPassword:$PASSWORD GetHosts:$HOSTS_PATH $TASK - else - fab SetPassword:$PASSWORD GetHosts:$HOSTS_PATH $TASK:$IPADDRESSES_PATH - fi -elif [ "$TASK" == "DeployVtslab" ] || [ "$TASK" == "DeployGCE" ]; then - if [ -z "$VTSLAB_PACKAGE_GCS_URL" ]; then - echo "Please specify vtslab package file URL using -f option." - exit - fi - fab SetPassword:$PASSWORD GetHosts:$HOSTS_PATH $TASK:$VTSLAB_PACKAGE_GCS_URL -else - fab SetPassword:$PASSWORD GetHosts:$HOSTS_PATH $TASK -fi diff --git a/script/run-unittest.sh b/script/run-unittest.sh deleted file mode 100755 index fb4fca0..0000000 --- a/script/run-unittest.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# -# Copyright 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -TEST_FRAMEWORK_DIR=`dirname $0` -TEST_FRAMEWORK_DIR=`dirname $TEST_FRAMEWORK_DIR` - -if [ -z "$ANDROID_BUILD_TOP" ]; then - echo "Missing ANDROID_BUILD_TOP env variable. Run 'lunch' first." - exit 1 -fi - -touch $ANDROID_BUILD_TOP/test/vti/__init__.py - -avoid_list=( - # known failures - "./harnesses/host_controller/console_test.py" - "./harnesses/host_controller/invocation_thread_test.py" - # not unit tests - "./harnesses/host_controller/command_processor/command_test.py" - ) - -####################################### -# Checks if a given file is included in the list of files to avoid -# Globals: -# Arguments: -# $1: list of files to avoid -# $2: the given file -# Returns: -# SUCCESS, if the given file exists in the list -# FAILURE, otherwise -####################################### -function contains_file() { - local -n arr=$1 - for avoid in "${arr[@]}"; do - if [ "$2" = "$avoid" ]; then - return # contains - fi - done - false # not contains -} - -# Runs all unit tests under test/framework. -for t in $(find $TEST_FRAMEWORK_DIR -type f -name "*_test.py"); -do - if contains_file avoid_list $t; then - continue - fi - echo "UNIT TEST", $t - echo PYTHONPATH=$ANDROID_BUILD_TOP/test/framework/harnesses:$ANDROID_BUILD_TOP/test python $t; - PYTHONPATH=$ANDROID_BUILD_TOP/test/framework/harnesses:$ANDROID_BUILD_TOP/test python $t; -done diff --git a/tools/host_controller/Android.bp b/tools/host_controller/Android.bp deleted file mode 100644 index 174ac4a..0000000 --- a/tools/host_controller/Android.bp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (C) 2017 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -sh_binary_host { - name: "run", - src: "run", -} diff --git a/tools/host_controller/make_screen b/tools/host_controller/make_screen deleted file mode 100755 index 10aed5d..0000000 --- a/tools/host_controller/make_screen +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Sets env var PATH and makes a detached screen instance -pushd ../bin -export PATH=`pwd`:$PATH -popd -screen -mdS $1 diff --git a/tools/host_controller/run b/tools/host_controller/run deleted file mode 100755 index 43b3707..0000000 --- a/tools/host_controller/run +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# Copyright (C) 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# launcher script for vts-hc (host controller) -# can be used from an Android build environment, or a standalone vts zip - -# get OS -HOST=`uname` -if [ "$HOST" == "Linux" ]; then - OS="linux-x86" -elif [ "$HOST" == "Darwin" ]; then - OS="darwin-x86" -else - echo "Unrecognized OS" - exit -fi - -# check if in Android build env -if [ ! -z "${ANDROID_BUILD_TOP}" ]; then - if [ ! -z "${ANDROID_HOST_OUT}" ]; then - VTS_ROOT=${ANDROID_HOST_OUT}/vtslab - else - VTS_ROOT=${ANDROID_BUILD_TOP}/${OUT_DIR:-out}/host/${OS}/vtslab - fi - if [ ! -d ${VTS_ROOT} ]; then - echo "Could not find $VTS_ROOT in Android build environment. Try 'make vts'" - exit - fi; -fi; - -if [ -z ${VTS_ROOT} ]; then - # assume we're in an extracted vts install - VTS_ROOT="$(dirname $(readlink -e $0))/../.." -fi; - -find ${VTS_ROOT} -name "*.pyc" -exec rm -f {} \; -cd ${VTS_ROOT}/android-vtslab/testcases/; python -m host_controller.main "$@" |