From 5a37f5e07007eee8f7ab3eecb684d66471f4b9b3 Mon Sep 17 00:00:00 2001 From: Hsin-Yi Chen Date: Tue, 4 Feb 2020 15:29:51 +0800 Subject: Convert vts_vndk_dependency_test to a host test TradeFed does not support running Python file on device. This commit converts vts_vndk_dependency_test to a python_test_host. Bug: 147454897 Test: ANDROID_SERIAL=ABCDEF \ VTS_DATA_FILE_PATH=$ANDROID_HOST_OUT/vts/android-vts/testcases \ LD_LIBRARY_PATH=$ANDROID_HOST_OUT_TESTCASES/../lib64 \ $ANDROID_HOST_OUT_TESTCASES/vts_vndk_dependency_test/x86_64/vts_vndk_dependency_test Change-Id: I9c4a3eb48864fb511bc89e643a0549d98a351bec --- Android.bp | 4 +-- dependency/vts_vndk_dependency_test.py | 51 ++++++++++++++++++++++++---------- utils.py | 15 ++++++++-- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/Android.bp b/Android.bp index 18a2b91..df43d92 100644 --- a/Android.bp +++ b/Android.bp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -python_library { +python_library_host { name: "vts_vndk_utils", pkg_path: "vts/testcases/vndk", srcs: [ @@ -50,7 +50,7 @@ python_defaults { } } -python_test { +python_test_host { name: "vts_vndk_dependency_test", defaults: ["vts_vndk_default"], main: "dependency/vts_vndk_dependency_test.py", diff --git a/dependency/vts_vndk_dependency_test.py b/dependency/vts_vndk_dependency_test.py index 6541a80..65d62c8 100644 --- a/dependency/vts_vndk_dependency_test.py +++ b/dependency/vts_vndk_dependency_test.py @@ -20,7 +20,10 @@ import collections import logging import os +import posixpath as target_path_module import re +import shutil +import tempfile import unittest from vts.testcases.vndk import utils @@ -34,6 +37,8 @@ class VtsVndkDependencyTest(unittest.TestCase): Attributes: _dut: The AndroidDevice under test. + _temp_dir: The temporary directory to which the odm and vendor + partitions are copied. _ll_ndk: Set of strings. The names of low-level NDK libraries in /system/lib[64]. _sp_hal: List of patterns. The names of the same-process HAL libraries @@ -44,6 +49,7 @@ class VtsVndkDependencyTest(unittest.TestCase): _VENDOR_LINK_PATHS: Format strings of vendor processes' link paths. """ _TARGET_DIR_SEP = "/" + _TARGET_ROOT_DIR = "/" _TARGET_ODM_DIR = "/odm" _TARGET_VENDOR_DIR = "/vendor" @@ -74,8 +80,8 @@ class VtsVndkDependencyTest(unittest.TestCase): def __init__(self, target_path, bitness, deps, runpaths): self.target_path = target_path - self.name = os.path.basename(target_path) - self.target_dir = os.path.dirname(target_path) + self.name = target_path_module.basename(target_path) + self.target_dir = target_path_module.dirname(target_path) self.bitness = bitness self.deps = deps # Format runpaths @@ -90,9 +96,19 @@ class VtsVndkDependencyTest(unittest.TestCase): def setUp(self): """Initializes device, temporary directory, and VNDK lists.""" - self._dut = utils.AndroidDevice() + serial_number = os.environ.get("ANDROID_SERIAL") + self.assertTrue(serial_number, "$ANDROID_SERIAL is empty.") + self._dut = utils.AndroidDevice(serial_number) self.assertTrue(self._dut.IsRoot(), "This test requires adb root.") + self._temp_dir = tempfile.mkdtemp() + for target_dir in (self._TARGET_ODM_DIR, self._TARGET_VENDOR_DIR): + if self._dut.IsDirectory(target_dir): + logging.info("adb pull %s %s", target_dir, self._temp_dir) + self._dut.AdbPull(target_dir, self._temp_dir) + else: + logging.info("Skip adb pull %s", target_dir) + vndk_lists = vndk_data.LoadVndkLibraryListsFromResources( self._dut.GetVndkVersion(), vndk_data.SP_HAL, @@ -110,6 +126,11 @@ class VtsVndkDependencyTest(unittest.TestCase): logging.debug("VNDK: %s", self._vndk) logging.debug("VNDK_SP: %s", self._vndk_sp) + def tearDown(self): + """Deletes the temporary directory.""" + logging.info("Delete %s", self._temp_dir) + shutil.rmtree(self._temp_dir, ignore_errors=True) + def _IsElfObjectForAp(self, elf, target_path, abi_list): """Checks whether an ELF object is for application processor. @@ -192,23 +213,28 @@ class VtsVndkDependencyTest(unittest.TestCase): for file_name in file_names: yield os.path.join(root_dir, file_name) - def _LoadElfObjects(self, target_dir, abi_list, elf_error_handler): + def _LoadElfObjects(self, host_dir, target_dir, abi_list, + elf_error_handler): """Scans a host directory recursively and loads all ELF files in it. Args: - target_dir: The host directory to scan. + host_dir: The host directory to scan. + target_dir: The path from which host_dir is copied. abi_list: A list of strings, the ABIs of the ELF files to load. elf_error_handler: A function that takes 2 arguments - (path, exception). It is called when + (target_path, exception). It is called when the parser fails to read an ELF file. Returns: List of ElfObject. """ objs = [] - for target_path in self._IterateFiles(target_dir): + for full_path in self._IterateFiles(host_dir): + rel_path = os.path.relpath(full_path, host_dir) + target_path = target_path_module.join( + target_dir, *rel_path.split(os.path.sep)) try: - elf = elf_parser.ElfParser(target_path) + elf = elf_parser.ElfParser(full_path) except elf_parser.ElfError: logging.debug("%s is not an ELF file", target_path) continue @@ -404,12 +430,9 @@ class VtsVndkDependencyTest(unittest.TestCase): """Tests vendor libraries/executables and SP-HAL dependencies.""" read_errors = [] abi_list = self._dut.GetCpuAbiList() - objs = [] - for target_dir in (self._TARGET_ODM_DIR, self._TARGET_VENDOR_DIR): - if self._dut.IsDirectory(target_dir): - objs.extend(self._LoadElfObjects( - target_dir, abi_list, - lambda p, e: read_errors.append((p, str(e))))) + objs = self._LoadElfObjects( + self._temp_dir, self._TARGET_ROOT_DIR, abi_list, + lambda p, e: read_errors.append((p, str(e)))) dep_errors = self._TestElfDependency(32, objs) if self._dut.GetCpuAbiList(64): diff --git a/utils.py b/utils.py index 8e6cc52..45a5951 100644 --- a/utils.py +++ b/utils.py @@ -24,7 +24,15 @@ import subprocess class AndroidDevice(object): """This class controls the device via adb commands.""" - def _ExecuteCommand(self, *cmd): + def __init__(self, serial_number): + self._serial_number = serial_number + + def AdbPull(self, src, dst): + cmd = ["adb", "-s", self._serial_number, "pull", src, dst] + subprocess.check_call(cmd, shell=False, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + def _ExecuteCommand(self, *args): """Executes a command. Args: @@ -34,6 +42,8 @@ class AndroidDevice(object): Stdout as a string, stderr as a string, and return code as an integer. """ + cmd = ["adb", "-s", self._serial_number, "shell"] + cmd.extend(args) proc = subprocess.Popen(cmd, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = proc.communicate() @@ -140,8 +150,7 @@ class AndroidDevice(object): def _Test(self, *args): """Tests file types and status.""" - out, err, return_code = self._ExecuteCommand("sh", "-c", - "test " + " ".join(args)) + out, err, return_code = self._ExecuteCommand("test", *args) if out.strip() or err.strip(): raise IOError("`test` args: %s\nstdout: %s\nstderr: %s" % (args, out, err)) -- cgit v1.2.3 From d23ea9199060361e365319ec2c45d94c6d3a7488 Mon Sep 17 00:00:00 2001 From: Hsin-Yi Chen Date: Sat, 29 Feb 2020 01:33:01 +0800 Subject: Add vts_vndk_files_test This commit copies VtsVndkFilesTest.py to vts_vndk_files_test.py, removes the dependency on VTS framework, and adds a python_test_host module for the new file. Bug: 147454897 Test: ANDROID_SERIAL=1234 \ LD_LIBRARY_PATH=$ANDROID_HOST_OUT/lib64 \ $ANDROID_HOST_OUT/nativetest64/vts_vndk_files_test/vts_vndk_files_test \ --verbose Change-Id: I4e8ee887a80fdaafed61456fee02405a145df5d3 --- Android.bp | 13 +++ files/vts_vndk_files_test.py | 183 ++++++++++++++++++++++++++++++++++++++++++ files/vts_vndk_files_test.xml | 24 ++++++ utils.py | 29 +++++++ 4 files changed, 249 insertions(+) create mode 100644 files/vts_vndk_files_test.py create mode 100644 files/vts_vndk_files_test.xml diff --git a/Android.bp b/Android.bp index df43d92..90cc368 100644 --- a/Android.bp +++ b/Android.bp @@ -58,3 +58,16 @@ python_test_host { "dependency/vts_vndk_dependency_test.py", ] } + +python_test_host { + name: "vts_vndk_files_test", + defaults: ["vts_vndk_default"], + main: "files/vts_vndk_files_test.py", + srcs: [ + "files/vts_vndk_files_test.py", + ], + test_suites: [ + "vts-core", + ], + test_config: "files/vts_vndk_files_test.xml", +} diff --git a/files/vts_vndk_files_test.py b/files/vts_vndk_files_test.py new file mode 100644 index 0000000..bcab2b7 --- /dev/null +++ b/files/vts_vndk_files_test.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import os +import posixpath as target_path_module +import unittest + +from vts.testcases.vndk import utils +from vts.testcases.vndk.golden import vndk_data +from vts.utils.python.vndk import vndk_utils + + +class VtsVndkFilesTest(unittest.TestCase): + """A test for VNDK files and directories. + + Attributes: + _dut: The AndroidDevice under test. + _vndk_version: The VNDK version of the device. + """ + # Some LL-NDK libraries may load the implementations with the same names + # from /vendor/lib. Since a vendor may install an implementation of an + # LL-NDK library with the same name, testNoLlndkInVendor doesn't raise + # errors on these LL-NDK libraries. + _LL_NDK_COLLIDING_NAMES = ("libEGL.so", "libGLESv1_CM.so", "libGLESv2.so", + "libGLESv3.so") + _TARGET_ODM_LIB = "/odm/{LIB}" + _TARGET_VENDOR_LIB = "/vendor/{LIB}" + + def setUp(self): + """Initializes attributes.""" + serial_number = os.environ.get("ANDROID_SERIAL") + self.assertTrue(serial_number, "$ANDROID_SERIAL is empty.") + self._dut = utils.AndroidDevice(serial_number) + self.assertTrue(self._dut.IsRoot(), "This test requires adb root.") + self._vndk_version = self._dut.GetVndkVersion() + + def _ListFiles(self, dir_path): + """Lists all files in a directory except subdirectories. + + Args: + dir_path: A string, path to the directory on device. + + Returns: + A list of strings, the file paths in the directory. + """ + if not self._dut.Exists(dir_path): + logging.info("%s not found", dir_path) + return [] + return self._dut.FindFiles(dir_path, "*", "!", "-type", "d") + + def _Fail(self, unexpected_paths): + """Logs error and fails current test. + + Args: + unexpected_paths: A list of strings, the paths to be shown in the + log message. + """ + logging.error("Unexpected files:\n%s", "\n".join(unexpected_paths)) + assert_lines = unexpected_paths[:20] + if len(unexpected_paths) > 20: + assert_lines.append("...") + assert_lines.append( + "Total number of errors: %d" % len(unexpected_paths)) + self.fail("\n".join(assert_lines)) + + def _TestVndkDirectory(self, vndk_dir, vndk_list_names): + """Verifies that the VNDK directory doesn't contain extra files. + + Args: + vndk_dir: The path to the VNDK directory on device. + vndk_list_names: Strings, the categories of the VNDK libraries + that can be in the directory. + """ + vndk_lists = vndk_data.LoadVndkLibraryListsFromResources( + self._vndk_version, *vndk_list_names) + self.assertTrue(vndk_lists, "Cannot load VNDK library lists.") + vndk_set = set().union(*vndk_lists) + logging.debug("vndk set: %s", vndk_set) + unexpected = [x for x in self._ListFiles(vndk_dir) if + target_path_module.basename(x) not in vndk_set] + if unexpected: + self._Fail(unexpected) + + def _TestNotInVndkDirecotory(self, vndk_dir, vndk_list_names, except_libs): + """Verifies that VNDK directory doesn't contain specific files. + + Args: + vndk_dir, The path to the VNDK directory on device. + vndk_list_names: A list of strings, the categories of the VNDK + libraries that should not be in the directory. + except_libs: A set of strings, the file names of the libraries that + are exceptions to this test. + """ + vndk_lists = vndk_data.LoadVndkLibraryListsFromResources( + self._vndk_version, *vndk_list_names) + self.assertTrue(vndk_lists, "Cannot load VNDK library lists.") + vndk_set = set().union(*vndk_lists) + vndk_set.difference_update(except_libs) + logging.debug("vndk set: %s", vndk_set) + unexpected = [x for x in self._ListFiles(vndk_dir) if + target_path_module.basename(x) in vndk_set] + if unexpected: + self._Fail(unexpected) + + def _TestVndkCoreDirectory(self, bitness): + """Verifies that VNDK directory doesn't contain extra files.""" + if not vndk_utils.IsVndkRuntimeEnforced(self._dut): + logging.info("Skip the test as VNDK runtime is not enforced on " + "the device.") + return + self._TestVndkDirectory( + vndk_utils.GetVndkDirectory(bitness, self._vndk_version), + (vndk_data.VNDK, vndk_data.VNDK_PRIVATE, vndk_data.VNDK_SP, + vndk_data.VNDK_SP_PRIVATE,)) + + def testVndkCoreDirectory32(self): + """Runs _TestVndkCoreDirectory for 32-bit libraries.""" + self._TestVndkCoreDirectory(32) + + def testVndkCoreDirectory64(self): + """Runs _TestVndkCoreDirectory for 64-bit libraries.""" + if self._dut.GetCpuAbiList(64): + self._TestVndkCoreDirectory(64) + else: + logging.info("Skip the test as the device doesn't support 64-bit " + "ABI.") + + def _TestNoLlndkInVendor(self, bitness): + """Verifies that vendor partition has no LL-NDK libraries.""" + self._TestNotInVndkDirecotory( + vndk_utils.FormatVndkPath(self._TARGET_VENDOR_LIB, bitness), + (vndk_data.LL_NDK,), + self._LL_NDK_COLLIDING_NAMES) + + def testNoLlndkInVendor32(self): + """Runs _TestNoLlndkInVendor for 32-bit libraries.""" + self._TestNoLlndkInVendor(32) + + def testNoLlndkInVendor64(self): + """Runs _TestNoLlndkInVendor for 64-bit libraries.""" + if self._dut.GetCpuAbiList(64): + self._TestNoLlndkInVendor(64) + else: + logging.info("Skip the test as the device doesn't support 64-bit " + "ABI.") + + def _TestNoLlndkInOdm(self, bitness): + """Verifies that odm partition has no LL-NDK libraries.""" + self._TestNotInVndkDirecotory( + vndk_utils.FormatVndkPath(self._TARGET_ODM_LIB, bitness), + (vndk_data.LL_NDK,), + self._LL_NDK_COLLIDING_NAMES) + + def testNoLlndkInOdm32(self): + """Runs _TestNoLlndkInOdm for 32-bit libraries.""" + self._TestNoLlndkInOdm(32) + + def testNoLlndkInOdm64(self): + """Runs _TestNoLlndkInOdm for 64-bit libraries.""" + if self._dut.GetCpuAbiList(64): + self._TestNoLlndkInOdm(64) + else: + logging.info("Skip the test as the device doesn't support 64-bit " + "ABI.") + + +if __name__ == "__main__": + unittest.main() diff --git a/files/vts_vndk_files_test.xml b/files/vts_vndk_files_test.xml new file mode 100644 index 0000000..4813916 --- /dev/null +++ b/files/vts_vndk_files_test.xml @@ -0,0 +1,24 @@ + + + + + + diff --git a/utils.py b/utils.py index 45a5951..0592bfa 100644 --- a/utils.py +++ b/utils.py @@ -156,6 +156,10 @@ class AndroidDevice(object): (args, out, err)) return return_code == 0 + def Exists(self, path): + """Returns whether a path on the device exists.""" + return self._Test("-e", path) + def IsDirectory(self, path): """Returns whether a path on the device is a directory.""" return self._Test("-d", path) @@ -172,3 +176,28 @@ class AndroidDevice(object): def IsExecutable(self, path): """Returns if execute permission is granted to a path on the device.""" return "x" in self._Stat("%A", path) + + def FindFiles(self, path, name_pattern, *options): + """Executes find command. + + Args: + path: A string, the path on the device. + name_pattern: A string, the pattern of the file name. + options: Strings, extra options passed to the command. + + Returns: + A list of strings, the paths to the found files. + + Raises: + ValueError if the pattern contains quotes. + IOError if the path does not exist. + """ + if '"' in name_pattern or "'" in name_pattern: + raise ValueError("File name pattern contains quotes.") + out, err, return_code = self._ExecuteCommand("find", path, "-name", + "'" + name_pattern + "'", + *options) + if return_code != 0 or err.strip(): + raise IOError("`find %s -name '%s' %s` stdout: %s\nstderr: %s" % + (path, name_pattern, " ".join(options), out, err)) + return out.strip().split("\n") -- cgit v1.2.3 From 6addb03ef46bc2d8be2ab9fdc01093de755d674c Mon Sep 17 00:00:00 2001 From: Hsin-Yi Chen Date: Tue, 25 Feb 2020 15:49:53 +0800 Subject: Add vts_vndk_dependency_test to vts-core Bug: 147454897 Test: atest vts_vndk_dependency_test Change-Id: I07c3e5ad692ad4819433851dc7a932061f8b8935 --- Android.bp | 6 +++++- dependency/vts_vndk_dependency_test.xml | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 dependency/vts_vndk_dependency_test.xml diff --git a/Android.bp b/Android.bp index 90cc368..9eae205 100644 --- a/Android.bp +++ b/Android.bp @@ -56,7 +56,11 @@ python_test_host { main: "dependency/vts_vndk_dependency_test.py", srcs: [ "dependency/vts_vndk_dependency_test.py", - ] + ], + test_suites: [ + "vts-core", + ], + test_config: "dependency/vts_vndk_dependency_test.xml", } python_test_host { diff --git a/dependency/vts_vndk_dependency_test.xml b/dependency/vts_vndk_dependency_test.xml new file mode 100644 index 0000000..f0f37c7 --- /dev/null +++ b/dependency/vts_vndk_dependency_test.xml @@ -0,0 +1,23 @@ + + + + + + + + -- cgit v1.2.3 From 1d1c640b24f9ec493e3094197f3e4cf79a0ddf79 Mon Sep 17 00:00:00 2001 From: Hsin-Yi Chen Date: Mon, 2 Mar 2020 17:40:19 +0800 Subject: Add vts_vndk_open_libraries_test This commit copies VtsVndkOpenLibrariesTest.py to vts_vndk_open_libraries_test.py, removes the dependency on VTS framework, and adds a python_test_host module for the new file. Bug: 147454897 Test: ANDROID_SERIAL=1234 \ LD_LIBRARY_PATH=$ANDROID_HOST_OUT/lib64 \ $ANDROID_HOST_OUT/nativetest64/vts_vndk_open_libraries_test/vts_vndk_open_libraries_test \ --verbose Change-Id: I0f35d8322045cedc0f1e43b3fb8eb7367c8e39b1 --- Android.bp | 13 ++ open_libraries/vts_vndk_open_libraries_test.py | 162 ++++++++++++++++++++++++ open_libraries/vts_vndk_open_libraries_test.xml | 23 ++++ utils.py | 17 ++- 4 files changed, 206 insertions(+), 9 deletions(-) create mode 100644 open_libraries/vts_vndk_open_libraries_test.py create mode 100644 open_libraries/vts_vndk_open_libraries_test.xml diff --git a/Android.bp b/Android.bp index 9eae205..176da18 100644 --- a/Android.bp +++ b/Android.bp @@ -75,3 +75,16 @@ python_test_host { ], test_config: "files/vts_vndk_files_test.xml", } + +python_test_host { + name: "vts_vndk_open_libraries_test", + defaults: ["vts_vndk_default"], + main: "open_libraries/vts_vndk_open_libraries_test.py", + srcs: [ + "open_libraries/vts_vndk_open_libraries_test.py", + ], + test_suites: [ + "vts-core", + ], + test_config: "open_libraries/vts_vndk_open_libraries_test.xml", +} diff --git a/open_libraries/vts_vndk_open_libraries_test.py b/open_libraries/vts_vndk_open_libraries_test.py new file mode 100644 index 0000000..aac1b66 --- /dev/null +++ b/open_libraries/vts_vndk_open_libraries_test.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import os +import posixpath as target_path_module +import re +import unittest + +from vts.testcases.vndk import utils +from vts.testcases.vndk.golden import vndk_data +from vts.utils.python.vndk import vndk_utils + + +class VtsVndkOpenLibrariesTest(unittest.TestCase): + """A test module to verify libraries opened by running processes. + + Attributes: + _dut: The AndroidDevice under test. + """ + + def setUp(self): + """Initializes attributes.""" + serial_number = os.environ.get("ANDROID_SERIAL") + self.assertTrue(serial_number, "$ANDROID_SERIAL is empty.") + self._dut = utils.AndroidDevice(serial_number) + + def _ListProcessCommands(self, cmd_filter): + """Finds current processes whose commands match the filter. + + Args: + cmd_filter: A function that takes a binary file path as argument + and returns whether the path matches the condition. + + Returns: + A dict of {pid: command} where pid and command are strings. + """ + ps_cmd = ["ps", "-Aw", "-o", "PID,COMMAND"] + out, err, return_code = self._dut.Execute(*ps_cmd) + if err.strip(): + logging.info("`%s` stderr: %s", " ".join(ps_cmd), err) + self.assertEqual(return_code, 0) + + lines = out.split("\n") + pid_end = lines[0].index("PID") + len("PID") + cmd_begin = lines[0].index("COMMAND", pid_end) + cmds = {} + for line in lines[1:]: + cmd = line[cmd_begin:] + if not cmd_filter(cmd): + continue + pid = line[:pid_end].lstrip() + cmds[pid] = cmd + return cmds + + def _ListOpenFiles(self, pids, file_filter): + """Finds open files whose names match the filter. + + Args: + pids: A collection of strings, the PIDs to list open files. + file_filter: A function that takes a file path as argument and + returns whether the path matches the condition. + + Returns: + A dict of {pid: [file, ...]} where pid and file are strings. + """ + lsof_cmd = ["lsof", "-p", ",".join(pids)] + out, err, return_code = self._dut.Execute(*lsof_cmd) + if err.strip(): + logging.info("`%s` stderr: %s", " ".join(lsof), err) + self.assertEqual(return_code, 0) + # The first line consists of the column names: + # COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME + # PID is right-justified. NAME is left-justified. + lines = out.split("\n") + pid_end = lines[0].index("PID") + len("PID") + name_begin = lines[0].index("NAME") + files = {} + for line in lines[1:]: + if not line.strip(): + continue + # On Android, COMMAND may exceed the column and causes the right + # columns to be misaligned. This program looks for digits in the + # PID column or on the right of the column. + try: + match_pid = next(match for match in + re.finditer(r"\s(\d+)\s", line) if + match.end(1) >= pid_end) + except StopIteration: + self.fail("Cannot parse PID from lsof output: " + line) + offset = match_pid.end(1) - pid_end + self.assertEqual(line[name_begin + offset - 1], " ", + "Cannot parse NAME from lsof output: " + line) + name = line[name_begin + offset:] + if not file_filter(name): + continue + pid = match_pid.group(1) + if pid in files: + files[pid].append(name) + else: + files[pid] = [name] + return files + + def testVendorProcessOpenLibraries(self): + """Checks if vendor processes load shared libraries on system.""" + if not vndk_utils.IsVndkRuntimeEnforced(self._dut): + logging.info("Skip the test as VNDK runtime is not enforced on " + "the device.") + return + vndk_lists = vndk_data.LoadVndkLibraryListsFromResources( + self._dut.GetVndkVersion(), + vndk_data.LL_NDK, + vndk_data.LL_NDK_PRIVATE, + vndk_data.VNDK, + vndk_data.VNDK_PRIVATE, + vndk_data.VNDK_SP, + vndk_data.VNDK_SP_PRIVATE) + self.assertTrue(vndk_lists, "Cannot load VNDK library lists.") + allowed_libs = set().union(*vndk_lists) + logging.debug("Allowed system libraries: %s", allowed_libs) + + self.assertTrue(self._dut.IsRoot(), + "Must be root to find all libraries in use.") + cmds = self._ListProcessCommands(lambda x: (x.startswith("/odm/") or + x.startswith("/vendor/"))) + + def _IsDisallowedSystemLib(lib_path): + return ((lib_path.startswith("/system/") or + lib_path.startswith("/apex/")) and + lib_path.endswith(".so") and + target_path_module.basename(lib_path) not in allowed_libs) + + deps = self._ListOpenFiles(cmds.keys(), _IsDisallowedSystemLib) + if deps: + error_lines = ["%s %s %s" % (pid, cmds[pid], libs) + for pid, libs in deps.items()] + logging.error("pid command libraries\n%s", "\n".join(error_lines)) + + assert_lines = ["pid command libraries"] + error_lines[:20] + if len(deps) > 20: + assert_lines.append("...") + assert_lines.append("Number of vendor processes using system " + "libraries: " + str(len(deps))) + self.fail("\n".join(assert_lines)) + + +if __name__ == "__main__": + unittest.main() diff --git a/open_libraries/vts_vndk_open_libraries_test.xml b/open_libraries/vts_vndk_open_libraries_test.xml new file mode 100644 index 0000000..664f60e --- /dev/null +++ b/open_libraries/vts_vndk_open_libraries_test.xml @@ -0,0 +1,23 @@ + + + + + + + + diff --git a/utils.py b/utils.py index 0592bfa..38efdce 100644 --- a/utils.py +++ b/utils.py @@ -32,7 +32,7 @@ class AndroidDevice(object): subprocess.check_call(cmd, shell=False, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - def _ExecuteCommand(self, *args): + def Execute(self, *args): """Executes a command. Args: @@ -66,7 +66,7 @@ class AndroidDevice(object): Raises: IOError if the command fails. """ - out, err, return_code = self._ExecuteCommand("getprop", name) + out, err, return_code = self.Execute("getprop", name) if err.strip() or return_code != 0: raise IOError("`getprop %s` stdout: %s\nstderr: %s" % (name, out, err)) @@ -143,14 +143,14 @@ class AndroidDevice(object): def IsRoot(self): """Returns whether adb has root privilege on the device.""" - out, err, return_code = self._ExecuteCommand("id") + out, err, return_code = self.Execute("id") if err.strip() or return_code != 0: raise IOError("`id` stdout: %s\nstderr: %s \n" % (out, err)) return "uid=0(root)" in out.strip() def _Test(self, *args): """Tests file types and status.""" - out, err, return_code = self._ExecuteCommand("test", *args) + out, err, return_code = self.Execute("test", *args) if out.strip() or err.strip(): raise IOError("`test` args: %s\nstdout: %s\nstderr: %s" % (args, out, err)) @@ -166,8 +166,7 @@ class AndroidDevice(object): def _Stat(self, fmt, path): """Executes stat command.""" - out, err, return_code = self._ExecuteCommand("stat", "--format", fmt, - path) + out, err, return_code = self.Execute("stat", "--format", fmt, path) if return_code != 0 or err.strip(): raise IOError("`stat --format %s %s` stdout: %s\nstderr: %s" % (fmt, path, out, err)) @@ -194,9 +193,9 @@ class AndroidDevice(object): """ if '"' in name_pattern or "'" in name_pattern: raise ValueError("File name pattern contains quotes.") - out, err, return_code = self._ExecuteCommand("find", path, "-name", - "'" + name_pattern + "'", - *options) + out, err, return_code = self.Execute("find", path, "-name", + "'" + name_pattern + "'", + *options) if return_code != 0 or err.strip(): raise IOError("`find %s -name '%s' %s` stdout: %s\nstderr: %s" % (path, name_pattern, " ".join(options), out, err)) -- cgit v1.2.3 From a75e7230754d8f1a5cc54187c8c928ae2f13fa0c Mon Sep 17 00:00:00 2001 From: Hsin-Yi Chen Date: Fri, 21 Feb 2020 22:57:15 +0800 Subject: Add vts_vndk_abi_test This commit copies VtsVndkAbiTest.py to vts_vndk_abi_test.py, removes the dependency on VTS framework, and adds a python_host_test module for the new file. The module also includes the dump data converted from prebuilts/abi-dumps/vndk. Bug: 147454897 Test: ANDROID_SERIAL=1234 \ LD_LIBRARY_PATH=$ANDROID_HOST_OUT/lib64 \ $ANDROID_HOST_OUT/nativetest64/vts_vndk_abi_test/vts_vndk_abi_test \ --verbose Test: LD_LIBRARY_PATH=$ANDROID_HOST_OUT/lib64 atest vts_vndk_abi_test Change-Id: Ie9fbe764d6b57e2c10fad8541bd6effe7c5c34b6 --- Android.bp | 16 ++ abi/vts_vndk_abi_test.py | 361 ++++++++++++++++++++++++++++++++++++++++++++++ abi/vts_vndk_abi_test.xml | 22 +++ golden/Android.bp | 10 ++ golden/extract_lsdump.py | 53 ++++++- golden/vndk_data.py | 82 ++++++++++- utils.py | 48 ++++++ 7 files changed, 581 insertions(+), 11 deletions(-) create mode 100644 abi/vts_vndk_abi_test.py create mode 100644 abi/vts_vndk_abi_test.xml diff --git a/Android.bp b/Android.bp index 176da18..7d6abec 100644 --- a/Android.bp +++ b/Android.bp @@ -50,6 +50,22 @@ python_defaults { } } +python_test_host { + name: "vts_vndk_abi_test", + defaults: ["vts_vndk_default"], + main: "abi/vts_vndk_abi_test.py", + srcs: [ + "abi/vts_vndk_abi_test.py", + ], + data: [ + ":vts_vndk_abi_dump_zip", + ], + test_suites: [ + "vts-core", + ], + test_config: "abi/vts_vndk_abi_test.xml", +} + python_test_host { name: "vts_vndk_dependency_test", defaults: ["vts_vndk_default"], diff --git a/abi/vts_vndk_abi_test.py b/abi/vts_vndk_abi_test.py new file mode 100644 index 0000000..56a5220 --- /dev/null +++ b/abi/vts_vndk_abi_test.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import logging +import os +import shutil +import tempfile +import unittest + +from vts.testcases.vndk import utils +from vts.testcases.vndk.golden import vndk_data +from vts.utils.python.library import elf_parser +from vts.utils.python.library.vtable import vtable_dumper +from vts.utils.python.vndk import vndk_utils + + +class VtsVndkAbiTest(unittest.TestCase): + """A test module to verify ABI compliance of vendor libraries. + + Attributes: + _dut: the AndroidDevice under test. + _temp_dir: The temporary directory for libraries copied from device. + """ + + def setUp(self): + """Initializes data file path, device, and temporary directory.""" + serial_number = os.environ.get("ANDROID_SERIAL", "") + self.assertTrue(serial_number, "$ANDROID_SERIAL is empty") + self._dut = utils.AndroidDevice(serial_number) + self._temp_dir = tempfile.mkdtemp() + + def tearDown(self): + """Deletes the temporary directory.""" + logging.info("Delete %s", self._temp_dir) + shutil.rmtree(self._temp_dir) + + def _PullOrCreateDir(self, target_dir, host_dir): + """Copies a directory from device. Creates an empty one if not exist. + + Args: + target_dir: The directory to copy from device. + host_dir: The directory to copy to host. + """ + if not self._dut.IsDirectory(target_dir): + logging.info("%s doesn't exist. Create %s.", target_dir, host_dir) + os.makedirs(host_dir) + return + parent_dir = os.path.dirname(host_dir) + if parent_dir and not os.path.isdir(parent_dir): + os.makedirs(parent_dir) + logging.info("adb pull %s %s", target_dir, host_dir) + self._dut.AdbPull(target_dir, host_dir) + + def _ToHostPath(self, target_path): + """Maps target path to host path in self._temp_dir.""" + return os.path.join(self._temp_dir, *target_path.strip("/").split("/")) + + @staticmethod + def _LoadGlobalSymbolsFromDump(dump_obj): + """Loads global symbols from a dump object. + + Args: + dump_obj: A dict, the dump in JSON format. + + Returns: + A set of strings, the symbol names. + """ + symbols = set() + for key in ("elf_functions", "elf_objects"): + symbols.update( + symbol.get("name", "") for symbol in dump_obj.get(key, []) if + symbol.get("binding", "global") == "global") + return symbols + + def _DiffElfSymbols(self, dump_obj, parser): + """Checks if a library includes all symbols in a dump. + + Args: + dump_obj: A dict, the dump in JSON format. + parser: An elf_parser.ElfParser that loads the library. + + Returns: + A list of strings, the global symbols that are in the dump but not + in the library. + + Raises: + elf_parser.ElfError if fails to load the library. + """ + dump_symbols = self._LoadGlobalSymbolsFromDump(dump_obj) + lib_symbols = parser.ListGlobalDynamicSymbols(include_weak=True) + return sorted(dump_symbols.difference(lib_symbols)) + + @staticmethod + def _DiffVtableComponent(offset, expected_symbol, vtable): + """Checks if a symbol is in a vtable entry. + + Args: + offset: An integer, the offset of the expected symbol. + expected_symbol: A string, the name of the expected symbol. + vtable: A dict of {offset: [entry]} where offset is an integer and + entry is an instance of vtable_dumper.VtableEntry. + + Returns: + A list of strings, the actual possible symbols if expected_symbol + does not match the vtable entry. + None if expected_symbol matches the entry. + """ + if offset not in vtable: + return [] + + entry = vtable[offset] + if not entry.names: + return [hex(entry.value).rstrip('L')] + + if expected_symbol not in entry.names: + return entry.names + + def _DiffVtableComponents(self, dump_obj, dumper): + """Checks if a library includes all vtable entries in a dump. + + Args: + dump_obj: A dict, the dump in JSON format. + dumper: An vtable_dumper.VtableDumper that loads the library. + bitness: 32 or 64, the size of the vtable entries. + + Returns: + A list of tuples (VTABLE, OFFSET, EXPECTED_SYMBOL, ACTUAL). + ACTUAL can be "missing", a list of symbol names, or an ELF virtual + address. + + Raises: + vtable_dumper.VtableError if fails to dump vtable from the library. + """ + function_kinds = [ + "function_pointer", + "complete_dtor_pointer", + "deleting_dtor_pointer" + ] + non_function_kinds = [ + "vcall_offset", + "vbase_offset", + "offset_to_top", + "rtti", + "unused_function_pointer" + ] + default_vtable_component_kind = "function_pointer" + + global_symbols = self._LoadGlobalSymbolsFromDump(dump_obj) + + lib_vtables = {vtable.name: vtable + for vtable in dumper.DumpVtables()} + logging.debug("\n\n".join(str(vtable) + for _, vtable in lib_vtables.items())) + + vtables_diff = [] + for record_type in dump_obj.get("record_types", []): + # Since Android R, unique_id has been replaced with linker_set_key. + # unique_id starts with "_ZTI"; linker_set_key starts with "_ZTS". + type_name_symbol = record_type.get("unique_id", "") + if type_name_symbol: + vtable_symbol = type_name_symbol.replace("_ZTS", "_ZTV", 1) + else: + type_name_symbol = record_type.get("linker_set_key", "") + vtable_symbol = type_name_symbol.replace("_ZTI", "_ZTV", 1) + + # Skip if the vtable symbol isn't global. + if vtable_symbol not in global_symbols: + continue + + # Collect vtable entries from library dump. + if vtable_symbol in lib_vtables: + lib_vtable = {entry.offset: entry + for entry in lib_vtables[vtable_symbol].entries} + else: + lib_vtable = dict() + + for index, entry in enumerate(record_type.get("vtable_components", + [])): + entry_offset = index * dumper.bitness // 8 + entry_kind = entry.get("kind", default_vtable_component_kind) + entry_symbol = entry.get("mangled_component_name", "") + entry_is_pure = entry.get("is_pure", False) + + if entry_kind in non_function_kinds: + continue + + if entry_kind not in function_kinds: + logging.warning("%s: Unexpected vtable entry kind %s", + vtable_symbol, entry_kind) + + if entry_symbol not in global_symbols: + # Itanium cxx abi doesn't specify pure virtual vtable + # entry's behaviour. However we can still do some checks + # based on compiler behaviour. + # Even though we don't check weak symbols, we can still + # issue a warning when a pure virtual function pointer + # is missing. + if entry_is_pure and entry_offset not in lib_vtable: + logging.warning("%s: Expected pure virtual function" + "in %s offset %s", + vtable_symbol, vtable_symbol, + entry_offset) + continue + + diff_symbols = self._DiffVtableComponent( + entry_offset, entry_symbol, lib_vtable) + if diff_symbols is None: + continue + + vtables_diff.append( + (vtable_symbol, str(entry_offset), entry_symbol, + (",".join(diff_symbols) if diff_symbols else "missing"))) + + return vtables_diff + + def _ScanLibDirs(self, dump_zip, dump_paths, lib_dirs, dump_version): + """Compares dump files with libraries copied from device. + + Args: + dump_zip: A zip_file.ZipFile object containing the dumps. + dump_paths: A dict of {library name: dump resource path}. + lib_dirs: The list of directories containing libraries. + dump_version: The VNDK version of the dump files. If the device has + no VNDK version or has extension in vendor partition, + this method compares the unversioned VNDK directories + with the dump directories of the given version. + + Returns: + A list of strings, the incompatible libraries. + """ + error_list = [] + lib_paths = dict() + for lib_dir in lib_dirs: + for parent_dir, dir_names, lib_names in os.walk(lib_dir): + for lib_name in lib_names: + if lib_name not in lib_paths: + lib_paths[lib_name] = os.path.join(parent_dir, + lib_name) + for lib_name, dump_path in dump_paths.items(): + if lib_name not in lib_paths: + logging.info("%s: Not found on target", lib_name) + continue + lib_path = lib_paths[lib_name] + rel_path = os.path.relpath(lib_path, self._temp_dir) + + has_exception = False + missing_symbols = [] + vtable_diff = [] + + try: + with dump_zip.open(dump_path, "r") as dump_file: + dump_obj = json.load(dump_file) + with vtable_dumper.VtableDumper(lib_path) as dumper: + missing_symbols = self._DiffElfSymbols( + dump_obj, dumper) + vtable_diff = self._DiffVtableComponents( + dump_obj, dumper) + except (IOError, + elf_parser.ElfError, + vtable_dumper.VtableError) as e: + logging.exception("%s: Cannot diff ABI", rel_path) + has_exception = True + + if missing_symbols: + logging.error("%s: Missing Symbols:\n%s", + rel_path, "\n".join(missing_symbols)) + if vtable_diff: + logging.error("%s: Vtable Difference:\n" + "vtable offset expected actual\n%s", + rel_path, + "\n".join(" ".join(e) for e in vtable_diff)) + if (has_exception or missing_symbols or vtable_diff): + error_list.append(rel_path) + else: + logging.info("%s: Pass", rel_path) + return error_list + + @staticmethod + def _GetLinkerSearchIndex(target_path): + """Returns the key for sorting linker search paths.""" + index = 0 + for prefix in ("/odm", "/vendor", "/apex"): + if target_path.startswith(prefix): + return index + index += 1 + return index + + def _TestAbiCompatibility(self, bitness): + """Checks ABI compliance of VNDK libraries. + + Args: + bitness: 32 or 64, the bitness of the tested libraries. + """ + self.assertTrue(self._dut.IsRoot(), "This test requires adb root.") + primary_abi = self._dut.GetCpuAbiList()[0] + binder_bitness = self._dut.GetBinderBitness() + self.assertTrue(binder_bitness, "Cannot determine binder bitness.") + dump_version = self._dut.GetVndkVersion() + self.assertTrue(dump_version, "Cannot determine VNDK version.") + + dump_paths = vndk_data.GetAbiDumpPathsFromResources( + dump_version, + binder_bitness, + primary_abi, + bitness) + self.assertTrue( + dump_paths, + "No dump files. version: %s ABI: %s bitness: %d" % ( + dump_version, primary_abi, bitness)) + + target_dirs = vndk_utils.GetVndkExtDirectories(bitness) + target_dirs += vndk_utils.GetVndkSpExtDirectories(bitness) + target_dirs += [vndk_utils.GetVndkDirectory(bitness, dump_version)] + target_dirs.sort(key=self._GetLinkerSearchIndex) + + host_dirs = [self._ToHostPath(x) for x in target_dirs] + for target_dir, host_dir in zip(target_dirs, host_dirs): + self._PullOrCreateDir(target_dir, host_dir) + + with vndk_data.AbiDumpResource() as dump_resource: + assert_lines = self._ScanLibDirs(dump_resource.zip_file, + dump_paths, host_dirs, + dump_version) + + if assert_lines: + error_count = len(assert_lines) + if error_count > 20: + assert_lines = assert_lines[:20] + ["..."] + assert_lines.append("Total number of errors: " + str(error_count)) + self.fail("\n".join(assert_lines)) + + def testAbiCompatibility32(self): + """Checks ABI compliance of 32-bit VNDK libraries.""" + self._TestAbiCompatibility(32) + + def testAbiCompatibility64(self): + """Checks ABI compliance of 64-bit VNDK libraries.""" + if self._dut.GetCpuAbiList(64): + self._TestAbiCompatibility(64) + else: + logging.info("Skip the test as the device doesn't support 64-bit " + "ABI.") + + +if __name__ == "__main__": + unittest.main() diff --git a/abi/vts_vndk_abi_test.xml b/abi/vts_vndk_abi_test.xml new file mode 100644 index 0000000..7fc7321 --- /dev/null +++ b/abi/vts_vndk_abi_test.xml @@ -0,0 +1,22 @@ + + + + + + + diff --git a/golden/Android.bp b/golden/Android.bp index 20222e1..c9abdaa 100644 --- a/golden/Android.bp +++ b/golden/Android.bp @@ -26,3 +26,13 @@ python_binary_host { }, } } + +// TODO(b/150663999): Replace with gensrcs when the build system is able to +// process the large file group. +genrule { + name: "vts_vndk_abi_dump_zip", + tools: ["extract_lsdump"], + cmd: "$(location extract_lsdump) $(in) $(out)", + srcs: [":vndk_abi_dump_zip"], + out: ["vts/testcases/vndk/abi_dump.zip"], +} diff --git a/golden/extract_lsdump.py b/golden/extract_lsdump.py index a6d95c7..fd42be6 100755 --- a/golden/extract_lsdump.py +++ b/golden/extract_lsdump.py @@ -19,6 +19,7 @@ import argparse import gzip import json import os +import zipfile class LsdumpError(Exception): @@ -205,35 +206,62 @@ def _ParseVtablesFromLsdump(lsdump, output_dump): output_dump.record_types.append(record_type) -def ParseLsdumpFile(input_path, output_path): - """Converts an lsdump file to a dump file for the ABI test. +def _ParseLsdumpFileObject(lsdump_file): + """Converts an lsdump file to a dump object. Args: - input_path: The path to the (gzipped) lsdump file. - output_path: The path to the output dump file. + lsdump_file: The file object of the input lsdump. + + Returns: + The AttrDict object converted from the lsdump file. Raises: LsdumpError if fails to create the dump file. """ try: - with _OpenFileOrGzipped(input_path) as lsdump_file: - lsdump = json.load(lsdump_file, object_hook=AttrDict) - except (IOError, ValueError) as e: + lsdump = json.load(lsdump_file, object_hook=AttrDict) + except ValueError: raise LsdumpError(e) try: output_dump = AttrDict() _ParseVtablesFromLsdump(lsdump, output_dump) _ParseSymbolsFromLsdump(lsdump, output_dump) + return output_dump except AttributeError as e: raise LsdumpError(e) + +def _ParseLsdumpZipFile(lsdump_zip, output_zip): + """Converts zipped lsdump files to the dump files for ABI test.""" + for name in lsdump_zip.namelist(): + if name.endswith(".lsdump"): + with lsdump_zip.open(name, mode="r") as lsdump_file: + output_dump = _ParseLsdumpFileObject(lsdump_file) + output_str = json.dumps(output_dump, indent=1, + separators=(',', ':')) + output_zip.writestr(name, output_str) + + +def ParseLsdumpFile(input_path, output_path): + """Converts an lsdump file to a dump file for the ABI test. + + Args: + input_path: The path to the (gzipped) lsdump file. + output_path: The path to the output dump file. + + Raises: + LsdumpError if fails to create the dump file. + """ abs_output_path = os.path.abspath(output_path) abs_output_dir = os.path.dirname(abs_output_path) try: if abs_output_dir and not os.path.exists(abs_output_dir): os.makedirs(abs_output_dir) + + with _OpenFileOrGzipped(input_path) as lsdump_file: + output_dump = _ParseLsdumpFileObject(lsdump_file) with open(output_path, 'wb') as output_file: json.dump(output_dump, output_file, indent=1, separators=(',', ':')) @@ -251,6 +279,17 @@ def main(): help='output dump file path.') args = arg_parser.parse_args() + # TODO(b/150663999): Remove this when the build system is able to process + # the large file group. + if zipfile.is_zipfile(args.input_path): + with zipfile.ZipFile(args.input_path, mode='r') as input_zip: + # The zip will be added to a Python package. It is not necessary + # to reduce the file size. + with zipfile.ZipFile(args.output_path, mode='w', + compression=zipfile.ZIP_STORED) as output_zip: + _ParseLsdumpZipFile(input_zip, output_zip) + exit(0) + try: ParseLsdumpFile(args.input_path, args.output_path) except LsdumpError as e: diff --git a/golden/vndk_data.py b/golden/vndk_data.py index 4eb127d..ffa1527 100644 --- a/golden/vndk_data.py +++ b/golden/vndk_data.py @@ -15,9 +15,11 @@ # import collections +import json import logging import os import re +import zipfile try: from importlib import resources @@ -47,9 +49,16 @@ VNDK_SP = "VNDK-SP" # VNDK-SP dependencies that vendor modules cannot directly access. VNDK_SP_PRIVATE = "VNDK-SP-private" -# The ABI dump directories. 64-bit comes before 32-bit in order to sequentially -# search for longest prefix. -_ABI_NAMES = ("arm64", "arm", "mips64", "mips", "x86_64", "x86") +# The tuples of (ABI name, bitness, arch name). 64-bit comes before 32-bit in +# order to sequentially search for longest prefix. +_ABI_LIST = ( + ("arm64", 64, "arm64_armv8-a"), + ("arm64", 32, "arm_armv8-a"), + ("arm", 32, "arm_armv7-a-neon"), + ("x86_64", 64, "x86_x86_64"), + ("x86_64", 32, "x86_64"), + ("x86", 32, "x86"), +) # The data directory. _GOLDEN_DIR = os.path.join("vts", "testcases", "vndk", "golden") @@ -57,6 +66,9 @@ _GOLDEN_DIR = os.path.join("vts", "testcases", "vndk", "golden") # The data package. _RESOURCE_PACKAGE = "vts.testcases.vndk"; +# The name of the zip file containing ABI dumps. +_ABI_DUMP_ZIP_NAME = "abi_dump.zip" + # Regular expression prefix for library name patterns. _REGEX_PREFIX = "[regex]" @@ -95,7 +107,7 @@ def GetAbiDumpDirectory(data_file_path, version, binder_bitness, abi_name, None if there is no directory for the version and ABI. """ try: - abi_dir = next(x for x in _ABI_NAMES if abi_name.startswith(x)) + abi_dir = next(x[0] for x in _ABI_LIST if abi_name.startswith(x[0])) except StopIteration: logging.warning("Unknown ABI %s.", abi_name) return None @@ -117,6 +129,68 @@ def GetAbiDumpDirectory(data_file_path, version, binder_bitness, abi_name, return dump_dir +class AbiDumpResource: + """The class for loading ABI dumps from the zip in resources.""" + + def __init__(self): + self._resource = None + self.zip_file = None + + def __enter__(self): + self._resource = resources.open_binary(_RESOURCE_PACKAGE, + _ABI_DUMP_ZIP_NAME) + self.zip_file = zipfile.ZipFile(self._resource, "r") + return self + + def __exit__(self, exc_type, exc_val, traceback): + if self._resource: + self._resource.close() + if self.zip_file: + self.zip_file.close() + + +def GetAbiDumpPathsFromResources(version, binder_bitness, abi_name, abi_bitness): + """Returns the VNDK dump paths in resources. + + Args: + version: A string, the VNDK version. + binder_bitness: A string or an integer, 32 or 64. + abi_name: A string, the ABI of the library dump. + abi_bitness: A string or an integer, 32 or 64. + + Returns: + A dict of {library name: dump resource path}. For example, + {"libbase.so": "R/64/arm64_armv8-a/source-based/libbase.so.lsdump"}. + If there is no dump for the version and ABI, this function returns an + empty dict. + """ + if not resources: + logging.error("Could not import resources module.") + return dict() + + abi_bitness = int(abi_bitness) + try: + arch_name = next(x[2] for x in _ABI_LIST if + abi_name.startswith(x[0]) and x[1] == abi_bitness) + except StopIteration: + logging.warning("Unknown %d-bit ABI %s.", abi_bitness, abi_name) + return dict() + + # The separator in zipped path is always "/". + dump_dir = "/".join((version, str(binder_bitness), arch_name, + "source-based")) + "/" + + dump_paths = dict() + + with AbiDumpResource() as dump_resource: + for path in dump_resource.zip_file.namelist(): + if path.startswith(dump_dir) and path.endswith(".lsdump"): + lib_name = path[len(dump_dir):-len(".lsdump")] + dump_paths[lib_name] = path + + return dump_paths + + def _LoadVndkLibraryListsFile(vndk_lists, tags, vndk_lib_list_file): """Load VNDK libraries from the file to the specified tuple. diff --git a/utils.py b/utils.py index 38efdce..65b2eee 100644 --- a/utils.py +++ b/utils.py @@ -18,8 +18,11 @@ # TODO(b/147454897): Keep the logic in sync with # test/vts/utils/python/controllers/android_device.py until # it is removed. +import gzip import logging +import os import subprocess +import tempfile class AndroidDevice(object): """This class controls the device via adb commands.""" @@ -141,6 +144,51 @@ class AndroidDevice(object): """Gets the VNDK version that the vendor partition requests.""" return self._GetProp("ro.vndk.version") + def GetKernelConfig(self, config_name): + """Gets kernel config from the device. + + Args: + config_name: A string, the name of the configuration. + + Returns: + "y" or "m" if the config is set. + "" if the config is not set. + None if fails to read config. + """ + line_prefix = config_name + "=" + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + config_path = temp_file.name + try: + logging.debug("Pull config.gz to %s", config_path) + self.AdbPull("/proc/config.gz", config_path) + with gzip.open(config_path, "rt") as config_file: + for line in config_file: + if line.strip().startswith(line_prefix): + logging.debug("Found config: %s", line) + return line.strip()[len(line_prefix):] + logging.debug("%s is not set.", config_name) + return "" + except (subprocess.CalledProcessError, IOError) as e: + logging.exception("Cannot read kernel config.", e) + return None + finally: + os.remove(config_path) + + def GetBinderBitness(self): + """Returns the value of BINDER_IPC_32BIT in kernel config. + + Returns: + 32 or 64, binder bitness of the device. + None if fails to read config. + """ + config_value = self.GetKernelConfig("CONFIG_ANDROID_BINDER_IPC_32BIT") + if config_value is None: + return None + elif config_value: + return 32 + else: + return 64 + def IsRoot(self): """Returns whether adb has root privilege on the device.""" out, err, return_code = self.Execute("id") -- cgit v1.2.3 From 97a68c9b6f25f67fc009afa3c31d828683c9b62c Mon Sep 17 00:00:00 2001 From: Bonian Chen Date: Wed, 11 Mar 2020 08:48:53 +0000 Subject: Revert "Add vts_vndk_abi_test" Revert "Add a module for all VNDK ABI dumps" Revert submission 1250008-vts_vndk_abi_test Reason for revert: Droidcop-triggered revert due to breakage https://android-build.googleplex.com/builds/quarterdeck?branch=git_qt-qpr1-dev-plus-aosp&target=aosp_merge_testing&lkgb=6283938&lkbb=6284140&fkbb=6283952, bug b/151200159 Reverted Changes: Ie9fbe764d:Add vts_vndk_abi_test I3c7adba97:Add a module for all VNDK ABI dumps Change-Id: Ifa346df3aaf950cc0c61a07d7eb6701ace290e09 BUG: 151200159 --- Android.bp | 16 -- abi/vts_vndk_abi_test.py | 361 ---------------------------------------------- abi/vts_vndk_abi_test.xml | 22 --- golden/Android.bp | 10 -- golden/extract_lsdump.py | 53 +------ golden/vndk_data.py | 82 +---------- utils.py | 48 ------ 7 files changed, 11 insertions(+), 581 deletions(-) delete mode 100644 abi/vts_vndk_abi_test.py delete mode 100644 abi/vts_vndk_abi_test.xml diff --git a/Android.bp b/Android.bp index 7d6abec..176da18 100644 --- a/Android.bp +++ b/Android.bp @@ -50,22 +50,6 @@ python_defaults { } } -python_test_host { - name: "vts_vndk_abi_test", - defaults: ["vts_vndk_default"], - main: "abi/vts_vndk_abi_test.py", - srcs: [ - "abi/vts_vndk_abi_test.py", - ], - data: [ - ":vts_vndk_abi_dump_zip", - ], - test_suites: [ - "vts-core", - ], - test_config: "abi/vts_vndk_abi_test.xml", -} - python_test_host { name: "vts_vndk_dependency_test", defaults: ["vts_vndk_default"], diff --git a/abi/vts_vndk_abi_test.py b/abi/vts_vndk_abi_test.py deleted file mode 100644 index 56a5220..0000000 --- a/abi/vts_vndk_abi_test.py +++ /dev/null @@ -1,361 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2020 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import logging -import os -import shutil -import tempfile -import unittest - -from vts.testcases.vndk import utils -from vts.testcases.vndk.golden import vndk_data -from vts.utils.python.library import elf_parser -from vts.utils.python.library.vtable import vtable_dumper -from vts.utils.python.vndk import vndk_utils - - -class VtsVndkAbiTest(unittest.TestCase): - """A test module to verify ABI compliance of vendor libraries. - - Attributes: - _dut: the AndroidDevice under test. - _temp_dir: The temporary directory for libraries copied from device. - """ - - def setUp(self): - """Initializes data file path, device, and temporary directory.""" - serial_number = os.environ.get("ANDROID_SERIAL", "") - self.assertTrue(serial_number, "$ANDROID_SERIAL is empty") - self._dut = utils.AndroidDevice(serial_number) - self._temp_dir = tempfile.mkdtemp() - - def tearDown(self): - """Deletes the temporary directory.""" - logging.info("Delete %s", self._temp_dir) - shutil.rmtree(self._temp_dir) - - def _PullOrCreateDir(self, target_dir, host_dir): - """Copies a directory from device. Creates an empty one if not exist. - - Args: - target_dir: The directory to copy from device. - host_dir: The directory to copy to host. - """ - if not self._dut.IsDirectory(target_dir): - logging.info("%s doesn't exist. Create %s.", target_dir, host_dir) - os.makedirs(host_dir) - return - parent_dir = os.path.dirname(host_dir) - if parent_dir and not os.path.isdir(parent_dir): - os.makedirs(parent_dir) - logging.info("adb pull %s %s", target_dir, host_dir) - self._dut.AdbPull(target_dir, host_dir) - - def _ToHostPath(self, target_path): - """Maps target path to host path in self._temp_dir.""" - return os.path.join(self._temp_dir, *target_path.strip("/").split("/")) - - @staticmethod - def _LoadGlobalSymbolsFromDump(dump_obj): - """Loads global symbols from a dump object. - - Args: - dump_obj: A dict, the dump in JSON format. - - Returns: - A set of strings, the symbol names. - """ - symbols = set() - for key in ("elf_functions", "elf_objects"): - symbols.update( - symbol.get("name", "") for symbol in dump_obj.get(key, []) if - symbol.get("binding", "global") == "global") - return symbols - - def _DiffElfSymbols(self, dump_obj, parser): - """Checks if a library includes all symbols in a dump. - - Args: - dump_obj: A dict, the dump in JSON format. - parser: An elf_parser.ElfParser that loads the library. - - Returns: - A list of strings, the global symbols that are in the dump but not - in the library. - - Raises: - elf_parser.ElfError if fails to load the library. - """ - dump_symbols = self._LoadGlobalSymbolsFromDump(dump_obj) - lib_symbols = parser.ListGlobalDynamicSymbols(include_weak=True) - return sorted(dump_symbols.difference(lib_symbols)) - - @staticmethod - def _DiffVtableComponent(offset, expected_symbol, vtable): - """Checks if a symbol is in a vtable entry. - - Args: - offset: An integer, the offset of the expected symbol. - expected_symbol: A string, the name of the expected symbol. - vtable: A dict of {offset: [entry]} where offset is an integer and - entry is an instance of vtable_dumper.VtableEntry. - - Returns: - A list of strings, the actual possible symbols if expected_symbol - does not match the vtable entry. - None if expected_symbol matches the entry. - """ - if offset not in vtable: - return [] - - entry = vtable[offset] - if not entry.names: - return [hex(entry.value).rstrip('L')] - - if expected_symbol not in entry.names: - return entry.names - - def _DiffVtableComponents(self, dump_obj, dumper): - """Checks if a library includes all vtable entries in a dump. - - Args: - dump_obj: A dict, the dump in JSON format. - dumper: An vtable_dumper.VtableDumper that loads the library. - bitness: 32 or 64, the size of the vtable entries. - - Returns: - A list of tuples (VTABLE, OFFSET, EXPECTED_SYMBOL, ACTUAL). - ACTUAL can be "missing", a list of symbol names, or an ELF virtual - address. - - Raises: - vtable_dumper.VtableError if fails to dump vtable from the library. - """ - function_kinds = [ - "function_pointer", - "complete_dtor_pointer", - "deleting_dtor_pointer" - ] - non_function_kinds = [ - "vcall_offset", - "vbase_offset", - "offset_to_top", - "rtti", - "unused_function_pointer" - ] - default_vtable_component_kind = "function_pointer" - - global_symbols = self._LoadGlobalSymbolsFromDump(dump_obj) - - lib_vtables = {vtable.name: vtable - for vtable in dumper.DumpVtables()} - logging.debug("\n\n".join(str(vtable) - for _, vtable in lib_vtables.items())) - - vtables_diff = [] - for record_type in dump_obj.get("record_types", []): - # Since Android R, unique_id has been replaced with linker_set_key. - # unique_id starts with "_ZTI"; linker_set_key starts with "_ZTS". - type_name_symbol = record_type.get("unique_id", "") - if type_name_symbol: - vtable_symbol = type_name_symbol.replace("_ZTS", "_ZTV", 1) - else: - type_name_symbol = record_type.get("linker_set_key", "") - vtable_symbol = type_name_symbol.replace("_ZTI", "_ZTV", 1) - - # Skip if the vtable symbol isn't global. - if vtable_symbol not in global_symbols: - continue - - # Collect vtable entries from library dump. - if vtable_symbol in lib_vtables: - lib_vtable = {entry.offset: entry - for entry in lib_vtables[vtable_symbol].entries} - else: - lib_vtable = dict() - - for index, entry in enumerate(record_type.get("vtable_components", - [])): - entry_offset = index * dumper.bitness // 8 - entry_kind = entry.get("kind", default_vtable_component_kind) - entry_symbol = entry.get("mangled_component_name", "") - entry_is_pure = entry.get("is_pure", False) - - if entry_kind in non_function_kinds: - continue - - if entry_kind not in function_kinds: - logging.warning("%s: Unexpected vtable entry kind %s", - vtable_symbol, entry_kind) - - if entry_symbol not in global_symbols: - # Itanium cxx abi doesn't specify pure virtual vtable - # entry's behaviour. However we can still do some checks - # based on compiler behaviour. - # Even though we don't check weak symbols, we can still - # issue a warning when a pure virtual function pointer - # is missing. - if entry_is_pure and entry_offset not in lib_vtable: - logging.warning("%s: Expected pure virtual function" - "in %s offset %s", - vtable_symbol, vtable_symbol, - entry_offset) - continue - - diff_symbols = self._DiffVtableComponent( - entry_offset, entry_symbol, lib_vtable) - if diff_symbols is None: - continue - - vtables_diff.append( - (vtable_symbol, str(entry_offset), entry_symbol, - (",".join(diff_symbols) if diff_symbols else "missing"))) - - return vtables_diff - - def _ScanLibDirs(self, dump_zip, dump_paths, lib_dirs, dump_version): - """Compares dump files with libraries copied from device. - - Args: - dump_zip: A zip_file.ZipFile object containing the dumps. - dump_paths: A dict of {library name: dump resource path}. - lib_dirs: The list of directories containing libraries. - dump_version: The VNDK version of the dump files. If the device has - no VNDK version or has extension in vendor partition, - this method compares the unversioned VNDK directories - with the dump directories of the given version. - - Returns: - A list of strings, the incompatible libraries. - """ - error_list = [] - lib_paths = dict() - for lib_dir in lib_dirs: - for parent_dir, dir_names, lib_names in os.walk(lib_dir): - for lib_name in lib_names: - if lib_name not in lib_paths: - lib_paths[lib_name] = os.path.join(parent_dir, - lib_name) - for lib_name, dump_path in dump_paths.items(): - if lib_name not in lib_paths: - logging.info("%s: Not found on target", lib_name) - continue - lib_path = lib_paths[lib_name] - rel_path = os.path.relpath(lib_path, self._temp_dir) - - has_exception = False - missing_symbols = [] - vtable_diff = [] - - try: - with dump_zip.open(dump_path, "r") as dump_file: - dump_obj = json.load(dump_file) - with vtable_dumper.VtableDumper(lib_path) as dumper: - missing_symbols = self._DiffElfSymbols( - dump_obj, dumper) - vtable_diff = self._DiffVtableComponents( - dump_obj, dumper) - except (IOError, - elf_parser.ElfError, - vtable_dumper.VtableError) as e: - logging.exception("%s: Cannot diff ABI", rel_path) - has_exception = True - - if missing_symbols: - logging.error("%s: Missing Symbols:\n%s", - rel_path, "\n".join(missing_symbols)) - if vtable_diff: - logging.error("%s: Vtable Difference:\n" - "vtable offset expected actual\n%s", - rel_path, - "\n".join(" ".join(e) for e in vtable_diff)) - if (has_exception or missing_symbols or vtable_diff): - error_list.append(rel_path) - else: - logging.info("%s: Pass", rel_path) - return error_list - - @staticmethod - def _GetLinkerSearchIndex(target_path): - """Returns the key for sorting linker search paths.""" - index = 0 - for prefix in ("/odm", "/vendor", "/apex"): - if target_path.startswith(prefix): - return index - index += 1 - return index - - def _TestAbiCompatibility(self, bitness): - """Checks ABI compliance of VNDK libraries. - - Args: - bitness: 32 or 64, the bitness of the tested libraries. - """ - self.assertTrue(self._dut.IsRoot(), "This test requires adb root.") - primary_abi = self._dut.GetCpuAbiList()[0] - binder_bitness = self._dut.GetBinderBitness() - self.assertTrue(binder_bitness, "Cannot determine binder bitness.") - dump_version = self._dut.GetVndkVersion() - self.assertTrue(dump_version, "Cannot determine VNDK version.") - - dump_paths = vndk_data.GetAbiDumpPathsFromResources( - dump_version, - binder_bitness, - primary_abi, - bitness) - self.assertTrue( - dump_paths, - "No dump files. version: %s ABI: %s bitness: %d" % ( - dump_version, primary_abi, bitness)) - - target_dirs = vndk_utils.GetVndkExtDirectories(bitness) - target_dirs += vndk_utils.GetVndkSpExtDirectories(bitness) - target_dirs += [vndk_utils.GetVndkDirectory(bitness, dump_version)] - target_dirs.sort(key=self._GetLinkerSearchIndex) - - host_dirs = [self._ToHostPath(x) for x in target_dirs] - for target_dir, host_dir in zip(target_dirs, host_dirs): - self._PullOrCreateDir(target_dir, host_dir) - - with vndk_data.AbiDumpResource() as dump_resource: - assert_lines = self._ScanLibDirs(dump_resource.zip_file, - dump_paths, host_dirs, - dump_version) - - if assert_lines: - error_count = len(assert_lines) - if error_count > 20: - assert_lines = assert_lines[:20] + ["..."] - assert_lines.append("Total number of errors: " + str(error_count)) - self.fail("\n".join(assert_lines)) - - def testAbiCompatibility32(self): - """Checks ABI compliance of 32-bit VNDK libraries.""" - self._TestAbiCompatibility(32) - - def testAbiCompatibility64(self): - """Checks ABI compliance of 64-bit VNDK libraries.""" - if self._dut.GetCpuAbiList(64): - self._TestAbiCompatibility(64) - else: - logging.info("Skip the test as the device doesn't support 64-bit " - "ABI.") - - -if __name__ == "__main__": - unittest.main() diff --git a/abi/vts_vndk_abi_test.xml b/abi/vts_vndk_abi_test.xml deleted file mode 100644 index 7fc7321..0000000 --- a/abi/vts_vndk_abi_test.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - diff --git a/golden/Android.bp b/golden/Android.bp index c9abdaa..20222e1 100644 --- a/golden/Android.bp +++ b/golden/Android.bp @@ -26,13 +26,3 @@ python_binary_host { }, } } - -// TODO(b/150663999): Replace with gensrcs when the build system is able to -// process the large file group. -genrule { - name: "vts_vndk_abi_dump_zip", - tools: ["extract_lsdump"], - cmd: "$(location extract_lsdump) $(in) $(out)", - srcs: [":vndk_abi_dump_zip"], - out: ["vts/testcases/vndk/abi_dump.zip"], -} diff --git a/golden/extract_lsdump.py b/golden/extract_lsdump.py index fd42be6..a6d95c7 100755 --- a/golden/extract_lsdump.py +++ b/golden/extract_lsdump.py @@ -19,7 +19,6 @@ import argparse import gzip import json import os -import zipfile class LsdumpError(Exception): @@ -206,62 +205,35 @@ def _ParseVtablesFromLsdump(lsdump, output_dump): output_dump.record_types.append(record_type) -def _ParseLsdumpFileObject(lsdump_file): - """Converts an lsdump file to a dump object. +def ParseLsdumpFile(input_path, output_path): + """Converts an lsdump file to a dump file for the ABI test. Args: - lsdump_file: The file object of the input lsdump. - - Returns: - The AttrDict object converted from the lsdump file. + input_path: The path to the (gzipped) lsdump file. + output_path: The path to the output dump file. Raises: LsdumpError if fails to create the dump file. """ try: - lsdump = json.load(lsdump_file, object_hook=AttrDict) - except ValueError: + with _OpenFileOrGzipped(input_path) as lsdump_file: + lsdump = json.load(lsdump_file, object_hook=AttrDict) + except (IOError, ValueError) as e: raise LsdumpError(e) try: output_dump = AttrDict() _ParseVtablesFromLsdump(lsdump, output_dump) _ParseSymbolsFromLsdump(lsdump, output_dump) - return output_dump except AttributeError as e: raise LsdumpError(e) - -def _ParseLsdumpZipFile(lsdump_zip, output_zip): - """Converts zipped lsdump files to the dump files for ABI test.""" - for name in lsdump_zip.namelist(): - if name.endswith(".lsdump"): - with lsdump_zip.open(name, mode="r") as lsdump_file: - output_dump = _ParseLsdumpFileObject(lsdump_file) - output_str = json.dumps(output_dump, indent=1, - separators=(',', ':')) - output_zip.writestr(name, output_str) - - -def ParseLsdumpFile(input_path, output_path): - """Converts an lsdump file to a dump file for the ABI test. - - Args: - input_path: The path to the (gzipped) lsdump file. - output_path: The path to the output dump file. - - Raises: - LsdumpError if fails to create the dump file. - """ abs_output_path = os.path.abspath(output_path) abs_output_dir = os.path.dirname(abs_output_path) try: if abs_output_dir and not os.path.exists(abs_output_dir): os.makedirs(abs_output_dir) - - with _OpenFileOrGzipped(input_path) as lsdump_file: - output_dump = _ParseLsdumpFileObject(lsdump_file) with open(output_path, 'wb') as output_file: json.dump(output_dump, output_file, indent=1, separators=(',', ':')) @@ -279,17 +251,6 @@ def main(): help='output dump file path.') args = arg_parser.parse_args() - # TODO(b/150663999): Remove this when the build system is able to process - # the large file group. - if zipfile.is_zipfile(args.input_path): - with zipfile.ZipFile(args.input_path, mode='r') as input_zip: - # The zip will be added to a Python package. It is not necessary - # to reduce the file size. - with zipfile.ZipFile(args.output_path, mode='w', - compression=zipfile.ZIP_STORED) as output_zip: - _ParseLsdumpZipFile(input_zip, output_zip) - exit(0) - try: ParseLsdumpFile(args.input_path, args.output_path) except LsdumpError as e: diff --git a/golden/vndk_data.py b/golden/vndk_data.py index ffa1527..4eb127d 100644 --- a/golden/vndk_data.py +++ b/golden/vndk_data.py @@ -15,11 +15,9 @@ # import collections -import json import logging import os import re -import zipfile try: from importlib import resources @@ -49,16 +47,9 @@ VNDK_SP = "VNDK-SP" # VNDK-SP dependencies that vendor modules cannot directly access. VNDK_SP_PRIVATE = "VNDK-SP-private" -# The tuples of (ABI name, bitness, arch name). 64-bit comes before 32-bit in -# order to sequentially search for longest prefix. -_ABI_LIST = ( - ("arm64", 64, "arm64_armv8-a"), - ("arm64", 32, "arm_armv8-a"), - ("arm", 32, "arm_armv7-a-neon"), - ("x86_64", 64, "x86_x86_64"), - ("x86_64", 32, "x86_64"), - ("x86", 32, "x86"), -) +# The ABI dump directories. 64-bit comes before 32-bit in order to sequentially +# search for longest prefix. +_ABI_NAMES = ("arm64", "arm", "mips64", "mips", "x86_64", "x86") # The data directory. _GOLDEN_DIR = os.path.join("vts", "testcases", "vndk", "golden") @@ -66,9 +57,6 @@ _GOLDEN_DIR = os.path.join("vts", "testcases", "vndk", "golden") # The data package. _RESOURCE_PACKAGE = "vts.testcases.vndk"; -# The name of the zip file containing ABI dumps. -_ABI_DUMP_ZIP_NAME = "abi_dump.zip" - # Regular expression prefix for library name patterns. _REGEX_PREFIX = "[regex]" @@ -107,7 +95,7 @@ def GetAbiDumpDirectory(data_file_path, version, binder_bitness, abi_name, None if there is no directory for the version and ABI. """ try: - abi_dir = next(x[0] for x in _ABI_LIST if abi_name.startswith(x[0])) + abi_dir = next(x for x in _ABI_NAMES if abi_name.startswith(x)) except StopIteration: logging.warning("Unknown ABI %s.", abi_name) return None @@ -129,68 +117,6 @@ def GetAbiDumpDirectory(data_file_path, version, binder_bitness, abi_name, return dump_dir -class AbiDumpResource: - """The class for loading ABI dumps from the zip in resources.""" - - def __init__(self): - self._resource = None - self.zip_file = None - - def __enter__(self): - self._resource = resources.open_binary(_RESOURCE_PACKAGE, - _ABI_DUMP_ZIP_NAME) - self.zip_file = zipfile.ZipFile(self._resource, "r") - return self - - def __exit__(self, exc_type, exc_val, traceback): - if self._resource: - self._resource.close() - if self.zip_file: - self.zip_file.close() - - -def GetAbiDumpPathsFromResources(version, binder_bitness, abi_name, abi_bitness): - """Returns the VNDK dump paths in resources. - - Args: - version: A string, the VNDK version. - binder_bitness: A string or an integer, 32 or 64. - abi_name: A string, the ABI of the library dump. - abi_bitness: A string or an integer, 32 or 64. - - Returns: - A dict of {library name: dump resource path}. For example, - {"libbase.so": "R/64/arm64_armv8-a/source-based/libbase.so.lsdump"}. - If there is no dump for the version and ABI, this function returns an - empty dict. - """ - if not resources: - logging.error("Could not import resources module.") - return dict() - - abi_bitness = int(abi_bitness) - try: - arch_name = next(x[2] for x in _ABI_LIST if - abi_name.startswith(x[0]) and x[1] == abi_bitness) - except StopIteration: - logging.warning("Unknown %d-bit ABI %s.", abi_bitness, abi_name) - return dict() - - # The separator in zipped path is always "/". - dump_dir = "/".join((version, str(binder_bitness), arch_name, - "source-based")) + "/" - - dump_paths = dict() - - with AbiDumpResource() as dump_resource: - for path in dump_resource.zip_file.namelist(): - if path.startswith(dump_dir) and path.endswith(".lsdump"): - lib_name = path[len(dump_dir):-len(".lsdump")] - dump_paths[lib_name] = path - - return dump_paths - - def _LoadVndkLibraryListsFile(vndk_lists, tags, vndk_lib_list_file): """Load VNDK libraries from the file to the specified tuple. diff --git a/utils.py b/utils.py index 65b2eee..38efdce 100644 --- a/utils.py +++ b/utils.py @@ -18,11 +18,8 @@ # TODO(b/147454897): Keep the logic in sync with # test/vts/utils/python/controllers/android_device.py until # it is removed. -import gzip import logging -import os import subprocess -import tempfile class AndroidDevice(object): """This class controls the device via adb commands.""" @@ -144,51 +141,6 @@ class AndroidDevice(object): """Gets the VNDK version that the vendor partition requests.""" return self._GetProp("ro.vndk.version") - def GetKernelConfig(self, config_name): - """Gets kernel config from the device. - - Args: - config_name: A string, the name of the configuration. - - Returns: - "y" or "m" if the config is set. - "" if the config is not set. - None if fails to read config. - """ - line_prefix = config_name + "=" - with tempfile.NamedTemporaryFile(delete=False) as temp_file: - config_path = temp_file.name - try: - logging.debug("Pull config.gz to %s", config_path) - self.AdbPull("/proc/config.gz", config_path) - with gzip.open(config_path, "rt") as config_file: - for line in config_file: - if line.strip().startswith(line_prefix): - logging.debug("Found config: %s", line) - return line.strip()[len(line_prefix):] - logging.debug("%s is not set.", config_name) - return "" - except (subprocess.CalledProcessError, IOError) as e: - logging.exception("Cannot read kernel config.", e) - return None - finally: - os.remove(config_path) - - def GetBinderBitness(self): - """Returns the value of BINDER_IPC_32BIT in kernel config. - - Returns: - 32 or 64, binder bitness of the device. - None if fails to read config. - """ - config_value = self.GetKernelConfig("CONFIG_ANDROID_BINDER_IPC_32BIT") - if config_value is None: - return None - elif config_value: - return 32 - else: - return 64 - def IsRoot(self): """Returns whether adb has root privilege on the device.""" out, err, return_code = self.Execute("id") -- cgit v1.2.3 From f5c3e3cffdf28ccea38cf7a627922c22bea90e7a Mon Sep 17 00:00:00 2001 From: Dan Shi Date: Wed, 11 Mar 2020 15:40:11 +0000 Subject: Revert^2 "Add vts_vndk_abi_test" 97a68c9b6f25f67fc009afa3c31d828683c9b62c Change-Id: If3286250553e08a45c849de0bccbafc41e82d5e8 --- Android.bp | 16 ++ abi/vts_vndk_abi_test.py | 361 ++++++++++++++++++++++++++++++++++++++++++++++ abi/vts_vndk_abi_test.xml | 22 +++ golden/Android.bp | 10 ++ golden/extract_lsdump.py | 53 ++++++- golden/vndk_data.py | 82 ++++++++++- utils.py | 48 ++++++ 7 files changed, 581 insertions(+), 11 deletions(-) create mode 100644 abi/vts_vndk_abi_test.py create mode 100644 abi/vts_vndk_abi_test.xml diff --git a/Android.bp b/Android.bp index 176da18..7d6abec 100644 --- a/Android.bp +++ b/Android.bp @@ -50,6 +50,22 @@ python_defaults { } } +python_test_host { + name: "vts_vndk_abi_test", + defaults: ["vts_vndk_default"], + main: "abi/vts_vndk_abi_test.py", + srcs: [ + "abi/vts_vndk_abi_test.py", + ], + data: [ + ":vts_vndk_abi_dump_zip", + ], + test_suites: [ + "vts-core", + ], + test_config: "abi/vts_vndk_abi_test.xml", +} + python_test_host { name: "vts_vndk_dependency_test", defaults: ["vts_vndk_default"], diff --git a/abi/vts_vndk_abi_test.py b/abi/vts_vndk_abi_test.py new file mode 100644 index 0000000..56a5220 --- /dev/null +++ b/abi/vts_vndk_abi_test.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import json +import logging +import os +import shutil +import tempfile +import unittest + +from vts.testcases.vndk import utils +from vts.testcases.vndk.golden import vndk_data +from vts.utils.python.library import elf_parser +from vts.utils.python.library.vtable import vtable_dumper +from vts.utils.python.vndk import vndk_utils + + +class VtsVndkAbiTest(unittest.TestCase): + """A test module to verify ABI compliance of vendor libraries. + + Attributes: + _dut: the AndroidDevice under test. + _temp_dir: The temporary directory for libraries copied from device. + """ + + def setUp(self): + """Initializes data file path, device, and temporary directory.""" + serial_number = os.environ.get("ANDROID_SERIAL", "") + self.assertTrue(serial_number, "$ANDROID_SERIAL is empty") + self._dut = utils.AndroidDevice(serial_number) + self._temp_dir = tempfile.mkdtemp() + + def tearDown(self): + """Deletes the temporary directory.""" + logging.info("Delete %s", self._temp_dir) + shutil.rmtree(self._temp_dir) + + def _PullOrCreateDir(self, target_dir, host_dir): + """Copies a directory from device. Creates an empty one if not exist. + + Args: + target_dir: The directory to copy from device. + host_dir: The directory to copy to host. + """ + if not self._dut.IsDirectory(target_dir): + logging.info("%s doesn't exist. Create %s.", target_dir, host_dir) + os.makedirs(host_dir) + return + parent_dir = os.path.dirname(host_dir) + if parent_dir and not os.path.isdir(parent_dir): + os.makedirs(parent_dir) + logging.info("adb pull %s %s", target_dir, host_dir) + self._dut.AdbPull(target_dir, host_dir) + + def _ToHostPath(self, target_path): + """Maps target path to host path in self._temp_dir.""" + return os.path.join(self._temp_dir, *target_path.strip("/").split("/")) + + @staticmethod + def _LoadGlobalSymbolsFromDump(dump_obj): + """Loads global symbols from a dump object. + + Args: + dump_obj: A dict, the dump in JSON format. + + Returns: + A set of strings, the symbol names. + """ + symbols = set() + for key in ("elf_functions", "elf_objects"): + symbols.update( + symbol.get("name", "") for symbol in dump_obj.get(key, []) if + symbol.get("binding", "global") == "global") + return symbols + + def _DiffElfSymbols(self, dump_obj, parser): + """Checks if a library includes all symbols in a dump. + + Args: + dump_obj: A dict, the dump in JSON format. + parser: An elf_parser.ElfParser that loads the library. + + Returns: + A list of strings, the global symbols that are in the dump but not + in the library. + + Raises: + elf_parser.ElfError if fails to load the library. + """ + dump_symbols = self._LoadGlobalSymbolsFromDump(dump_obj) + lib_symbols = parser.ListGlobalDynamicSymbols(include_weak=True) + return sorted(dump_symbols.difference(lib_symbols)) + + @staticmethod + def _DiffVtableComponent(offset, expected_symbol, vtable): + """Checks if a symbol is in a vtable entry. + + Args: + offset: An integer, the offset of the expected symbol. + expected_symbol: A string, the name of the expected symbol. + vtable: A dict of {offset: [entry]} where offset is an integer and + entry is an instance of vtable_dumper.VtableEntry. + + Returns: + A list of strings, the actual possible symbols if expected_symbol + does not match the vtable entry. + None if expected_symbol matches the entry. + """ + if offset not in vtable: + return [] + + entry = vtable[offset] + if not entry.names: + return [hex(entry.value).rstrip('L')] + + if expected_symbol not in entry.names: + return entry.names + + def _DiffVtableComponents(self, dump_obj, dumper): + """Checks if a library includes all vtable entries in a dump. + + Args: + dump_obj: A dict, the dump in JSON format. + dumper: An vtable_dumper.VtableDumper that loads the library. + bitness: 32 or 64, the size of the vtable entries. + + Returns: + A list of tuples (VTABLE, OFFSET, EXPECTED_SYMBOL, ACTUAL). + ACTUAL can be "missing", a list of symbol names, or an ELF virtual + address. + + Raises: + vtable_dumper.VtableError if fails to dump vtable from the library. + """ + function_kinds = [ + "function_pointer", + "complete_dtor_pointer", + "deleting_dtor_pointer" + ] + non_function_kinds = [ + "vcall_offset", + "vbase_offset", + "offset_to_top", + "rtti", + "unused_function_pointer" + ] + default_vtable_component_kind = "function_pointer" + + global_symbols = self._LoadGlobalSymbolsFromDump(dump_obj) + + lib_vtables = {vtable.name: vtable + for vtable in dumper.DumpVtables()} + logging.debug("\n\n".join(str(vtable) + for _, vtable in lib_vtables.items())) + + vtables_diff = [] + for record_type in dump_obj.get("record_types", []): + # Since Android R, unique_id has been replaced with linker_set_key. + # unique_id starts with "_ZTI"; linker_set_key starts with "_ZTS". + type_name_symbol = record_type.get("unique_id", "") + if type_name_symbol: + vtable_symbol = type_name_symbol.replace("_ZTS", "_ZTV", 1) + else: + type_name_symbol = record_type.get("linker_set_key", "") + vtable_symbol = type_name_symbol.replace("_ZTI", "_ZTV", 1) + + # Skip if the vtable symbol isn't global. + if vtable_symbol not in global_symbols: + continue + + # Collect vtable entries from library dump. + if vtable_symbol in lib_vtables: + lib_vtable = {entry.offset: entry + for entry in lib_vtables[vtable_symbol].entries} + else: + lib_vtable = dict() + + for index, entry in enumerate(record_type.get("vtable_components", + [])): + entry_offset = index * dumper.bitness // 8 + entry_kind = entry.get("kind", default_vtable_component_kind) + entry_symbol = entry.get("mangled_component_name", "") + entry_is_pure = entry.get("is_pure", False) + + if entry_kind in non_function_kinds: + continue + + if entry_kind not in function_kinds: + logging.warning("%s: Unexpected vtable entry kind %s", + vtable_symbol, entry_kind) + + if entry_symbol not in global_symbols: + # Itanium cxx abi doesn't specify pure virtual vtable + # entry's behaviour. However we can still do some checks + # based on compiler behaviour. + # Even though we don't check weak symbols, we can still + # issue a warning when a pure virtual function pointer + # is missing. + if entry_is_pure and entry_offset not in lib_vtable: + logging.warning("%s: Expected pure virtual function" + "in %s offset %s", + vtable_symbol, vtable_symbol, + entry_offset) + continue + + diff_symbols = self._DiffVtableComponent( + entry_offset, entry_symbol, lib_vtable) + if diff_symbols is None: + continue + + vtables_diff.append( + (vtable_symbol, str(entry_offset), entry_symbol, + (",".join(diff_symbols) if diff_symbols else "missing"))) + + return vtables_diff + + def _ScanLibDirs(self, dump_zip, dump_paths, lib_dirs, dump_version): + """Compares dump files with libraries copied from device. + + Args: + dump_zip: A zip_file.ZipFile object containing the dumps. + dump_paths: A dict of {library name: dump resource path}. + lib_dirs: The list of directories containing libraries. + dump_version: The VNDK version of the dump files. If the device has + no VNDK version or has extension in vendor partition, + this method compares the unversioned VNDK directories + with the dump directories of the given version. + + Returns: + A list of strings, the incompatible libraries. + """ + error_list = [] + lib_paths = dict() + for lib_dir in lib_dirs: + for parent_dir, dir_names, lib_names in os.walk(lib_dir): + for lib_name in lib_names: + if lib_name not in lib_paths: + lib_paths[lib_name] = os.path.join(parent_dir, + lib_name) + for lib_name, dump_path in dump_paths.items(): + if lib_name not in lib_paths: + logging.info("%s: Not found on target", lib_name) + continue + lib_path = lib_paths[lib_name] + rel_path = os.path.relpath(lib_path, self._temp_dir) + + has_exception = False + missing_symbols = [] + vtable_diff = [] + + try: + with dump_zip.open(dump_path, "r") as dump_file: + dump_obj = json.load(dump_file) + with vtable_dumper.VtableDumper(lib_path) as dumper: + missing_symbols = self._DiffElfSymbols( + dump_obj, dumper) + vtable_diff = self._DiffVtableComponents( + dump_obj, dumper) + except (IOError, + elf_parser.ElfError, + vtable_dumper.VtableError) as e: + logging.exception("%s: Cannot diff ABI", rel_path) + has_exception = True + + if missing_symbols: + logging.error("%s: Missing Symbols:\n%s", + rel_path, "\n".join(missing_symbols)) + if vtable_diff: + logging.error("%s: Vtable Difference:\n" + "vtable offset expected actual\n%s", + rel_path, + "\n".join(" ".join(e) for e in vtable_diff)) + if (has_exception or missing_symbols or vtable_diff): + error_list.append(rel_path) + else: + logging.info("%s: Pass", rel_path) + return error_list + + @staticmethod + def _GetLinkerSearchIndex(target_path): + """Returns the key for sorting linker search paths.""" + index = 0 + for prefix in ("/odm", "/vendor", "/apex"): + if target_path.startswith(prefix): + return index + index += 1 + return index + + def _TestAbiCompatibility(self, bitness): + """Checks ABI compliance of VNDK libraries. + + Args: + bitness: 32 or 64, the bitness of the tested libraries. + """ + self.assertTrue(self._dut.IsRoot(), "This test requires adb root.") + primary_abi = self._dut.GetCpuAbiList()[0] + binder_bitness = self._dut.GetBinderBitness() + self.assertTrue(binder_bitness, "Cannot determine binder bitness.") + dump_version = self._dut.GetVndkVersion() + self.assertTrue(dump_version, "Cannot determine VNDK version.") + + dump_paths = vndk_data.GetAbiDumpPathsFromResources( + dump_version, + binder_bitness, + primary_abi, + bitness) + self.assertTrue( + dump_paths, + "No dump files. version: %s ABI: %s bitness: %d" % ( + dump_version, primary_abi, bitness)) + + target_dirs = vndk_utils.GetVndkExtDirectories(bitness) + target_dirs += vndk_utils.GetVndkSpExtDirectories(bitness) + target_dirs += [vndk_utils.GetVndkDirectory(bitness, dump_version)] + target_dirs.sort(key=self._GetLinkerSearchIndex) + + host_dirs = [self._ToHostPath(x) for x in target_dirs] + for target_dir, host_dir in zip(target_dirs, host_dirs): + self._PullOrCreateDir(target_dir, host_dir) + + with vndk_data.AbiDumpResource() as dump_resource: + assert_lines = self._ScanLibDirs(dump_resource.zip_file, + dump_paths, host_dirs, + dump_version) + + if assert_lines: + error_count = len(assert_lines) + if error_count > 20: + assert_lines = assert_lines[:20] + ["..."] + assert_lines.append("Total number of errors: " + str(error_count)) + self.fail("\n".join(assert_lines)) + + def testAbiCompatibility32(self): + """Checks ABI compliance of 32-bit VNDK libraries.""" + self._TestAbiCompatibility(32) + + def testAbiCompatibility64(self): + """Checks ABI compliance of 64-bit VNDK libraries.""" + if self._dut.GetCpuAbiList(64): + self._TestAbiCompatibility(64) + else: + logging.info("Skip the test as the device doesn't support 64-bit " + "ABI.") + + +if __name__ == "__main__": + unittest.main() diff --git a/abi/vts_vndk_abi_test.xml b/abi/vts_vndk_abi_test.xml new file mode 100644 index 0000000..7fc7321 --- /dev/null +++ b/abi/vts_vndk_abi_test.xml @@ -0,0 +1,22 @@ + + + + + + + diff --git a/golden/Android.bp b/golden/Android.bp index 20222e1..c9abdaa 100644 --- a/golden/Android.bp +++ b/golden/Android.bp @@ -26,3 +26,13 @@ python_binary_host { }, } } + +// TODO(b/150663999): Replace with gensrcs when the build system is able to +// process the large file group. +genrule { + name: "vts_vndk_abi_dump_zip", + tools: ["extract_lsdump"], + cmd: "$(location extract_lsdump) $(in) $(out)", + srcs: [":vndk_abi_dump_zip"], + out: ["vts/testcases/vndk/abi_dump.zip"], +} diff --git a/golden/extract_lsdump.py b/golden/extract_lsdump.py index a6d95c7..fd42be6 100755 --- a/golden/extract_lsdump.py +++ b/golden/extract_lsdump.py @@ -19,6 +19,7 @@ import argparse import gzip import json import os +import zipfile class LsdumpError(Exception): @@ -205,35 +206,62 @@ def _ParseVtablesFromLsdump(lsdump, output_dump): output_dump.record_types.append(record_type) -def ParseLsdumpFile(input_path, output_path): - """Converts an lsdump file to a dump file for the ABI test. +def _ParseLsdumpFileObject(lsdump_file): + """Converts an lsdump file to a dump object. Args: - input_path: The path to the (gzipped) lsdump file. - output_path: The path to the output dump file. + lsdump_file: The file object of the input lsdump. + + Returns: + The AttrDict object converted from the lsdump file. Raises: LsdumpError if fails to create the dump file. """ try: - with _OpenFileOrGzipped(input_path) as lsdump_file: - lsdump = json.load(lsdump_file, object_hook=AttrDict) - except (IOError, ValueError) as e: + lsdump = json.load(lsdump_file, object_hook=AttrDict) + except ValueError: raise LsdumpError(e) try: output_dump = AttrDict() _ParseVtablesFromLsdump(lsdump, output_dump) _ParseSymbolsFromLsdump(lsdump, output_dump) + return output_dump except AttributeError as e: raise LsdumpError(e) + +def _ParseLsdumpZipFile(lsdump_zip, output_zip): + """Converts zipped lsdump files to the dump files for ABI test.""" + for name in lsdump_zip.namelist(): + if name.endswith(".lsdump"): + with lsdump_zip.open(name, mode="r") as lsdump_file: + output_dump = _ParseLsdumpFileObject(lsdump_file) + output_str = json.dumps(output_dump, indent=1, + separators=(',', ':')) + output_zip.writestr(name, output_str) + + +def ParseLsdumpFile(input_path, output_path): + """Converts an lsdump file to a dump file for the ABI test. + + Args: + input_path: The path to the (gzipped) lsdump file. + output_path: The path to the output dump file. + + Raises: + LsdumpError if fails to create the dump file. + """ abs_output_path = os.path.abspath(output_path) abs_output_dir = os.path.dirname(abs_output_path) try: if abs_output_dir and not os.path.exists(abs_output_dir): os.makedirs(abs_output_dir) + + with _OpenFileOrGzipped(input_path) as lsdump_file: + output_dump = _ParseLsdumpFileObject(lsdump_file) with open(output_path, 'wb') as output_file: json.dump(output_dump, output_file, indent=1, separators=(',', ':')) @@ -251,6 +279,17 @@ def main(): help='output dump file path.') args = arg_parser.parse_args() + # TODO(b/150663999): Remove this when the build system is able to process + # the large file group. + if zipfile.is_zipfile(args.input_path): + with zipfile.ZipFile(args.input_path, mode='r') as input_zip: + # The zip will be added to a Python package. It is not necessary + # to reduce the file size. + with zipfile.ZipFile(args.output_path, mode='w', + compression=zipfile.ZIP_STORED) as output_zip: + _ParseLsdumpZipFile(input_zip, output_zip) + exit(0) + try: ParseLsdumpFile(args.input_path, args.output_path) except LsdumpError as e: diff --git a/golden/vndk_data.py b/golden/vndk_data.py index 4eb127d..ffa1527 100644 --- a/golden/vndk_data.py +++ b/golden/vndk_data.py @@ -15,9 +15,11 @@ # import collections +import json import logging import os import re +import zipfile try: from importlib import resources @@ -47,9 +49,16 @@ VNDK_SP = "VNDK-SP" # VNDK-SP dependencies that vendor modules cannot directly access. VNDK_SP_PRIVATE = "VNDK-SP-private" -# The ABI dump directories. 64-bit comes before 32-bit in order to sequentially -# search for longest prefix. -_ABI_NAMES = ("arm64", "arm", "mips64", "mips", "x86_64", "x86") +# The tuples of (ABI name, bitness, arch name). 64-bit comes before 32-bit in +# order to sequentially search for longest prefix. +_ABI_LIST = ( + ("arm64", 64, "arm64_armv8-a"), + ("arm64", 32, "arm_armv8-a"), + ("arm", 32, "arm_armv7-a-neon"), + ("x86_64", 64, "x86_x86_64"), + ("x86_64", 32, "x86_64"), + ("x86", 32, "x86"), +) # The data directory. _GOLDEN_DIR = os.path.join("vts", "testcases", "vndk", "golden") @@ -57,6 +66,9 @@ _GOLDEN_DIR = os.path.join("vts", "testcases", "vndk", "golden") # The data package. _RESOURCE_PACKAGE = "vts.testcases.vndk"; +# The name of the zip file containing ABI dumps. +_ABI_DUMP_ZIP_NAME = "abi_dump.zip" + # Regular expression prefix for library name patterns. _REGEX_PREFIX = "[regex]" @@ -95,7 +107,7 @@ def GetAbiDumpDirectory(data_file_path, version, binder_bitness, abi_name, None if there is no directory for the version and ABI. """ try: - abi_dir = next(x for x in _ABI_NAMES if abi_name.startswith(x)) + abi_dir = next(x[0] for x in _ABI_LIST if abi_name.startswith(x[0])) except StopIteration: logging.warning("Unknown ABI %s.", abi_name) return None @@ -117,6 +129,68 @@ def GetAbiDumpDirectory(data_file_path, version, binder_bitness, abi_name, return dump_dir +class AbiDumpResource: + """The class for loading ABI dumps from the zip in resources.""" + + def __init__(self): + self._resource = None + self.zip_file = None + + def __enter__(self): + self._resource = resources.open_binary(_RESOURCE_PACKAGE, + _ABI_DUMP_ZIP_NAME) + self.zip_file = zipfile.ZipFile(self._resource, "r") + return self + + def __exit__(self, exc_type, exc_val, traceback): + if self._resource: + self._resource.close() + if self.zip_file: + self.zip_file.close() + + +def GetAbiDumpPathsFromResources(version, binder_bitness, abi_name, abi_bitness): + """Returns the VNDK dump paths in resources. + + Args: + version: A string, the VNDK version. + binder_bitness: A string or an integer, 32 or 64. + abi_name: A string, the ABI of the library dump. + abi_bitness: A string or an integer, 32 or 64. + + Returns: + A dict of {library name: dump resource path}. For example, + {"libbase.so": "R/64/arm64_armv8-a/source-based/libbase.so.lsdump"}. + If there is no dump for the version and ABI, this function returns an + empty dict. + """ + if not resources: + logging.error("Could not import resources module.") + return dict() + + abi_bitness = int(abi_bitness) + try: + arch_name = next(x[2] for x in _ABI_LIST if + abi_name.startswith(x[0]) and x[1] == abi_bitness) + except StopIteration: + logging.warning("Unknown %d-bit ABI %s.", abi_bitness, abi_name) + return dict() + + # The separator in zipped path is always "/". + dump_dir = "/".join((version, str(binder_bitness), arch_name, + "source-based")) + "/" + + dump_paths = dict() + + with AbiDumpResource() as dump_resource: + for path in dump_resource.zip_file.namelist(): + if path.startswith(dump_dir) and path.endswith(".lsdump"): + lib_name = path[len(dump_dir):-len(".lsdump")] + dump_paths[lib_name] = path + + return dump_paths + + def _LoadVndkLibraryListsFile(vndk_lists, tags, vndk_lib_list_file): """Load VNDK libraries from the file to the specified tuple. diff --git a/utils.py b/utils.py index 38efdce..65b2eee 100644 --- a/utils.py +++ b/utils.py @@ -18,8 +18,11 @@ # TODO(b/147454897): Keep the logic in sync with # test/vts/utils/python/controllers/android_device.py until # it is removed. +import gzip import logging +import os import subprocess +import tempfile class AndroidDevice(object): """This class controls the device via adb commands.""" @@ -141,6 +144,51 @@ class AndroidDevice(object): """Gets the VNDK version that the vendor partition requests.""" return self._GetProp("ro.vndk.version") + def GetKernelConfig(self, config_name): + """Gets kernel config from the device. + + Args: + config_name: A string, the name of the configuration. + + Returns: + "y" or "m" if the config is set. + "" if the config is not set. + None if fails to read config. + """ + line_prefix = config_name + "=" + with tempfile.NamedTemporaryFile(delete=False) as temp_file: + config_path = temp_file.name + try: + logging.debug("Pull config.gz to %s", config_path) + self.AdbPull("/proc/config.gz", config_path) + with gzip.open(config_path, "rt") as config_file: + for line in config_file: + if line.strip().startswith(line_prefix): + logging.debug("Found config: %s", line) + return line.strip()[len(line_prefix):] + logging.debug("%s is not set.", config_name) + return "" + except (subprocess.CalledProcessError, IOError) as e: + logging.exception("Cannot read kernel config.", e) + return None + finally: + os.remove(config_path) + + def GetBinderBitness(self): + """Returns the value of BINDER_IPC_32BIT in kernel config. + + Returns: + 32 or 64, binder bitness of the device. + None if fails to read config. + """ + config_value = self.GetKernelConfig("CONFIG_ANDROID_BINDER_IPC_32BIT") + if config_value is None: + return None + elif config_value: + return 32 + else: + return 64 + def IsRoot(self): """Returns whether adb has root privilege on the device.""" out, err, return_code = self.Execute("id") -- cgit v1.2.3 From 0db0129115c2c28d887a7174e724b8ff1a23bb7d Mon Sep 17 00:00:00 2001 From: Dan Shi Date: Thu, 26 Mar 2020 11:40:55 -0700 Subject: Rename vts-core to vts Bug: 151896491 Test: local build Exempt-From-Owner-Approval: This CL adds all tests in vts to a new suite vts10. vts10 will be the new name of existing vts suite. This CL won't change test logic or behavior. Change-Id: Ie08837255cd1503cf0bf8b7cc600c1a7c117d660 Merged-In: Ie08837255cd1503cf0bf8b7cc600c1a7c117d660 --- Android.bp | 8 ++++---- files/vts_vndk_files_test.xml | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Android.bp b/Android.bp index 7d6abec..3d8ab5a 100644 --- a/Android.bp +++ b/Android.bp @@ -61,7 +61,7 @@ python_test_host { ":vts_vndk_abi_dump_zip", ], test_suites: [ - "vts-core", + "vts", ], test_config: "abi/vts_vndk_abi_test.xml", } @@ -74,7 +74,7 @@ python_test_host { "dependency/vts_vndk_dependency_test.py", ], test_suites: [ - "vts-core", + "vts", ], test_config: "dependency/vts_vndk_dependency_test.xml", } @@ -87,7 +87,7 @@ python_test_host { "files/vts_vndk_files_test.py", ], test_suites: [ - "vts-core", + "vts", ], test_config: "files/vts_vndk_files_test.xml", } @@ -100,7 +100,7 @@ python_test_host { "open_libraries/vts_vndk_open_libraries_test.py", ], test_suites: [ - "vts-core", + "vts", ], test_config: "open_libraries/vts_vndk_open_libraries_test.xml", } diff --git a/files/vts_vndk_files_test.xml b/files/vts_vndk_files_test.xml index 4813916..c2a4d9a 100644 --- a/files/vts_vndk_files_test.xml +++ b/files/vts_vndk_files_test.xml @@ -15,7 +15,6 @@ --> - -- cgit v1.2.3