From 9221e3393ca0e4f577de3f21016e2e7345e0ecca Mon Sep 17 00:00:00 2001 From: Hsin-Yi Chen Date: Wed, 3 May 2017 10:29:41 +0800 Subject: Update VTS vndk-sp test Rewrite test for /system/lib/vndk-sp design. - Add ELF parser to replace readelf command. - Check same-process HAL and its dependencies. Bug: 36556372 Test: vts-tradefed run commandAndExit vts -m VtsVndkDependencyTest Change-Id: I74804f9ee98bff8e3932ff143790e1a10d3f4c1a --- dependency/VtsVndkDependencyTest.py | 323 ++++++++++++++++++++++++------------ dependency/elf_parser.py | 276 +++++++++++++++++++++++++----- 2 files changed, 454 insertions(+), 145 deletions(-) diff --git a/dependency/VtsVndkDependencyTest.py b/dependency/VtsVndkDependencyTest.py index ba800b5..26eb6ae 100644 --- a/dependency/VtsVndkDependencyTest.py +++ b/dependency/VtsVndkDependencyTest.py @@ -24,173 +24,288 @@ import tempfile from vts.runners.host import asserts from vts.runners.host import base_test from vts.runners.host import test_runner +from vts.runners.host import utils from vts.utils.python.controllers import android_device +from vts.utils.python.file import file_utils +from vts.utils.python.os import path_utils from vts.testcases.vndk.dependency import elf_parser + class VtsVndkDependencyTest(base_test.BaseTestClass): """A test case to verify vendor library dependency. Attributes: + _dut: The AndroidDevice under test. + _shell: The ShellMirrorObject to execute commands _temp_dir: The temporary directory to which the vendor partition is - copied. - _vendor_libs: Collection of strings. The names of the shared libraries - on vendor partition. + copied. + _LOW_LEVEL_NDK: List of strings. The names of low-level NDK libraries in + /system/lib[64]. + _SAME_PROCESS_HAL: List of patterns. The names of same-process HAL + libraries expected to be in /vendor/lib[64]. + _SAME_PROCESS_NDK: List if strings. The names of same-process NDK + libraries in /system/lib[64]. """ - _SHELL_NAME = "vendor_dep_test_shell" - _VENDOR_PATH = "/vendor" - _SAME_PROCESS_DIR_32 = "/vendor/lib/sameprocess" - _SAME_PROCESS_DIR_64 = "/vendor/lib64/sameprocess" + _TARGET_VENDOR_DIR = "/vendor" + _TARGET_VNDK_SP_DIR_32 = "/system/lib/vndk-sp" + _TARGET_VNDK_SP_DIR_64 = "/system/lib64/vndk-sp" + + # copied from build/soong/cc/config/global.go _LOW_LEVEL_NDK = [ + "ld-android.so", "libc.so", - "libm.so", - "libz.so", - "liblog.so", "libdl.so", - "libstdc++.so" + "liblog.so", + "libm.so" ] - _SAME_PROCESS_NDK = [re.compile(p) for p in [ + # copied from development/vndk/tools/definition-tool/vndk_definition_tool.py + _SAME_PROCESS_HAL = [re.compile(p) for p in [ + "android\\.hardware\\.graphics\\.mapper@\\d+\\.\\d+-impl\\.so$", + "gralloc\\..*\\.so$", "libEGL_.*\\.so$", + "libGLES_.*\\.so$", "libGLESv1_CM_.*\\.so$", "libGLESv2_.*\\.so$", "libGLESv3_.*\\.so$", - "vulkan.*\\.so$", - "libRSDriver.*\\.so$", "libPVRRS\\.so$", - "android\\.hardware\\.graphics\\.mapper@\\d+\\.\\d+-impl\\.so$" + "libRSDriver.*\\.so$", + "vulkan.*\\.so$" ]] + _SAME_PROCESS_NDK = [ + "libEGL.so", + "libGLESv1_CM.so", + "libGLESv2.so", + "libGLESv3.so", + "libnativewindow.so", + "libsync.so", + "libvulkan.so" + ] + _SP_HAL_LINK_PATHS_32 = [ + "/vendor/lib/egl", + "/vendor/lib/hw", + "/vendor/lib" + ] + _SP_HAL_LINK_PATHS_64 = [ + "/vendor/lib64/egl", + "/vendor/lib64/hw", + "/vendor/lib64" + ] + + class ElfObject(object): + """Contains dependencies of an ELF file on target device. + + Attributes: + target_path: String. The path to the ELF file on target. + name: String. File name of the ELF. + target_dir: String. The directory containing the ELF file on target. + bitness: Integer. Bitness of the ELF. + deps: List of strings. The names of the depended libraries. + """ + def __init__(self, target_path, bitness, deps): + self.target_path = target_path + self.name = path_utils.TargetBaseName(target_path) + self.target_dir = path_utils.TargetDirName(target_path) + self.bitness = bitness + self.deps = deps def setUpClass(self): """Initializes device and temporary directory.""" - self.dut = self.registerController(android_device)[0] - self.dut.shell.InvokeTerminal(self._SHELL_NAME) + self._dut = self.registerController(android_device)[0] + self._dut.shell.InvokeTerminal("one") + self._shell = self._dut.shell.one self._temp_dir = tempfile.mkdtemp() - logging.info("adb pull %s %s", self._VENDOR_PATH, self._temp_dir) - pull_output = self.dut.adb.pull(self._VENDOR_PATH, self._temp_dir) + logging.info("adb pull %s %s", self._TARGET_VENDOR_DIR, self._temp_dir) + pull_output = self._dut.adb.pull( + self._TARGET_VENDOR_DIR, self._temp_dir) logging.debug(pull_output) - self._vendor_libs = self._listSharedLibraries(self._temp_dir) - logging.info("Vendor libraries: " + str(self._vendor_libs)) def tearDownClass(self): """Deletes the temporary directory.""" logging.info("Delete %s", self._temp_dir) shutil.rmtree(self._temp_dir) - def _isSameProcessLibrary(self, lib_name): - """Checks whether a library is same-process. + def _loadElfObjects(self, host_dir, target_dir, elf_error_handler): + """Scans a host directory recursively and loads all ELF files in it. Args: - lib_name: String. The name of the library. + host_dir: The host directory to scan. + target_dir: The path from which host_dir is copied. + elf_error_handler: A function that takes 2 arguments + (target_path, exception). It is called when + the parser fails to read an ELF file. Returns: - A boolean representing whether the library is same-process. + List of ElfObject. """ - for pattern in self._SAME_PROCESS_NDK: - if pattern.match(lib_name): - return True - return False + objs = [] + for root_dir, file_name in utils.iterate_files(host_dir): + full_path = os.path.join(root_dir, file_name) + rel_path = os.path.relpath(full_path, host_dir) + target_path = path_utils.JoinTargetPath( + target_dir, *rel_path.split(os.path.sep)); + try: + elf = elf_parser.ElfParser(full_path) + except elf_parser.ElfError: + logging.debug("%s is not an ELF file", target_path) + continue + try: + deps = elf.listDependencies() + except elf_parser.ElfError as e: + elf_error_handler(target_path, e) + continue + finally: + elf.close() + + logging.info("%s depends on: %s", target_path, ", ".join(deps)) + objs.append(self.ElfObject(target_path, elf.bitness, deps)) + return objs - def _isAllowedDependency(self, lib_name): - """Checks whether a library dependency is allowed. + def _isAllowedSpHalDependency(self, lib_name, vndk_sp_names, linkable_libs): + """Checks whether a same-process HAL library dependency is allowed. - A vendor library/executable is only allowed to depend on + A same-process HAL library is allowed to depend on - Low-level NDK - Same-process NDK - - Other libraries on vendor partition + - vndk-sp + - Other libraries in vendor/lib[64] Args: lib_name: String. The name of the depended library. + vndk_sp_names: Set of strings. The names of the libraries in + vndk-sp directory. + linkable_libs: Dictionary. The keys are the names of the libraries + which can be linked to same-process HAL. Returns: - A boolean representing whether the library is allowed. + A boolean representing whether the dependency is allowed. """ - if lib_name in self._vendor_libs or lib_name in self._LOW_LEVEL_NDK: - return True - if self._isSameProcessLibrary(lib_name): + if (lib_name in self._LOW_LEVEL_NDK or + lib_name in self._SAME_PROCESS_NDK or + lib_name in vndk_sp_names or + lib_name in linkable_libs): return True return False - @staticmethod - def _iterateFiles(dir_path): - """A generator yielding regular files in a directory recursively. + def _getTargetVndkSpDir(self, bitness): + """Returns 32/64-bit vndk-sp directory path on target device.""" + return getattr(self, "_TARGET_VNDK_SP_DIR_" + str(bitness)) + + def _getSpHalLinkPaths(self, bitness): + """Returns 32/64-bit same-process HAL link paths""" + return getattr(self, "_SP_HAL_LINK_PATHS_" + str(bitness)) + + def _isInSpHalLinkPaths(self, lib): + """Checks whether a library can be linked to same-process HAL. Args: - dir_path: String. The path to search. + lib: ElfObject. The library to check. - Yields: - A tuple of strings (directory, file). The directory containing - the file and the file name. + Returns: + True if can be linked to same-process HAL; False otherwise. """ - for root_dir, dir_names, file_names in os.walk(dir_path): - for file_name in file_names: - yield root_dir, file_name + return lib.target_dir in self._getSpHalLinkPaths(lib.bitness) - @staticmethod - def _listSharedLibraries(path): - """Finds all shared libraries under a directory. + def _spHalLinkOrder(self, lib): + """Returns the key for sorting libraries in linker search order. Args: - path: String. The path to search. + lib: ElfObject. Returns: - Set of strings. The names of the found libraries. + An integer representing linker search order. """ - results = set() - for root_dir, file_name in VtsVndkDependencyTest._iterateFiles(path): - if file_name.endswith(".so"): - results.add(file_name) - return results - - def testSameProcessLibrary(self): - """Checks if same-process directory contains only allowed libraries.""" - dev_sp_dirs = [self._SAME_PROCESS_DIR_32] - if self.dut.is64Bit: - dev_sp_dirs.append(self._SAME_PROCESS_DIR_64) - error_count = 0 - for dev_sp_dir in dev_sp_dirs: - sp_dir = os.path.join(self._temp_dir, dev_sp_dir) - if not os.path.isdir(sp_dir): - logging.warning("%s is not a directory", sp_dir) - continue - logging.info("Enter %s", sp_dir) - for root_dir, file_name in self._iterateFiles(sp_dir): - full_path = os.path.join(root_dir, file_name) - if self._isSameProcessLibrary(file_name): - logging.info("%s is a same-process lib", full_path) - continue - error_count += 1 - logging.error("%s is not a same-process lib", full_path) - asserts.assertEqual(error_count, 0, - "Total number of errors: " + str(error_count)) + link_paths = self._getSpHalLinkPaths(lib.bitness) + for order in range(len(link_paths)): + if lib.target_dir == link_paths[order]: + return order + order = len(link_paths) + if lib.name in self._LOW_LEVEL_NDK: + return order + order += 1 + if lib.name in self._SAME_PROCESS_NDK: + return order + order += 1 + return order + + def _dfsDependencies(self, lib, searched, searchable): + """Depth-first-search for library dependencies. + + Args: + lib: ElfObject. The library to search dependencies. + searched: The set of searched libraries. + searchable: The dictionary that maps file names to libraries. + """ + if lib in searched: + return + searched.add(lib) + for dep_name in lib.deps: + if dep_name in searchable: + self._dfsDependencies( + searchable[dep_name], searched, searchable) + + def _testSpHalDependency(self, bitness, objs): + """Scans same-process HAL dependency on vendor partition. + + Returns: + List of tuples (path, dependency_names). The library with + disallowed dependencies and list of the dependencies. + """ + vndk_sp_dir = self._getTargetVndkSpDir(bitness) + vndk_sp_paths = file_utils.FindFiles(self._shell, vndk_sp_dir, "*.so") + vndk_sp_names = set(path_utils.TargetBaseName(x) for x in vndk_sp_paths) + logging.info("%s libraries: %s" % ( + vndk_sp_dir, ", ".join(vndk_sp_names))) + # map file names to libraries which can be linked to same-process HAL + linkable_libs = dict() + for obj in [x for x in objs + if x.bitness == bitness and self._isInSpHalLinkPaths(x)]: + if obj.name not in linkable_libs: + linkable_libs[obj.name] = obj + else: + linkable_libs[obj.name] = min(linkable_libs[obj.name], obj, + key=self._spHalLinkOrder) + # find same-process HAL and dependencies + sp_hal_libs = set() + for file_name, obj in linkable_libs.iteritems(): + if any([x.match(file_name) for x in self._SAME_PROCESS_HAL]): + self._dfsDependencies(obj, sp_hal_libs, linkable_libs) + logging.info("%d-bit SP HAL libraries: %s" % ( + bitness, ", ".join([x.name for x in sp_hal_libs]))) + # check disallowed dependencies + dep_errors = [] + for obj in sp_hal_libs: + disallowed_libs = [x for x in obj.deps + if not self._isAllowedSpHalDependency(x, vndk_sp_names, + linkable_libs)] + if disallowed_libs: + dep_errors.append((obj.target_path, disallowed_libs)) + return dep_errors def testElfDependency(self): """Scans library/executable dependency on vendor partition.""" - if not elf_parser.ElfParser.isSupported(): - asserts.fail("readelf is not available") - error_count = 0 - for root_dir, file_name in self._iterateFiles(self._temp_dir): - file_path = os.path.join(root_dir, file_name) - elf = elf_parser.ElfParser(file_path) - if not elf.isValid(): - logging.info("%s is not an ELF file", file_path) - continue - try: - dep_libs = elf.listDependencies() - except OSError as e: - error_count += 1 - logging.exception("Cannot read %s: %s", file_path, str(e)) - continue - logging.info("%s depends on: %s", file_path, str(dep_libs)) - disallowed_libs = filter( - lambda x: not self._isAllowedDependency(x), dep_libs) - if not disallowed_libs: - continue - error_count += 1 - logging.error("%s depends on disallowed libs: %s", - file_path.replace(self._temp_dir, "", 1), - str(disallowed_libs)) + read_errors = [] + objs = self._loadElfObjects( + self._temp_dir, + path_utils.TargetDirName(self._TARGET_VENDOR_DIR), + lambda p, e: read_errors.append((p, str(e)))) + + dep_errors = self._testSpHalDependency(32, objs) + if self._dut.is64Bit: + dep_errors.extend(self._testSpHalDependency(64, objs)) + # TODO(hsinyichen): check other vendor libraries + + if read_errors: + logging.error("%d read errors:", len(read_errors)) + for x in read_errors: + logging.error("%s: %s", x[0], x[1]) + if dep_errors: + logging.error("%d disallowed dependencies:", len(dep_errors)) + for x in dep_errors: + logging.error("%s: %s", x[0], ", ".join(x[1])) + error_count = len(read_errors) + len(dep_errors) asserts.assertEqual(error_count, 0, "Total number of errors: " + str(error_count)) + if __name__ == "__main__": test_runner.main() - diff --git a/dependency/elf_parser.py b/dependency/elf_parser.py index 50e2584..1d3c7e5 100644 --- a/dependency/elf_parser.py +++ b/dependency/elf_parser.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.4 +#!/usr/bin/env python # # Copyright (C) 2017 The Android Open Source Project # @@ -16,66 +16,260 @@ # import os -import re +import struct + + +class ElfError(Exception): + """The exception raised by ElfParser.""" + pass -from vts.runners.host import utils class ElfParser(object): - """This class reads an ELF file by parsing output of the command readelf. + """The class reads information from an ELF file. Attributes: - _file_path: The path to the ELF file. + _file: The ELF file object. + _file_size: Size of the ELF. + bitness: Bitness of the ELF. + _address_size: Size of address or offset in the ELF. + _offsets: Offset of each entry in the ELF. + _seek_read_address: The function to read an address or offset entry + from the ELF. + _sh_offset: Offset of section header table in the file. + _sh_size: Size of section header table entry. + _sh_count: Number of section header table entries. + _section_headers: List of SectionHeader objects read from the ELF. """ + _MAGIC_OFFSET = 0 + _MAGIC_BYTES = b"\x7fELF" + _BITNESS_OFFSET = 4 + _BITNESS_32 = 1 + _BITNESS_64 = 2 + # Section type + _SHT_DYNAMIC = 6 + # Tag in dynamic section + _DT_NULL = 0 + _DT_NEEDED = 1 + _DT_STRTAB = 5 + + class ElfOffsets32(object): + """Offset of each entry in 32-bit ELF""" + # offset from ELF header + SECTION_HEADER_OFFSET = 0x20 + SECTION_HEADER_SIZE = 0x2e + SECTION_HEADER_COUNT = 0x30 + # offset from section header + SECTION_TYPE = 0x04 + SECTION_ADDRESS = 0x0c + SECTION_OFFSET = 0x10 + + class ElfOffsets64(object): + """Offset of each entry in 64-bit ELF""" + # offset from ELF header + SECTION_HEADER_OFFSET = 0x28 + SECTION_HEADER_SIZE = 0x3a + SECTION_HEADER_COUNT = 0x3c + # offset from section header + SECTION_TYPE = 0x04 + SECTION_ADDRESS = 0x10 + SECTION_OFFSET = 0x18 + + class SectionHeader(object): + """Contains section header entries as attributes. + + Attributes: + type: Type of the section. + address: The virtual memory address where the section is loaded. + offset: The offset of the section in the ELF file. + """ + def __init__(self, type, address, offset): + self.type = type + self.address = address + self.offset = offset def __init__(self, file_path): - self._file_path = file_path + """Creates a parser to open and read an ELF file. + + Args: + file_path: The path to the ELF. - @staticmethod - def isSupported(): - """Checks whether readelf is available.""" + Raises: + ElfError if the file is not a valid ELF. + """ + try: + self._file = open(file_path, "rb") + except IOError as e: + raise ElfError(e) try: - utils.exe_cmd("readelf", "--version") - return True - except OSError: - return False + self._loadElfHeader() + self._section_headers = [ + self._loadSectionHeader(self._sh_offset + i * self._sh_size) + for i in range(self._sh_count)] + except: + self._file.close() + raise + + def __del__(self): + """Closes the ELF file.""" + self.close() - def isValid(self): - """Checks size and first 4 bytes of the ELF file. + def close(self): + """Closes the ELF file.""" + self._file.close() + + def _seekRead(self, offset, read_size): + """Reads a byte string at specific offset in the file. + + Args: + offset: An integer, the offset from the beginning of the file. + read_size: An integer, number of bytes to read. Returns: - A boolean representing whether _file_path is a valid ELF. + A byte string which is the file content. + + Raises: + ElfError if fails to seek and read. """ + if offset + read_size > self._file_size: + raise ElfError("Read beyond end of file.") try: - size = os.path.getsize(self._file_path) - # must be larger than 32-bit file header - if size < 52: - return False - except OSError: - return False + self._file.seek(offset) + return self._file.read(read_size) + except IOError as e: + raise ElfError(e) + + def _seekRead8(self, offset): + """Reads an 1-byte integer from file.""" + return struct.unpack("B", self._seekRead(offset, 1))[0] + + def _seekRead16(self, offset): + """Reads a 2-byte integer from file.""" + return struct.unpack("H", self._seekRead(offset, 2))[0] + + def _seekRead32(self, offset): + """Reads a 4-byte integer from file.""" + return struct.unpack("I", self._seekRead(offset, 4))[0] + + def _seekRead64(self, offset): + """Reads an 8-byte integer from file.""" + return struct.unpack("Q", self._seekRead(offset, 8))[0] + + def _seekReadString(self, offset): + """Reads a null-terminated string starting from specific offset. + + Args: + offset: The offset from the beginning of the file. + + Returns: + A byte string, excluding the null character. + """ + ret = "" + buf_size = 16 + self._file.seek(offset) + while True: + try: + buf = self._file.read(buf_size) + except IOError as e: + raise ElfError(e) + end_index = buf.find('\0') + if end_index < 0: + ret += buf + else: + ret += buf[:end_index] + return ret + if len(buf) != buf_size: + raise ElfError("Null-terminated string reaches end of file.") + + def _loadElfHeader(self): + """Loads ElfHeader and initializes attributes""" try: - with open(self._file_path, "rb") as f: - magic = f.read(4) - if list(bytearray(magic)) != [0x7f, 0x45, 0x4c, 0x46]: - return False - except IOError: - return False + self._file_size = os.fstat(self._file.fileno()).st_size + except OSError as e: + raise ElfError(e) + + magic = self._seekRead(self._MAGIC_OFFSET, 4) + if magic != self._MAGIC_BYTES: + raise ElfError("Wrong magic bytes.") + bitness = self._seekRead8(self._BITNESS_OFFSET) + if bitness == self._BITNESS_32: + self.bitness = 32 + self._address_size = 4 + self._offsets = self.ElfOffsets32 + self._seek_read_address = self._seekRead32 + elif bitness == self._BITNESS_64: + self.bitness = 64 + self._address_size = 8 + self._offsets = self.ElfOffsets64 + self._seek_read_address = self._seekRead64 + else: + raise ElfError("Wrong bitness value.") + + self._sh_offset = self._seek_read_address( + self._offsets.SECTION_HEADER_OFFSET) + self._sh_size = self._seekRead16(self._offsets.SECTION_HEADER_SIZE) + self._sh_count = self._seekRead16(self._offsets.SECTION_HEADER_COUNT) return True - def listDependencies(self): - """Lists the shared libraries that the ELF depends on. + def _loadSectionHeader(self, offset): + """Loads a section header from ELF file. + + Args: + offset: The starting offset of the section header. Returns: - List of strings. The names of the depended libraries. + An instance of SectionHeader. + """ + return self.SectionHeader( + self._seekRead32(offset + self._offsets.SECTION_TYPE), + self._seek_read_address(offset + self._offsets.SECTION_ADDRESS), + self._seek_read_address(offset + self._offsets.SECTION_OFFSET)) - Raises: - OSError if readelf fails. + def _loadDtNeeded(self, offset): + """Reads DT_NEEDED entries from dynamic section. + + Args: + offset: The offset of the dynamic section from the beginning of + the file + + Returns: + A list of strings, the names of libraries. """ - pattern = re.compile("\\(NEEDED\\)\\s*Shared library: \[(.+)\]") - output = utils.exe_cmd("readelf", "--dynamic", self._file_path) - results = [] - for line in output.split("\n"): - match = pattern.search(line) - if match: - results.append(match.group(1)) - return results + strtab_address = None + name_offsets = [] + while True: + tag = self._seek_read_address(offset) + offset += self._address_size + value = self._seek_read_address(offset) + offset += self._address_size + + if tag == self._DT_NULL: + break + if tag == self._DT_NEEDED: + name_offsets.append(value) + if tag == self._DT_STRTAB: + strtab_address = value + + if strtab_address is None: + raise ElfError("Cannot find string table offset in dynamic section") + + try: + strtab_offset = next(x.offset for x in self._section_headers + if x.address == strtab_address) + except StopIteration: + raise ElfError("Cannot find dynamic string table.") + + names = [self._seekReadString(strtab_offset + x) + for x in name_offsets] + return names + def listDependencies(self): + """Lists the shared libraries that the ELF depends on. + + Returns: + A list of strings, the names of the depended libraries. + """ + deps = [] + for sh in self._section_headers: + if sh.type == self._SHT_DYNAMIC: + deps.extend(self._loadDtNeeded(sh.offset)) + return deps -- cgit v1.2.3