diff options
author | Elliott Hughes <enh@google.com> | 2024-04-16 00:37:38 +0000 |
---|---|---|
committer | Elliott Hughes <enh@google.com> | 2024-04-16 15:05:55 +0000 |
commit | f978a85cc35935a1b1a86e865b4898960cfb9075 (patch) | |
tree | 2e2caffa26877c7a1cb94e67b8b2ff871348b245 | |
parent | a778b83a5339ddeaf18861fb722bf4a8b500f718 (diff) | |
download | bionic-f978a85cc35935a1b1a86e865b4898960cfb9075.tar.gz |
Simplify Oryon ifunc resolvers.
Mainly just factoring out the code, but there are two functional
changes here too:
1. The inline assembler was missing `volatile`, making the hwcap
check ineffective (because the compiler would sometimes move the
MIDR_EL1 read above the hwcap check).
2. The previous code accepted variants 0x0 to 0x5 while the comment
said 0x1 to 0x5. The comment was correct.
I resisted the temptation to actually have a table to search on the assumption that it'll be a while before we need such a thing.
Bug: https://issuetracker.google.com/330105715
Change-Id: I9fdc1e70e49b26ef32794b55ca5e5fd37f1163f9
-rw-r--r-- | libc/arch-arm64/dynamic_function_dispatch.cpp | 111 |
1 files changed, 23 insertions, 88 deletions
diff --git a/libc/arch-arm64/dynamic_function_dispatch.cpp b/libc/arch-arm64/dynamic_function_dispatch.cpp index db002b864..a42c36118 100644 --- a/libc/arch-arm64/dynamic_function_dispatch.cpp +++ b/libc/arch-arm64/dynamic_function_dispatch.cpp @@ -30,18 +30,22 @@ #include <stddef.h> #include <sys/auxv.h> -#define MIDR_IMPL_ID_SHIFT 24u -#define MIDR_IMPL_ID_MASK 0xFF -#define CPU_VARIANT_SHIFT 20u -#define CPU_VARIANT_MASK 0xF +static inline bool __bionic_is_oryon(unsigned long hwcap) { + if (!(hwcap & HWCAP_CPUID)) return false; -/* Macro to identify CPU implementer */ -#define QCOM_IMPL_ID 0x51 + // Extract the implementor and variant bits from MIDR_EL1. + // https://www.kernel.org/doc/html/latest/arch/arm64/cpu-feature-registers.html#list-of-registers-with-visible-features + unsigned long midr; + __asm__ __volatile__("mrs %0, MIDR_EL1" : "=r"(midr)); + uint16_t cpu = (midr >> 20) & 0xfff; -/* Macro to indentify qualcomm CPU variants which supports - * __memcpy_aarch64_nt routine - */ -#define QCOM_ORYON_CPU_VARIANTS 0x5 + auto make_cpu = [](unsigned implementor, unsigned variant) { + return (implementor << 4) | variant; + }; + + // Check for implementor Qualcomm's variants 0x1..0x5 (Oryon). + return cpu >= make_cpu('Q', 0x1) && cpu <= make_cpu('Q', 0x5); +} extern "C" { @@ -62,68 +66,20 @@ DEFINE_IFUNC_FOR(memcmp) { typedef void* memcpy_func(void*, const void*, size_t); DEFINE_IFUNC_FOR(memcpy) { - unsigned long midr; - unsigned int impl_id, cpu_variant; - - /* Check if hardware capability CPUID is available */ - if (arg->_hwcap & HWCAP_CPUID) { - /* Read the MIDR register */ - asm("mrs %0, MIDR_EL1 \n\t" : "=r"(midr)); - - /* Extract the CPU Implementer ID */ - impl_id = (midr >> MIDR_IMPL_ID_SHIFT) & (MIDR_IMPL_ID_MASK); - - /* Check for Qualcomm implementer ID */ - if (impl_id == QCOM_IMPL_ID) { - cpu_variant = (midr >> CPU_VARIANT_SHIFT) & CPU_VARIANT_MASK; - - /* Check for Qualcomm Oryon CPU variants: 0x1, 0x2, 0x3, 0x4, 0x5 */ - if (cpu_variant <= QCOM_ORYON_CPU_VARIANTS) { + if (__bionic_is_oryon(arg->_hwcap)) { RETURN_FUNC(memcpy_func, __memcpy_aarch64_nt); - } else { + } else if (arg->_hwcap & HWCAP_ASIMD) { + RETURN_FUNC(memcpy_func, __memcpy_aarch64_simd); + } else { RETURN_FUNC(memcpy_func, __memcpy_aarch64); - } } - } - /* If CPU implementer is not Qualcomm, choose the custom - * implementation based on CPU architecture feature - * */ - if (arg->_hwcap & HWCAP_ASIMD) { - RETURN_FUNC(memcpy_func, __memcpy_aarch64_simd); - } else { - RETURN_FUNC(memcpy_func, __memcpy_aarch64); - } } typedef void* memmove_func(void*, const void*, size_t); DEFINE_IFUNC_FOR(memmove) { - unsigned long midr; - unsigned int impl_id, cpu_variant; - - /* Check if hardware capability CPUID is available */ - if (arg->_hwcap & HWCAP_CPUID) { - /* Read the MIDR register */ - asm("mrs %0, MIDR_EL1 \n\t" : "=r"(midr)); - - /* Extract the CPU Implementer ID */ - impl_id = (midr >> MIDR_IMPL_ID_SHIFT) & (MIDR_IMPL_ID_MASK); - - /* Check for Qualcomm implementer ID */ - if (impl_id == QCOM_IMPL_ID) { - cpu_variant = (midr >> CPU_VARIANT_SHIFT) & CPU_VARIANT_MASK; - - /* Check for Qualcomm Oryon CPU variants: 0x1, 0x2, 0x3, 0x4, 0x5 */ - if (cpu_variant <= QCOM_ORYON_CPU_VARIANTS) { - RETURN_FUNC(memcpy_func, __memmove_aarch64_nt); - } else { - RETURN_FUNC(memcpy_func, __memmove_aarch64); - } - } - } - /* If CPU implementer is not Qualcomm, choose the custom - * implementation based on CPU architecture feature - * */ - if (arg->_hwcap & HWCAP_ASIMD) { + if (__bionic_is_oryon(arg->_hwcap)) { + RETURN_FUNC(memcpy_func, __memmove_aarch64_nt); + } else if (arg->_hwcap & HWCAP_ASIMD) { RETURN_FUNC(memmove_func, __memmove_aarch64_simd); } else { RETURN_FUNC(memmove_func, __memmove_aarch64); @@ -137,32 +93,11 @@ DEFINE_IFUNC_FOR(memrchr) { typedef int memset_func(void*, int, size_t); DEFINE_IFUNC_FOR(memset) { - unsigned long midr; - unsigned int impl_id, cpu_variant; - - if (arg->_hwcap & HWCAP_CPUID) { - /* Read the MIDR register */ - asm("mrs %0, MIDR_EL1 \n\t" : "=r"(midr)); - - /* Extract the CPU Implementer ID */ - impl_id = (midr >> MIDR_IMPL_ID_SHIFT) & (MIDR_IMPL_ID_MASK); - - /* Check for Qualcomm implementer ID */ - if (impl_id == QCOM_IMPL_ID) { - cpu_variant = (midr >> CPU_VARIANT_SHIFT) & CPU_VARIANT_MASK; - - /* Check for Qualcomm Oryon CPU variants: 0x1, 0x2, 0x3, 0x4, 0x5 */ - if (cpu_variant <= QCOM_ORYON_CPU_VARIANTS) { + if (__bionic_is_oryon(arg->_hwcap)) { RETURN_FUNC(memset_func, __memset_aarch64_nt); - } else { - RETURN_FUNC(memset_func, __memset_aarch64); - } } else { - RETURN_FUNC(memset_func, __memset_aarch64); + RETURN_FUNC(memset_func, __memset_aarch64); } - } else { - RETURN_FUNC(memset_func, __memset_aarch64); - } } typedef char* stpcpy_func(char*, const char*, size_t); |