diff options
Diffstat (limited to 'third_party/boringssl/linux-aarch64')
10 files changed, 179 insertions, 25 deletions
diff --git a/third_party/boringssl/linux-aarch64/crypto/chacha/chacha-armv8.S b/third_party/boringssl/linux-aarch64/crypto/chacha/chacha-armv8.S index 49449bf5..e7f019ce 100644 --- a/third_party/boringssl/linux-aarch64/crypto/chacha/chacha-armv8.S +++ b/third_party/boringssl/linux-aarch64/crypto/chacha/chacha-armv8.S @@ -16,6 +16,7 @@ #include <openssl/arm_arch.h> +.hidden OPENSSL_armcap_P .section .rodata @@ -34,6 +35,7 @@ .type ChaCha20_ctr32,%function .align 5 ChaCha20_ctr32: + AARCH64_VALID_CALL_TARGET cbz x2,.Labort #if __has_feature(hwaddress_sanitizer) && __clang_major__ >= 10 adrp x5,:pg_hi21_nc:OPENSSL_armcap_P @@ -47,6 +49,7 @@ ChaCha20_ctr32: b.ne ChaCha20_neon .Lshort: + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -259,6 +262,7 @@ ChaCha20_ctr32: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 + AARCH64_VALIDATE_LINK_REGISTER .Labort: ret @@ -315,12 +319,14 @@ ChaCha20_ctr32: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 + AARCH64_VALIDATE_LINK_REGISTER ret .size ChaCha20_ctr32,.-ChaCha20_ctr32 .type ChaCha20_neon,%function .align 5 ChaCha20_neon: + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -701,6 +707,7 @@ ChaCha20_neon: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 + AARCH64_VALIDATE_LINK_REGISTER ret .Ltail_neon: @@ -810,11 +817,13 @@ ChaCha20_neon: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 + AARCH64_VALIDATE_LINK_REGISTER ret .size ChaCha20_neon,.-ChaCha20_neon .type ChaCha20_512_neon,%function .align 5 ChaCha20_512_neon: + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -1978,6 +1987,7 @@ ChaCha20_512_neon: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 + AARCH64_VALIDATE_LINK_REGISTER ret .size ChaCha20_512_neon,.-ChaCha20_512_neon #endif diff --git a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/aesv8-armx64.S b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/aesv8-armx64.S index 60c70a24..f8cd03df 100644 --- a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/aesv8-armx64.S +++ b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/aesv8-armx64.S @@ -33,6 +33,8 @@ .align 5 aes_hw_set_encrypt_key: .Lenc_key: + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. + AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! add x29,sp,#0 mov x3,#-1 @@ -201,6 +203,7 @@ aes_hw_set_encrypt_key: .type aes_hw_set_decrypt_key,%function .align 5 aes_hw_set_decrypt_key: + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 bl .Lenc_key @@ -234,6 +237,7 @@ aes_hw_set_decrypt_key: eor x0,x0,x0 // return value .Ldec_key_abort: ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER ret .size aes_hw_set_decrypt_key,.-aes_hw_set_decrypt_key .globl aes_hw_encrypt @@ -241,6 +245,7 @@ aes_hw_set_decrypt_key: .type aes_hw_encrypt,%function .align 5 aes_hw_encrypt: + AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] ld1 {v0.4s},[x2],#16 ld1 {v2.16b},[x0] @@ -271,6 +276,7 @@ aes_hw_encrypt: .type aes_hw_decrypt,%function .align 5 aes_hw_decrypt: + AARCH64_VALID_CALL_TARGET ldr w3,[x2,#240] ld1 {v0.4s},[x2],#16 ld1 {v2.16b},[x0] @@ -301,6 +307,8 @@ aes_hw_decrypt: .type aes_hw_cbc_encrypt,%function .align 5 aes_hw_cbc_encrypt: + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. + AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! add x29,sp,#0 subs x2,x2,#16 @@ -592,6 +600,8 @@ aes_hw_cbc_encrypt: .type aes_hw_ctr32_encrypt_blocks,%function .align 5 aes_hw_ctr32_encrypt_blocks: + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. + AARCH64_VALID_CALL_TARGET stp x29,x30,[sp,#-16]! add x29,sp,#0 ldr w5,[x3,#240] @@ -611,20 +621,34 @@ aes_hw_ctr32_encrypt_blocks: add x7,x3,#32 mov w6,w5 csel x12,xzr,x12,lo + + // ARM Cortex-A57 and Cortex-A72 cores running in 32-bit mode are + // affected by silicon errata #1742098 [0] and #1655431 [1], + // respectively, where the second instruction of an aese/aesmc + // instruction pair may execute twice if an interrupt is taken right + // after the first instruction consumes an input register of which a + // single 32-bit lane has been updated the last time it was modified. + // + // This function uses a counter in one 32-bit lane. The vmov lines + // could write to v1.16b and v18.16b directly, but that trips this bugs. + // We write to v6.16b and copy to the final register as a workaround. + // + // [0] ARM-EPM-049219 v23 Cortex-A57 MPCore Software Developers Errata Notice + // [1] ARM-EPM-012079 v11.0 Cortex-A72 MPCore Software Developers Errata Notice #ifndef __ARMEB__ rev w8, w8 #endif - orr v1.16b,v0.16b,v0.16b add w10, w8, #1 - orr v18.16b,v0.16b,v0.16b - add w8, w8, #2 orr v6.16b,v0.16b,v0.16b rev w10, w10 - mov v1.s[3],w10 + mov v6.s[3],w10 + add w8, w8, #2 + orr v1.16b,v6.16b,v6.16b b.ls .Lctr32_tail rev w12, w8 + mov v6.s[3],w12 sub x2,x2,#3 // bias - mov v18.s[3],w12 + orr v18.16b,v6.16b,v6.16b b .Loop3x_ctr32 .align 4 @@ -651,11 +675,11 @@ aes_hw_ctr32_encrypt_blocks: aese v1.16b,v16.16b aesmc v5.16b,v1.16b ld1 {v2.16b},[x0],#16 - orr v0.16b,v6.16b,v6.16b + add w9,w8,#1 aese v18.16b,v16.16b aesmc v18.16b,v18.16b ld1 {v3.16b},[x0],#16 - orr v1.16b,v6.16b,v6.16b + rev w9,w9 aese v4.16b,v17.16b aesmc v4.16b,v4.16b aese v5.16b,v17.16b @@ -664,8 +688,6 @@ aes_hw_ctr32_encrypt_blocks: mov x7,x3 aese v18.16b,v17.16b aesmc v17.16b,v18.16b - orr v18.16b,v6.16b,v6.16b - add w9,w8,#1 aese v4.16b,v20.16b aesmc v4.16b,v4.16b aese v5.16b,v20.16b @@ -680,21 +702,26 @@ aes_hw_ctr32_encrypt_blocks: aesmc v4.16b,v4.16b aese v5.16b,v21.16b aesmc v5.16b,v5.16b + // Note the logic to update v0.16b, v1.16b, and v1.16b is written to work + // around a bug in ARM Cortex-A57 and Cortex-A72 cores running in + // 32-bit mode. See the comment above. eor v19.16b,v19.16b,v7.16b - rev w9,w9 + mov v6.s[3], w9 aese v17.16b,v21.16b aesmc v17.16b,v17.16b - mov v0.s[3], w9 + orr v0.16b,v6.16b,v6.16b rev w10,w10 aese v4.16b,v22.16b aesmc v4.16b,v4.16b + mov v6.s[3], w10 + rev w12,w8 aese v5.16b,v22.16b aesmc v5.16b,v5.16b - mov v1.s[3], w10 - rev w12,w8 + orr v1.16b,v6.16b,v6.16b + mov v6.s[3], w12 aese v17.16b,v22.16b aesmc v17.16b,v17.16b - mov v18.s[3], w12 + orr v18.16b,v6.16b,v6.16b subs x2,x2,#3 aese v4.16b,v23.16b aese v5.16b,v23.16b diff --git a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/armv8-mont.S b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/armv8-mont.S index 360bf4c7..db89859a 100644 --- a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/armv8-mont.S +++ b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/armv8-mont.S @@ -13,6 +13,8 @@ #if defined(BORINGSSL_PREFIX) #include <boringssl_prefix_symbols_asm.h> #endif +#include <openssl/arm_arch.h> + .text .globl bn_mul_mont @@ -20,6 +22,7 @@ .type bn_mul_mont,%function .align 5 bn_mul_mont: + AARCH64_SIGN_LINK_REGISTER tst x5,#7 b.eq __bn_sqr8x_mont tst x5,#3 @@ -217,11 +220,14 @@ bn_mul_mont: mov x0,#1 ldp x23,x24,[x29,#48] ldr x29,[sp],#64 + AARCH64_VALIDATE_LINK_REGISTER ret .size bn_mul_mont,.-bn_mul_mont .type __bn_sqr8x_mont,%function .align 5 __bn_sqr8x_mont: + // Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_sqr8x_mont is jumped to + // only from bn_mul_mont which has already signed the return address. cmp x1,x2 b.ne __bn_mul4x_mont .Lsqr8x_mont: @@ -975,11 +981,16 @@ __bn_sqr8x_mont: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldr x29,[sp],#128 + // x30 is popped earlier + AARCH64_VALIDATE_LINK_REGISTER ret .size __bn_sqr8x_mont,.-__bn_sqr8x_mont .type __bn_mul4x_mont,%function .align 5 __bn_mul4x_mont: + // Not adding AARCH64_SIGN_LINK_REGISTER here because __bn_mul4x_mont is jumped to + // only from bn_mul_mont or __bn_mul8x_mont which have already signed the + // return address. stp x29,x30,[sp,#-128]! add x29,sp,#0 stp x19,x20,[sp,#16] @@ -1413,6 +1424,8 @@ __bn_mul4x_mont: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldr x29,[sp],#128 + // x30 is popped earlier + AARCH64_VALIDATE_LINK_REGISTER ret .size __bn_mul4x_mont,.-__bn_mul4x_mont .byte 77,111,110,116,103,111,109,101,114,121,32,77,117,108,116,105,112,108,105,99,97,116,105,111,110,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 diff --git a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/ghash-neon-armv8.S b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/ghash-neon-armv8.S index f876db3f..098967b5 100644 --- a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/ghash-neon-armv8.S +++ b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/ghash-neon-armv8.S @@ -13,6 +13,8 @@ #if defined(BORINGSSL_PREFIX) #include <boringssl_prefix_symbols_asm.h> #endif +#include <openssl/arm_arch.h> + .text .globl gcm_init_neon @@ -20,6 +22,7 @@ .type gcm_init_neon,%function .align 4 gcm_init_neon: + AARCH64_VALID_CALL_TARGET // This function is adapted from gcm_init_v8. xC2 is t3. ld1 {v17.2d}, [x1] // load H movi v19.16b, #0xe1 @@ -45,6 +48,7 @@ gcm_init_neon: .type gcm_gmult_neon,%function .align 4 gcm_gmult_neon: + AARCH64_VALID_CALL_TARGET ld1 {v3.16b}, [x0] // load Xi ld1 {v5.1d}, [x1], #8 // load twisted H ld1 {v6.1d}, [x1] @@ -64,6 +68,7 @@ gcm_gmult_neon: .type gcm_ghash_neon,%function .align 4 gcm_ghash_neon: + AARCH64_VALID_CALL_TARGET ld1 {v0.16b}, [x0] // load Xi ld1 {v5.1d}, [x1], #8 // load twisted H ld1 {v6.1d}, [x1] diff --git a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/ghashv8-armx64.S b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/ghashv8-armx64.S index 37d97317..62e58842 100644 --- a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/ghashv8-armx64.S +++ b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/ghashv8-armx64.S @@ -22,6 +22,7 @@ .type gcm_init_v8,%function .align 4 gcm_init_v8: + AARCH64_VALID_CALL_TARGET ld1 {v17.2d},[x1] //load input H movi v19.16b,#0xe1 shl v19.2d,v19.2d,#57 //0xc2.0 @@ -73,6 +74,7 @@ gcm_init_v8: .type gcm_gmult_v8,%function .align 4 gcm_gmult_v8: + AARCH64_VALID_CALL_TARGET ld1 {v17.2d},[x0] //load Xi movi v19.16b,#0xe1 ld1 {v20.2d,v21.2d},[x1] //load twisted H, ... @@ -115,6 +117,7 @@ gcm_gmult_v8: .type gcm_ghash_v8,%function .align 4 gcm_ghash_v8: + AARCH64_VALID_CALL_TARGET ld1 {v0.2d},[x0] //load [rotated] Xi //"[rotated]" means that //loaded value would have diff --git a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/sha1-armv8.S b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/sha1-armv8.S index f681b998..838ec329 100644 --- a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/sha1-armv8.S +++ b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/sha1-armv8.S @@ -18,11 +18,14 @@ .text +.hidden OPENSSL_armcap_P .globl sha1_block_data_order .hidden sha1_block_data_order .type sha1_block_data_order,%function .align 6 sha1_block_data_order: + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. + AARCH64_VALID_CALL_TARGET #if __has_feature(hwaddress_sanitizer) && __clang_major__ >= 10 adrp x16,:pg_hi21_nc:OPENSSL_armcap_P #else @@ -1090,6 +1093,8 @@ sha1_block_data_order: .type sha1_block_armv8,%function .align 6 sha1_block_armv8: + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. + AARCH64_VALID_CALL_TARGET .Lv8_entry: stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1228,8 +1233,6 @@ sha1_block_armv8: .byte 83,72,65,49,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 .align 2 .align 2 -.comm OPENSSL_armcap_P,4,4 -.hidden OPENSSL_armcap_P #endif #endif // !OPENSSL_NO_ASM .section .note.GNU-stack,"",%progbits diff --git a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/sha256-armv8.S b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/sha256-armv8.S index 6e09f69a..a4f170ec 100644 --- a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/sha256-armv8.S +++ b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/sha256-armv8.S @@ -59,11 +59,13 @@ .text +.hidden OPENSSL_armcap_P .globl sha256_block_data_order .hidden sha256_block_data_order .type sha256_block_data_order,%function .align 6 sha256_block_data_order: + AARCH64_VALID_CALL_TARGET #ifndef __KERNEL__ #if __has_feature(hwaddress_sanitizer) && __clang_major__ >= 10 adrp x16,:pg_hi21_nc:OPENSSL_armcap_P @@ -74,6 +76,7 @@ sha256_block_data_order: tst w16,#ARMV8_SHA256 b.ne .Lv8_entry #endif + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-128]! add x29,sp,#0 @@ -1034,6 +1037,7 @@ sha256_block_data_order: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 + AARCH64_VALIDATE_LINK_REGISTER ret .size sha256_block_data_order,.-sha256_block_data_order @@ -1068,6 +1072,7 @@ sha256_block_data_order: .align 6 sha256_block_armv8: .Lv8_entry: + // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -1204,10 +1209,6 @@ sha256_block_armv8: ret .size sha256_block_armv8,.-sha256_block_armv8 #endif -#ifndef __KERNEL__ -.comm OPENSSL_armcap_P,4,4 -.hidden OPENSSL_armcap_P -#endif #endif #endif // !OPENSSL_NO_ASM .section .note.GNU-stack,"",%progbits diff --git a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/sha512-armv8.S b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/sha512-armv8.S index 7b9b22a0..98b7a7e2 100644 --- a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/sha512-armv8.S +++ b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/sha512-armv8.S @@ -59,11 +59,13 @@ .text +.hidden OPENSSL_armcap_P .globl sha512_block_data_order .hidden sha512_block_data_order .type sha512_block_data_order,%function .align 6 sha512_block_data_order: + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-128]! add x29,sp,#0 @@ -1024,6 +1026,7 @@ sha512_block_data_order: ldp x25,x26,[x29,#64] ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#128 + AARCH64_VALIDATE_LINK_REGISTER ret .size sha512_block_data_order,.-sha512_block_data_order @@ -1076,10 +1079,6 @@ sha512_block_data_order: .byte 83,72,65,53,49,50,32,98,108,111,99,107,32,116,114,97,110,115,102,111,114,109,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 .align 2 .align 2 -#ifndef __KERNEL__ -.comm OPENSSL_armcap_P,4,4 -.hidden OPENSSL_armcap_P -#endif #endif #endif // !OPENSSL_NO_ASM .section .note.GNU-stack,"",%progbits diff --git a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S index f57b7b51..c02be40f 100644 --- a/third_party/boringssl/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S +++ b/third_party/boringssl/linux-aarch64/crypto/fipsmodule/vpaes-armv8.S @@ -13,6 +13,8 @@ #if defined(BORINGSSL_PREFIX) #include <boringssl_prefix_symbols_asm.h> #endif +#include <openssl/arm_arch.h> + .section .rodata .type _vpaes_consts,%object @@ -215,6 +217,7 @@ _vpaes_encrypt_core: .type vpaes_encrypt,%function .align 4 vpaes_encrypt: + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -224,6 +227,7 @@ vpaes_encrypt: st1 {v0.16b}, [x1] ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_encrypt,.-vpaes_encrypt @@ -452,6 +456,7 @@ _vpaes_decrypt_core: .type vpaes_decrypt,%function .align 4 vpaes_decrypt: + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 @@ -461,6 +466,7 @@ vpaes_decrypt: st1 {v0.16b}, [x1] ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_decrypt,.-vpaes_decrypt @@ -630,6 +636,7 @@ _vpaes_key_preheat: .type _vpaes_schedule_core,%function .align 4 _vpaes_schedule_core: + AARCH64_SIGN_LINK_REGISTER stp x29, x30, [sp,#-16]! add x29,sp,#0 @@ -799,6 +806,7 @@ _vpaes_schedule_core: eor v6.16b, v6.16b, v6.16b // vpxor %xmm6, %xmm6, %xmm6 eor v7.16b, v7.16b, v7.16b // vpxor %xmm7, %xmm7, %xmm7 ldp x29, x30, [sp],#16 + AARCH64_VALIDATE_LINK_REGISTER ret .size _vpaes_schedule_core,.-_vpaes_schedule_core @@ -1001,7 +1009,7 @@ _vpaes_schedule_mangle: .Lschedule_mangle_both: tbl v3.16b, {v3.16b}, v1.16b // vpshufb %xmm1, %xmm3, %xmm3 - add x8, x8, #64-16 // add $-16, %r8 + add x8, x8, #48 // add $-16, %r8 and x8, x8, #~(1<<6) // and $0x30, %r8 st1 {v3.2d}, [x2] // vmovdqu %xmm3, (%rdx) ret @@ -1012,6 +1020,7 @@ _vpaes_schedule_mangle: .type vpaes_set_encrypt_key,%function .align 4 vpaes_set_encrypt_key: + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1027,6 +1036,7 @@ vpaes_set_encrypt_key: ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_set_encrypt_key,.-vpaes_set_encrypt_key @@ -1035,6 +1045,7 @@ vpaes_set_encrypt_key: .type vpaes_set_decrypt_key,%function .align 4 vpaes_set_decrypt_key: + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1054,6 +1065,7 @@ vpaes_set_decrypt_key: ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_set_decrypt_key,.-vpaes_set_decrypt_key .globl vpaes_cbc_encrypt @@ -1061,6 +1073,7 @@ vpaes_set_decrypt_key: .type vpaes_cbc_encrypt,%function .align 4 vpaes_cbc_encrypt: + AARCH64_SIGN_LINK_REGISTER cbz x2, .Lcbc_abort cmp w5, #0 // check direction b.eq vpaes_cbc_decrypt @@ -1087,6 +1100,7 @@ vpaes_cbc_encrypt: st1 {v0.16b}, [x4] // write ivec ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER .Lcbc_abort: ret .size vpaes_cbc_encrypt,.-vpaes_cbc_encrypt @@ -1094,6 +1108,8 @@ vpaes_cbc_encrypt: .type vpaes_cbc_decrypt,%function .align 4 vpaes_cbc_decrypt: + // Not adding AARCH64_SIGN_LINK_REGISTER here because vpaes_cbc_decrypt is jumped to + // only from vpaes_cbc_encrypt which has already signed the return address. stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1135,6 +1151,7 @@ vpaes_cbc_decrypt: ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_cbc_decrypt,.-vpaes_cbc_decrypt .globl vpaes_ctr32_encrypt_blocks @@ -1142,6 +1159,7 @@ vpaes_cbc_decrypt: .type vpaes_ctr32_encrypt_blocks,%function .align 4 vpaes_ctr32_encrypt_blocks: + AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-16]! add x29,sp,#0 stp d8,d9,[sp,#-16]! // ABI spec says so @@ -1209,6 +1227,7 @@ vpaes_ctr32_encrypt_blocks: ldp d10,d11,[sp],#16 ldp d8,d9,[sp],#16 ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER ret .size vpaes_ctr32_encrypt_blocks,.-vpaes_ctr32_encrypt_blocks #endif diff --git a/third_party/boringssl/linux-aarch64/crypto/test/trampoline-armv8.S b/third_party/boringssl/linux-aarch64/crypto/test/trampoline-armv8.S index f7bbfc29..8928d7f5 100644 --- a/third_party/boringssl/linux-aarch64/crypto/test/trampoline-armv8.S +++ b/third_party/boringssl/linux-aarch64/crypto/test/trampoline-armv8.S @@ -13,6 +13,8 @@ #if defined(BORINGSSL_PREFIX) #include <boringssl_prefix_symbols_asm.h> #endif +#include <openssl/arm_arch.h> + .text // abi_test_trampoline loads callee-saved registers from |state|, calls |func| @@ -26,6 +28,8 @@ .hidden abi_test_trampoline .align 4 abi_test_trampoline: +.Labi_test_trampoline_begin: + AARCH64_SIGN_LINK_REGISTER // Stack layout (low to high addresses) // x29,x30 (16 bytes) // d8-d15 (64 bytes) @@ -128,6 +132,7 @@ abi_test_trampoline: ldp x27, x28, [sp, #144] ldp x29, x30, [sp], #176 + AARCH64_VALIDATE_LINK_REGISTER ret .size abi_test_trampoline,.-abi_test_trampoline .type abi_test_clobber_x0, %function @@ -135,6 +140,7 @@ abi_test_trampoline: .hidden abi_test_clobber_x0 .align 4 abi_test_clobber_x0: + AARCH64_VALID_CALL_TARGET mov x0, xzr ret .size abi_test_clobber_x0,.-abi_test_clobber_x0 @@ -143,6 +149,7 @@ abi_test_clobber_x0: .hidden abi_test_clobber_x1 .align 4 abi_test_clobber_x1: + AARCH64_VALID_CALL_TARGET mov x1, xzr ret .size abi_test_clobber_x1,.-abi_test_clobber_x1 @@ -151,6 +158,7 @@ abi_test_clobber_x1: .hidden abi_test_clobber_x2 .align 4 abi_test_clobber_x2: + AARCH64_VALID_CALL_TARGET mov x2, xzr ret .size abi_test_clobber_x2,.-abi_test_clobber_x2 @@ -159,6 +167,7 @@ abi_test_clobber_x2: .hidden abi_test_clobber_x3 .align 4 abi_test_clobber_x3: + AARCH64_VALID_CALL_TARGET mov x3, xzr ret .size abi_test_clobber_x3,.-abi_test_clobber_x3 @@ -167,6 +176,7 @@ abi_test_clobber_x3: .hidden abi_test_clobber_x4 .align 4 abi_test_clobber_x4: + AARCH64_VALID_CALL_TARGET mov x4, xzr ret .size abi_test_clobber_x4,.-abi_test_clobber_x4 @@ -175,6 +185,7 @@ abi_test_clobber_x4: .hidden abi_test_clobber_x5 .align 4 abi_test_clobber_x5: + AARCH64_VALID_CALL_TARGET mov x5, xzr ret .size abi_test_clobber_x5,.-abi_test_clobber_x5 @@ -183,6 +194,7 @@ abi_test_clobber_x5: .hidden abi_test_clobber_x6 .align 4 abi_test_clobber_x6: + AARCH64_VALID_CALL_TARGET mov x6, xzr ret .size abi_test_clobber_x6,.-abi_test_clobber_x6 @@ -191,6 +203,7 @@ abi_test_clobber_x6: .hidden abi_test_clobber_x7 .align 4 abi_test_clobber_x7: + AARCH64_VALID_CALL_TARGET mov x7, xzr ret .size abi_test_clobber_x7,.-abi_test_clobber_x7 @@ -199,6 +212,7 @@ abi_test_clobber_x7: .hidden abi_test_clobber_x8 .align 4 abi_test_clobber_x8: + AARCH64_VALID_CALL_TARGET mov x8, xzr ret .size abi_test_clobber_x8,.-abi_test_clobber_x8 @@ -207,6 +221,7 @@ abi_test_clobber_x8: .hidden abi_test_clobber_x9 .align 4 abi_test_clobber_x9: + AARCH64_VALID_CALL_TARGET mov x9, xzr ret .size abi_test_clobber_x9,.-abi_test_clobber_x9 @@ -215,6 +230,7 @@ abi_test_clobber_x9: .hidden abi_test_clobber_x10 .align 4 abi_test_clobber_x10: + AARCH64_VALID_CALL_TARGET mov x10, xzr ret .size abi_test_clobber_x10,.-abi_test_clobber_x10 @@ -223,6 +239,7 @@ abi_test_clobber_x10: .hidden abi_test_clobber_x11 .align 4 abi_test_clobber_x11: + AARCH64_VALID_CALL_TARGET mov x11, xzr ret .size abi_test_clobber_x11,.-abi_test_clobber_x11 @@ -231,6 +248,7 @@ abi_test_clobber_x11: .hidden abi_test_clobber_x12 .align 4 abi_test_clobber_x12: + AARCH64_VALID_CALL_TARGET mov x12, xzr ret .size abi_test_clobber_x12,.-abi_test_clobber_x12 @@ -239,6 +257,7 @@ abi_test_clobber_x12: .hidden abi_test_clobber_x13 .align 4 abi_test_clobber_x13: + AARCH64_VALID_CALL_TARGET mov x13, xzr ret .size abi_test_clobber_x13,.-abi_test_clobber_x13 @@ -247,6 +266,7 @@ abi_test_clobber_x13: .hidden abi_test_clobber_x14 .align 4 abi_test_clobber_x14: + AARCH64_VALID_CALL_TARGET mov x14, xzr ret .size abi_test_clobber_x14,.-abi_test_clobber_x14 @@ -255,6 +275,7 @@ abi_test_clobber_x14: .hidden abi_test_clobber_x15 .align 4 abi_test_clobber_x15: + AARCH64_VALID_CALL_TARGET mov x15, xzr ret .size abi_test_clobber_x15,.-abi_test_clobber_x15 @@ -263,6 +284,7 @@ abi_test_clobber_x15: .hidden abi_test_clobber_x16 .align 4 abi_test_clobber_x16: + AARCH64_VALID_CALL_TARGET mov x16, xzr ret .size abi_test_clobber_x16,.-abi_test_clobber_x16 @@ -271,6 +293,7 @@ abi_test_clobber_x16: .hidden abi_test_clobber_x17 .align 4 abi_test_clobber_x17: + AARCH64_VALID_CALL_TARGET mov x17, xzr ret .size abi_test_clobber_x17,.-abi_test_clobber_x17 @@ -279,6 +302,7 @@ abi_test_clobber_x17: .hidden abi_test_clobber_x19 .align 4 abi_test_clobber_x19: + AARCH64_VALID_CALL_TARGET mov x19, xzr ret .size abi_test_clobber_x19,.-abi_test_clobber_x19 @@ -287,6 +311,7 @@ abi_test_clobber_x19: .hidden abi_test_clobber_x20 .align 4 abi_test_clobber_x20: + AARCH64_VALID_CALL_TARGET mov x20, xzr ret .size abi_test_clobber_x20,.-abi_test_clobber_x20 @@ -295,6 +320,7 @@ abi_test_clobber_x20: .hidden abi_test_clobber_x21 .align 4 abi_test_clobber_x21: + AARCH64_VALID_CALL_TARGET mov x21, xzr ret .size abi_test_clobber_x21,.-abi_test_clobber_x21 @@ -303,6 +329,7 @@ abi_test_clobber_x21: .hidden abi_test_clobber_x22 .align 4 abi_test_clobber_x22: + AARCH64_VALID_CALL_TARGET mov x22, xzr ret .size abi_test_clobber_x22,.-abi_test_clobber_x22 @@ -311,6 +338,7 @@ abi_test_clobber_x22: .hidden abi_test_clobber_x23 .align 4 abi_test_clobber_x23: + AARCH64_VALID_CALL_TARGET mov x23, xzr ret .size abi_test_clobber_x23,.-abi_test_clobber_x23 @@ -319,6 +347,7 @@ abi_test_clobber_x23: .hidden abi_test_clobber_x24 .align 4 abi_test_clobber_x24: + AARCH64_VALID_CALL_TARGET mov x24, xzr ret .size abi_test_clobber_x24,.-abi_test_clobber_x24 @@ -327,6 +356,7 @@ abi_test_clobber_x24: .hidden abi_test_clobber_x25 .align 4 abi_test_clobber_x25: + AARCH64_VALID_CALL_TARGET mov x25, xzr ret .size abi_test_clobber_x25,.-abi_test_clobber_x25 @@ -335,6 +365,7 @@ abi_test_clobber_x25: .hidden abi_test_clobber_x26 .align 4 abi_test_clobber_x26: + AARCH64_VALID_CALL_TARGET mov x26, xzr ret .size abi_test_clobber_x26,.-abi_test_clobber_x26 @@ -343,6 +374,7 @@ abi_test_clobber_x26: .hidden abi_test_clobber_x27 .align 4 abi_test_clobber_x27: + AARCH64_VALID_CALL_TARGET mov x27, xzr ret .size abi_test_clobber_x27,.-abi_test_clobber_x27 @@ -351,6 +383,7 @@ abi_test_clobber_x27: .hidden abi_test_clobber_x28 .align 4 abi_test_clobber_x28: + AARCH64_VALID_CALL_TARGET mov x28, xzr ret .size abi_test_clobber_x28,.-abi_test_clobber_x28 @@ -359,6 +392,7 @@ abi_test_clobber_x28: .hidden abi_test_clobber_x29 .align 4 abi_test_clobber_x29: + AARCH64_VALID_CALL_TARGET mov x29, xzr ret .size abi_test_clobber_x29,.-abi_test_clobber_x29 @@ -367,6 +401,7 @@ abi_test_clobber_x29: .hidden abi_test_clobber_d0 .align 4 abi_test_clobber_d0: + AARCH64_VALID_CALL_TARGET fmov d0, xzr ret .size abi_test_clobber_d0,.-abi_test_clobber_d0 @@ -375,6 +410,7 @@ abi_test_clobber_d0: .hidden abi_test_clobber_d1 .align 4 abi_test_clobber_d1: + AARCH64_VALID_CALL_TARGET fmov d1, xzr ret .size abi_test_clobber_d1,.-abi_test_clobber_d1 @@ -383,6 +419,7 @@ abi_test_clobber_d1: .hidden abi_test_clobber_d2 .align 4 abi_test_clobber_d2: + AARCH64_VALID_CALL_TARGET fmov d2, xzr ret .size abi_test_clobber_d2,.-abi_test_clobber_d2 @@ -391,6 +428,7 @@ abi_test_clobber_d2: .hidden abi_test_clobber_d3 .align 4 abi_test_clobber_d3: + AARCH64_VALID_CALL_TARGET fmov d3, xzr ret .size abi_test_clobber_d3,.-abi_test_clobber_d3 @@ -399,6 +437,7 @@ abi_test_clobber_d3: .hidden abi_test_clobber_d4 .align 4 abi_test_clobber_d4: + AARCH64_VALID_CALL_TARGET fmov d4, xzr ret .size abi_test_clobber_d4,.-abi_test_clobber_d4 @@ -407,6 +446,7 @@ abi_test_clobber_d4: .hidden abi_test_clobber_d5 .align 4 abi_test_clobber_d5: + AARCH64_VALID_CALL_TARGET fmov d5, xzr ret .size abi_test_clobber_d5,.-abi_test_clobber_d5 @@ -415,6 +455,7 @@ abi_test_clobber_d5: .hidden abi_test_clobber_d6 .align 4 abi_test_clobber_d6: + AARCH64_VALID_CALL_TARGET fmov d6, xzr ret .size abi_test_clobber_d6,.-abi_test_clobber_d6 @@ -423,6 +464,7 @@ abi_test_clobber_d6: .hidden abi_test_clobber_d7 .align 4 abi_test_clobber_d7: + AARCH64_VALID_CALL_TARGET fmov d7, xzr ret .size abi_test_clobber_d7,.-abi_test_clobber_d7 @@ -431,6 +473,7 @@ abi_test_clobber_d7: .hidden abi_test_clobber_d8 .align 4 abi_test_clobber_d8: + AARCH64_VALID_CALL_TARGET fmov d8, xzr ret .size abi_test_clobber_d8,.-abi_test_clobber_d8 @@ -439,6 +482,7 @@ abi_test_clobber_d8: .hidden abi_test_clobber_d9 .align 4 abi_test_clobber_d9: + AARCH64_VALID_CALL_TARGET fmov d9, xzr ret .size abi_test_clobber_d9,.-abi_test_clobber_d9 @@ -447,6 +491,7 @@ abi_test_clobber_d9: .hidden abi_test_clobber_d10 .align 4 abi_test_clobber_d10: + AARCH64_VALID_CALL_TARGET fmov d10, xzr ret .size abi_test_clobber_d10,.-abi_test_clobber_d10 @@ -455,6 +500,7 @@ abi_test_clobber_d10: .hidden abi_test_clobber_d11 .align 4 abi_test_clobber_d11: + AARCH64_VALID_CALL_TARGET fmov d11, xzr ret .size abi_test_clobber_d11,.-abi_test_clobber_d11 @@ -463,6 +509,7 @@ abi_test_clobber_d11: .hidden abi_test_clobber_d12 .align 4 abi_test_clobber_d12: + AARCH64_VALID_CALL_TARGET fmov d12, xzr ret .size abi_test_clobber_d12,.-abi_test_clobber_d12 @@ -471,6 +518,7 @@ abi_test_clobber_d12: .hidden abi_test_clobber_d13 .align 4 abi_test_clobber_d13: + AARCH64_VALID_CALL_TARGET fmov d13, xzr ret .size abi_test_clobber_d13,.-abi_test_clobber_d13 @@ -479,6 +527,7 @@ abi_test_clobber_d13: .hidden abi_test_clobber_d14 .align 4 abi_test_clobber_d14: + AARCH64_VALID_CALL_TARGET fmov d14, xzr ret .size abi_test_clobber_d14,.-abi_test_clobber_d14 @@ -487,6 +536,7 @@ abi_test_clobber_d14: .hidden abi_test_clobber_d15 .align 4 abi_test_clobber_d15: + AARCH64_VALID_CALL_TARGET fmov d15, xzr ret .size abi_test_clobber_d15,.-abi_test_clobber_d15 @@ -495,6 +545,7 @@ abi_test_clobber_d15: .hidden abi_test_clobber_d16 .align 4 abi_test_clobber_d16: + AARCH64_VALID_CALL_TARGET fmov d16, xzr ret .size abi_test_clobber_d16,.-abi_test_clobber_d16 @@ -503,6 +554,7 @@ abi_test_clobber_d16: .hidden abi_test_clobber_d17 .align 4 abi_test_clobber_d17: + AARCH64_VALID_CALL_TARGET fmov d17, xzr ret .size abi_test_clobber_d17,.-abi_test_clobber_d17 @@ -511,6 +563,7 @@ abi_test_clobber_d17: .hidden abi_test_clobber_d18 .align 4 abi_test_clobber_d18: + AARCH64_VALID_CALL_TARGET fmov d18, xzr ret .size abi_test_clobber_d18,.-abi_test_clobber_d18 @@ -519,6 +572,7 @@ abi_test_clobber_d18: .hidden abi_test_clobber_d19 .align 4 abi_test_clobber_d19: + AARCH64_VALID_CALL_TARGET fmov d19, xzr ret .size abi_test_clobber_d19,.-abi_test_clobber_d19 @@ -527,6 +581,7 @@ abi_test_clobber_d19: .hidden abi_test_clobber_d20 .align 4 abi_test_clobber_d20: + AARCH64_VALID_CALL_TARGET fmov d20, xzr ret .size abi_test_clobber_d20,.-abi_test_clobber_d20 @@ -535,6 +590,7 @@ abi_test_clobber_d20: .hidden abi_test_clobber_d21 .align 4 abi_test_clobber_d21: + AARCH64_VALID_CALL_TARGET fmov d21, xzr ret .size abi_test_clobber_d21,.-abi_test_clobber_d21 @@ -543,6 +599,7 @@ abi_test_clobber_d21: .hidden abi_test_clobber_d22 .align 4 abi_test_clobber_d22: + AARCH64_VALID_CALL_TARGET fmov d22, xzr ret .size abi_test_clobber_d22,.-abi_test_clobber_d22 @@ -551,6 +608,7 @@ abi_test_clobber_d22: .hidden abi_test_clobber_d23 .align 4 abi_test_clobber_d23: + AARCH64_VALID_CALL_TARGET fmov d23, xzr ret .size abi_test_clobber_d23,.-abi_test_clobber_d23 @@ -559,6 +617,7 @@ abi_test_clobber_d23: .hidden abi_test_clobber_d24 .align 4 abi_test_clobber_d24: + AARCH64_VALID_CALL_TARGET fmov d24, xzr ret .size abi_test_clobber_d24,.-abi_test_clobber_d24 @@ -567,6 +626,7 @@ abi_test_clobber_d24: .hidden abi_test_clobber_d25 .align 4 abi_test_clobber_d25: + AARCH64_VALID_CALL_TARGET fmov d25, xzr ret .size abi_test_clobber_d25,.-abi_test_clobber_d25 @@ -575,6 +635,7 @@ abi_test_clobber_d25: .hidden abi_test_clobber_d26 .align 4 abi_test_clobber_d26: + AARCH64_VALID_CALL_TARGET fmov d26, xzr ret .size abi_test_clobber_d26,.-abi_test_clobber_d26 @@ -583,6 +644,7 @@ abi_test_clobber_d26: .hidden abi_test_clobber_d27 .align 4 abi_test_clobber_d27: + AARCH64_VALID_CALL_TARGET fmov d27, xzr ret .size abi_test_clobber_d27,.-abi_test_clobber_d27 @@ -591,6 +653,7 @@ abi_test_clobber_d27: .hidden abi_test_clobber_d28 .align 4 abi_test_clobber_d28: + AARCH64_VALID_CALL_TARGET fmov d28, xzr ret .size abi_test_clobber_d28,.-abi_test_clobber_d28 @@ -599,6 +662,7 @@ abi_test_clobber_d28: .hidden abi_test_clobber_d29 .align 4 abi_test_clobber_d29: + AARCH64_VALID_CALL_TARGET fmov d29, xzr ret .size abi_test_clobber_d29,.-abi_test_clobber_d29 @@ -607,6 +671,7 @@ abi_test_clobber_d29: .hidden abi_test_clobber_d30 .align 4 abi_test_clobber_d30: + AARCH64_VALID_CALL_TARGET fmov d30, xzr ret .size abi_test_clobber_d30,.-abi_test_clobber_d30 @@ -615,6 +680,7 @@ abi_test_clobber_d30: .hidden abi_test_clobber_d31 .align 4 abi_test_clobber_d31: + AARCH64_VALID_CALL_TARGET fmov d31, xzr ret .size abi_test_clobber_d31,.-abi_test_clobber_d31 @@ -623,6 +689,7 @@ abi_test_clobber_d31: .hidden abi_test_clobber_v8_upper .align 4 abi_test_clobber_v8_upper: + AARCH64_VALID_CALL_TARGET fmov v8.d[1], xzr ret .size abi_test_clobber_v8_upper,.-abi_test_clobber_v8_upper @@ -631,6 +698,7 @@ abi_test_clobber_v8_upper: .hidden abi_test_clobber_v9_upper .align 4 abi_test_clobber_v9_upper: + AARCH64_VALID_CALL_TARGET fmov v9.d[1], xzr ret .size abi_test_clobber_v9_upper,.-abi_test_clobber_v9_upper @@ -639,6 +707,7 @@ abi_test_clobber_v9_upper: .hidden abi_test_clobber_v10_upper .align 4 abi_test_clobber_v10_upper: + AARCH64_VALID_CALL_TARGET fmov v10.d[1], xzr ret .size abi_test_clobber_v10_upper,.-abi_test_clobber_v10_upper @@ -647,6 +716,7 @@ abi_test_clobber_v10_upper: .hidden abi_test_clobber_v11_upper .align 4 abi_test_clobber_v11_upper: + AARCH64_VALID_CALL_TARGET fmov v11.d[1], xzr ret .size abi_test_clobber_v11_upper,.-abi_test_clobber_v11_upper @@ -655,6 +725,7 @@ abi_test_clobber_v11_upper: .hidden abi_test_clobber_v12_upper .align 4 abi_test_clobber_v12_upper: + AARCH64_VALID_CALL_TARGET fmov v12.d[1], xzr ret .size abi_test_clobber_v12_upper,.-abi_test_clobber_v12_upper @@ -663,6 +734,7 @@ abi_test_clobber_v12_upper: .hidden abi_test_clobber_v13_upper .align 4 abi_test_clobber_v13_upper: + AARCH64_VALID_CALL_TARGET fmov v13.d[1], xzr ret .size abi_test_clobber_v13_upper,.-abi_test_clobber_v13_upper @@ -671,6 +743,7 @@ abi_test_clobber_v13_upper: .hidden abi_test_clobber_v14_upper .align 4 abi_test_clobber_v14_upper: + AARCH64_VALID_CALL_TARGET fmov v14.d[1], xzr ret .size abi_test_clobber_v14_upper,.-abi_test_clobber_v14_upper @@ -679,6 +752,7 @@ abi_test_clobber_v14_upper: .hidden abi_test_clobber_v15_upper .align 4 abi_test_clobber_v15_upper: + AARCH64_VALID_CALL_TARGET fmov v15.d[1], xzr ret .size abi_test_clobber_v15_upper,.-abi_test_clobber_v15_upper |