diff options
| author | Haibo Huang <hhb@google.com> | 2021-02-19 21:03:50 +0000 |
|---|---|---|
| committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2021-02-19 21:03:50 +0000 |
| commit | 79ceed3b69c3e5af2fc3670827993d063fb106ad (patch) | |
| tree | 2ac81406c26743024a258437ddc505c6e7a79084 | |
| parent | 571857abde20b567dfd216f031d240e128420798 (diff) | |
| parent | bc56560bd9731cdb76048f10757f9d2a5dbb3db1 (diff) | |
| download | arm-optimized-routines-79ceed3b69c3e5af2fc3670827993d063fb106ad.tar.gz | |
Upgrade arm-optimized-routines to v21.02 am: ef1728935a am: ce035f0976 am: bc56560bd9
Original change: https://android-review.googlesource.com/c/platform/external/arm-optimized-routines/+/1595408
MUST ONLY BE SUBMITTED BY AUTOMERGER
Change-Id: Iee751e13b05b67c433288b6ce8c598294fa3e7fe
104 files changed, 683 insertions, 153 deletions
@@ -9,11 +9,11 @@ third_party { type: GIT value: "https://github.com/ARM-software/optimized-routines.git" } - version: "v20.11" + version: "v21.02" license_type: NOTICE last_upgrade_date { - year: 2020 - month: 11 - day: 30 + year: 2021 + month: 2 + day: 18 } } @@ -1,6 +1,6 @@ # Makefile - requires GNU make # -# Copyright (c) 2018-2019, Arm Limited. +# Copyright (c) 2018-2020, Arm Limited. # SPDX-License-Identifier: MIT srcdir = . diff --git a/config.mk.dist b/config.mk.dist index 3e55c98..177e1ac 100644 --- a/config.mk.dist +++ b/config.mk.dist @@ -1,6 +1,6 @@ # Example config.mk # -# Copyright (c) 2018-2019, Arm Limited. +# Copyright (c) 2018-2020, Arm Limited. # SPDX-License-Identifier: MIT # Subprojects to build diff --git a/math/cosf.c b/math/cosf.c index 831b39e..f29f194 100644 --- a/math/cosf.c +++ b/math/cosf.c @@ -1,7 +1,7 @@ /* * Single-precision cos function. * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2019, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -1,7 +1,7 @@ /* * Double-precision e^x function. * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/exp2.c b/math/exp2.c index 47aa479..35ab39f 100644 --- a/math/exp2.c +++ b/math/exp2.c @@ -1,7 +1,7 @@ /* * Double-precision 2^x function. * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/expf.c b/math/expf.c index 0fe1f7d..9b2f0c3 100644 --- a/math/expf.c +++ b/math/expf.c @@ -1,7 +1,7 @@ /* * Single-precision e^x function. * - * Copyright (c) 2017-2018, Arm Limited. + * Copyright (c) 2017-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/include/mathlib.h b/math/include/mathlib.h index 4493008..279d829 100644 --- a/math/include/mathlib.h +++ b/math/include/mathlib.h @@ -1,7 +1,7 @@ /* * Public API. * - * Copyright (c) 2015-2019, Arm Limited. + * Copyright (c) 2015-2020, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -1,7 +1,7 @@ /* * Double-precision log(x) function. * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/log2.c b/math/log2.c index 804fb85..55102b7 100644 --- a/math/log2.c +++ b/math/log2.c @@ -1,7 +1,7 @@ /* * Double-precision log2(x) function. * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/logf.c b/math/logf.c index ee3120a..cfbaee1 100644 --- a/math/logf.c +++ b/math/logf.c @@ -1,7 +1,7 @@ /* * Single-precision log function. * - * Copyright (c) 2017-2018, Arm Limited. + * Copyright (c) 2017-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/logf_data.c b/math/logf_data.c index 53c5f62..e8973ce 100644 --- a/math/logf_data.c +++ b/math/logf_data.c @@ -1,7 +1,7 @@ /* * Data definition for logf. * - * Copyright (c) 2017-2018, Arm Limited. + * Copyright (c) 2017-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/math_config.h b/math/math_config.h index 09755ad..e851043 100644 --- a/math/math_config.h +++ b/math/math_config.h @@ -1,7 +1,7 @@ /* * Configuration for math routines. * - * Copyright (c) 2017-2018, Arm Limited. + * Copyright (c) 2017-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/math_errf.c b/math/math_errf.c index 771d982..d5350b8 100644 --- a/math/math_errf.c +++ b/math/math_errf.c @@ -1,7 +1,7 @@ /* * Single-precision math error handling. * - * Copyright (c) 2017-2018, Arm Limited. + * Copyright (c) 2017-2020, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -1,7 +1,7 @@ /* * Double-precision x^y function. * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/powf.c b/math/powf.c index 1534a09..6ba45d3 100644 --- a/math/powf.c +++ b/math/powf.c @@ -1,7 +1,7 @@ /* * Single-precision pow function. * - * Copyright (c) 2017-2018, Arm Limited. + * Copyright (c) 2017-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/powf_log2_data.c b/math/powf_log2_data.c index b9fbdc4..97e0d98 100644 --- a/math/powf_log2_data.c +++ b/math/powf_log2_data.c @@ -1,7 +1,7 @@ /* * Data definition for powf. * - * Copyright (c) 2017-2018, Arm Limited. + * Copyright (c) 2017-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/sincosf.c b/math/sincosf.c index e6cd41e..9746f1c 100644 --- a/math/sincosf.c +++ b/math/sincosf.c @@ -1,7 +1,7 @@ /* * Single-precision sin/cos function. * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/sincosf_data.c b/math/sincosf_data.c index 5d0b58e..ab4ac47 100644 --- a/math/sincosf_data.c +++ b/math/sincosf_data.c @@ -1,7 +1,7 @@ /* * Data definition for sinf, cosf and sincosf. * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/sinf.c b/math/sinf.c index 770b294..ddbc1da 100644 --- a/math/sinf.c +++ b/math/sinf.c @@ -1,7 +1,7 @@ /* * Single-precision sin function. * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/mathbench.c b/math/test/mathbench.c index 2e8aaf8..0c17826 100644 --- a/math/test/mathbench.c +++ b/math/test/mathbench.c @@ -1,7 +1,7 @@ /* * Microbenchmark for math functions. * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/mathtest.c b/math/test/mathtest.c index 2ff8c3f..3108967 100644 --- a/math/test/mathtest.c +++ b/math/test/mathtest.c @@ -1,7 +1,7 @@ /* * mathtest.c - test rig for mathlib * - * Copyright (c) 1998-2018, Arm Limited. + * Copyright (c) 1998-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/rtest/dotest.c b/math/test/rtest/dotest.c index f416477..6be79e1 100644 --- a/math/test/rtest/dotest.c +++ b/math/test/rtest/dotest.c @@ -1,7 +1,7 @@ /* * dotest.c - actually generate mathlib test cases * - * Copyright (c) 1999-2018, Arm Limited. + * Copyright (c) 1999-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/rtest/intern.h b/math/test/rtest/intern.h index af574b0..12a9c74 100644 --- a/math/test/rtest/intern.h +++ b/math/test/rtest/intern.h @@ -1,7 +1,7 @@ /* * intern.h * - * Copyright (c) 1999-2018, Arm Limited. + * Copyright (c) 1999-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/rtest/main.c b/math/test/rtest/main.c index e94e455..0d8ead8 100644 --- a/math/test/rtest/main.c +++ b/math/test/rtest/main.c @@ -1,7 +1,7 @@ /* * main.c * - * Copyright (c) 1999-2018, Arm Limited. + * Copyright (c) 1999-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/rtest/random.c b/math/test/rtest/random.c index e97a8c6..5612396 100644 --- a/math/test/rtest/random.c +++ b/math/test/rtest/random.c @@ -1,7 +1,7 @@ /* * random.c - random number generator for producing mathlib test cases * - * Copyright (c) 1998-2018, Arm Limited. + * Copyright (c) 1998-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/rtest/random.h b/math/test/rtest/random.h index c1ce956..b4b22df 100644 --- a/math/test/rtest/random.h +++ b/math/test/rtest/random.h @@ -1,7 +1,7 @@ /* * random.h - header for random.c * - * Copyright (c) 2009-2018, Arm Limited. + * Copyright (c) 2009-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/rtest/semi.c b/math/test/rtest/semi.c index 938dc3a..c9f0daf 100644 --- a/math/test/rtest/semi.c +++ b/math/test/rtest/semi.c @@ -1,7 +1,7 @@ /* * semi.c: test implementations of mathlib seminumerical functions * - * Copyright (c) 1999-2018, Arm Limited. + * Copyright (c) 1999-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/rtest/semi.h b/math/test/rtest/semi.h index da473a2..17dc415 100644 --- a/math/test/rtest/semi.h +++ b/math/test/rtest/semi.h @@ -1,7 +1,7 @@ /* * semi.h: header for semi.c * - * Copyright (c) 1999-2018, Arm Limited. + * Copyright (c) 1999-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/rtest/types.h b/math/test/rtest/types.h index 1a76c2e..53cd557 100644 --- a/math/test/rtest/types.h +++ b/math/test/rtest/types.h @@ -1,7 +1,7 @@ /* * types.h * - * Copyright (c) 2005-2018, Arm Limited. + * Copyright (c) 2005-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/rtest/wrappers.c b/math/test/rtest/wrappers.c index acaf671..de45ac5 100644 --- a/math/test/rtest/wrappers.c +++ b/math/test/rtest/wrappers.c @@ -1,7 +1,7 @@ /* * wrappers.c - wrappers to modify output of MPFR/MPC test functions * - * Copyright (c) 2014-2018, Arm Limited. + * Copyright (c) 2014-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/rtest/wrappers.h b/math/test/rtest/wrappers.h index 5804935..7b09c85 100644 --- a/math/test/rtest/wrappers.h +++ b/math/test/rtest/wrappers.h @@ -1,7 +1,7 @@ /* * wrappers.h - wrappers to modify output of MPFR/MPC test functions * - * Copyright (c) 2014-2018, Arm Limited. + * Copyright (c) 2014-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/test/runulp.sh b/math/test/runulp.sh index 1d87a81..0190d9a 100755 --- a/math/test/runulp.sh +++ b/math/test/runulp.sh @@ -2,7 +2,7 @@ # ULP error check script. # -# Copyright (c) 2019, Arm Limited. +# Copyright (c) 2019-2020, Arm Limited. # SPDX-License-Identifier: MIT #set -x diff --git a/math/test/testcases/directed/cosf.tst b/math/test/testcases/directed/cosf.tst index 5dc0994..7916044 100644 --- a/math/test/testcases/directed/cosf.tst +++ b/math/test/testcases/directed/cosf.tst @@ -1,6 +1,6 @@ ; cosf.tst - Directed test cases for SP cosine ; -; Copyright (c) 2007-2018, Arm Limited. +; Copyright (c) 2007-2019, Arm Limited. ; SPDX-License-Identifier: MIT func=cosf op1=7fc00001 result=7fc00001 errno=0 diff --git a/math/test/testcases/directed/exp.tst b/math/test/testcases/directed/exp.tst index addfc0a..85d556c 100644 --- a/math/test/testcases/directed/exp.tst +++ b/math/test/testcases/directed/exp.tst @@ -1,6 +1,6 @@ ; Directed test cases for exp ; -; Copyright (c) 2018, Arm Limited. +; Copyright (c) 2018-2019, Arm Limited. ; SPDX-License-Identifier: MIT func=exp op1=7ff80000.00000001 result=7ff80000.00000001 errno=0 diff --git a/math/test/testcases/directed/exp2.tst b/math/test/testcases/directed/exp2.tst index 04a5a50..fa56c9f 100644 --- a/math/test/testcases/directed/exp2.tst +++ b/math/test/testcases/directed/exp2.tst @@ -1,6 +1,6 @@ ; Directed test cases for exp2 ; -; Copyright (c) 2018, Arm Limited. +; Copyright (c) 2018-2019, Arm Limited. ; SPDX-License-Identifier: MIT func=exp2 op1=7ff80000.00000001 result=7ff80000.00000001 errno=0 diff --git a/math/test/testcases/directed/exp2f.tst b/math/test/testcases/directed/exp2f.tst index 2b6a9b5..38cfc3f 100644 --- a/math/test/testcases/directed/exp2f.tst +++ b/math/test/testcases/directed/exp2f.tst @@ -1,6 +1,6 @@ ; exp2f.tst - Directed test cases for exp2f ; -; Copyright (c) 2017-2018, Arm Limited. +; Copyright (c) 2017-2019, Arm Limited. ; SPDX-License-Identifier: MIT func=exp2f op1=7fc00001 result=7fc00001 errno=0 diff --git a/math/test/testcases/directed/expf.tst b/math/test/testcases/directed/expf.tst index 74664c7..ff0f671 100644 --- a/math/test/testcases/directed/expf.tst +++ b/math/test/testcases/directed/expf.tst @@ -1,6 +1,6 @@ ; expf.tst - Directed test cases for expf ; -; Copyright (c) 2007-2018, Arm Limited. +; Copyright (c) 2007-2019, Arm Limited. ; SPDX-License-Identifier: MIT func=expf op1=7fc00001 result=7fc00001 errno=0 diff --git a/math/test/testcases/directed/log.tst b/math/test/testcases/directed/log.tst index eeb762c..a0aa398 100644 --- a/math/test/testcases/directed/log.tst +++ b/math/test/testcases/directed/log.tst @@ -1,6 +1,6 @@ ; Directed test cases for log ; -; Copyright (c) 2018, Arm Limited. +; Copyright (c) 2018-2019, Arm Limited. ; SPDX-License-Identifier: MIT func=log op1=7ff80000.00000001 result=7ff80000.00000001 errno=0 diff --git a/math/test/testcases/directed/log2.tst b/math/test/testcases/directed/log2.tst index e0765d8..ff1286c 100644 --- a/math/test/testcases/directed/log2.tst +++ b/math/test/testcases/directed/log2.tst @@ -1,6 +1,6 @@ ; Directed test cases for log2 ; -; Copyright (c) 2018, Arm Limited. +; Copyright (c) 2018-2019, Arm Limited. ; SPDX-License-Identifier: MIT func=log2 op1=7ff80000.00000001 result=7ff80000.00000001 errno=0 diff --git a/math/test/testcases/directed/log2f.tst b/math/test/testcases/directed/log2f.tst index 8d685ba..5832c4f 100644 --- a/math/test/testcases/directed/log2f.tst +++ b/math/test/testcases/directed/log2f.tst @@ -1,6 +1,6 @@ ; log2f.tst - Directed test cases for log2f ; -; Copyright (c) 2017-2018, Arm Limited. +; Copyright (c) 2017-2019, Arm Limited. ; SPDX-License-Identifier: MIT func=log2f op1=7fc00001 result=7fc00001 errno=0 diff --git a/math/test/testcases/directed/logf.tst b/math/test/testcases/directed/logf.tst index 7ccc873..6e68a36 100644 --- a/math/test/testcases/directed/logf.tst +++ b/math/test/testcases/directed/logf.tst @@ -1,6 +1,6 @@ ; logf.tst - Directed test cases for logf ; -; Copyright (c) 2007-2018, Arm Limited. +; Copyright (c) 2007-2019, Arm Limited. ; SPDX-License-Identifier: MIT func=logf op1=7fc00001 result=7fc00001 errno=0 diff --git a/math/test/testcases/directed/pow.tst b/math/test/testcases/directed/pow.tst index a4c42be..1966581 100644 --- a/math/test/testcases/directed/pow.tst +++ b/math/test/testcases/directed/pow.tst @@ -1,6 +1,6 @@ ; Directed test cases for pow ; -; Copyright (c) 2018, Arm Limited. +; Copyright (c) 2018-2019, Arm Limited. ; SPDX-License-Identifier: MIT func=pow op1=00000000.00000000 op2=00000000.00000000 result=3ff00000.00000000 errno=0 diff --git a/math/test/testcases/directed/powf.tst b/math/test/testcases/directed/powf.tst index efd1dd5..3fa8b11 100644 --- a/math/test/testcases/directed/powf.tst +++ b/math/test/testcases/directed/powf.tst @@ -1,6 +1,6 @@ ; powf.tst - Directed test cases for powf ; -; Copyright (c) 2007-2018, Arm Limited. +; Copyright (c) 2007-2019, Arm Limited. ; SPDX-License-Identifier: MIT func=powf op1=7f800001 op2=7f800001 result=7fc00001 errno=0 status=i diff --git a/math/test/testcases/directed/sincosf.tst b/math/test/testcases/directed/sincosf.tst index b4b2526..4b33d22 100644 --- a/math/test/testcases/directed/sincosf.tst +++ b/math/test/testcases/directed/sincosf.tst @@ -1,6 +1,6 @@ ; Directed test cases for SP sincos ; -; Copyright (c) 2007-2018, Arm Limited. +; Copyright (c) 2007-2019, Arm Limited. ; SPDX-License-Identifier: MIT diff --git a/math/test/testcases/directed/sinf.tst b/math/test/testcases/directed/sinf.tst index 13cfdca..ded80b1 100644 --- a/math/test/testcases/directed/sinf.tst +++ b/math/test/testcases/directed/sinf.tst @@ -1,6 +1,6 @@ ; sinf.tst - Directed test cases for SP sine ; -; Copyright (c) 2007-2018, Arm Limited. +; Copyright (c) 2007-2019, Arm Limited. ; SPDX-License-Identifier: MIT diff --git a/math/test/testcases/random/double.tst b/math/test/testcases/random/double.tst index c37e837..c24ff80 100644 --- a/math/test/testcases/random/double.tst +++ b/math/test/testcases/random/double.tst @@ -1,6 +1,6 @@ !! double.tst - Random test case specification for DP functions !! -!! Copyright (c) 1999-2018, Arm Limited. +!! Copyright (c) 1999-2019, Arm Limited. !! SPDX-License-Identifier: MIT test exp 10000 diff --git a/math/test/testcases/random/float.tst b/math/test/testcases/random/float.tst index baf62b9..d02a227 100644 --- a/math/test/testcases/random/float.tst +++ b/math/test/testcases/random/float.tst @@ -1,6 +1,6 @@ !! single.tst - Random test case specification for SP functions !! -!! Copyright (c) 1999-2018, Arm Limited. +!! Copyright (c) 1999-2019, Arm Limited. !! SPDX-License-Identifier: MIT test sinf 10000 diff --git a/math/test/ulp.c b/math/test/ulp.c index 05a5321..51479b8 100644 --- a/math/test/ulp.c +++ b/math/test/ulp.c @@ -1,7 +1,7 @@ /* * ULP error checking tool for math functions. * - * Copyright (c) 2019, Arm Limited. + * Copyright (c) 2019-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/math/tools/remez.jl b/math/tools/remez.jl index f479fc5..2ff436f 100755 --- a/math/tools/remez.jl +++ b/math/tools/remez.jl @@ -3,7 +3,7 @@ # remez.jl - implementation of the Remez algorithm for polynomial approximation # -# Copyright (c) 2015-2018, Arm Limited. +# Copyright (c) 2015-2019, Arm Limited. # SPDX-License-Identifier: MIT import Base.\ diff --git a/math/v_math.h b/math/v_math.h index 3db22e5..f2cc467 100644 --- a/math/v_math.h +++ b/math/v_math.h @@ -1,7 +1,7 @@ /* * Vector math abstractions. * - * Copyright (c) 2019, Arm Limited. + * Copyright (c) 2019-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/string/Dir.mk b/string/Dir.mk index 7817357..cf3453f 100644 --- a/string/Dir.mk +++ b/string/Dir.mk @@ -1,6 +1,6 @@ # Makefile fragment - requires GNU make # -# Copyright (c) 2019-2020, Arm Limited. +# Copyright (c) 2019-2021, Arm Limited. # SPDX-License-Identifier: MIT S := $(srcdir)/string @@ -29,6 +29,8 @@ string-tests := \ build/bin/test/memchr \ build/bin/test/memrchr \ build/bin/test/memcmp \ + build/bin/test/__mtag_tag_region \ + build/bin/test/__mtag_tag_zero_region \ build/bin/test/strcpy \ build/bin/test/stpcpy \ build/bin/test/strcmp \ diff --git a/string/aarch64/__mtag_tag_region.S b/string/aarch64/__mtag_tag_region.S new file mode 100644 index 0000000..84339f7 --- /dev/null +++ b/string/aarch64/__mtag_tag_region.S @@ -0,0 +1,100 @@ +/* + * __mtag_tag_region - tag memory + * + * Copyright (c) 2021, Arm Limited. + * SPDX-License-Identifier: MIT + */ + +/* Assumptions: + * + * ARMv8-a, AArch64, MTE, LP64 ABI. + * + * Interface contract: + * Address is 16 byte aligned and size is multiple of 16. + * Returns the passed pointer. + * The memory region may remain untagged if tagging is not enabled. + */ + +#include "../asmdefs.h" + +#if __ARM_FEATURE_MEMORY_TAGGING + +#define dstin x0 +#define count x1 +#define dst x2 +#define dstend x3 +#define tmp x4 +#define zva_val x4 + +ENTRY (__mtag_tag_region) + PTR_ARG (0) + SIZE_ARG (1) + + add dstend, dstin, count + + cmp count, 96 + b.hi L(set_long) + + tbnz count, 6, L(set96) + + /* Set 0, 16, 32, or 48 bytes. */ + lsr tmp, count, 5 + add tmp, dstin, tmp, lsl 4 + cbz count, L(end) + stg dstin, [dstin] + stg dstin, [tmp] + stg dstin, [dstend, -16] +L(end): + ret + + .p2align 4 + /* Set 64..96 bytes. Write 64 bytes from the start and + 32 bytes from the end. */ +L(set96): + st2g dstin, [dstin] + st2g dstin, [dstin, 32] + st2g dstin, [dstend, -32] + ret + + .p2align 4 + /* Size is > 96 bytes. */ +L(set_long): + cmp count, 160 + b.lo L(no_zva) + +#ifndef SKIP_ZVA_CHECK + mrs zva_val, dczid_el0 + and zva_val, zva_val, 31 + cmp zva_val, 4 /* ZVA size is 64 bytes. */ + b.ne L(no_zva) +#endif + st2g dstin, [dstin] + st2g dstin, [dstin, 32] + bic dst, dstin, 63 + sub count, dstend, dst /* Count is now 64 too large. */ + sub count, count, 128 /* Adjust count and bias for loop. */ + + .p2align 4 +L(zva_loop): + add dst, dst, 64 + dc gva, dst + subs count, count, 64 + b.hi L(zva_loop) + st2g dstin, [dstend, -64] + st2g dstin, [dstend, -32] + ret + +L(no_zva): + sub dst, dstin, 32 /* Dst is biased by -32. */ + sub count, count, 64 /* Adjust count for loop. */ +L(no_zva_loop): + st2g dstin, [dst, 32] + st2g dstin, [dst, 64]! + subs count, count, 64 + b.hi L(no_zva_loop) + st2g dstin, [dstend, -64] + st2g dstin, [dstend, -32] + ret + +END (__mtag_tag_region) +#endif diff --git a/string/aarch64/__mtag_tag_zero_region.S b/string/aarch64/__mtag_tag_zero_region.S new file mode 100644 index 0000000..f58364c --- /dev/null +++ b/string/aarch64/__mtag_tag_zero_region.S @@ -0,0 +1,100 @@ +/* + * __mtag_tag_zero_region - tag memory and fill it with zero bytes + * + * Copyright (c) 2021, Arm Limited. + * SPDX-License-Identifier: MIT + */ + +/* Assumptions: + * + * ARMv8-a, AArch64, MTE, LP64 ABI. + * + * Interface contract: + * Address is 16 byte aligned and size is multiple of 16. + * Returns the passed pointer. + * The memory region may remain untagged if tagging is not enabled. + */ + +#include "../asmdefs.h" + +#if __ARM_FEATURE_MEMORY_TAGGING + +#define dstin x0 +#define count x1 +#define dst x2 +#define dstend x3 +#define tmp x4 +#define zva_val x4 + +ENTRY (__mtag_tag_zero_region) + PTR_ARG (0) + SIZE_ARG (1) + + add dstend, dstin, count + + cmp count, 96 + b.hi L(set_long) + + tbnz count, 6, L(set96) + + /* Set 0, 16, 32, or 48 bytes. */ + lsr tmp, count, 5 + add tmp, dstin, tmp, lsl 4 + cbz count, L(end) + stzg dstin, [dstin] + stzg dstin, [tmp] + stzg dstin, [dstend, -16] +L(end): + ret + + .p2align 4 + /* Set 64..96 bytes. Write 64 bytes from the start and + 32 bytes from the end. */ +L(set96): + stz2g dstin, [dstin] + stz2g dstin, [dstin, 32] + stz2g dstin, [dstend, -32] + ret + + .p2align 4 + /* Size is > 96 bytes. */ +L(set_long): + cmp count, 160 + b.lo L(no_zva) + +#ifndef SKIP_ZVA_CHECK + mrs zva_val, dczid_el0 + and zva_val, zva_val, 31 + cmp zva_val, 4 /* ZVA size is 64 bytes. */ + b.ne L(no_zva) +#endif + stz2g dstin, [dstin] + stz2g dstin, [dstin, 32] + bic dst, dstin, 63 + sub count, dstend, dst /* Count is now 64 too large. */ + sub count, count, 128 /* Adjust count and bias for loop. */ + + .p2align 4 +L(zva_loop): + add dst, dst, 64 + dc gzva, dst + subs count, count, 64 + b.hi L(zva_loop) + stz2g dstin, [dstend, -64] + stz2g dstin, [dstend, -32] + ret + +L(no_zva): + sub dst, dstin, 32 /* Dst is biased by -32. */ + sub count, count, 64 /* Adjust count for loop. */ +L(no_zva_loop): + stz2g dstin, [dst, 32] + stz2g dstin, [dst, 64]! + subs count, count, 64 + b.hi L(no_zva_loop) + stz2g dstin, [dstend, -64] + stz2g dstin, [dstend, -32] + ret + +END (__mtag_tag_zero_region) +#endif diff --git a/string/aarch64/memchr-mte.S b/string/aarch64/memchr-mte.S index 31ad050..c2e967d 100644 --- a/string/aarch64/memchr-mte.S +++ b/string/aarch64/memchr-mte.S @@ -44,6 +44,8 @@ string, counting trailing zeros identifies exactly which byte matched. */ ENTRY (__memchr_aarch64_mte) + PTR_ARG (0) + SIZE_ARG (2) bic src, srcin, 15 cbz cntin, L(nomatch) ld1 {vdata.16b}, [src] diff --git a/string/aarch64/memchr-sve.S b/string/aarch64/memchr-sve.S index 4a5c726..c22e659 100644 --- a/string/aarch64/memchr-sve.S +++ b/string/aarch64/memchr-sve.S @@ -1,7 +1,7 @@ /* * memchr - find a character in a memory zone * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -14,15 +14,14 @@ * SVE Available. */ - .arch armv8-a+sve - .text - -ENTRY_ALIGN(__memchr_aarch64_sve, 4) +ENTRY (__memchr_aarch64_sve) + PTR_ARG (0) + SIZE_ARG (2) dup z1.b, w1 /* duplicate c to a vector */ setffr /* initialize FFR */ mov x3, 0 /* initialize off */ - nop + .p2align 4 0: whilelo p1.b, x3, x2 /* make sure off < max */ b.none 9f diff --git a/string/aarch64/memchr.S b/string/aarch64/memchr.S index dfba79f..353f0d1 100644 --- a/string/aarch64/memchr.S +++ b/string/aarch64/memchr.S @@ -1,7 +1,7 @@ /* * memchr - find a character in a memory zone * - * Copyright (c) 2014-2019, Arm Limited. + * Copyright (c) 2014-2020, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -47,6 +47,8 @@ */ ENTRY (__memchr_aarch64) + PTR_ARG (0) + SIZE_ARG (2) /* Do not dereference srcin if no bytes to compare. */ cbz cntin, L(zero_length) /* diff --git a/string/aarch64/memcmp-sve.S b/string/aarch64/memcmp-sve.S index 8a0a2ea..78c5eca 100644 --- a/string/aarch64/memcmp-sve.S +++ b/string/aarch64/memcmp-sve.S @@ -1,7 +1,7 @@ /* * memcmp - compare memory * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -14,10 +14,10 @@ * SVE Available. */ - .arch armv8-a+sve - .text - -ENTRY_ALIGN (__memcmp_aarch64_sve, 4) +ENTRY (__memcmp_aarch64_sve) + PTR_ARG (0) + PTR_ARG (1) + SIZE_ARG (2) mov x3, 0 /* initialize off */ 0: whilelo p0.b, x3, x2 /* while off < max */ diff --git a/string/aarch64/memcmp.S b/string/aarch64/memcmp.S index dac9147..3b10266 100644 --- a/string/aarch64/memcmp.S +++ b/string/aarch64/memcmp.S @@ -1,6 +1,6 @@ /* memcmp - compare memory * - * Copyright (c) 2013, Arm Limited. + * Copyright (c) 2013-2020, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -28,6 +28,9 @@ #define tmp2 x8 ENTRY (__memcmp_aarch64) + PTR_ARG (0) + PTR_ARG (1) + SIZE_ARG (2) subs limit, limit, 8 b.lo L(less8) diff --git a/string/aarch64/memcpy-advsimd.S b/string/aarch64/memcpy-advsimd.S index 23545a3..f97f2c3 100644 --- a/string/aarch64/memcpy-advsimd.S +++ b/string/aarch64/memcpy-advsimd.S @@ -52,6 +52,9 @@ ENTRY_ALIAS (__memmove_aarch64_simd) ENTRY (__memcpy_aarch64_simd) + PTR_ARG (0) + PTR_ARG (1) + SIZE_ARG (2) add srcend, src, count add dstend, dstin, count cmp count, 128 diff --git a/string/aarch64/memcpy.S b/string/aarch64/memcpy.S index 157bb0d..dd254f6 100644 --- a/string/aarch64/memcpy.S +++ b/string/aarch64/memcpy.S @@ -55,6 +55,9 @@ ENTRY_ALIAS (__memmove_aarch64) ENTRY (__memcpy_aarch64) + PTR_ARG (0) + PTR_ARG (1) + SIZE_ARG (2) add srcend, src, count add dstend, dstin, count cmp count, 128 diff --git a/string/aarch64/memrchr.S b/string/aarch64/memrchr.S index ad42b49..7b4be84 100644 --- a/string/aarch64/memrchr.S +++ b/string/aarch64/memrchr.S @@ -46,6 +46,7 @@ string, counting trailing zeros identifies exactly which byte matched. */ ENTRY (__memrchr_aarch64) + PTR_ARG (0) add end, srcin, cntin sub endm1, end, 1 bic src, endm1, 15 diff --git a/string/aarch64/memset.S b/string/aarch64/memset.S index 27743f1..9fcd975 100644 --- a/string/aarch64/memset.S +++ b/string/aarch64/memset.S @@ -1,7 +1,7 @@ /* * memset - fill memory with a constant byte * - * Copyright (c) 2012-2020, Arm Limited. + * Copyright (c) 2012-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -22,6 +22,8 @@ #define zva_val x5 ENTRY (__memset_aarch64) + PTR_ARG (0) + SIZE_ARG (2) dup v0.16B, valw add dstend, dstin, count @@ -37,7 +39,7 @@ ENTRY (__memset_aarch64) str val, [dstin] str val, [dstend, -8] ret - nop + .p2align 4 1: tbz count, 2, 2f str valw, [dstin] str valw, [dstend, -4] diff --git a/string/aarch64/strchr-mte.S b/string/aarch64/strchr-mte.S index 577752e..dcb0e46 100644 --- a/string/aarch64/strchr-mte.S +++ b/string/aarch64/strchr-mte.S @@ -43,6 +43,7 @@ string, counting trailing zeros identifies exactly which byte matched. */ ENTRY (__strchr_aarch64_mte) + PTR_ARG (0) bic src, srcin, 15 dup vrepchr.16b, chrin ld1 {vdata.16b}, [src] diff --git a/string/aarch64/strchr-sve.S b/string/aarch64/strchr-sve.S index 495beda..13ba9f4 100644 --- a/string/aarch64/strchr-sve.S +++ b/string/aarch64/strchr-sve.S @@ -1,7 +1,7 @@ /* * strchr/strchrnul - find a character in a string * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -14,9 +14,6 @@ * SVE Available. */ - .arch armv8-a+sve - .text - /* To build as strchrnul, define BUILD_STRCHRNUL before compiling this file. */ #ifdef BUILD_STRCHRNUL #define FUNC __strchrnul_aarch64_sve @@ -24,7 +21,8 @@ #define FUNC __strchr_aarch64_sve #endif -ENTRY_ALIGN (FUNC, 4) +ENTRY (FUNC) + PTR_ARG (0) dup z1.b, w1 /* replicate byte across vector */ setffr /* initialize FFR */ ptrue p1.b /* all ones; loop invariant */ diff --git a/string/aarch64/strchr.S b/string/aarch64/strchr.S index 8d8e3fc..1063cbf 100644 --- a/string/aarch64/strchr.S +++ b/string/aarch64/strchr.S @@ -1,7 +1,7 @@ /* * strchr - find a character in a string * - * Copyright (c) 2014-2019, Arm Limited. + * Copyright (c) 2014-2020, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -51,6 +51,7 @@ /* Locals and temporaries. */ ENTRY (__strchr_aarch64) + PTR_ARG (0) /* Magic constant 0xc0300c03 to allow us to identify which lane matches the requested byte. Even bits are set if the character matches, odd bits if either the char is NUL or matches. */ diff --git a/string/aarch64/strchrnul-mte.S b/string/aarch64/strchrnul-mte.S index 0dbf0dc..1b0d0a6 100644 --- a/string/aarch64/strchrnul-mte.S +++ b/string/aarch64/strchrnul-mte.S @@ -41,6 +41,7 @@ string, counting trailing zeros identifies exactly which byte matched. */ ENTRY (__strchrnul_aarch64_mte) + PTR_ARG (0) bic src, srcin, 15 dup vrepchr.16b, chrin ld1 {vdata.16b}, [src] diff --git a/string/aarch64/strchrnul-sve.S b/string/aarch64/strchrnul-sve.S index 5140e59..428ff1a 100644 --- a/string/aarch64/strchrnul-sve.S +++ b/string/aarch64/strchrnul-sve.S @@ -1,7 +1,7 @@ /* * strchrnul - find a character or nul in a string * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2019, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/string/aarch64/strchrnul.S b/string/aarch64/strchrnul.S index 45be15c..a4230d9 100644 --- a/string/aarch64/strchrnul.S +++ b/string/aarch64/strchrnul.S @@ -1,7 +1,7 @@ /* * strchrnul - find a character or nul in a string * - * Copyright (c) 2014-2019, Arm Limited. + * Copyright (c) 2014-2020, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -47,6 +47,7 @@ /* Locals and temporaries. */ ENTRY (__strchrnul_aarch64) + PTR_ARG (0) /* Magic constant 0x40100401 to allow us to identify which lane matches the termination condition. */ mov wtmp2, #0x0401 diff --git a/string/aarch64/strcmp-mte.S b/string/aarch64/strcmp-mte.S index 1b6db42..12d1a6b 100644 --- a/string/aarch64/strcmp-mte.S +++ b/string/aarch64/strcmp-mte.S @@ -51,6 +51,8 @@ ENTRY (__strcmp_aarch64_mte) + PTR_ARG (0) + PTR_ARG (1) sub off2, src2, src1 mov zeroones, REP8_01 and tmp, src1, 7 diff --git a/string/aarch64/strcmp-sve.S b/string/aarch64/strcmp-sve.S index dc5b769..e6d2da5 100644 --- a/string/aarch64/strcmp-sve.S +++ b/string/aarch64/strcmp-sve.S @@ -1,7 +1,7 @@ /* * __strcmp_aarch64_sve - compare two strings * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -14,16 +14,15 @@ * SVE Available. */ - .arch armv8-a+sve - .text - -ENTRY_ALIGN (__strcmp_aarch64_sve, 4) +ENTRY (__strcmp_aarch64_sve) + PTR_ARG (0) + PTR_ARG (1) setffr /* initialize FFR */ ptrue p1.b, all /* all ones; loop invariant */ mov x2, 0 /* initialize offset */ - nop /* Read a vector's worth of bytes, stopping on first fault. */ + .p2align 4 0: ldff1b z0.b, p1/z, [x0, x2] ldff1b z1.b, p1/z, [x1, x2] rdffrs p0.b, p1/z diff --git a/string/aarch64/strcmp.S b/string/aarch64/strcmp.S index ee95958..7714ebf 100644 --- a/string/aarch64/strcmp.S +++ b/string/aarch64/strcmp.S @@ -37,6 +37,8 @@ /* Start of performance-critical section -- one 64B cache line. */ ENTRY (__strcmp_aarch64) + PTR_ARG (0) + PTR_ARG (1) eor tmp1, src1, src2 mov zeroones, #REP8_01 tst tmp1, #7 diff --git a/string/aarch64/strcpy-mte.S b/string/aarch64/strcpy-mte.S index 7c8629e..88c222d 100644 --- a/string/aarch64/strcpy-mte.S +++ b/string/aarch64/strcpy-mte.S @@ -55,6 +55,8 @@ string, counting trailing zeros identifies exactly which byte matched. */ ENTRY (STRCPY) + PTR_ARG (0) + PTR_ARG (1) bic src, srcin, 15 mov wtmp, 0xf00f ld1 {vdata.16b}, [src] diff --git a/string/aarch64/strcpy-sve.S b/string/aarch64/strcpy-sve.S index a785d45..f515462 100644 --- a/string/aarch64/strcpy-sve.S +++ b/string/aarch64/strcpy-sve.S @@ -1,7 +1,7 @@ /* * strcpy/stpcpy - copy a string returning pointer to start/end. * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -14,9 +14,6 @@ * SVE Available. */ - .arch armv8-a+sve - .text - /* To build as stpcpy, define BUILD_STPCPY before compiling this file. */ #ifdef BUILD_STPCPY #define FUNC __stpcpy_aarch64_sve @@ -24,7 +21,9 @@ #define FUNC __strcpy_aarch64_sve #endif -ENTRY_ALIGN (FUNC, 4) +ENTRY (FUNC) + PTR_ARG (0) + PTR_ARG (1) setffr /* initialize FFR */ ptrue p2.b, all /* all ones; loop invariant */ mov x2, 0 /* initialize offset */ diff --git a/string/aarch64/strcpy.S b/string/aarch64/strcpy.S index a6090c8..6e9ed42 100644 --- a/string/aarch64/strcpy.S +++ b/string/aarch64/strcpy.S @@ -1,7 +1,7 @@ /* * strcpy/stpcpy - copy a string returning pointer to start/end. * - * Copyright (c) 2013-2019, Arm Limited. + * Copyright (c) 2013-2020, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -80,6 +80,8 @@ #define MIN_PAGE_SIZE (1 << MIN_PAGE_P2) ENTRY (STRCPY) + PTR_ARG (0) + PTR_ARG (1) /* For moderately short strings, the fastest way to do the copy is to calculate the length of the string in the same way as strlen, then essentially do a memcpy of the result. This avoids the need for diff --git a/string/aarch64/strlen-mte.S b/string/aarch64/strlen-mte.S index 6a99340..7cf41d5 100644 --- a/string/aarch64/strlen-mte.S +++ b/string/aarch64/strlen-mte.S @@ -39,6 +39,7 @@ string, counting trailing zeros identifies exactly which byte matched. */ ENTRY (__strlen_aarch64_mte) + PTR_ARG (0) bic src, srcin, 15 mov wtmp, 0xf00f ld1 {vdata.16b}, [src] diff --git a/string/aarch64/strlen-sve.S b/string/aarch64/strlen-sve.S index 9a9a359..2392493 100644 --- a/string/aarch64/strlen-sve.S +++ b/string/aarch64/strlen-sve.S @@ -1,7 +1,7 @@ /* * __strlen_aarch64_sve - compute the length of a string * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -14,18 +14,15 @@ * SVE Available. */ - .arch armv8-a+sve - .text - -ENTRY_ALIGN (__strlen_aarch64_sve, 4) +ENTRY (__strlen_aarch64_sve) + PTR_ARG (0) setffr /* initialize FFR */ ptrue p2.b /* all ones; loop invariant */ mov x1, 0 /* initialize length */ - nop /* Read a vector's worth of bytes, stopping on first fault. */ + .p2align 4 0: ldff1b z0.b, p2/z, [x0, x1] - nop rdffrs p0.b, p2/z b.nlast 2f diff --git a/string/aarch64/strlen.S b/string/aarch64/strlen.S index b20eaeb..a1b164a 100644 --- a/string/aarch64/strlen.S +++ b/string/aarch64/strlen.S @@ -74,6 +74,7 @@ character, return the length, if not, continue in the main loop. */ ENTRY (__strlen_aarch64) + PTR_ARG (0) and tmp1, srcin, MIN_PAGE_SIZE - 1 cmp tmp1, MIN_PAGE_SIZE - 32 b.hi L(page_cross) diff --git a/string/aarch64/strncmp-mte.S b/string/aarch64/strncmp-mte.S index 46765d6..c9d6fc8 100644 --- a/string/aarch64/strncmp-mte.S +++ b/string/aarch64/strncmp-mte.S @@ -1,7 +1,7 @@ /* * strncmp - compare two strings * - * Copyright (c) 2013-2020, Arm Limited. + * Copyright (c) 2013-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -53,12 +53,10 @@ #define LS_BK lsl #endif - .text - .p2align 6 - .rep 9 - nop /* Pad so that the loop below fits a cache line. */ - .endr -ENTRY_ALIGN (__strncmp_aarch64_mte, 0) +ENTRY (__strncmp_aarch64_mte) + PTR_ARG (0) + PTR_ARG (1) + SIZE_ARG (2) cbz limit, L(ret0) eor tmp1, src1, src2 mov zeroones, #REP8_01 @@ -70,7 +68,7 @@ ENTRY_ALIGN (__strncmp_aarch64_mte, 0) /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and can be done in parallel across the entire word. */ - /* Start of performance-critical section -- one 64B cache line. */ + .p2align 4 L(loop_aligned): ldr data1, [src1], #8 ldr data2, [src2], #8 @@ -83,7 +81,7 @@ L(start_realigned): bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ ccmp endloop, #0, #0, eq b.eq L(loop_aligned) - /* End of performance-critical section -- one 64B cache line. */ + /* End of main loop */ L(full_check): #ifndef __AARCH64EB__ @@ -175,7 +173,7 @@ L(mutual_align): orr data2, data2, tmp2 b L(start_realigned) - .p2align 6 + .p2align 4 /* Don't bother with dwords for up to 16 bytes. */ L(misaligned8): cmp limit, #16 diff --git a/string/aarch64/strncmp-sve.S b/string/aarch64/strncmp-sve.S index fdbe7ae..234190e 100644 --- a/string/aarch64/strncmp-sve.S +++ b/string/aarch64/strncmp-sve.S @@ -1,7 +1,7 @@ /* * strncmp - compare two strings with limit * - * Copyright (c) 2018, Arm Limited. + * Copyright (c) 2018-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -14,10 +14,10 @@ * SVE Available. */ - .arch armv8-a+sve - .text - -ENTRY_ALIGN (__strncmp_aarch64_sve, 4) +ENTRY (__strncmp_aarch64_sve) + PTR_ARG (0) + PTR_ARG (1) + SIZE_ARG (2) setffr /* initialize FFR */ mov x3, 0 /* initialize off */ diff --git a/string/aarch64/strncmp.S b/string/aarch64/strncmp.S index 584c54a..738b653 100644 --- a/string/aarch64/strncmp.S +++ b/string/aarch64/strncmp.S @@ -1,7 +1,7 @@ /* * strncmp - compare two strings * - * Copyright (c) 2013, Arm Limited. + * Copyright (c) 2013-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -40,12 +40,10 @@ #define endloop x15 #define count mask - .text - .p2align 6 - .rep 6 - nop /* Pad so that the loop below fits a cache line. */ - .endr -ENTRY_ALIGN (__strncmp_aarch64, 0) +ENTRY (__strncmp_aarch64) + PTR_ARG (0) + PTR_ARG (1) + SIZE_ARG (2) cbz limit, L(ret0) eor tmp1, src1, src2 mov zeroones, #REP8_01 @@ -60,7 +58,7 @@ ENTRY_ALIGN (__strncmp_aarch64, 0) /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and can be done in parallel across the entire word. */ - /* Start of performance-critical section -- one 64B cache line. */ + .p2align 4 L(loop_aligned): ldr data1, [src1], #8 ldr data2, [src2], #8 @@ -73,7 +71,7 @@ L(start_realigned): bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ ccmp endloop, #0, #0, eq b.eq L(loop_aligned) - /* End of performance-critical section -- one 64B cache line. */ + /* End of main loop */ /* Not reached the limit, must have found the end or a diff. */ tbz limit_wd, #63, L(not_limit) @@ -178,7 +176,7 @@ L(mutual_align): add limit_wd, limit_wd, tmp3, lsr #3 b L(start_realigned) - .p2align 6 + .p2align 4 /* Don't bother with dwords for up to 16 bytes. */ L(misaligned8): cmp limit, #16 diff --git a/string/aarch64/strnlen-sve.S b/string/aarch64/strnlen-sve.S index 5ad40d3..5b9ebf7 100644 --- a/string/aarch64/strnlen-sve.S +++ b/string/aarch64/strnlen-sve.S @@ -1,7 +1,7 @@ /* * strnlen - calculate the length of a string with limit. * - * Copyright (c) 2019, Arm Limited. + * Copyright (c) 2019-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -14,10 +14,9 @@ * SVE Available. */ - .arch armv8-a+sve - .text - -ENTRY_ALIGN (__strnlen_aarch64_sve, 4) +ENTRY (__strnlen_aarch64_sve) + PTR_ARG (0) + SIZE_ARG (1) setffr /* initialize FFR */ mov x2, 0 /* initialize len */ b 1f @@ -66,7 +65,7 @@ ENTRY_ALIGN (__strnlen_aarch64_sve, 4) b 1b /* End of count. Return max. */ -9: mov x0, x2 +9: mov x0, x1 ret END (__strnlen_aarch64_sve) diff --git a/string/aarch64/strnlen.S b/string/aarch64/strnlen.S index 4852edc..48d2495 100644 --- a/string/aarch64/strnlen.S +++ b/string/aarch64/strnlen.S @@ -42,6 +42,8 @@ string, counting trailing zeros identifies exactly which byte matched. */ ENTRY (__strnlen_aarch64) + PTR_ARG (0) + SIZE_ARG (1) bic src, srcin, 15 mov wtmp, 0xf00f cbz cntin, L(nomatch) diff --git a/string/aarch64/strrchr-mte.S b/string/aarch64/strrchr-mte.S index 5a409b9..1e4fb1a 100644 --- a/string/aarch64/strrchr-mte.S +++ b/string/aarch64/strrchr-mte.S @@ -44,6 +44,7 @@ if the relevant byte matched the NUL end of string. */ ENTRY (__strrchr_aarch64_mte) + PTR_ARG (0) bic src, srcin, 15 dup vrepchr.16b, chrin mov wtmp, 0x3003 diff --git a/string/aarch64/strrchr-sve.S b/string/aarch64/strrchr-sve.S index dbb9bfd..d36d69a 100644 --- a/string/aarch64/strrchr-sve.S +++ b/string/aarch64/strrchr-sve.S @@ -1,7 +1,7 @@ /* * strrchr - find the last of a character in a string * - * Copyright (c) 2019, Arm Limited. + * Copyright (c) 2019-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -14,10 +14,8 @@ * SVE Available. */ - .arch armv8-a+sve - .text - -ENTRY_ALIGN (__strrchr_aarch64_sve, 4) +ENTRY (__strrchr_aarch64_sve) + PTR_ARG (0) dup z1.b, w1 /* replicate byte across vector */ setffr /* initialize FFR */ ptrue p1.b /* all ones; loop invariant */ diff --git a/string/aarch64/strrchr.S b/string/aarch64/strrchr.S index f3d22d4..56185ff 100644 --- a/string/aarch64/strrchr.S +++ b/string/aarch64/strrchr.S @@ -55,6 +55,7 @@ identify exactly which byte is causing the termination, and why. */ ENTRY (__strrchr_aarch64) + PTR_ARG (0) /* Magic constant 0x40100401 to allow us to identify which lane matches the requested byte. Magic constant 0x80200802 used similarly for NUL termination. */ diff --git a/string/arm/memchr.S b/string/arm/memchr.S index 2eff4d1..3f1ac4d 100644 --- a/string/arm/memchr.S +++ b/string/arm/memchr.S @@ -1,7 +1,7 @@ /* * memchr - scan memory for a character * - * Copyright (c) 2010, Arm Limited. + * Copyright (c) 2010-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -31,7 +31,6 @@ #else #define CHARTSTMASK(c) 1<<(c*8) #endif - .text .thumb @ --------------------------------------------------------------------------- diff --git a/string/arm/memcpy.S b/string/arm/memcpy.S index 4a34142..86e6493 100644 --- a/string/arm/memcpy.S +++ b/string/arm/memcpy.S @@ -1,7 +1,7 @@ /* * memcpy - copy memory area * - * Copyright (c) 2013, Arm Limited. + * Copyright (c) 2013-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/string/arm/memset.S b/string/arm/memset.S index 3ee5238..11e9273 100644 --- a/string/arm/memset.S +++ b/string/arm/memset.S @@ -1,7 +1,7 @@ /* * memset - fill memory with a constant * - * Copyright (c) 2010, Arm Limited. + * Copyright (c) 2010-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -25,7 +25,6 @@ #else #define CHARTSTMASK(c) 1<<(c*8) #endif - .text .thumb @ --------------------------------------------------------------------------- diff --git a/string/arm/strcmp-armv6m.S b/string/arm/strcmp-armv6m.S index 3e54519..b75d414 100644 --- a/string/arm/strcmp-armv6m.S +++ b/string/arm/strcmp-armv6m.S @@ -1,7 +1,7 @@ /* * strcmp for ARMv6-M (optimized for performance, not size) * - * Copyright (c) 2014-2019, Arm Limited. + * Copyright (c) 2014-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/string/arm/strcmp.S b/string/arm/strcmp.S index f939da2..51443e3 100644 --- a/string/arm/strcmp.S +++ b/string/arm/strcmp.S @@ -1,7 +1,7 @@ /* * strcmp for ARMv7 * - * Copyright (c) 2012-2019, Arm Limited. + * Copyright (c) 2012-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -125,7 +125,6 @@ #endif .endm - .text .p2align 5 L(strcmp_start_addr): #if STRCMP_NO_PRECHECK == 0 diff --git a/string/arm/strcpy.c b/string/arm/strcpy.c index 2d8834b..02cf94f 100644 --- a/string/arm/strcpy.c +++ b/string/arm/strcpy.c @@ -1,7 +1,7 @@ /* * strcpy * - * Copyright (c) 2008-2019, Arm Limited. + * Copyright (c) 2008-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/string/arm/strlen-armv6t2.S b/string/arm/strlen-armv6t2.S index c50de9b..5ad30c9 100644 --- a/string/arm/strlen-armv6t2.S +++ b/string/arm/strlen-armv6t2.S @@ -1,7 +1,7 @@ /* * strlen - calculate the length of a string * - * Copyright (c) 2010, Arm Limited. + * Copyright (c) 2010-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/string/asmdefs.h b/string/asmdefs.h index 31c0f9d..340b427 100644 --- a/string/asmdefs.h +++ b/string/asmdefs.h @@ -1,7 +1,7 @@ /* * Macros for asm code. * - * Copyright (c) 2019, Arm Limited. + * Copyright (c) 2019-2020, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -81,4 +81,18 @@ GNU_PROPERTY (FEATURE_1_AND, FEATURE_1_BTI|FEATURE_1_PAC) #define L(l) .L ## l +#ifdef __ILP32__ + /* Sanitize padding bits of pointer arguments as per aapcs64 */ +#define PTR_ARG(n) mov w##n, w##n +#else +#define PTR_ARG(n) +#endif + +#ifdef __ILP32__ + /* Sanitize padding bits of size arguments as per aapcs64 */ +#define SIZE_ARG(n) mov w##n, w##n +#else +#define SIZE_ARG(n) +#endif + #endif diff --git a/string/include/stringlib.h b/string/include/stringlib.h index 841a7bb..378c3cd 100644 --- a/string/include/stringlib.h +++ b/string/include/stringlib.h @@ -1,7 +1,7 @@ /* * Public API. * - * Copyright (c) 2019-2020, Arm Limited. + * Copyright (c) 2019-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -54,6 +54,10 @@ size_t __strlen_aarch64_sve (const char *); size_t __strnlen_aarch64_sve (const char *, size_t); int __strncmp_aarch64_sve (const char *, const char *, size_t); # endif +# if __ARM_FEATURE_MEMORY_TAGGING +void *__mtag_tag_region (void *, size_t); +void *__mtag_tag_zero_region (void *, size_t); +# endif #elif __arm__ void *__memcpy_arm (void *__restrict, const void *__restrict, size_t); void *__memset_arm (void *, int, size_t); diff --git a/string/test/__mtag_tag_region.c b/string/test/__mtag_tag_region.c new file mode 100644 index 0000000..d8c02d9 --- /dev/null +++ b/string/test/__mtag_tag_region.c @@ -0,0 +1,147 @@ +/* + * __mtag_tag_region test. + * + * Copyright (c) 2021, Arm Limited. + * SPDX-License-Identifier: MIT + */ + +#if __ARM_FEATURE_MEMORY_TAGGING && WANT_MTE_TEST +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "mte.h" +#include "stringlib.h" +#include "stringtest.h" + +static void +mtag_quoteat (const char *prefix, void *p, int len, int at) +{ + /* Print tag, untag and quote the context. */ + printf ("location: %p\n", __arm_mte_get_tag ((char *) p + at)); + untag_buffer (p, len, 1); + p = untag_pointer (p); + quoteat (prefix, p, len, at); +} + +#define F(x) {#x, x}, + +static const struct fun +{ + const char *name; + void *(*fun) (void *s, size_t n); +} funtab[] = { +// clang-format off +#if __aarch64__ + F(__mtag_tag_region) +#endif + {0, 0} + // clang-format on +}; +#undef F + +#define A 64 +#define LEN 250000 +static unsigned char *sbuf; + +static void * +alignup (void *p) +{ + return (void *) (((uintptr_t) p + A - 1) & -A); +} + +static void +test (const struct fun *fun, int salign, int len) +{ + unsigned char *src = alignup (sbuf); + unsigned char *s = src + salign; + void *p; + int i; + + if (err_count >= ERR_LIMIT) + return; + if (len > LEN || salign >= A) + abort (); + for (i = 0; i < len + 2 * A; i++) + src[i] = '?'; + for (i = 0; i < len; i++) + s[i] = 'a'; + + src = tag_buffer (src, len + 2 * A, 1); + s = src + salign; + /* Use different tag. */ + s = __arm_mte_increment_tag (s, 1); + p = fun->fun (s, len); + + if (p != s) + ERR ("%s(%p,..) returned %p\n", fun->name, s, p); + + for (i = 0; i < salign; i++) + { + if (src[i] != '?') + { + ERR ("%s(align %d, %d) failed\n", fun->name, salign, len); + mtag_quoteat ("got head", src, len + 2 * A, i); + return; + } + } + + for (; i < salign + len; i++) + { + if (s[i - salign] != 'a') + { + ERR ("%s(align %d, %d) failed\n", fun->name, salign, len); + mtag_quoteat ("got body", src, len + 2 * A, i); + return; + } + } + + for (; i < len + 2 * A; i++) + { + if (src[i] != '?') + { + ERR ("%s(align %d, %d) failed\n", fun->name, salign, len); + mtag_quoteat ("got tail", src, len + 2 * A, i); + return; + } + } + + untag_buffer (src, len + 2 * A, 1); +} + +int +main () +{ + if (!mte_enabled ()) + return 0; + + sbuf = mte_mmap (LEN + 3 * A); + int r = 0; + for (int i = 0; funtab[i].name; i++) + { + err_count = 0; + for (int s = 0; s < A; s += 16) + { + int n; + for (n = 0; n < 200; n += 16) + { + test (funtab + i, s, n); + } + for (; n < LEN; n *= 2) + { + test (funtab + i, s, n); + } + } + printf ("%s %s\n", err_count ? "FAIL" : "PASS", funtab[i].name); + if (err_count) + r = -1; + } + return r; +} +#else +int +main () +{ + return 0; +} +#endif diff --git a/string/test/__mtag_tag_zero_region.c b/string/test/__mtag_tag_zero_region.c new file mode 100644 index 0000000..221c223 --- /dev/null +++ b/string/test/__mtag_tag_zero_region.c @@ -0,0 +1,147 @@ +/* + * __mtag_tag_zero_region test. + * + * Copyright (c) 2021, Arm Limited. + * SPDX-License-Identifier: MIT + */ + +#if __ARM_FEATURE_MEMORY_TAGGING && WANT_MTE_TEST +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "mte.h" +#include "stringlib.h" +#include "stringtest.h" + +static void +mtag_quoteat (const char *prefix, void *p, int len, int at) +{ + /* Print tag, untag and quote the context. */ + printf ("location: %p\n", __arm_mte_get_tag ((char *) p + at)); + untag_buffer (p, len, 1); + p = untag_pointer (p); + quoteat (prefix, p, len, at); +} + +#define F(x) {#x, x}, + +static const struct fun +{ + const char *name; + void *(*fun) (void *s, size_t n); +} funtab[] = { +// clang-format off +#if __aarch64__ + F(__mtag_tag_zero_region) +#endif + {0, 0} + // clang-format on +}; +#undef F + +#define A 64 +#define LEN 250000 +static unsigned char *sbuf; + +static void * +alignup (void *p) +{ + return (void *) (((uintptr_t) p + A - 1) & -A); +} + +static void +test (const struct fun *fun, int salign, int len) +{ + unsigned char *src = alignup (sbuf); + unsigned char *s = src + salign; + void *p; + int i; + + if (err_count >= ERR_LIMIT) + return; + if (len > LEN || salign >= A) + abort (); + for (i = 0; i < len + 2 * A; i++) + src[i] = '?'; + for (i = 0; i < len; i++) + s[i] = 'a' + i % 23; + + src = tag_buffer (src, len + 2 * A, 1); + s = src + salign; + /* Use different tag. */ + s = __arm_mte_increment_tag (s, 1); + p = fun->fun (s, len); + + if (p != s) + ERR ("%s(%p,..) returned %p\n", fun->name, s, p); + + for (i = 0; i < salign; i++) + { + if (src[i] != '?') + { + ERR ("%s(align %d, %d) failed\n", fun->name, salign, len); + mtag_quoteat ("got head", src, len + 2 * A, i); + return; + } + } + + for (; i < salign + len; i++) + { + if (s[i - salign] != 0) + { + ERR ("%s(align %d, %d) failed\n", fun->name, salign, len); + mtag_quoteat ("got body", src, len + 2 * A, i); + return; + } + } + + for (; i < len + 2 * A; i++) + { + if (src[i] != '?') + { + ERR ("%s(align %d, %d) failed\n", fun->name, salign, len); + mtag_quoteat ("got tail", src, len + 2 * A, i); + return; + } + } + + untag_buffer (src, len + 2 * A, 1); +} + +int +main () +{ + if (!mte_enabled ()) + return 0; + + sbuf = mte_mmap (LEN + 3 * A); + int r = 0; + for (int i = 0; funtab[i].name; i++) + { + err_count = 0; + for (int s = 0; s < A; s += 16) + { + int n; + for (n = 0; n < 200; n += 16) + { + test (funtab + i, s, n); + } + for (; n < LEN; n *= 2) + { + test (funtab + i, s, n); + } + } + printf ("%s %s\n", err_count ? "FAIL" : "PASS", funtab[i].name); + if (err_count) + r = -1; + } + return r; +} +#else +int +main () +{ + return 0; +} +#endif diff --git a/string/test/memcmp.c b/string/test/memcmp.c index dd93698..7a7cf9c 100644 --- a/string/test/memcmp.c +++ b/string/test/memcmp.c @@ -1,7 +1,7 @@ /* * memcmp test. * - * Copyright (c) 2019, Arm Limited. + * Copyright (c) 2019-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/string/test/memcpy.c b/string/test/memcpy.c index 346d920..ce0ceee 100644 --- a/string/test/memcpy.c +++ b/string/test/memcpy.c @@ -1,7 +1,7 @@ /* * memcpy test. * - * Copyright (c) 2019, Arm Limited. + * Copyright (c) 2019-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/string/test/memmove.c b/string/test/memmove.c index af92fe3..689b68c 100644 --- a/string/test/memmove.c +++ b/string/test/memmove.c @@ -1,7 +1,7 @@ /* * memmove test. * - * Copyright (c) 2019, Arm Limited. + * Copyright (c) 2019-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/string/test/memset.c b/string/test/memset.c index cebe9ad..f172144 100644 --- a/string/test/memset.c +++ b/string/test/memset.c @@ -1,7 +1,7 @@ /* * memset test. * - * Copyright (c) 2019, Arm Limited. + * Copyright (c) 2019-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/string/test/strcmp.c b/string/test/strcmp.c index 4e718e3..d57b54e 100644 --- a/string/test/strcmp.c +++ b/string/test/strcmp.c @@ -1,7 +1,7 @@ /* * strcmp test. * - * Copyright (c) 2019, Arm Limited. + * Copyright (c) 2019-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/string/test/strncmp.c b/string/test/strncmp.c index 23fbb0a..018a8a4 100644 --- a/string/test/strncmp.c +++ b/string/test/strncmp.c @@ -1,7 +1,7 @@ /* * strncmp test. * - * Copyright (c) 2019, Arm Limited. + * Copyright (c) 2019-2020, Arm Limited. * SPDX-License-Identifier: MIT */ diff --git a/string/test/strrchr.c b/string/test/strrchr.c index b968457..fedbdc5 100644 --- a/string/test/strrchr.c +++ b/string/test/strrchr.c @@ -1,7 +1,7 @@ /* * strrchr test. * - * Copyright (c) 2019-2020, Arm Limited. + * Copyright (c) 2019-2021, Arm Limited. * SPDX-License-Identifier: MIT */ @@ -91,7 +91,7 @@ test (const struct fun *fun, int align, int seekpos, int len) if (p != s + len) { ERR ("%s (%p, 0x%02x) len %d returned %p, expected %p pos %d\n", - fun->name, s, 0, len, p, f, len); + fun->name, s, 0, len, p, s + len, len); quote ("input", s, len); } } |