diff options
author | Marat Dukhan <maratek@google.com> | 2020-07-16 23:50:10 -0700 |
---|---|---|
committer | XNNPACK Team <xnnpack-github-robot@google.com> | 2020-07-16 23:50:49 -0700 |
commit | bbf7f3f9be24b707392805c490a9b2dc884e9bb4 (patch) | |
tree | 5dba42cddc57ef8606910e76f78ff5bf1b564365 | |
parent | 115d3e2f98d279e7c2f05aadfdf10f69a2d2875a (diff) | |
download | XNNPACK-bbf7f3f9be24b707392805c490a9b2dc884e9bb4.tar.gz |
Remove PSIMD variants of DWCONV microkernels
PiperOrigin-RevId: 321723576
-rw-r--r-- | BUILD.bazel | 12 | ||||
-rwxr-xr-x | CMakeLists.txt | 12 | ||||
-rw-r--r-- | bench/f32-dwconv-e2e.cc | 31 | ||||
-rw-r--r-- | bench/f32-dwconv.cc | 19 | ||||
-rwxr-xr-x | scripts/generate-f32-dwconv.sh | 16 | ||||
-rw-r--r-- | src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c | 451 | ||||
-rw-r--r-- | src/f32-dwconv/gen/up4x25-minmax-psimd.c | 447 | ||||
-rw-r--r-- | src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c | 136 | ||||
-rw-r--r-- | src/f32-dwconv/gen/up4x4-minmax-psimd.c | 132 | ||||
-rw-r--r-- | src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c | 211 | ||||
-rw-r--r-- | src/f32-dwconv/gen/up4x9-minmax-psimd.c | 207 | ||||
-rw-r--r-- | src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c | 695 | ||||
-rw-r--r-- | src/f32-dwconv/gen/up8x25-minmax-psimd.c | 688 | ||||
-rw-r--r-- | src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c | 191 | ||||
-rw-r--r-- | src/f32-dwconv/gen/up8x4-minmax-psimd.c | 184 | ||||
-rw-r--r-- | src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c | 311 | ||||
-rw-r--r-- | src/f32-dwconv/gen/up8x9-minmax-psimd.c | 304 | ||||
-rw-r--r-- | src/f32-dwconv/up-psimd.c.in | 154 | ||||
-rw-r--r-- | src/xnnpack/dwconv.h | 12 | ||||
-rw-r--r-- | test/f32-dwconv-minmax.cc | 2232 | ||||
-rw-r--r-- | test/f32-dwconv-minmax.yaml | 12 |
21 files changed, 0 insertions, 6457 deletions
diff --git a/BUILD.bazel b/BUILD.bazel index 9956a8ebb..f816af28b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -845,18 +845,6 @@ PSIMD_FASTMATH_UKERNELS = [ "src/f32-dwconv-chw/3x3s2p1-psimd.c", "src/f32-dwconv-chw/5x5p2-psimd.c", "src/f32-dwconv-chw/5x5s2p2-psimd.c", - "src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c", - "src/f32-dwconv/gen/up4x25-minmax-psimd.c", - "src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c", - "src/f32-dwconv/gen/up4x4-minmax-psimd.c", - "src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c", - "src/f32-dwconv/gen/up4x9-minmax-psimd.c", - "src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c", - "src/f32-dwconv/gen/up8x25-minmax-psimd.c", - "src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c", - "src/f32-dwconv/gen/up8x4-minmax-psimd.c", - "src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c", - "src/f32-dwconv/gen/up8x9-minmax-psimd.c", "src/f32-gavgpool-cw/psimd-x4.c", "src/f32-gavgpool/7p7x-minmax-psimd-c4.c", "src/f32-gavgpool/7x-minmax-psimd-c4.c", diff --git a/CMakeLists.txt b/CMakeLists.txt index c6899cd4b..3779507fe 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -525,18 +525,6 @@ SET(XNNPACK_PSIMD_FASTMATH_MICROKERNEL_SRCS src/f32-dwconv-chw/3x3s2p1-psimd.c src/f32-dwconv-chw/5x5p2-psimd.c src/f32-dwconv-chw/5x5s2p2-psimd.c - src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c - src/f32-dwconv/gen/up4x25-minmax-psimd.c - src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c - src/f32-dwconv/gen/up4x4-minmax-psimd.c - src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c - src/f32-dwconv/gen/up4x9-minmax-psimd.c - src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c - src/f32-dwconv/gen/up8x25-minmax-psimd.c - src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c - src/f32-dwconv/gen/up8x4-minmax-psimd.c - src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c - src/f32-dwconv/gen/up8x9-minmax-psimd.c src/f32-gavgpool-cw/psimd-x4.c src/f32-gavgpool/7p7x-minmax-psimd-c4.c src/f32-gavgpool/7x-minmax-psimd-c4.c diff --git a/bench/f32-dwconv-e2e.cc b/bench/f32-dwconv-e2e.cc index 2d1e7d2ab..367f908a1 100644 --- a/bench/f32-dwconv-e2e.cc +++ b/bench/f32-dwconv-e2e.cc @@ -310,37 +310,6 @@ static void DWConvEnd2EndBenchmark( BENCHMARK_FP32_END2END(f32_dwconv_up8x9__wasmsimd_acc2_x86); #endif // XNN_ARCH_WASMSIMD -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - static void f32_dwconv_up4x9__psimd(benchmark::State& state, models::ExecutionPlanFactory model) { - DWConvEnd2EndBenchmark(state, model, - xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, - 4 /* cr */, 9 /* mr */); - } - - static void f32_dwconv_up4x9__psimd_acc2(benchmark::State& state, models::ExecutionPlanFactory model) { - DWConvEnd2EndBenchmark(state, model, - xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, - 4 /* cr */, 9 /* mr */); - } - - static void f32_dwconv_up8x9__psimd(benchmark::State& state, models::ExecutionPlanFactory model) { - DWConvEnd2EndBenchmark(state, model, - xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, - 8 /* cr */, 9 /* mr */); - } - - static void f32_dwconv_up8x9__psimd_acc2(benchmark::State& state, models::ExecutionPlanFactory model) { - DWConvEnd2EndBenchmark(state, model, - xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, - 8 /* cr */, 9 /* mr */); - } - - BENCHMARK_FP32_END2END(f32_dwconv_up4x9__psimd); - BENCHMARK_FP32_END2END(f32_dwconv_up4x9__psimd_acc2); - BENCHMARK_FP32_END2END(f32_dwconv_up8x9__psimd); - BENCHMARK_FP32_END2END(f32_dwconv_up8x9__psimd_acc2); -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - static void f32_dwconv_up1x9__scalar(benchmark::State& state, models::ExecutionPlanFactory model) { DWConvEnd2EndBenchmark(state, model, xnn_f32_dwconv_minmax_ukernel_up1x9__scalar, diff --git a/bench/f32-dwconv.cc b/bench/f32-dwconv.cc index be22685f1..8121e5202 100644 --- a/bench/f32-dwconv.cc +++ b/bench/f32-dwconv.cc @@ -333,25 +333,6 @@ static void DWConvBenchmark(benchmark::State& state, #endif // XNN_ARCH_X86 || XNN_ARCH_X86_64 -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - static void f32_dwconv_4x4__psimd(benchmark::State& state, const char* net) { - DWConvBenchmark(state, xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, 4, 4); - } - - static void f32_dwconv_4x9__psimd(benchmark::State& state, const char* net) { - DWConvBenchmark(state, xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, 4, 9); - } - - static void f32_dwconv_4x25__psimd(benchmark::State& state, const char* net) { - DWConvBenchmark(state, xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, 4, 25); - } - - BENCHMARK_DWCONV(f32_dwconv_4x4__psimd) - BENCHMARK_DWCONV(f32_dwconv_4x9__psimd) - BENCHMARK_DWCONV(f32_dwconv_4x25__psimd) -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - static void f32_dwconv_1x4__scalar(benchmark::State& state, const char* net) { DWConvBenchmark(state, xnn_f32_dwconv_minmax_ukernel_up1x4__scalar, 1, 4); } diff --git a/scripts/generate-f32-dwconv.sh b/scripts/generate-f32-dwconv.sh index 7abef21ef..d6b9c9e72 100755 --- a/scripts/generate-f32-dwconv.sh +++ b/scripts/generate-f32-dwconv.sh @@ -201,22 +201,6 @@ tools/xngen src/f32-dwconv/up-avx512.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=25 - tools/xngen src/f32-dwconv/up-avx512.c.in -D CHANNEL_TILE=32 -D KERNEL_TILE=25 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up32x25-minmax-avx512f.c tools/xngen src/f32-dwconv/up-avx512.c.in -D CHANNEL_TILE=32 -D KERNEL_TILE=25 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up32x25-minmax-avx512f-acc2.c -#################################### PSIMD #################################### -tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=4 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up4x4-minmax-psimd.c -tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=4 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c -tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=4 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up8x4-minmax-psimd.c -tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=4 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c - -tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=9 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up4x9-minmax-psimd.c -tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=9 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c -tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up8x9-minmax-psimd.c -tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c - -tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=25 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up4x25-minmax-psimd.c -tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=25 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c -tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up8x25-minmax-psimd.c -tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c - ################################## Unit tests ################################# tools/generate-dwconv-test.py --spec test/f32-dwconv.yaml --output test/f32-dwconv.cc tools/generate-dwconv-test.py --spec test/f32-dwconv-minmax.yaml --output test/f32-dwconv-minmax.cc diff --git a/src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c b/src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c deleted file mode 100644 index 236cef2ad..000000000 --- a/src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c +++ /dev/null @@ -1,451 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/f32-dwconv/up-psimd.c.in -// Generator: tools/xngen -// -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - const float* i0 = input[0]; - assert(i0 != NULL); - if XNN_UNPREDICTABLE(i0 != zero) { - i0 = (const float*) ((uintptr_t) i0 + input_offset); - } - const float* i1 = input[1]; - assert(i1 != NULL); - if XNN_UNPREDICTABLE(i1 != zero) { - i1 = (const float*) ((uintptr_t) i1 + input_offset); - } - const float* i2 = input[2]; - assert(i2 != NULL); - if XNN_UNPREDICTABLE(i2 != zero) { - i2 = (const float*) ((uintptr_t) i2 + input_offset); - } - const float* i3 = input[3]; - assert(i3 != NULL); - if XNN_UNPREDICTABLE(i3 != zero) { - i3 = (const float*) ((uintptr_t) i3 + input_offset); - } - const float* i4 = input[4]; - assert(i4 != NULL); - if XNN_UNPREDICTABLE(i4 != zero) { - i4 = (const float*) ((uintptr_t) i4 + input_offset); - } - const float* i5 = input[5]; - assert(i5 != NULL); - if XNN_UNPREDICTABLE(i5 != zero) { - i5 = (const float*) ((uintptr_t) i5 + input_offset); - } - const float* i6 = input[6]; - assert(i6 != NULL); - if XNN_UNPREDICTABLE(i6 != zero) { - i6 = (const float*) ((uintptr_t) i6 + input_offset); - } - const float* i7 = input[7]; - assert(i7 != NULL); - if XNN_UNPREDICTABLE(i7 != zero) { - i7 = (const float*) ((uintptr_t) i7 + input_offset); - } - const float* i8 = input[8]; - assert(i8 != NULL); - if XNN_UNPREDICTABLE(i8 != zero) { - i8 = (const float*) ((uintptr_t) i8 + input_offset); - } - const float* i9 = input[9]; - assert(i9 != NULL); - if XNN_UNPREDICTABLE(i9 != zero) { - i9 = (const float*) ((uintptr_t) i9 + input_offset); - } - const float* i10 = input[10]; - assert(i10 != NULL); - if XNN_UNPREDICTABLE(i10 != zero) { - i10 = (const float*) ((uintptr_t) i10 + input_offset); - } - const float* i11 = input[11]; - assert(i11 != NULL); - if XNN_UNPREDICTABLE(i11 != zero) { - i11 = (const float*) ((uintptr_t) i11 + input_offset); - } - const float* i12 = input[12]; - assert(i12 != NULL); - if XNN_UNPREDICTABLE(i12 != zero) { - i12 = (const float*) ((uintptr_t) i12 + input_offset); - } - const float* i13 = input[13]; - assert(i13 != NULL); - if XNN_UNPREDICTABLE(i13 != zero) { - i13 = (const float*) ((uintptr_t) i13 + input_offset); - } - const float* i14 = input[14]; - assert(i14 != NULL); - if XNN_UNPREDICTABLE(i14 != zero) { - i14 = (const float*) ((uintptr_t) i14 + input_offset); - } - const float* i15 = input[15]; - assert(i15 != NULL); - if XNN_UNPREDICTABLE(i15 != zero) { - i15 = (const float*) ((uintptr_t) i15 + input_offset); - } - const float* i16 = input[16]; - assert(i16 != NULL); - if XNN_UNPREDICTABLE(i16 != zero) { - i16 = (const float*) ((uintptr_t) i16 + input_offset); - } - const float* i17 = input[17]; - assert(i17 != NULL); - if XNN_UNPREDICTABLE(i17 != zero) { - i17 = (const float*) ((uintptr_t) i17 + input_offset); - } - const float* i18 = input[18]; - assert(i18 != NULL); - if XNN_UNPREDICTABLE(i18 != zero) { - i18 = (const float*) ((uintptr_t) i18 + input_offset); - } - const float* i19 = input[19]; - assert(i19 != NULL); - if XNN_UNPREDICTABLE(i19 != zero) { - i19 = (const float*) ((uintptr_t) i19 + input_offset); - } - const float* i20 = input[20]; - assert(i20 != NULL); - if XNN_UNPREDICTABLE(i20 != zero) { - i20 = (const float*) ((uintptr_t) i20 + input_offset); - } - const float* i21 = input[21]; - assert(i21 != NULL); - if XNN_UNPREDICTABLE(i21 != zero) { - i21 = (const float*) ((uintptr_t) i21 + input_offset); - } - const float* i22 = input[22]; - assert(i22 != NULL); - if XNN_UNPREDICTABLE(i22 != zero) { - i22 = (const float*) ((uintptr_t) i22 + input_offset); - } - const float* i23 = input[23]; - assert(i23 != NULL); - if XNN_UNPREDICTABLE(i23 != zero) { - i23 = (const float*) ((uintptr_t) i23 + input_offset); - } - const float* i24 = input[24]; - assert(i24 != NULL); - if XNN_UNPREDICTABLE(i24 != zero) { - i24 = (const float*) ((uintptr_t) i24 + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - i0 += 4; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 4); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - i1 += 4; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 8); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - i2 += 4; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - i3 += 4; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 16); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - i4 += 4; - - const psimd_f32 vk4x0123 = psimd_load_f32(w + 20); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - i5 += 4; - - const psimd_f32 vk5x0123 = psimd_load_f32(w + 24); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - i6 += 4; - - const psimd_f32 vk6x0123 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - i7 += 4; - - const psimd_f32 vk7x0123 = psimd_load_f32(w + 32); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - i8 += 4; - - const psimd_f32 vk8x0123 = psimd_load_f32(w + 36); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - const psimd_f32 vi9x0123 = psimd_load_f32(i9); - i9 += 4; - - const psimd_f32 vk9x0123 = psimd_load_f32(w + 40); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi9x0123, vk9x0123); - - const psimd_f32 vi10x0123 = psimd_load_f32(i10); - i10 += 4; - - const psimd_f32 vk10x0123 = psimd_load_f32(w + 44); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123); - - const psimd_f32 vi11x0123 = psimd_load_f32(i11); - i11 += 4; - - const psimd_f32 vk11x0123 = psimd_load_f32(w + 48); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi11x0123, vk11x0123); - - const psimd_f32 vi12x0123 = psimd_load_f32(i12); - i12 += 4; - - const psimd_f32 vk12x0123 = psimd_load_f32(w + 52); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123); - - const psimd_f32 vi13x0123 = psimd_load_f32(i13); - i13 += 4; - - const psimd_f32 vk13x0123 = psimd_load_f32(w + 56); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi13x0123, vk13x0123); - - const psimd_f32 vi14x0123 = psimd_load_f32(i14); - i14 += 4; - - const psimd_f32 vk14x0123 = psimd_load_f32(w + 60); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123); - - const psimd_f32 vi15x0123 = psimd_load_f32(i15); - i15 += 4; - - const psimd_f32 vk15x0123 = psimd_load_f32(w + 64); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi15x0123, vk15x0123); - - const psimd_f32 vi16x0123 = psimd_load_f32(i16); - i16 += 4; - - const psimd_f32 vk16x0123 = psimd_load_f32(w + 68); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123); - - const psimd_f32 vi17x0123 = psimd_load_f32(i17); - i17 += 4; - - const psimd_f32 vk17x0123 = psimd_load_f32(w + 72); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi17x0123, vk17x0123); - - const psimd_f32 vi18x0123 = psimd_load_f32(i18); - i18 += 4; - - const psimd_f32 vk18x0123 = psimd_load_f32(w + 76); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123); - - const psimd_f32 vi19x0123 = psimd_load_f32(i19); - i19 += 4; - - const psimd_f32 vk19x0123 = psimd_load_f32(w + 80); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi19x0123, vk19x0123); - - const psimd_f32 vi20x0123 = psimd_load_f32(i20); - i20 += 4; - - const psimd_f32 vk20x0123 = psimd_load_f32(w + 84); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123); - - const psimd_f32 vi21x0123 = psimd_load_f32(i21); - i21 += 4; - - const psimd_f32 vk21x0123 = psimd_load_f32(w + 88); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi21x0123, vk21x0123); - - const psimd_f32 vi22x0123 = psimd_load_f32(i22); - i22 += 4; - - const psimd_f32 vk22x0123 = psimd_load_f32(w + 92); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123); - - const psimd_f32 vi23x0123 = psimd_load_f32(i23); - i23 += 4; - - const psimd_f32 vk23x0123 = psimd_load_f32(w + 96); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi23x0123, vk23x0123); - - const psimd_f32 vi24x0123 = psimd_load_f32(i24); - i24 += 4; - - const psimd_f32 vk24x0123 = psimd_load_f32(w + 100); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123); - - w += 104; - - // Add up all accumulators to vacc0123p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vk0x0123 = psimd_load_f32(w + 4); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vk1x0123 = psimd_load_f32(w + 8); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vk2x0123 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vk3x0123 = psimd_load_f32(w + 16); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - const psimd_f32 vk4x0123 = psimd_load_f32(w + 20); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - const psimd_f32 vk5x0123 = psimd_load_f32(w + 24); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - const psimd_f32 vk6x0123 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - const psimd_f32 vk7x0123 = psimd_load_f32(w + 32); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - const psimd_f32 vk8x0123 = psimd_load_f32(w + 36); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - const psimd_f32 vi9x0123 = psimd_load_f32(i9); - const psimd_f32 vk9x0123 = psimd_load_f32(w + 40); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi9x0123, vk9x0123); - - const psimd_f32 vi10x0123 = psimd_load_f32(i10); - const psimd_f32 vk10x0123 = psimd_load_f32(w + 44); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123); - - const psimd_f32 vi11x0123 = psimd_load_f32(i11); - const psimd_f32 vk11x0123 = psimd_load_f32(w + 48); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi11x0123, vk11x0123); - - const psimd_f32 vi12x0123 = psimd_load_f32(i12); - const psimd_f32 vk12x0123 = psimd_load_f32(w + 52); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123); - - const psimd_f32 vi13x0123 = psimd_load_f32(i13); - const psimd_f32 vk13x0123 = psimd_load_f32(w + 56); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi13x0123, vk13x0123); - - const psimd_f32 vi14x0123 = psimd_load_f32(i14); - const psimd_f32 vk14x0123 = psimd_load_f32(w + 60); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123); - - const psimd_f32 vi15x0123 = psimd_load_f32(i15); - const psimd_f32 vk15x0123 = psimd_load_f32(w + 64); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi15x0123, vk15x0123); - - const psimd_f32 vi16x0123 = psimd_load_f32(i16); - const psimd_f32 vk16x0123 = psimd_load_f32(w + 68); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123); - - const psimd_f32 vi17x0123 = psimd_load_f32(i17); - const psimd_f32 vk17x0123 = psimd_load_f32(w + 72); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi17x0123, vk17x0123); - - const psimd_f32 vi18x0123 = psimd_load_f32(i18); - const psimd_f32 vk18x0123 = psimd_load_f32(w + 76); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123); - - const psimd_f32 vi19x0123 = psimd_load_f32(i19); - const psimd_f32 vk19x0123 = psimd_load_f32(w + 80); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi19x0123, vk19x0123); - - const psimd_f32 vi20x0123 = psimd_load_f32(i20); - const psimd_f32 vk20x0123 = psimd_load_f32(w + 84); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123); - - const psimd_f32 vi21x0123 = psimd_load_f32(i21); - const psimd_f32 vk21x0123 = psimd_load_f32(w + 88); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi21x0123, vk21x0123); - - const psimd_f32 vi22x0123 = psimd_load_f32(i22); - const psimd_f32 vk22x0123 = psimd_load_f32(w + 92); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123); - - const psimd_f32 vi23x0123 = psimd_load_f32(i23); - const psimd_f32 vk23x0123 = psimd_load_f32(w + 96); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi23x0123, vk23x0123); - - const psimd_f32 vi24x0123 = psimd_load_f32(i24); - const psimd_f32 vk24x0123 = psimd_load_f32(w + 100); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123); - - // Add up all accumulators to vacc0123p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/f32-dwconv/gen/up4x25-minmax-psimd.c b/src/f32-dwconv/gen/up4x25-minmax-psimd.c deleted file mode 100644 index a0c3adaf9..000000000 --- a/src/f32-dwconv/gen/up4x25-minmax-psimd.c +++ /dev/null @@ -1,447 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/f32-dwconv/up-psimd.c.in -// Generator: tools/xngen -// -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up4x25__psimd( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - const float* i0 = input[0]; - assert(i0 != NULL); - if XNN_UNPREDICTABLE(i0 != zero) { - i0 = (const float*) ((uintptr_t) i0 + input_offset); - } - const float* i1 = input[1]; - assert(i1 != NULL); - if XNN_UNPREDICTABLE(i1 != zero) { - i1 = (const float*) ((uintptr_t) i1 + input_offset); - } - const float* i2 = input[2]; - assert(i2 != NULL); - if XNN_UNPREDICTABLE(i2 != zero) { - i2 = (const float*) ((uintptr_t) i2 + input_offset); - } - const float* i3 = input[3]; - assert(i3 != NULL); - if XNN_UNPREDICTABLE(i3 != zero) { - i3 = (const float*) ((uintptr_t) i3 + input_offset); - } - const float* i4 = input[4]; - assert(i4 != NULL); - if XNN_UNPREDICTABLE(i4 != zero) { - i4 = (const float*) ((uintptr_t) i4 + input_offset); - } - const float* i5 = input[5]; - assert(i5 != NULL); - if XNN_UNPREDICTABLE(i5 != zero) { - i5 = (const float*) ((uintptr_t) i5 + input_offset); - } - const float* i6 = input[6]; - assert(i6 != NULL); - if XNN_UNPREDICTABLE(i6 != zero) { - i6 = (const float*) ((uintptr_t) i6 + input_offset); - } - const float* i7 = input[7]; - assert(i7 != NULL); - if XNN_UNPREDICTABLE(i7 != zero) { - i7 = (const float*) ((uintptr_t) i7 + input_offset); - } - const float* i8 = input[8]; - assert(i8 != NULL); - if XNN_UNPREDICTABLE(i8 != zero) { - i8 = (const float*) ((uintptr_t) i8 + input_offset); - } - const float* i9 = input[9]; - assert(i9 != NULL); - if XNN_UNPREDICTABLE(i9 != zero) { - i9 = (const float*) ((uintptr_t) i9 + input_offset); - } - const float* i10 = input[10]; - assert(i10 != NULL); - if XNN_UNPREDICTABLE(i10 != zero) { - i10 = (const float*) ((uintptr_t) i10 + input_offset); - } - const float* i11 = input[11]; - assert(i11 != NULL); - if XNN_UNPREDICTABLE(i11 != zero) { - i11 = (const float*) ((uintptr_t) i11 + input_offset); - } - const float* i12 = input[12]; - assert(i12 != NULL); - if XNN_UNPREDICTABLE(i12 != zero) { - i12 = (const float*) ((uintptr_t) i12 + input_offset); - } - const float* i13 = input[13]; - assert(i13 != NULL); - if XNN_UNPREDICTABLE(i13 != zero) { - i13 = (const float*) ((uintptr_t) i13 + input_offset); - } - const float* i14 = input[14]; - assert(i14 != NULL); - if XNN_UNPREDICTABLE(i14 != zero) { - i14 = (const float*) ((uintptr_t) i14 + input_offset); - } - const float* i15 = input[15]; - assert(i15 != NULL); - if XNN_UNPREDICTABLE(i15 != zero) { - i15 = (const float*) ((uintptr_t) i15 + input_offset); - } - const float* i16 = input[16]; - assert(i16 != NULL); - if XNN_UNPREDICTABLE(i16 != zero) { - i16 = (const float*) ((uintptr_t) i16 + input_offset); - } - const float* i17 = input[17]; - assert(i17 != NULL); - if XNN_UNPREDICTABLE(i17 != zero) { - i17 = (const float*) ((uintptr_t) i17 + input_offset); - } - const float* i18 = input[18]; - assert(i18 != NULL); - if XNN_UNPREDICTABLE(i18 != zero) { - i18 = (const float*) ((uintptr_t) i18 + input_offset); - } - const float* i19 = input[19]; - assert(i19 != NULL); - if XNN_UNPREDICTABLE(i19 != zero) { - i19 = (const float*) ((uintptr_t) i19 + input_offset); - } - const float* i20 = input[20]; - assert(i20 != NULL); - if XNN_UNPREDICTABLE(i20 != zero) { - i20 = (const float*) ((uintptr_t) i20 + input_offset); - } - const float* i21 = input[21]; - assert(i21 != NULL); - if XNN_UNPREDICTABLE(i21 != zero) { - i21 = (const float*) ((uintptr_t) i21 + input_offset); - } - const float* i22 = input[22]; - assert(i22 != NULL); - if XNN_UNPREDICTABLE(i22 != zero) { - i22 = (const float*) ((uintptr_t) i22 + input_offset); - } - const float* i23 = input[23]; - assert(i23 != NULL); - if XNN_UNPREDICTABLE(i23 != zero) { - i23 = (const float*) ((uintptr_t) i23 + input_offset); - } - const float* i24 = input[24]; - assert(i24 != NULL); - if XNN_UNPREDICTABLE(i24 != zero) { - i24 = (const float*) ((uintptr_t) i24 + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - i0 += 4; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 4); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - i1 += 4; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - i2 += 4; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - i3 += 4; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 16); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - i4 += 4; - - const psimd_f32 vk4x0123 = psimd_load_f32(w + 20); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - i5 += 4; - - const psimd_f32 vk5x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - i6 += 4; - - const psimd_f32 vk6x0123 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - i7 += 4; - - const psimd_f32 vk7x0123 = psimd_load_f32(w + 32); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - i8 += 4; - - const psimd_f32 vk8x0123 = psimd_load_f32(w + 36); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - const psimd_f32 vi9x0123 = psimd_load_f32(i9); - i9 += 4; - - const psimd_f32 vk9x0123 = psimd_load_f32(w + 40); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi9x0123, vk9x0123); - - const psimd_f32 vi10x0123 = psimd_load_f32(i10); - i10 += 4; - - const psimd_f32 vk10x0123 = psimd_load_f32(w + 44); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123); - - const psimd_f32 vi11x0123 = psimd_load_f32(i11); - i11 += 4; - - const psimd_f32 vk11x0123 = psimd_load_f32(w + 48); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi11x0123, vk11x0123); - - const psimd_f32 vi12x0123 = psimd_load_f32(i12); - i12 += 4; - - const psimd_f32 vk12x0123 = psimd_load_f32(w + 52); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123); - - const psimd_f32 vi13x0123 = psimd_load_f32(i13); - i13 += 4; - - const psimd_f32 vk13x0123 = psimd_load_f32(w + 56); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi13x0123, vk13x0123); - - const psimd_f32 vi14x0123 = psimd_load_f32(i14); - i14 += 4; - - const psimd_f32 vk14x0123 = psimd_load_f32(w + 60); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123); - - const psimd_f32 vi15x0123 = psimd_load_f32(i15); - i15 += 4; - - const psimd_f32 vk15x0123 = psimd_load_f32(w + 64); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi15x0123, vk15x0123); - - const psimd_f32 vi16x0123 = psimd_load_f32(i16); - i16 += 4; - - const psimd_f32 vk16x0123 = psimd_load_f32(w + 68); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123); - - const psimd_f32 vi17x0123 = psimd_load_f32(i17); - i17 += 4; - - const psimd_f32 vk17x0123 = psimd_load_f32(w + 72); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi17x0123, vk17x0123); - - const psimd_f32 vi18x0123 = psimd_load_f32(i18); - i18 += 4; - - const psimd_f32 vk18x0123 = psimd_load_f32(w + 76); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123); - - const psimd_f32 vi19x0123 = psimd_load_f32(i19); - i19 += 4; - - const psimd_f32 vk19x0123 = psimd_load_f32(w + 80); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi19x0123, vk19x0123); - - const psimd_f32 vi20x0123 = psimd_load_f32(i20); - i20 += 4; - - const psimd_f32 vk20x0123 = psimd_load_f32(w + 84); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123); - - const psimd_f32 vi21x0123 = psimd_load_f32(i21); - i21 += 4; - - const psimd_f32 vk21x0123 = psimd_load_f32(w + 88); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi21x0123, vk21x0123); - - const psimd_f32 vi22x0123 = psimd_load_f32(i22); - i22 += 4; - - const psimd_f32 vk22x0123 = psimd_load_f32(w + 92); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123); - - const psimd_f32 vi23x0123 = psimd_load_f32(i23); - i23 += 4; - - const psimd_f32 vk23x0123 = psimd_load_f32(w + 96); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi23x0123, vk23x0123); - - const psimd_f32 vi24x0123 = psimd_load_f32(i24); - i24 += 4; - - const psimd_f32 vk24x0123 = psimd_load_f32(w + 100); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123); - - w += 104; - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vk0x0123 = psimd_load_f32(w + 4); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vk1x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vk2x0123 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vk3x0123 = psimd_load_f32(w + 16); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - const psimd_f32 vk4x0123 = psimd_load_f32(w + 20); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - const psimd_f32 vk5x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - const psimd_f32 vk6x0123 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - const psimd_f32 vk7x0123 = psimd_load_f32(w + 32); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - const psimd_f32 vk8x0123 = psimd_load_f32(w + 36); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - const psimd_f32 vi9x0123 = psimd_load_f32(i9); - const psimd_f32 vk9x0123 = psimd_load_f32(w + 40); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi9x0123, vk9x0123); - - const psimd_f32 vi10x0123 = psimd_load_f32(i10); - const psimd_f32 vk10x0123 = psimd_load_f32(w + 44); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123); - - const psimd_f32 vi11x0123 = psimd_load_f32(i11); - const psimd_f32 vk11x0123 = psimd_load_f32(w + 48); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi11x0123, vk11x0123); - - const psimd_f32 vi12x0123 = psimd_load_f32(i12); - const psimd_f32 vk12x0123 = psimd_load_f32(w + 52); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123); - - const psimd_f32 vi13x0123 = psimd_load_f32(i13); - const psimd_f32 vk13x0123 = psimd_load_f32(w + 56); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi13x0123, vk13x0123); - - const psimd_f32 vi14x0123 = psimd_load_f32(i14); - const psimd_f32 vk14x0123 = psimd_load_f32(w + 60); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123); - - const psimd_f32 vi15x0123 = psimd_load_f32(i15); - const psimd_f32 vk15x0123 = psimd_load_f32(w + 64); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi15x0123, vk15x0123); - - const psimd_f32 vi16x0123 = psimd_load_f32(i16); - const psimd_f32 vk16x0123 = psimd_load_f32(w + 68); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123); - - const psimd_f32 vi17x0123 = psimd_load_f32(i17); - const psimd_f32 vk17x0123 = psimd_load_f32(w + 72); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi17x0123, vk17x0123); - - const psimd_f32 vi18x0123 = psimd_load_f32(i18); - const psimd_f32 vk18x0123 = psimd_load_f32(w + 76); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123); - - const psimd_f32 vi19x0123 = psimd_load_f32(i19); - const psimd_f32 vk19x0123 = psimd_load_f32(w + 80); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi19x0123, vk19x0123); - - const psimd_f32 vi20x0123 = psimd_load_f32(i20); - const psimd_f32 vk20x0123 = psimd_load_f32(w + 84); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123); - - const psimd_f32 vi21x0123 = psimd_load_f32(i21); - const psimd_f32 vk21x0123 = psimd_load_f32(w + 88); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi21x0123, vk21x0123); - - const psimd_f32 vi22x0123 = psimd_load_f32(i22); - const psimd_f32 vk22x0123 = psimd_load_f32(w + 92); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123); - - const psimd_f32 vi23x0123 = psimd_load_f32(i23); - const psimd_f32 vk23x0123 = psimd_load_f32(w + 96); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi23x0123, vk23x0123); - - const psimd_f32 vi24x0123 = psimd_load_f32(i24); - const psimd_f32 vk24x0123 = psimd_load_f32(w + 100); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123); - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c b/src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c deleted file mode 100644 index 9c6a98389..000000000 --- a/src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c +++ /dev/null @@ -1,136 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/f32-dwconv/up-psimd.c.in -// Generator: tools/xngen -// -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - const float* i0 = input[0]; - assert(i0 != NULL); - if XNN_UNPREDICTABLE(i0 != zero) { - i0 = (const float*) ((uintptr_t) i0 + input_offset); - } - const float* i1 = input[1]; - assert(i1 != NULL); - if XNN_UNPREDICTABLE(i1 != zero) { - i1 = (const float*) ((uintptr_t) i1 + input_offset); - } - const float* i2 = input[2]; - assert(i2 != NULL); - if XNN_UNPREDICTABLE(i2 != zero) { - i2 = (const float*) ((uintptr_t) i2 + input_offset); - } - const float* i3 = input[3]; - assert(i3 != NULL); - if XNN_UNPREDICTABLE(i3 != zero) { - i3 = (const float*) ((uintptr_t) i3 + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - i0 += 4; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 4); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - i1 += 4; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 8); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - i2 += 4; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - i3 += 4; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 16); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - - w += 20; - - // Add up all accumulators to vacc0123p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vk0x0123 = psimd_load_f32(w + 4); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vk1x0123 = psimd_load_f32(w + 8); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vk2x0123 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vk3x0123 = psimd_load_f32(w + 16); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - - // Add up all accumulators to vacc0123p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/f32-dwconv/gen/up4x4-minmax-psimd.c b/src/f32-dwconv/gen/up4x4-minmax-psimd.c deleted file mode 100644 index 3814c025e..000000000 --- a/src/f32-dwconv/gen/up4x4-minmax-psimd.c +++ /dev/null @@ -1,132 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/f32-dwconv/up-psimd.c.in -// Generator: tools/xngen -// -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up4x4__psimd( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - const float* i0 = input[0]; - assert(i0 != NULL); - if XNN_UNPREDICTABLE(i0 != zero) { - i0 = (const float*) ((uintptr_t) i0 + input_offset); - } - const float* i1 = input[1]; - assert(i1 != NULL); - if XNN_UNPREDICTABLE(i1 != zero) { - i1 = (const float*) ((uintptr_t) i1 + input_offset); - } - const float* i2 = input[2]; - assert(i2 != NULL); - if XNN_UNPREDICTABLE(i2 != zero) { - i2 = (const float*) ((uintptr_t) i2 + input_offset); - } - const float* i3 = input[3]; - assert(i3 != NULL); - if XNN_UNPREDICTABLE(i3 != zero) { - i3 = (const float*) ((uintptr_t) i3 + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - i0 += 4; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 4); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - i1 += 4; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - i2 += 4; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - i3 += 4; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 16); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - - w += 20; - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vk0x0123 = psimd_load_f32(w + 4); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vk1x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vk2x0123 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vk3x0123 = psimd_load_f32(w + 16); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c b/src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c deleted file mode 100644 index e4071864f..000000000 --- a/src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c +++ /dev/null @@ -1,211 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/f32-dwconv/up-psimd.c.in -// Generator: tools/xngen -// -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - const float* i0 = input[0]; - assert(i0 != NULL); - if XNN_UNPREDICTABLE(i0 != zero) { - i0 = (const float*) ((uintptr_t) i0 + input_offset); - } - const float* i1 = input[1]; - assert(i1 != NULL); - if XNN_UNPREDICTABLE(i1 != zero) { - i1 = (const float*) ((uintptr_t) i1 + input_offset); - } - const float* i2 = input[2]; - assert(i2 != NULL); - if XNN_UNPREDICTABLE(i2 != zero) { - i2 = (const float*) ((uintptr_t) i2 + input_offset); - } - const float* i3 = input[3]; - assert(i3 != NULL); - if XNN_UNPREDICTABLE(i3 != zero) { - i3 = (const float*) ((uintptr_t) i3 + input_offset); - } - const float* i4 = input[4]; - assert(i4 != NULL); - if XNN_UNPREDICTABLE(i4 != zero) { - i4 = (const float*) ((uintptr_t) i4 + input_offset); - } - const float* i5 = input[5]; - assert(i5 != NULL); - if XNN_UNPREDICTABLE(i5 != zero) { - i5 = (const float*) ((uintptr_t) i5 + input_offset); - } - const float* i6 = input[6]; - assert(i6 != NULL); - if XNN_UNPREDICTABLE(i6 != zero) { - i6 = (const float*) ((uintptr_t) i6 + input_offset); - } - const float* i7 = input[7]; - assert(i7 != NULL); - if XNN_UNPREDICTABLE(i7 != zero) { - i7 = (const float*) ((uintptr_t) i7 + input_offset); - } - const float* i8 = input[8]; - assert(i8 != NULL); - if XNN_UNPREDICTABLE(i8 != zero) { - i8 = (const float*) ((uintptr_t) i8 + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - i0 += 4; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 4); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - i1 += 4; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 8); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - i2 += 4; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - i3 += 4; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 16); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - i4 += 4; - - const psimd_f32 vk4x0123 = psimd_load_f32(w + 20); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - i5 += 4; - - const psimd_f32 vk5x0123 = psimd_load_f32(w + 24); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - i6 += 4; - - const psimd_f32 vk6x0123 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - i7 += 4; - - const psimd_f32 vk7x0123 = psimd_load_f32(w + 32); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - i8 += 4; - - const psimd_f32 vk8x0123 = psimd_load_f32(w + 36); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - w += 40; - - // Add up all accumulators to vacc0123p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vk0x0123 = psimd_load_f32(w + 4); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vk1x0123 = psimd_load_f32(w + 8); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vk2x0123 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vk3x0123 = psimd_load_f32(w + 16); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - const psimd_f32 vk4x0123 = psimd_load_f32(w + 20); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - const psimd_f32 vk5x0123 = psimd_load_f32(w + 24); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - const psimd_f32 vk6x0123 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - const psimd_f32 vk7x0123 = psimd_load_f32(w + 32); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - const psimd_f32 vk8x0123 = psimd_load_f32(w + 36); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - // Add up all accumulators to vacc0123p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/f32-dwconv/gen/up4x9-minmax-psimd.c b/src/f32-dwconv/gen/up4x9-minmax-psimd.c deleted file mode 100644 index 0bc1115ba..000000000 --- a/src/f32-dwconv/gen/up4x9-minmax-psimd.c +++ /dev/null @@ -1,207 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/f32-dwconv/up-psimd.c.in -// Generator: tools/xngen -// -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up4x9__psimd( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - const float* i0 = input[0]; - assert(i0 != NULL); - if XNN_UNPREDICTABLE(i0 != zero) { - i0 = (const float*) ((uintptr_t) i0 + input_offset); - } - const float* i1 = input[1]; - assert(i1 != NULL); - if XNN_UNPREDICTABLE(i1 != zero) { - i1 = (const float*) ((uintptr_t) i1 + input_offset); - } - const float* i2 = input[2]; - assert(i2 != NULL); - if XNN_UNPREDICTABLE(i2 != zero) { - i2 = (const float*) ((uintptr_t) i2 + input_offset); - } - const float* i3 = input[3]; - assert(i3 != NULL); - if XNN_UNPREDICTABLE(i3 != zero) { - i3 = (const float*) ((uintptr_t) i3 + input_offset); - } - const float* i4 = input[4]; - assert(i4 != NULL); - if XNN_UNPREDICTABLE(i4 != zero) { - i4 = (const float*) ((uintptr_t) i4 + input_offset); - } - const float* i5 = input[5]; - assert(i5 != NULL); - if XNN_UNPREDICTABLE(i5 != zero) { - i5 = (const float*) ((uintptr_t) i5 + input_offset); - } - const float* i6 = input[6]; - assert(i6 != NULL); - if XNN_UNPREDICTABLE(i6 != zero) { - i6 = (const float*) ((uintptr_t) i6 + input_offset); - } - const float* i7 = input[7]; - assert(i7 != NULL); - if XNN_UNPREDICTABLE(i7 != zero) { - i7 = (const float*) ((uintptr_t) i7 + input_offset); - } - const float* i8 = input[8]; - assert(i8 != NULL); - if XNN_UNPREDICTABLE(i8 != zero) { - i8 = (const float*) ((uintptr_t) i8 + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - i0 += 4; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 4); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - i1 += 4; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - i2 += 4; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - i3 += 4; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 16); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - i4 += 4; - - const psimd_f32 vk4x0123 = psimd_load_f32(w + 20); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - i5 += 4; - - const psimd_f32 vk5x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - i6 += 4; - - const psimd_f32 vk6x0123 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - i7 += 4; - - const psimd_f32 vk7x0123 = psimd_load_f32(w + 32); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - i8 += 4; - - const psimd_f32 vk8x0123 = psimd_load_f32(w + 36); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - w += 40; - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vk0x0123 = psimd_load_f32(w + 4); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vk1x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vk2x0123 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vk3x0123 = psimd_load_f32(w + 16); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - const psimd_f32 vk4x0123 = psimd_load_f32(w + 20); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - const psimd_f32 vk5x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - const psimd_f32 vk6x0123 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - const psimd_f32 vk7x0123 = psimd_load_f32(w + 32); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - const psimd_f32 vk8x0123 = psimd_load_f32(w + 36); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c b/src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c deleted file mode 100644 index 2da6c1482..000000000 --- a/src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c +++ /dev/null @@ -1,695 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/f32-dwconv/up-psimd.c.in -// Generator: tools/xngen -// -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - const float* i0 = input[0]; - assert(i0 != NULL); - if XNN_UNPREDICTABLE(i0 != zero) { - i0 = (const float*) ((uintptr_t) i0 + input_offset); - } - const float* i1 = input[1]; - assert(i1 != NULL); - if XNN_UNPREDICTABLE(i1 != zero) { - i1 = (const float*) ((uintptr_t) i1 + input_offset); - } - const float* i2 = input[2]; - assert(i2 != NULL); - if XNN_UNPREDICTABLE(i2 != zero) { - i2 = (const float*) ((uintptr_t) i2 + input_offset); - } - const float* i3 = input[3]; - assert(i3 != NULL); - if XNN_UNPREDICTABLE(i3 != zero) { - i3 = (const float*) ((uintptr_t) i3 + input_offset); - } - const float* i4 = input[4]; - assert(i4 != NULL); - if XNN_UNPREDICTABLE(i4 != zero) { - i4 = (const float*) ((uintptr_t) i4 + input_offset); - } - const float* i5 = input[5]; - assert(i5 != NULL); - if XNN_UNPREDICTABLE(i5 != zero) { - i5 = (const float*) ((uintptr_t) i5 + input_offset); - } - const float* i6 = input[6]; - assert(i6 != NULL); - if XNN_UNPREDICTABLE(i6 != zero) { - i6 = (const float*) ((uintptr_t) i6 + input_offset); - } - const float* i7 = input[7]; - assert(i7 != NULL); - if XNN_UNPREDICTABLE(i7 != zero) { - i7 = (const float*) ((uintptr_t) i7 + input_offset); - } - const float* i8 = input[8]; - assert(i8 != NULL); - if XNN_UNPREDICTABLE(i8 != zero) { - i8 = (const float*) ((uintptr_t) i8 + input_offset); - } - const float* i9 = input[9]; - assert(i9 != NULL); - if XNN_UNPREDICTABLE(i9 != zero) { - i9 = (const float*) ((uintptr_t) i9 + input_offset); - } - const float* i10 = input[10]; - assert(i10 != NULL); - if XNN_UNPREDICTABLE(i10 != zero) { - i10 = (const float*) ((uintptr_t) i10 + input_offset); - } - const float* i11 = input[11]; - assert(i11 != NULL); - if XNN_UNPREDICTABLE(i11 != zero) { - i11 = (const float*) ((uintptr_t) i11 + input_offset); - } - const float* i12 = input[12]; - assert(i12 != NULL); - if XNN_UNPREDICTABLE(i12 != zero) { - i12 = (const float*) ((uintptr_t) i12 + input_offset); - } - const float* i13 = input[13]; - assert(i13 != NULL); - if XNN_UNPREDICTABLE(i13 != zero) { - i13 = (const float*) ((uintptr_t) i13 + input_offset); - } - const float* i14 = input[14]; - assert(i14 != NULL); - if XNN_UNPREDICTABLE(i14 != zero) { - i14 = (const float*) ((uintptr_t) i14 + input_offset); - } - const float* i15 = input[15]; - assert(i15 != NULL); - if XNN_UNPREDICTABLE(i15 != zero) { - i15 = (const float*) ((uintptr_t) i15 + input_offset); - } - const float* i16 = input[16]; - assert(i16 != NULL); - if XNN_UNPREDICTABLE(i16 != zero) { - i16 = (const float*) ((uintptr_t) i16 + input_offset); - } - const float* i17 = input[17]; - assert(i17 != NULL); - if XNN_UNPREDICTABLE(i17 != zero) { - i17 = (const float*) ((uintptr_t) i17 + input_offset); - } - const float* i18 = input[18]; - assert(i18 != NULL); - if XNN_UNPREDICTABLE(i18 != zero) { - i18 = (const float*) ((uintptr_t) i18 + input_offset); - } - const float* i19 = input[19]; - assert(i19 != NULL); - if XNN_UNPREDICTABLE(i19 != zero) { - i19 = (const float*) ((uintptr_t) i19 + input_offset); - } - const float* i20 = input[20]; - assert(i20 != NULL); - if XNN_UNPREDICTABLE(i20 != zero) { - i20 = (const float*) ((uintptr_t) i20 + input_offset); - } - const float* i21 = input[21]; - assert(i21 != NULL); - if XNN_UNPREDICTABLE(i21 != zero) { - i21 = (const float*) ((uintptr_t) i21 + input_offset); - } - const float* i22 = input[22]; - assert(i22 != NULL); - if XNN_UNPREDICTABLE(i22 != zero) { - i22 = (const float*) ((uintptr_t) i22 + input_offset); - } - const float* i23 = input[23]; - assert(i23 != NULL); - if XNN_UNPREDICTABLE(i23 != zero) { - i23 = (const float*) ((uintptr_t) i23 + input_offset); - } - const float* i24 = input[24]; - assert(i24 != NULL); - if XNN_UNPREDICTABLE(i24 != zero) { - i24 = (const float*) ((uintptr_t) i24 + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= 8; c -= 8) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - psimd_f32 vacc4567p0 = psimd_load_f32(w + 4); - - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vi0x4567 = psimd_load_f32(i0 + 4); - i0 += 8; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - const psimd_f32 vk0x4567 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi0x4567, vk0x4567); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vi1x4567 = psimd_load_f32(i1 + 4); - i1 += 8; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - const psimd_f32 vk1x4567 = psimd_load_f32(w + 20); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - psimd_f32 vacc4567p1 = psimd_mul_f32(vi1x4567, vk1x4567); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vi2x4567 = psimd_load_f32(i2 + 4); - i2 += 8; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - const psimd_f32 vk2x4567 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi2x4567, vk2x4567); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vi3x4567 = psimd_load_f32(i3 + 4); - i3 += 8; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - const psimd_f32 vk3x4567 = psimd_load_f32(w + 36); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi3x4567, vk3x4567); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - const psimd_f32 vi4x4567 = psimd_load_f32(i4 + 4); - i4 += 8; - - const psimd_f32 vk4x0123 = psimd_load_f32(w + 40); - const psimd_f32 vk4x4567 = psimd_load_f32(w + 44); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi4x4567, vk4x4567); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - const psimd_f32 vi5x4567 = psimd_load_f32(i5 + 4); - i5 += 8; - - const psimd_f32 vk5x0123 = psimd_load_f32(w + 48); - const psimd_f32 vk5x4567 = psimd_load_f32(w + 52); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi5x4567, vk5x4567); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - const psimd_f32 vi6x4567 = psimd_load_f32(i6 + 4); - i6 += 8; - - const psimd_f32 vk6x0123 = psimd_load_f32(w + 56); - const psimd_f32 vk6x4567 = psimd_load_f32(w + 60); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi6x4567, vk6x4567); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - const psimd_f32 vi7x4567 = psimd_load_f32(i7 + 4); - i7 += 8; - - const psimd_f32 vk7x0123 = psimd_load_f32(w + 64); - const psimd_f32 vk7x4567 = psimd_load_f32(w + 68); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi7x4567, vk7x4567); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - const psimd_f32 vi8x4567 = psimd_load_f32(i8 + 4); - i8 += 8; - - const psimd_f32 vk8x0123 = psimd_load_f32(w + 72); - const psimd_f32 vk8x4567 = psimd_load_f32(w + 76); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi8x4567, vk8x4567); - - const psimd_f32 vi9x0123 = psimd_load_f32(i9); - const psimd_f32 vi9x4567 = psimd_load_f32(i9 + 4); - i9 += 8; - - const psimd_f32 vk9x0123 = psimd_load_f32(w + 80); - const psimd_f32 vk9x4567 = psimd_load_f32(w + 84); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi9x0123, vk9x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi9x4567, vk9x4567); - - const psimd_f32 vi10x0123 = psimd_load_f32(i10); - const psimd_f32 vi10x4567 = psimd_load_f32(i10 + 4); - i10 += 8; - - const psimd_f32 vk10x0123 = psimd_load_f32(w + 88); - const psimd_f32 vk10x4567 = psimd_load_f32(w + 92); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi10x4567, vk10x4567); - - const psimd_f32 vi11x0123 = psimd_load_f32(i11); - const psimd_f32 vi11x4567 = psimd_load_f32(i11 + 4); - i11 += 8; - - const psimd_f32 vk11x0123 = psimd_load_f32(w + 96); - const psimd_f32 vk11x4567 = psimd_load_f32(w + 100); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi11x0123, vk11x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi11x4567, vk11x4567); - - const psimd_f32 vi12x0123 = psimd_load_f32(i12); - const psimd_f32 vi12x4567 = psimd_load_f32(i12 + 4); - i12 += 8; - - const psimd_f32 vk12x0123 = psimd_load_f32(w + 104); - const psimd_f32 vk12x4567 = psimd_load_f32(w + 108); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi12x4567, vk12x4567); - - const psimd_f32 vi13x0123 = psimd_load_f32(i13); - const psimd_f32 vi13x4567 = psimd_load_f32(i13 + 4); - i13 += 8; - - const psimd_f32 vk13x0123 = psimd_load_f32(w + 112); - const psimd_f32 vk13x4567 = psimd_load_f32(w + 116); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi13x0123, vk13x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi13x4567, vk13x4567); - - const psimd_f32 vi14x0123 = psimd_load_f32(i14); - const psimd_f32 vi14x4567 = psimd_load_f32(i14 + 4); - i14 += 8; - - const psimd_f32 vk14x0123 = psimd_load_f32(w + 120); - const psimd_f32 vk14x4567 = psimd_load_f32(w + 124); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi14x4567, vk14x4567); - - const psimd_f32 vi15x0123 = psimd_load_f32(i15); - const psimd_f32 vi15x4567 = psimd_load_f32(i15 + 4); - i15 += 8; - - const psimd_f32 vk15x0123 = psimd_load_f32(w + 128); - const psimd_f32 vk15x4567 = psimd_load_f32(w + 132); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi15x0123, vk15x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi15x4567, vk15x4567); - - const psimd_f32 vi16x0123 = psimd_load_f32(i16); - const psimd_f32 vi16x4567 = psimd_load_f32(i16 + 4); - i16 += 8; - - const psimd_f32 vk16x0123 = psimd_load_f32(w + 136); - const psimd_f32 vk16x4567 = psimd_load_f32(w + 140); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi16x4567, vk16x4567); - - const psimd_f32 vi17x0123 = psimd_load_f32(i17); - const psimd_f32 vi17x4567 = psimd_load_f32(i17 + 4); - i17 += 8; - - const psimd_f32 vk17x0123 = psimd_load_f32(w + 144); - const psimd_f32 vk17x4567 = psimd_load_f32(w + 148); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi17x0123, vk17x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi17x4567, vk17x4567); - - const psimd_f32 vi18x0123 = psimd_load_f32(i18); - const psimd_f32 vi18x4567 = psimd_load_f32(i18 + 4); - i18 += 8; - - const psimd_f32 vk18x0123 = psimd_load_f32(w + 152); - const psimd_f32 vk18x4567 = psimd_load_f32(w + 156); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi18x4567, vk18x4567); - - const psimd_f32 vi19x0123 = psimd_load_f32(i19); - const psimd_f32 vi19x4567 = psimd_load_f32(i19 + 4); - i19 += 8; - - const psimd_f32 vk19x0123 = psimd_load_f32(w + 160); - const psimd_f32 vk19x4567 = psimd_load_f32(w + 164); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi19x0123, vk19x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi19x4567, vk19x4567); - - const psimd_f32 vi20x0123 = psimd_load_f32(i20); - const psimd_f32 vi20x4567 = psimd_load_f32(i20 + 4); - i20 += 8; - - const psimd_f32 vk20x0123 = psimd_load_f32(w + 168); - const psimd_f32 vk20x4567 = psimd_load_f32(w + 172); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi20x4567, vk20x4567); - - const psimd_f32 vi21x0123 = psimd_load_f32(i21); - const psimd_f32 vi21x4567 = psimd_load_f32(i21 + 4); - i21 += 8; - - const psimd_f32 vk21x0123 = psimd_load_f32(w + 176); - const psimd_f32 vk21x4567 = psimd_load_f32(w + 180); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi21x0123, vk21x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi21x4567, vk21x4567); - - const psimd_f32 vi22x0123 = psimd_load_f32(i22); - const psimd_f32 vi22x4567 = psimd_load_f32(i22 + 4); - i22 += 8; - - const psimd_f32 vk22x0123 = psimd_load_f32(w + 184); - const psimd_f32 vk22x4567 = psimd_load_f32(w + 188); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi22x4567, vk22x4567); - - const psimd_f32 vi23x0123 = psimd_load_f32(i23); - const psimd_f32 vi23x4567 = psimd_load_f32(i23 + 4); - i23 += 8; - - const psimd_f32 vk23x0123 = psimd_load_f32(w + 192); - const psimd_f32 vk23x4567 = psimd_load_f32(w + 196); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi23x0123, vk23x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi23x4567, vk23x4567); - - const psimd_f32 vi24x0123 = psimd_load_f32(i24); - const psimd_f32 vi24x4567 = psimd_load_f32(i24 + 4); - i24 += 8; - - const psimd_f32 vk24x0123 = psimd_load_f32(w + 200); - const psimd_f32 vk24x4567 = psimd_load_f32(w + 204); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi24x4567, vk24x4567); - - w += 208; - - // Add up all accumulators to vacc01234567p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - vacc4567p0 = psimd_add_f32(vacc4567p0, vacc4567p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - psimd_f32 vacc4567 = psimd_max_f32(vacc4567p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - vacc4567 = psimd_min_f32(vacc4567, vmax); - - psimd_store_f32(output, vacc0123); - psimd_store_f32(output + 4, vacc4567); - output += 8; - } - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - i0 += 4; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - i1 += 4; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - i2 += 4; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - i3 += 4; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - i4 += 4; - - const psimd_f32 vk4x0123 = psimd_load_f32(w + 40); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - i5 += 4; - - const psimd_f32 vk5x0123 = psimd_load_f32(w + 48); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - i6 += 4; - - const psimd_f32 vk6x0123 = psimd_load_f32(w + 56); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - i7 += 4; - - const psimd_f32 vk7x0123 = psimd_load_f32(w + 64); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - i8 += 4; - - const psimd_f32 vk8x0123 = psimd_load_f32(w + 72); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - const psimd_f32 vi9x0123 = psimd_load_f32(i9); - i9 += 4; - - const psimd_f32 vk9x0123 = psimd_load_f32(w + 80); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi9x0123, vk9x0123); - - const psimd_f32 vi10x0123 = psimd_load_f32(i10); - i10 += 4; - - const psimd_f32 vk10x0123 = psimd_load_f32(w + 88); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123); - - const psimd_f32 vi11x0123 = psimd_load_f32(i11); - i11 += 4; - - const psimd_f32 vk11x0123 = psimd_load_f32(w + 96); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi11x0123, vk11x0123); - - const psimd_f32 vi12x0123 = psimd_load_f32(i12); - i12 += 4; - - const psimd_f32 vk12x0123 = psimd_load_f32(w + 104); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123); - - const psimd_f32 vi13x0123 = psimd_load_f32(i13); - i13 += 4; - - const psimd_f32 vk13x0123 = psimd_load_f32(w + 112); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi13x0123, vk13x0123); - - const psimd_f32 vi14x0123 = psimd_load_f32(i14); - i14 += 4; - - const psimd_f32 vk14x0123 = psimd_load_f32(w + 120); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123); - - const psimd_f32 vi15x0123 = psimd_load_f32(i15); - i15 += 4; - - const psimd_f32 vk15x0123 = psimd_load_f32(w + 128); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi15x0123, vk15x0123); - - const psimd_f32 vi16x0123 = psimd_load_f32(i16); - i16 += 4; - - const psimd_f32 vk16x0123 = psimd_load_f32(w + 136); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123); - - const psimd_f32 vi17x0123 = psimd_load_f32(i17); - i17 += 4; - - const psimd_f32 vk17x0123 = psimd_load_f32(w + 144); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi17x0123, vk17x0123); - - const psimd_f32 vi18x0123 = psimd_load_f32(i18); - i18 += 4; - - const psimd_f32 vk18x0123 = psimd_load_f32(w + 152); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123); - - const psimd_f32 vi19x0123 = psimd_load_f32(i19); - i19 += 4; - - const psimd_f32 vk19x0123 = psimd_load_f32(w + 160); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi19x0123, vk19x0123); - - const psimd_f32 vi20x0123 = psimd_load_f32(i20); - i20 += 4; - - const psimd_f32 vk20x0123 = psimd_load_f32(w + 168); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123); - - const psimd_f32 vi21x0123 = psimd_load_f32(i21); - i21 += 4; - - const psimd_f32 vk21x0123 = psimd_load_f32(w + 176); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi21x0123, vk21x0123); - - const psimd_f32 vi22x0123 = psimd_load_f32(i22); - i22 += 4; - - const psimd_f32 vk22x0123 = psimd_load_f32(w + 184); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123); - - const psimd_f32 vi23x0123 = psimd_load_f32(i23); - i23 += 4; - - const psimd_f32 vk23x0123 = psimd_load_f32(w + 192); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi23x0123, vk23x0123); - - const psimd_f32 vi24x0123 = psimd_load_f32(i24); - i24 += 4; - - const psimd_f32 vk24x0123 = psimd_load_f32(w + 200); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123); - - w += 4; - - // Add up all accumulators to vacc01234567p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - const psimd_f32 vk4x0123 = psimd_load_f32(w + 40); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - const psimd_f32 vk5x0123 = psimd_load_f32(w + 48); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - const psimd_f32 vk6x0123 = psimd_load_f32(w + 56); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - const psimd_f32 vk7x0123 = psimd_load_f32(w + 64); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - const psimd_f32 vk8x0123 = psimd_load_f32(w + 72); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - const psimd_f32 vi9x0123 = psimd_load_f32(i9); - const psimd_f32 vk9x0123 = psimd_load_f32(w + 80); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi9x0123, vk9x0123); - - const psimd_f32 vi10x0123 = psimd_load_f32(i10); - const psimd_f32 vk10x0123 = psimd_load_f32(w + 88); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123); - - const psimd_f32 vi11x0123 = psimd_load_f32(i11); - const psimd_f32 vk11x0123 = psimd_load_f32(w + 96); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi11x0123, vk11x0123); - - const psimd_f32 vi12x0123 = psimd_load_f32(i12); - const psimd_f32 vk12x0123 = psimd_load_f32(w + 104); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123); - - const psimd_f32 vi13x0123 = psimd_load_f32(i13); - const psimd_f32 vk13x0123 = psimd_load_f32(w + 112); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi13x0123, vk13x0123); - - const psimd_f32 vi14x0123 = psimd_load_f32(i14); - const psimd_f32 vk14x0123 = psimd_load_f32(w + 120); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123); - - const psimd_f32 vi15x0123 = psimd_load_f32(i15); - const psimd_f32 vk15x0123 = psimd_load_f32(w + 128); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi15x0123, vk15x0123); - - const psimd_f32 vi16x0123 = psimd_load_f32(i16); - const psimd_f32 vk16x0123 = psimd_load_f32(w + 136); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123); - - const psimd_f32 vi17x0123 = psimd_load_f32(i17); - const psimd_f32 vk17x0123 = psimd_load_f32(w + 144); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi17x0123, vk17x0123); - - const psimd_f32 vi18x0123 = psimd_load_f32(i18); - const psimd_f32 vk18x0123 = psimd_load_f32(w + 152); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123); - - const psimd_f32 vi19x0123 = psimd_load_f32(i19); - const psimd_f32 vk19x0123 = psimd_load_f32(w + 160); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi19x0123, vk19x0123); - - const psimd_f32 vi20x0123 = psimd_load_f32(i20); - const psimd_f32 vk20x0123 = psimd_load_f32(w + 168); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123); - - const psimd_f32 vi21x0123 = psimd_load_f32(i21); - const psimd_f32 vk21x0123 = psimd_load_f32(w + 176); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi21x0123, vk21x0123); - - const psimd_f32 vi22x0123 = psimd_load_f32(i22); - const psimd_f32 vk22x0123 = psimd_load_f32(w + 184); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123); - - const psimd_f32 vi23x0123 = psimd_load_f32(i23); - const psimd_f32 vk23x0123 = psimd_load_f32(w + 192); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi23x0123, vk23x0123); - - const psimd_f32 vi24x0123 = psimd_load_f32(i24); - const psimd_f32 vk24x0123 = psimd_load_f32(w + 200); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123); - - // Add up all accumulators to vacc01234567p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/f32-dwconv/gen/up8x25-minmax-psimd.c b/src/f32-dwconv/gen/up8x25-minmax-psimd.c deleted file mode 100644 index 2126f5eff..000000000 --- a/src/f32-dwconv/gen/up8x25-minmax-psimd.c +++ /dev/null @@ -1,688 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/f32-dwconv/up-psimd.c.in -// Generator: tools/xngen -// -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up8x25__psimd( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - const float* i0 = input[0]; - assert(i0 != NULL); - if XNN_UNPREDICTABLE(i0 != zero) { - i0 = (const float*) ((uintptr_t) i0 + input_offset); - } - const float* i1 = input[1]; - assert(i1 != NULL); - if XNN_UNPREDICTABLE(i1 != zero) { - i1 = (const float*) ((uintptr_t) i1 + input_offset); - } - const float* i2 = input[2]; - assert(i2 != NULL); - if XNN_UNPREDICTABLE(i2 != zero) { - i2 = (const float*) ((uintptr_t) i2 + input_offset); - } - const float* i3 = input[3]; - assert(i3 != NULL); - if XNN_UNPREDICTABLE(i3 != zero) { - i3 = (const float*) ((uintptr_t) i3 + input_offset); - } - const float* i4 = input[4]; - assert(i4 != NULL); - if XNN_UNPREDICTABLE(i4 != zero) { - i4 = (const float*) ((uintptr_t) i4 + input_offset); - } - const float* i5 = input[5]; - assert(i5 != NULL); - if XNN_UNPREDICTABLE(i5 != zero) { - i5 = (const float*) ((uintptr_t) i5 + input_offset); - } - const float* i6 = input[6]; - assert(i6 != NULL); - if XNN_UNPREDICTABLE(i6 != zero) { - i6 = (const float*) ((uintptr_t) i6 + input_offset); - } - const float* i7 = input[7]; - assert(i7 != NULL); - if XNN_UNPREDICTABLE(i7 != zero) { - i7 = (const float*) ((uintptr_t) i7 + input_offset); - } - const float* i8 = input[8]; - assert(i8 != NULL); - if XNN_UNPREDICTABLE(i8 != zero) { - i8 = (const float*) ((uintptr_t) i8 + input_offset); - } - const float* i9 = input[9]; - assert(i9 != NULL); - if XNN_UNPREDICTABLE(i9 != zero) { - i9 = (const float*) ((uintptr_t) i9 + input_offset); - } - const float* i10 = input[10]; - assert(i10 != NULL); - if XNN_UNPREDICTABLE(i10 != zero) { - i10 = (const float*) ((uintptr_t) i10 + input_offset); - } - const float* i11 = input[11]; - assert(i11 != NULL); - if XNN_UNPREDICTABLE(i11 != zero) { - i11 = (const float*) ((uintptr_t) i11 + input_offset); - } - const float* i12 = input[12]; - assert(i12 != NULL); - if XNN_UNPREDICTABLE(i12 != zero) { - i12 = (const float*) ((uintptr_t) i12 + input_offset); - } - const float* i13 = input[13]; - assert(i13 != NULL); - if XNN_UNPREDICTABLE(i13 != zero) { - i13 = (const float*) ((uintptr_t) i13 + input_offset); - } - const float* i14 = input[14]; - assert(i14 != NULL); - if XNN_UNPREDICTABLE(i14 != zero) { - i14 = (const float*) ((uintptr_t) i14 + input_offset); - } - const float* i15 = input[15]; - assert(i15 != NULL); - if XNN_UNPREDICTABLE(i15 != zero) { - i15 = (const float*) ((uintptr_t) i15 + input_offset); - } - const float* i16 = input[16]; - assert(i16 != NULL); - if XNN_UNPREDICTABLE(i16 != zero) { - i16 = (const float*) ((uintptr_t) i16 + input_offset); - } - const float* i17 = input[17]; - assert(i17 != NULL); - if XNN_UNPREDICTABLE(i17 != zero) { - i17 = (const float*) ((uintptr_t) i17 + input_offset); - } - const float* i18 = input[18]; - assert(i18 != NULL); - if XNN_UNPREDICTABLE(i18 != zero) { - i18 = (const float*) ((uintptr_t) i18 + input_offset); - } - const float* i19 = input[19]; - assert(i19 != NULL); - if XNN_UNPREDICTABLE(i19 != zero) { - i19 = (const float*) ((uintptr_t) i19 + input_offset); - } - const float* i20 = input[20]; - assert(i20 != NULL); - if XNN_UNPREDICTABLE(i20 != zero) { - i20 = (const float*) ((uintptr_t) i20 + input_offset); - } - const float* i21 = input[21]; - assert(i21 != NULL); - if XNN_UNPREDICTABLE(i21 != zero) { - i21 = (const float*) ((uintptr_t) i21 + input_offset); - } - const float* i22 = input[22]; - assert(i22 != NULL); - if XNN_UNPREDICTABLE(i22 != zero) { - i22 = (const float*) ((uintptr_t) i22 + input_offset); - } - const float* i23 = input[23]; - assert(i23 != NULL); - if XNN_UNPREDICTABLE(i23 != zero) { - i23 = (const float*) ((uintptr_t) i23 + input_offset); - } - const float* i24 = input[24]; - assert(i24 != NULL); - if XNN_UNPREDICTABLE(i24 != zero) { - i24 = (const float*) ((uintptr_t) i24 + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= 8; c -= 8) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - psimd_f32 vacc4567p0 = psimd_load_f32(w + 4); - - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vi0x4567 = psimd_load_f32(i0 + 4); - i0 += 8; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - const psimd_f32 vk0x4567 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi0x4567, vk0x4567); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vi1x4567 = psimd_load_f32(i1 + 4); - i1 += 8; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - const psimd_f32 vk1x4567 = psimd_load_f32(w + 20); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi1x4567, vk1x4567); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vi2x4567 = psimd_load_f32(i2 + 4); - i2 += 8; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - const psimd_f32 vk2x4567 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi2x4567, vk2x4567); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vi3x4567 = psimd_load_f32(i3 + 4); - i3 += 8; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - const psimd_f32 vk3x4567 = psimd_load_f32(w + 36); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi3x4567, vk3x4567); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - const psimd_f32 vi4x4567 = psimd_load_f32(i4 + 4); - i4 += 8; - - const psimd_f32 vk4x0123 = psimd_load_f32(w + 40); - const psimd_f32 vk4x4567 = psimd_load_f32(w + 44); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi4x4567, vk4x4567); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - const psimd_f32 vi5x4567 = psimd_load_f32(i5 + 4); - i5 += 8; - - const psimd_f32 vk5x0123 = psimd_load_f32(w + 48); - const psimd_f32 vk5x4567 = psimd_load_f32(w + 52); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi5x4567, vk5x4567); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - const psimd_f32 vi6x4567 = psimd_load_f32(i6 + 4); - i6 += 8; - - const psimd_f32 vk6x0123 = psimd_load_f32(w + 56); - const psimd_f32 vk6x4567 = psimd_load_f32(w + 60); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi6x4567, vk6x4567); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - const psimd_f32 vi7x4567 = psimd_load_f32(i7 + 4); - i7 += 8; - - const psimd_f32 vk7x0123 = psimd_load_f32(w + 64); - const psimd_f32 vk7x4567 = psimd_load_f32(w + 68); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi7x4567, vk7x4567); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - const psimd_f32 vi8x4567 = psimd_load_f32(i8 + 4); - i8 += 8; - - const psimd_f32 vk8x0123 = psimd_load_f32(w + 72); - const psimd_f32 vk8x4567 = psimd_load_f32(w + 76); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi8x4567, vk8x4567); - - const psimd_f32 vi9x0123 = psimd_load_f32(i9); - const psimd_f32 vi9x4567 = psimd_load_f32(i9 + 4); - i9 += 8; - - const psimd_f32 vk9x0123 = psimd_load_f32(w + 80); - const psimd_f32 vk9x4567 = psimd_load_f32(w + 84); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi9x0123, vk9x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi9x4567, vk9x4567); - - const psimd_f32 vi10x0123 = psimd_load_f32(i10); - const psimd_f32 vi10x4567 = psimd_load_f32(i10 + 4); - i10 += 8; - - const psimd_f32 vk10x0123 = psimd_load_f32(w + 88); - const psimd_f32 vk10x4567 = psimd_load_f32(w + 92); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi10x4567, vk10x4567); - - const psimd_f32 vi11x0123 = psimd_load_f32(i11); - const psimd_f32 vi11x4567 = psimd_load_f32(i11 + 4); - i11 += 8; - - const psimd_f32 vk11x0123 = psimd_load_f32(w + 96); - const psimd_f32 vk11x4567 = psimd_load_f32(w + 100); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi11x0123, vk11x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi11x4567, vk11x4567); - - const psimd_f32 vi12x0123 = psimd_load_f32(i12); - const psimd_f32 vi12x4567 = psimd_load_f32(i12 + 4); - i12 += 8; - - const psimd_f32 vk12x0123 = psimd_load_f32(w + 104); - const psimd_f32 vk12x4567 = psimd_load_f32(w + 108); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi12x4567, vk12x4567); - - const psimd_f32 vi13x0123 = psimd_load_f32(i13); - const psimd_f32 vi13x4567 = psimd_load_f32(i13 + 4); - i13 += 8; - - const psimd_f32 vk13x0123 = psimd_load_f32(w + 112); - const psimd_f32 vk13x4567 = psimd_load_f32(w + 116); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi13x0123, vk13x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi13x4567, vk13x4567); - - const psimd_f32 vi14x0123 = psimd_load_f32(i14); - const psimd_f32 vi14x4567 = psimd_load_f32(i14 + 4); - i14 += 8; - - const psimd_f32 vk14x0123 = psimd_load_f32(w + 120); - const psimd_f32 vk14x4567 = psimd_load_f32(w + 124); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi14x4567, vk14x4567); - - const psimd_f32 vi15x0123 = psimd_load_f32(i15); - const psimd_f32 vi15x4567 = psimd_load_f32(i15 + 4); - i15 += 8; - - const psimd_f32 vk15x0123 = psimd_load_f32(w + 128); - const psimd_f32 vk15x4567 = psimd_load_f32(w + 132); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi15x0123, vk15x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi15x4567, vk15x4567); - - const psimd_f32 vi16x0123 = psimd_load_f32(i16); - const psimd_f32 vi16x4567 = psimd_load_f32(i16 + 4); - i16 += 8; - - const psimd_f32 vk16x0123 = psimd_load_f32(w + 136); - const psimd_f32 vk16x4567 = psimd_load_f32(w + 140); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi16x4567, vk16x4567); - - const psimd_f32 vi17x0123 = psimd_load_f32(i17); - const psimd_f32 vi17x4567 = psimd_load_f32(i17 + 4); - i17 += 8; - - const psimd_f32 vk17x0123 = psimd_load_f32(w + 144); - const psimd_f32 vk17x4567 = psimd_load_f32(w + 148); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi17x0123, vk17x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi17x4567, vk17x4567); - - const psimd_f32 vi18x0123 = psimd_load_f32(i18); - const psimd_f32 vi18x4567 = psimd_load_f32(i18 + 4); - i18 += 8; - - const psimd_f32 vk18x0123 = psimd_load_f32(w + 152); - const psimd_f32 vk18x4567 = psimd_load_f32(w + 156); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi18x4567, vk18x4567); - - const psimd_f32 vi19x0123 = psimd_load_f32(i19); - const psimd_f32 vi19x4567 = psimd_load_f32(i19 + 4); - i19 += 8; - - const psimd_f32 vk19x0123 = psimd_load_f32(w + 160); - const psimd_f32 vk19x4567 = psimd_load_f32(w + 164); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi19x0123, vk19x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi19x4567, vk19x4567); - - const psimd_f32 vi20x0123 = psimd_load_f32(i20); - const psimd_f32 vi20x4567 = psimd_load_f32(i20 + 4); - i20 += 8; - - const psimd_f32 vk20x0123 = psimd_load_f32(w + 168); - const psimd_f32 vk20x4567 = psimd_load_f32(w + 172); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi20x4567, vk20x4567); - - const psimd_f32 vi21x0123 = psimd_load_f32(i21); - const psimd_f32 vi21x4567 = psimd_load_f32(i21 + 4); - i21 += 8; - - const psimd_f32 vk21x0123 = psimd_load_f32(w + 176); - const psimd_f32 vk21x4567 = psimd_load_f32(w + 180); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi21x0123, vk21x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi21x4567, vk21x4567); - - const psimd_f32 vi22x0123 = psimd_load_f32(i22); - const psimd_f32 vi22x4567 = psimd_load_f32(i22 + 4); - i22 += 8; - - const psimd_f32 vk22x0123 = psimd_load_f32(w + 184); - const psimd_f32 vk22x4567 = psimd_load_f32(w + 188); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi22x4567, vk22x4567); - - const psimd_f32 vi23x0123 = psimd_load_f32(i23); - const psimd_f32 vi23x4567 = psimd_load_f32(i23 + 4); - i23 += 8; - - const psimd_f32 vk23x0123 = psimd_load_f32(w + 192); - const psimd_f32 vk23x4567 = psimd_load_f32(w + 196); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi23x0123, vk23x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi23x4567, vk23x4567); - - const psimd_f32 vi24x0123 = psimd_load_f32(i24); - const psimd_f32 vi24x4567 = psimd_load_f32(i24 + 4); - i24 += 8; - - const psimd_f32 vk24x0123 = psimd_load_f32(w + 200); - const psimd_f32 vk24x4567 = psimd_load_f32(w + 204); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi24x4567, vk24x4567); - - w += 208; - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - psimd_f32 vacc4567 = psimd_max_f32(vacc4567p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - vacc4567 = psimd_min_f32(vacc4567, vmax); - - psimd_store_f32(output, vacc0123); - psimd_store_f32(output + 4, vacc4567); - output += 8; - } - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - i0 += 4; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - i1 += 4; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - i2 += 4; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - i3 += 4; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - i4 += 4; - - const psimd_f32 vk4x0123 = psimd_load_f32(w + 40); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - i5 += 4; - - const psimd_f32 vk5x0123 = psimd_load_f32(w + 48); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - i6 += 4; - - const psimd_f32 vk6x0123 = psimd_load_f32(w + 56); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - i7 += 4; - - const psimd_f32 vk7x0123 = psimd_load_f32(w + 64); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - i8 += 4; - - const psimd_f32 vk8x0123 = psimd_load_f32(w + 72); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - const psimd_f32 vi9x0123 = psimd_load_f32(i9); - i9 += 4; - - const psimd_f32 vk9x0123 = psimd_load_f32(w + 80); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi9x0123, vk9x0123); - - const psimd_f32 vi10x0123 = psimd_load_f32(i10); - i10 += 4; - - const psimd_f32 vk10x0123 = psimd_load_f32(w + 88); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123); - - const psimd_f32 vi11x0123 = psimd_load_f32(i11); - i11 += 4; - - const psimd_f32 vk11x0123 = psimd_load_f32(w + 96); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi11x0123, vk11x0123); - - const psimd_f32 vi12x0123 = psimd_load_f32(i12); - i12 += 4; - - const psimd_f32 vk12x0123 = psimd_load_f32(w + 104); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123); - - const psimd_f32 vi13x0123 = psimd_load_f32(i13); - i13 += 4; - - const psimd_f32 vk13x0123 = psimd_load_f32(w + 112); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi13x0123, vk13x0123); - - const psimd_f32 vi14x0123 = psimd_load_f32(i14); - i14 += 4; - - const psimd_f32 vk14x0123 = psimd_load_f32(w + 120); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123); - - const psimd_f32 vi15x0123 = psimd_load_f32(i15); - i15 += 4; - - const psimd_f32 vk15x0123 = psimd_load_f32(w + 128); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi15x0123, vk15x0123); - - const psimd_f32 vi16x0123 = psimd_load_f32(i16); - i16 += 4; - - const psimd_f32 vk16x0123 = psimd_load_f32(w + 136); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123); - - const psimd_f32 vi17x0123 = psimd_load_f32(i17); - i17 += 4; - - const psimd_f32 vk17x0123 = psimd_load_f32(w + 144); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi17x0123, vk17x0123); - - const psimd_f32 vi18x0123 = psimd_load_f32(i18); - i18 += 4; - - const psimd_f32 vk18x0123 = psimd_load_f32(w + 152); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123); - - const psimd_f32 vi19x0123 = psimd_load_f32(i19); - i19 += 4; - - const psimd_f32 vk19x0123 = psimd_load_f32(w + 160); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi19x0123, vk19x0123); - - const psimd_f32 vi20x0123 = psimd_load_f32(i20); - i20 += 4; - - const psimd_f32 vk20x0123 = psimd_load_f32(w + 168); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123); - - const psimd_f32 vi21x0123 = psimd_load_f32(i21); - i21 += 4; - - const psimd_f32 vk21x0123 = psimd_load_f32(w + 176); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi21x0123, vk21x0123); - - const psimd_f32 vi22x0123 = psimd_load_f32(i22); - i22 += 4; - - const psimd_f32 vk22x0123 = psimd_load_f32(w + 184); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123); - - const psimd_f32 vi23x0123 = psimd_load_f32(i23); - i23 += 4; - - const psimd_f32 vk23x0123 = psimd_load_f32(w + 192); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi23x0123, vk23x0123); - - const psimd_f32 vi24x0123 = psimd_load_f32(i24); - i24 += 4; - - const psimd_f32 vk24x0123 = psimd_load_f32(w + 200); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123); - - w += 4; - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - const psimd_f32 vk4x0123 = psimd_load_f32(w + 40); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - const psimd_f32 vk5x0123 = psimd_load_f32(w + 48); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - const psimd_f32 vk6x0123 = psimd_load_f32(w + 56); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - const psimd_f32 vk7x0123 = psimd_load_f32(w + 64); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - const psimd_f32 vk8x0123 = psimd_load_f32(w + 72); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - const psimd_f32 vi9x0123 = psimd_load_f32(i9); - const psimd_f32 vk9x0123 = psimd_load_f32(w + 80); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi9x0123, vk9x0123); - - const psimd_f32 vi10x0123 = psimd_load_f32(i10); - const psimd_f32 vk10x0123 = psimd_load_f32(w + 88); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123); - - const psimd_f32 vi11x0123 = psimd_load_f32(i11); - const psimd_f32 vk11x0123 = psimd_load_f32(w + 96); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi11x0123, vk11x0123); - - const psimd_f32 vi12x0123 = psimd_load_f32(i12); - const psimd_f32 vk12x0123 = psimd_load_f32(w + 104); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123); - - const psimd_f32 vi13x0123 = psimd_load_f32(i13); - const psimd_f32 vk13x0123 = psimd_load_f32(w + 112); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi13x0123, vk13x0123); - - const psimd_f32 vi14x0123 = psimd_load_f32(i14); - const psimd_f32 vk14x0123 = psimd_load_f32(w + 120); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123); - - const psimd_f32 vi15x0123 = psimd_load_f32(i15); - const psimd_f32 vk15x0123 = psimd_load_f32(w + 128); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi15x0123, vk15x0123); - - const psimd_f32 vi16x0123 = psimd_load_f32(i16); - const psimd_f32 vk16x0123 = psimd_load_f32(w + 136); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123); - - const psimd_f32 vi17x0123 = psimd_load_f32(i17); - const psimd_f32 vk17x0123 = psimd_load_f32(w + 144); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi17x0123, vk17x0123); - - const psimd_f32 vi18x0123 = psimd_load_f32(i18); - const psimd_f32 vk18x0123 = psimd_load_f32(w + 152); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123); - - const psimd_f32 vi19x0123 = psimd_load_f32(i19); - const psimd_f32 vk19x0123 = psimd_load_f32(w + 160); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi19x0123, vk19x0123); - - const psimd_f32 vi20x0123 = psimd_load_f32(i20); - const psimd_f32 vk20x0123 = psimd_load_f32(w + 168); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123); - - const psimd_f32 vi21x0123 = psimd_load_f32(i21); - const psimd_f32 vk21x0123 = psimd_load_f32(w + 176); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi21x0123, vk21x0123); - - const psimd_f32 vi22x0123 = psimd_load_f32(i22); - const psimd_f32 vk22x0123 = psimd_load_f32(w + 184); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123); - - const psimd_f32 vi23x0123 = psimd_load_f32(i23); - const psimd_f32 vk23x0123 = psimd_load_f32(w + 192); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi23x0123, vk23x0123); - - const psimd_f32 vi24x0123 = psimd_load_f32(i24); - const psimd_f32 vk24x0123 = psimd_load_f32(w + 200); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123); - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c b/src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c deleted file mode 100644 index 03faf9037..000000000 --- a/src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c +++ /dev/null @@ -1,191 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/f32-dwconv/up-psimd.c.in -// Generator: tools/xngen -// -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - const float* i0 = input[0]; - assert(i0 != NULL); - if XNN_UNPREDICTABLE(i0 != zero) { - i0 = (const float*) ((uintptr_t) i0 + input_offset); - } - const float* i1 = input[1]; - assert(i1 != NULL); - if XNN_UNPREDICTABLE(i1 != zero) { - i1 = (const float*) ((uintptr_t) i1 + input_offset); - } - const float* i2 = input[2]; - assert(i2 != NULL); - if XNN_UNPREDICTABLE(i2 != zero) { - i2 = (const float*) ((uintptr_t) i2 + input_offset); - } - const float* i3 = input[3]; - assert(i3 != NULL); - if XNN_UNPREDICTABLE(i3 != zero) { - i3 = (const float*) ((uintptr_t) i3 + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= 8; c -= 8) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - psimd_f32 vacc4567p0 = psimd_load_f32(w + 4); - - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vi0x4567 = psimd_load_f32(i0 + 4); - i0 += 8; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - const psimd_f32 vk0x4567 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi0x4567, vk0x4567); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vi1x4567 = psimd_load_f32(i1 + 4); - i1 += 8; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - const psimd_f32 vk1x4567 = psimd_load_f32(w + 20); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - psimd_f32 vacc4567p1 = psimd_mul_f32(vi1x4567, vk1x4567); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vi2x4567 = psimd_load_f32(i2 + 4); - i2 += 8; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - const psimd_f32 vk2x4567 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi2x4567, vk2x4567); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vi3x4567 = psimd_load_f32(i3 + 4); - i3 += 8; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - const psimd_f32 vk3x4567 = psimd_load_f32(w + 36); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi3x4567, vk3x4567); - - w += 40; - - // Add up all accumulators to vacc01234567p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - vacc4567p0 = psimd_add_f32(vacc4567p0, vacc4567p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - psimd_f32 vacc4567 = psimd_max_f32(vacc4567p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - vacc4567 = psimd_min_f32(vacc4567, vmax); - - psimd_store_f32(output, vacc0123); - psimd_store_f32(output + 4, vacc4567); - output += 8; - } - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - i0 += 4; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - i1 += 4; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - i2 += 4; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - i3 += 4; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - - w += 4; - - // Add up all accumulators to vacc01234567p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - - // Add up all accumulators to vacc01234567p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/f32-dwconv/gen/up8x4-minmax-psimd.c b/src/f32-dwconv/gen/up8x4-minmax-psimd.c deleted file mode 100644 index 99c2f7468..000000000 --- a/src/f32-dwconv/gen/up8x4-minmax-psimd.c +++ /dev/null @@ -1,184 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/f32-dwconv/up-psimd.c.in -// Generator: tools/xngen -// -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up8x4__psimd( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - const float* i0 = input[0]; - assert(i0 != NULL); - if XNN_UNPREDICTABLE(i0 != zero) { - i0 = (const float*) ((uintptr_t) i0 + input_offset); - } - const float* i1 = input[1]; - assert(i1 != NULL); - if XNN_UNPREDICTABLE(i1 != zero) { - i1 = (const float*) ((uintptr_t) i1 + input_offset); - } - const float* i2 = input[2]; - assert(i2 != NULL); - if XNN_UNPREDICTABLE(i2 != zero) { - i2 = (const float*) ((uintptr_t) i2 + input_offset); - } - const float* i3 = input[3]; - assert(i3 != NULL); - if XNN_UNPREDICTABLE(i3 != zero) { - i3 = (const float*) ((uintptr_t) i3 + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= 8; c -= 8) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - psimd_f32 vacc4567p0 = psimd_load_f32(w + 4); - - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vi0x4567 = psimd_load_f32(i0 + 4); - i0 += 8; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - const psimd_f32 vk0x4567 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi0x4567, vk0x4567); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vi1x4567 = psimd_load_f32(i1 + 4); - i1 += 8; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - const psimd_f32 vk1x4567 = psimd_load_f32(w + 20); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi1x4567, vk1x4567); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vi2x4567 = psimd_load_f32(i2 + 4); - i2 += 8; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - const psimd_f32 vk2x4567 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi2x4567, vk2x4567); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vi3x4567 = psimd_load_f32(i3 + 4); - i3 += 8; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - const psimd_f32 vk3x4567 = psimd_load_f32(w + 36); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi3x4567, vk3x4567); - - w += 40; - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - psimd_f32 vacc4567 = psimd_max_f32(vacc4567p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - vacc4567 = psimd_min_f32(vacc4567, vmax); - - psimd_store_f32(output, vacc0123); - psimd_store_f32(output + 4, vacc4567); - output += 8; - } - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - i0 += 4; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - i1 += 4; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - i2 += 4; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - i3 += 4; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - - w += 4; - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c b/src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c deleted file mode 100644 index cb5ee907e..000000000 --- a/src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c +++ /dev/null @@ -1,311 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/f32-dwconv/up-psimd.c.in -// Generator: tools/xngen -// -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - const float* i0 = input[0]; - assert(i0 != NULL); - if XNN_UNPREDICTABLE(i0 != zero) { - i0 = (const float*) ((uintptr_t) i0 + input_offset); - } - const float* i1 = input[1]; - assert(i1 != NULL); - if XNN_UNPREDICTABLE(i1 != zero) { - i1 = (const float*) ((uintptr_t) i1 + input_offset); - } - const float* i2 = input[2]; - assert(i2 != NULL); - if XNN_UNPREDICTABLE(i2 != zero) { - i2 = (const float*) ((uintptr_t) i2 + input_offset); - } - const float* i3 = input[3]; - assert(i3 != NULL); - if XNN_UNPREDICTABLE(i3 != zero) { - i3 = (const float*) ((uintptr_t) i3 + input_offset); - } - const float* i4 = input[4]; - assert(i4 != NULL); - if XNN_UNPREDICTABLE(i4 != zero) { - i4 = (const float*) ((uintptr_t) i4 + input_offset); - } - const float* i5 = input[5]; - assert(i5 != NULL); - if XNN_UNPREDICTABLE(i5 != zero) { - i5 = (const float*) ((uintptr_t) i5 + input_offset); - } - const float* i6 = input[6]; - assert(i6 != NULL); - if XNN_UNPREDICTABLE(i6 != zero) { - i6 = (const float*) ((uintptr_t) i6 + input_offset); - } - const float* i7 = input[7]; - assert(i7 != NULL); - if XNN_UNPREDICTABLE(i7 != zero) { - i7 = (const float*) ((uintptr_t) i7 + input_offset); - } - const float* i8 = input[8]; - assert(i8 != NULL); - if XNN_UNPREDICTABLE(i8 != zero) { - i8 = (const float*) ((uintptr_t) i8 + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= 8; c -= 8) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - psimd_f32 vacc4567p0 = psimd_load_f32(w + 4); - - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vi0x4567 = psimd_load_f32(i0 + 4); - i0 += 8; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - const psimd_f32 vk0x4567 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi0x4567, vk0x4567); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vi1x4567 = psimd_load_f32(i1 + 4); - i1 += 8; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - const psimd_f32 vk1x4567 = psimd_load_f32(w + 20); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - psimd_f32 vacc4567p1 = psimd_mul_f32(vi1x4567, vk1x4567); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vi2x4567 = psimd_load_f32(i2 + 4); - i2 += 8; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - const psimd_f32 vk2x4567 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi2x4567, vk2x4567); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vi3x4567 = psimd_load_f32(i3 + 4); - i3 += 8; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - const psimd_f32 vk3x4567 = psimd_load_f32(w + 36); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi3x4567, vk3x4567); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - const psimd_f32 vi4x4567 = psimd_load_f32(i4 + 4); - i4 += 8; - - const psimd_f32 vk4x0123 = psimd_load_f32(w + 40); - const psimd_f32 vk4x4567 = psimd_load_f32(w + 44); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi4x4567, vk4x4567); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - const psimd_f32 vi5x4567 = psimd_load_f32(i5 + 4); - i5 += 8; - - const psimd_f32 vk5x0123 = psimd_load_f32(w + 48); - const psimd_f32 vk5x4567 = psimd_load_f32(w + 52); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi5x4567, vk5x4567); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - const psimd_f32 vi6x4567 = psimd_load_f32(i6 + 4); - i6 += 8; - - const psimd_f32 vk6x0123 = psimd_load_f32(w + 56); - const psimd_f32 vk6x4567 = psimd_load_f32(w + 60); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi6x4567, vk6x4567); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - const psimd_f32 vi7x4567 = psimd_load_f32(i7 + 4); - i7 += 8; - - const psimd_f32 vk7x0123 = psimd_load_f32(w + 64); - const psimd_f32 vk7x4567 = psimd_load_f32(w + 68); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123); - vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi7x4567, vk7x4567); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - const psimd_f32 vi8x4567 = psimd_load_f32(i8 + 4); - i8 += 8; - - const psimd_f32 vk8x0123 = psimd_load_f32(w + 72); - const psimd_f32 vk8x4567 = psimd_load_f32(w + 76); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi8x4567, vk8x4567); - - w += 80; - - // Add up all accumulators to vacc01234567p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - vacc4567p0 = psimd_add_f32(vacc4567p0, vacc4567p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - psimd_f32 vacc4567 = psimd_max_f32(vacc4567p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - vacc4567 = psimd_min_f32(vacc4567, vmax); - - psimd_store_f32(output, vacc0123); - psimd_store_f32(output + 4, vacc4567); - output += 8; - } - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - i0 += 4; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - i1 += 4; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - i2 += 4; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - i3 += 4; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - i4 += 4; - - const psimd_f32 vk4x0123 = psimd_load_f32(w + 40); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - i5 += 4; - - const psimd_f32 vk5x0123 = psimd_load_f32(w + 48); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - i6 += 4; - - const psimd_f32 vk6x0123 = psimd_load_f32(w + 56); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - i7 += 4; - - const psimd_f32 vk7x0123 = psimd_load_f32(w + 64); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - i8 += 4; - - const psimd_f32 vk8x0123 = psimd_load_f32(w + 72); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - w += 4; - - // Add up all accumulators to vacc01234567p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - const psimd_f32 vk4x0123 = psimd_load_f32(w + 40); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - const psimd_f32 vk5x0123 = psimd_load_f32(w + 48); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - const psimd_f32 vk6x0123 = psimd_load_f32(w + 56); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - const psimd_f32 vk7x0123 = psimd_load_f32(w + 64); - vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - const psimd_f32 vk8x0123 = psimd_load_f32(w + 72); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - // Add up all accumulators to vacc01234567p0 - vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1); - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/f32-dwconv/gen/up8x9-minmax-psimd.c b/src/f32-dwconv/gen/up8x9-minmax-psimd.c deleted file mode 100644 index 13163a721..000000000 --- a/src/f32-dwconv/gen/up8x9-minmax-psimd.c +++ /dev/null @@ -1,304 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/f32-dwconv/up-psimd.c.in -// Generator: tools/xngen -// -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up8x9__psimd( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - const float* i0 = input[0]; - assert(i0 != NULL); - if XNN_UNPREDICTABLE(i0 != zero) { - i0 = (const float*) ((uintptr_t) i0 + input_offset); - } - const float* i1 = input[1]; - assert(i1 != NULL); - if XNN_UNPREDICTABLE(i1 != zero) { - i1 = (const float*) ((uintptr_t) i1 + input_offset); - } - const float* i2 = input[2]; - assert(i2 != NULL); - if XNN_UNPREDICTABLE(i2 != zero) { - i2 = (const float*) ((uintptr_t) i2 + input_offset); - } - const float* i3 = input[3]; - assert(i3 != NULL); - if XNN_UNPREDICTABLE(i3 != zero) { - i3 = (const float*) ((uintptr_t) i3 + input_offset); - } - const float* i4 = input[4]; - assert(i4 != NULL); - if XNN_UNPREDICTABLE(i4 != zero) { - i4 = (const float*) ((uintptr_t) i4 + input_offset); - } - const float* i5 = input[5]; - assert(i5 != NULL); - if XNN_UNPREDICTABLE(i5 != zero) { - i5 = (const float*) ((uintptr_t) i5 + input_offset); - } - const float* i6 = input[6]; - assert(i6 != NULL); - if XNN_UNPREDICTABLE(i6 != zero) { - i6 = (const float*) ((uintptr_t) i6 + input_offset); - } - const float* i7 = input[7]; - assert(i7 != NULL); - if XNN_UNPREDICTABLE(i7 != zero) { - i7 = (const float*) ((uintptr_t) i7 + input_offset); - } - const float* i8 = input[8]; - assert(i8 != NULL); - if XNN_UNPREDICTABLE(i8 != zero) { - i8 = (const float*) ((uintptr_t) i8 + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= 8; c -= 8) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - psimd_f32 vacc4567p0 = psimd_load_f32(w + 4); - - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vi0x4567 = psimd_load_f32(i0 + 4); - i0 += 8; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - const psimd_f32 vk0x4567 = psimd_load_f32(w + 12); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi0x4567, vk0x4567); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vi1x4567 = psimd_load_f32(i1 + 4); - i1 += 8; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - const psimd_f32 vk1x4567 = psimd_load_f32(w + 20); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi1x4567, vk1x4567); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vi2x4567 = psimd_load_f32(i2 + 4); - i2 += 8; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - const psimd_f32 vk2x4567 = psimd_load_f32(w + 28); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi2x4567, vk2x4567); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vi3x4567 = psimd_load_f32(i3 + 4); - i3 += 8; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - const psimd_f32 vk3x4567 = psimd_load_f32(w + 36); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi3x4567, vk3x4567); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - const psimd_f32 vi4x4567 = psimd_load_f32(i4 + 4); - i4 += 8; - - const psimd_f32 vk4x0123 = psimd_load_f32(w + 40); - const psimd_f32 vk4x4567 = psimd_load_f32(w + 44); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi4x4567, vk4x4567); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - const psimd_f32 vi5x4567 = psimd_load_f32(i5 + 4); - i5 += 8; - - const psimd_f32 vk5x0123 = psimd_load_f32(w + 48); - const psimd_f32 vk5x4567 = psimd_load_f32(w + 52); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi5x4567, vk5x4567); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - const psimd_f32 vi6x4567 = psimd_load_f32(i6 + 4); - i6 += 8; - - const psimd_f32 vk6x0123 = psimd_load_f32(w + 56); - const psimd_f32 vk6x4567 = psimd_load_f32(w + 60); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi6x4567, vk6x4567); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - const psimd_f32 vi7x4567 = psimd_load_f32(i7 + 4); - i7 += 8; - - const psimd_f32 vk7x0123 = psimd_load_f32(w + 64); - const psimd_f32 vk7x4567 = psimd_load_f32(w + 68); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi7x4567, vk7x4567); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - const psimd_f32 vi8x4567 = psimd_load_f32(i8 + 4); - i8 += 8; - - const psimd_f32 vk8x0123 = psimd_load_f32(w + 72); - const psimd_f32 vk8x4567 = psimd_load_f32(w + 76); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi8x4567, vk8x4567); - - w += 80; - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - psimd_f32 vacc4567 = psimd_max_f32(vacc4567p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - vacc4567 = psimd_min_f32(vacc4567, vmax); - - psimd_store_f32(output, vacc0123); - psimd_store_f32(output + 4, vacc4567); - output += 8; - } - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - i0 += 4; - - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - i1 += 4; - - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - i2 += 4; - - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - i3 += 4; - - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - i4 += 4; - - const psimd_f32 vk4x0123 = psimd_load_f32(w + 40); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - i5 += 4; - - const psimd_f32 vk5x0123 = psimd_load_f32(w + 48); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - i6 += 4; - - const psimd_f32 vk6x0123 = psimd_load_f32(w + 56); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - i7 += 4; - - const psimd_f32 vk7x0123 = psimd_load_f32(w + 64); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - i8 += 4; - - const psimd_f32 vk8x0123 = psimd_load_f32(w + 72); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - w += 4; - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - - const psimd_f32 vi0x0123 = psimd_load_f32(i0); - const psimd_f32 vk0x0123 = psimd_load_f32(w + 8); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123); - - const psimd_f32 vi1x0123 = psimd_load_f32(i1); - const psimd_f32 vk1x0123 = psimd_load_f32(w + 16); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123); - - const psimd_f32 vi2x0123 = psimd_load_f32(i2); - const psimd_f32 vk2x0123 = psimd_load_f32(w + 24); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123); - - const psimd_f32 vi3x0123 = psimd_load_f32(i3); - const psimd_f32 vk3x0123 = psimd_load_f32(w + 32); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123); - - const psimd_f32 vi4x0123 = psimd_load_f32(i4); - const psimd_f32 vk4x0123 = psimd_load_f32(w + 40); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123); - - const psimd_f32 vi5x0123 = psimd_load_f32(i5); - const psimd_f32 vk5x0123 = psimd_load_f32(w + 48); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123); - - const psimd_f32 vi6x0123 = psimd_load_f32(i6); - const psimd_f32 vk6x0123 = psimd_load_f32(w + 56); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123); - - const psimd_f32 vi7x0123 = psimd_load_f32(i7); - const psimd_f32 vk7x0123 = psimd_load_f32(w + 64); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123); - - const psimd_f32 vi8x0123 = psimd_load_f32(i8); - const psimd_f32 vk8x0123 = psimd_load_f32(w + 72); - vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123); - - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/f32-dwconv/up-psimd.c.in b/src/f32-dwconv/up-psimd.c.in deleted file mode 100644 index 0b7cee90c..000000000 --- a/src/f32-dwconv/up-psimd.c.in +++ /dev/null @@ -1,154 +0,0 @@ -// Copyright 2019 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -$assert CHANNEL_TILE % 4 == 0 -$assert KERNEL_TILE >= 2 -$assert ACCUMULATORS >= 1 -$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" -#include <assert.h> - -#include <psimd.h> - -#include <xnnpack/dwconv.h> - - -void xnn_f32_dwconv_minmax_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__psimd${"" if ACCUMULATORS == 1 else "_acc%d" % ACCUMULATORS}( - size_t channels, - size_t output_width, - const float** input, - const float* weights, - float* output, - size_t input_stride, - size_t output_increment, - size_t input_offset, - const float* zero, - const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(channels != 0); - assert(output_width != 0); - - const psimd_f32 vmax = psimd_load_splat_f32(¶ms->scalar.max); - const psimd_f32 vmin = psimd_load_splat_f32(¶ms->scalar.min); - do { - $for K in range(KERNEL_TILE): - const float* i${K} = input[${K}]; - assert(i${K} != NULL); - if XNN_UNPREDICTABLE(i${K} != zero) { - i${K} = (const float*) ((uintptr_t) i${K} + input_offset); - } - input = (const float**) ((uintptr_t) input + input_stride); - - size_t c = channels; - const float* w = weights; - for (; c >= ${CHANNEL_TILE}; c -= ${CHANNEL_TILE}) { - psimd_f32 vacc${ABC[0:4]}p0 = psimd_load_f32(w); - $for C in range(4, CHANNEL_TILE, 4): - psimd_f32 vacc${ABC[C:C+4]}p0 = psimd_load_f32(w + ${C}); - - $for K in range(KERNEL_TILE): - - const psimd_f32 vi${K}x${ABC[0:4]} = psimd_load_f32(i${K}); - $for C in range(4, CHANNEL_TILE, 4): - const psimd_f32 vi${K}x${ABC[C:C+4]} = psimd_load_f32(i${K} + ${C}); - i${K} += ${CHANNEL_TILE}; - - $for C in range(0, CHANNEL_TILE, 4): - const psimd_f32 vk${K}x${ABC[C:C+4]} = psimd_load_f32(w + ${(K + 1) * CHANNEL_TILE + C}); - $for C in range(0, CHANNEL_TILE, 4): - $if 1 <= K < ACCUMULATORS: - psimd_f32 vacc${ABC[C:C+4]}p${K} = psimd_mul_f32(vi${K}x${ABC[C:C+4]}, vk${K}x${ABC[C:C+4]}); - $else: - vacc${ABC[C:C+4]}p${K % ACCUMULATORS} = psimd_qfma_f32(vacc${ABC[C:C+4]}p${K % ACCUMULATORS}, vi${K}x${ABC[C:C+4]}, vk${K}x${ABC[C:C+4]}); - - w += ${(KERNEL_TILE + 1) * CHANNEL_TILE}; - - $if ACCUMULATORS > 1: - // Add up all accumulators to vacc${ABC[0:CHANNEL_TILE]}p0 - $ACC_SLICE = 1 - $while ACC_SLICE < ACCUMULATORS: - $for A in range(0, ACCUMULATORS, ACC_SLICE * 2): - $if A + ACC_SLICE < ACCUMULATORS: - $for C in range(0, CHANNEL_TILE, 4): - vacc${ABC[C:C+4]}p${A} = psimd_add_f32(vacc${ABC[C:C+4]}p${A}, vacc${ABC[C:C+4]}p${A + ACC_SLICE}); - $ACC_SLICE *= 2 - - $for C in range(0, CHANNEL_TILE, 4): - psimd_f32 vacc${ABC[C:C+4]} = psimd_max_f32(vacc${ABC[C:C+4]}p0, vmin); - $for C in range(0, CHANNEL_TILE, 4): - vacc${ABC[C:C+4]} = psimd_min_f32(vacc${ABC[C:C+4]}, vmax); - - psimd_store_f32(output, vacc${ABC[0:4]}); - $for C in range(4, CHANNEL_TILE, 4): - psimd_store_f32(output + ${C}, vacc${ABC[C:C+4]}); - output += ${CHANNEL_TILE}; - } - $if CHANNEL_TILE > 4: - for (; c >= 4; c -= 4) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - $for K in range(KERNEL_TILE): - - const psimd_f32 vi${K}x0123 = psimd_load_f32(i${K}); - i${K} += 4; - - const psimd_f32 vk${K}x0123 = psimd_load_f32(w + ${(K + 1) * CHANNEL_TILE}); - $if 1 <= K < ACCUMULATORS: - psimd_f32 vacc0123p${K} = psimd_mul_f32(vi${K}x0123, vk${K}x0123); - $else: - vacc0123p${K % ACCUMULATORS} = psimd_qfma_f32(vacc0123p${K % ACCUMULATORS}, vi${K}x0123, vk${K}x0123); - - w += 4; - - $if ACCUMULATORS > 1: - // Add up all accumulators to vacc${ABC[0:CHANNEL_TILE]}p0 - $ACC_SLICE = 1 - $while ACC_SLICE < ACCUMULATORS: - $for A in range(0, ACCUMULATORS, ACC_SLICE * 2): - $if A + ACC_SLICE < ACCUMULATORS: - vacc0123p${A} = psimd_add_f32(vacc0123p${A}, vacc0123p${A + ACC_SLICE}); - $ACC_SLICE *= 2 - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - psimd_store_f32(output, vacc0123); - output += 4; - } - if XNN_UNLIKELY(c != 0) { - psimd_f32 vacc0123p0 = psimd_load_f32(w); - $for K in range(KERNEL_TILE): - - const psimd_f32 vi${K}x0123 = psimd_load_f32(i${K}); - const psimd_f32 vk${K}x0123 = psimd_load_f32(w + ${(K+1) * CHANNEL_TILE}); - $if 1 <= K < ACCUMULATORS: - psimd_f32 vacc0123p${K} = psimd_mul_f32(vi${K}x0123, vk${K}x0123); - $else: - vacc0123p${K % ACCUMULATORS} = psimd_qfma_f32(vacc0123p${K % ACCUMULATORS}, vi${K}x0123, vk${K}x0123); - - $if ACCUMULATORS > 1: - // Add up all accumulators to vacc${ABC[0:CHANNEL_TILE]}p0 - $ACC_SLICE = 1 - $while ACC_SLICE < ACCUMULATORS: - $for A in range(0, ACCUMULATORS, ACC_SLICE * 2): - $if A + ACC_SLICE < ACCUMULATORS: - vacc0123p${A} = psimd_add_f32(vacc0123p${A}, vacc0123p${A + ACC_SLICE}); - $ACC_SLICE *= 2 - - psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin); - vacc0123 = psimd_min_f32(vacc0123, vmax); - - if (c & 2) { - psimd_store2_f32(output, vacc0123); - vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123); - output += 2; - } - if (c & 1) { - psimd_store1_f32(output, vacc0123); - output += 1; - } - } - - output = (float*) ((uintptr_t) output + output_increment); - } while (--output_width != 0); -} diff --git a/src/xnnpack/dwconv.h b/src/xnnpack/dwconv.h index 0a2df8238..46d86097d 100644 --- a/src/xnnpack/dwconv.h +++ b/src/xnnpack/dwconv.h @@ -52,10 +52,6 @@ DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__neonfma_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__neonfma) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__neonfma_acc2) -DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd) -DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2) -DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd) -DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__sse) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__sse_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__sse) @@ -83,10 +79,6 @@ DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__neonfma_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__aarch64_neonfma) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__aarch64_neonfma_cortex_a55) -DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd) -DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2) -DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd) -DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__sse) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__sse_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__sse) @@ -112,10 +104,6 @@ DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__neonfma_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__neonfma) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__neonfma_acc2) -DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd) -DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2) -DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd) -DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__sse) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__sse_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__sse) diff --git a/test/f32-dwconv-minmax.cc b/test/f32-dwconv-minmax.cc index cb2418a93..d312c59e5 100644 --- a/test/f32-dwconv-minmax.cc +++ b/test/f32-dwconv-minmax.cc @@ -13793,2238 +13793,6 @@ #endif // XNN_ARCH_X86 || XNN_ARCH_X86_64 -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_eq_4) { - TEST_REQUIRES_PSIMD; - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(4) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_div_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_div_4_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_div_4_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_lt_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 1; channels < 4; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_gt_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_gt_4_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_gt_4_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, multipixel) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .width(3) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, multipixel_with_step) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - for (size_t step = 2; step <= 25; step++) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .width(3) - .step(step) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, multipixel_with_output_stride) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(4) - .width(5) - .output_stride(23) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, multipixel_with_qmin) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .width(3) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, multipixel_with_qmax) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .width(3) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, input_offset) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .input_offset(112) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, zero) { - TEST_REQUIRES_PSIMD; - for (uint32_t mz = 0; mz < 25; mz++) { - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .input_offset(112) - .zero_index(mz) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - } -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_eq_4) { - TEST_REQUIRES_PSIMD; - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(4) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_div_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_div_4_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_div_4_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_lt_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 1; channels < 4; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_gt_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_gt_4_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_gt_4_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, multipixel) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .width(3) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, multipixel_with_step) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - for (size_t step = 2; step <= 25; step++) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .width(3) - .step(step) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, multipixel_with_output_stride) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(4) - .width(5) - .output_stride(23) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, multipixel_with_qmin) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .width(3) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, multipixel_with_qmax) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .width(3) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, input_offset) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .input_offset(112) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, zero) { - TEST_REQUIRES_PSIMD; - for (uint32_t mz = 0; mz < 25; mz++) { - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(25) - .channels(channels) - .input_offset(112) - .zero_index(mz) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - } -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_eq_8) { - TEST_REQUIRES_PSIMD; - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(8) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_div_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_div_8_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_div_8_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_lt_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 1; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_gt_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_gt_8_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_gt_8_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, multipixel) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .width(3) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, multipixel_with_step) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - for (size_t step = 2; step <= 25; step++) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .width(3) - .step(step) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, multipixel_with_output_stride) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(8) - .width(5) - .output_stride(43) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, multipixel_with_qmin) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .width(3) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, multipixel_with_qmax) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .width(3) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, input_offset) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .input_offset(176) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, zero) { - TEST_REQUIRES_PSIMD; - for (uint32_t mz = 0; mz < 25; mz++) { - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .input_offset(176) - .zero_index(mz) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - } -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_eq_8) { - TEST_REQUIRES_PSIMD; - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(8) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_div_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_div_8_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_div_8_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_lt_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 1; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_gt_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_gt_8_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_gt_8_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, multipixel) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .width(3) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, multipixel_with_step) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - for (size_t step = 2; step <= 25; step++) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .width(3) - .step(step) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, multipixel_with_output_stride) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(8) - .width(5) - .output_stride(43) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, multipixel_with_qmin) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .width(3) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, multipixel_with_qmax) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .width(3) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, input_offset) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .input_offset(176) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, zero) { - TEST_REQUIRES_PSIMD; - for (uint32_t mz = 0; mz < 25; mz++) { - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(25) - .channels(channels) - .input_offset(176) - .zero_index(mz) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - } -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_eq_4) { - TEST_REQUIRES_PSIMD; - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(4) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_div_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_div_4_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_div_4_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_lt_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 1; channels < 4; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_gt_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_gt_4_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_gt_4_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, multipixel) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .width(3) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, multipixel_with_step) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - for (size_t step = 2; step <= 9; step++) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .width(3) - .step(step) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, multipixel_with_output_stride) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(4) - .width(5) - .output_stride(23) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, multipixel_with_qmin) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .width(3) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, multipixel_with_qmax) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .width(3) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, input_offset) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .input_offset(112) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, zero) { - TEST_REQUIRES_PSIMD; - for (uint32_t mz = 0; mz < 9; mz++) { - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .input_offset(112) - .zero_index(mz) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - } -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_eq_4) { - TEST_REQUIRES_PSIMD; - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(4) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_div_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_div_4_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_div_4_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_lt_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 1; channels < 4; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_gt_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_gt_4_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_gt_4_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, multipixel) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .width(3) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, multipixel_with_step) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - for (size_t step = 2; step <= 9; step++) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .width(3) - .step(step) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, multipixel_with_output_stride) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(4) - .width(5) - .output_stride(23) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, multipixel_with_qmin) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .width(3) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, multipixel_with_qmax) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .width(3) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, input_offset) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .input_offset(112) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, zero) { - TEST_REQUIRES_PSIMD; - for (uint32_t mz = 0; mz < 9; mz++) { - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(9) - .channels(channels) - .input_offset(112) - .zero_index(mz) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - } -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_eq_8) { - TEST_REQUIRES_PSIMD; - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(8) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_div_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_div_8_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_div_8_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_lt_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 1; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_gt_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_gt_8_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_gt_8_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, multipixel) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .width(3) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, multipixel_with_step) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - for (size_t step = 2; step <= 9; step++) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .width(3) - .step(step) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, multipixel_with_output_stride) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(8) - .width(5) - .output_stride(43) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, multipixel_with_qmin) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .width(3) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, multipixel_with_qmax) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .width(3) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, input_offset) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .input_offset(176) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, zero) { - TEST_REQUIRES_PSIMD; - for (uint32_t mz = 0; mz < 9; mz++) { - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .input_offset(176) - .zero_index(mz) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - } -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_eq_8) { - TEST_REQUIRES_PSIMD; - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(8) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_div_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_div_8_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_div_8_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_lt_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 1; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_gt_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_gt_8_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_gt_8_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, multipixel) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .width(3) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, multipixel_with_step) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - for (size_t step = 2; step <= 9; step++) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .width(3) - .step(step) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, multipixel_with_output_stride) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(8) - .width(5) - .output_stride(43) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, multipixel_with_qmin) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .width(3) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, multipixel_with_qmax) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .width(3) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, input_offset) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .input_offset(176) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, zero) { - TEST_REQUIRES_PSIMD; - for (uint32_t mz = 0; mz < 9; mz++) { - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(9) - .channels(channels) - .input_offset(176) - .zero_index(mz) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - } -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_eq_4) { - TEST_REQUIRES_PSIMD; - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(4) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_div_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_div_4_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_div_4_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_lt_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 1; channels < 4; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_gt_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_gt_4_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_gt_4_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, multipixel) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .width(3) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, multipixel_with_step) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - for (size_t step = 2; step <= 4; step++) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .width(3) - .step(step) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, multipixel_with_output_stride) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(4) - .width(5) - .output_stride(23) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, multipixel_with_qmin) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .width(3) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, multipixel_with_qmax) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .width(3) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, input_offset) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .input_offset(112) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, zero) { - TEST_REQUIRES_PSIMD; - for (uint32_t mz = 0; mz < 4; mz++) { - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .input_offset(112) - .zero_index(mz) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - } -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_eq_4) { - TEST_REQUIRES_PSIMD; - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(4) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_div_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_div_4_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_div_4_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_lt_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 1; channels < 4; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_gt_4) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_gt_4_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_gt_4_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 5; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, multipixel) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .width(3) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, multipixel_with_step) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - for (size_t step = 2; step <= 4; step++) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .width(3) - .step(step) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, multipixel_with_output_stride) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(4) - .width(5) - .output_stride(23) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, multipixel_with_qmin) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .width(3) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, multipixel_with_qmax) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 20; channels += 3) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .width(3) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, input_offset) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .input_offset(112) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, zero) { - TEST_REQUIRES_PSIMD; - for (uint32_t mz = 0; mz < 4; mz++) { - for (uint32_t channels = 8; channels < 64; channels += 12) { - DWConvMicrokernelTester() - .cr(4) - .kr(4) - .channels(channels) - .input_offset(112) - .zero_index(mz) - .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - } -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_eq_8) { - TEST_REQUIRES_PSIMD; - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(8) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_div_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_div_8_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_div_8_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_lt_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 1; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_gt_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_gt_8_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_gt_8_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, multipixel) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .width(3) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, multipixel_with_step) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - for (size_t step = 2; step <= 4; step++) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .width(3) - .step(step) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, multipixel_with_output_stride) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(8) - .width(5) - .output_stride(43) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, multipixel_with_qmin) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .width(3) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, multipixel_with_qmax) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .width(3) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, input_offset) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .input_offset(176) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, zero) { - TEST_REQUIRES_PSIMD; - for (uint32_t mz = 0; mz < 4; mz++) { - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .input_offset(176) - .zero_index(mz) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar); - } - } - } -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - -#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_eq_8) { - TEST_REQUIRES_PSIMD; - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(8) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_div_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_div_8_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_div_8_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_lt_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 1; channels < 8; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_gt_8) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_gt_8_with_qmin) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_gt_8_with_qmax) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 9; channels < 16; channels++) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, multipixel) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .width(3) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, multipixel_with_step) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - for (size_t step = 2; step <= 4; step++) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .width(3) - .step(step) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, multipixel_with_output_stride) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(8) - .width(5) - .output_stride(43) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, multipixel_with_qmin) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .width(3) - .qmin(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, multipixel_with_qmax) { - TEST_REQUIRES_PSIMD; - for (size_t channels = 1; channels <= 40; channels += 7) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .width(3) - .qmax(128) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, input_offset) { - TEST_REQUIRES_PSIMD; - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .input_offset(176) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - - TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, zero) { - TEST_REQUIRES_PSIMD; - for (uint32_t mz = 0; mz < 4; mz++) { - for (uint32_t channels = 16; channels < 128; channels += 24) { - DWConvMicrokernelTester() - .cr(8) - .kr(4) - .channels(channels) - .input_offset(176) - .zero_index(mz) - .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar); - } - } - } -#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC - - #if XNN_ARCH_WASMSIMD TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, c_eq_4) { DWConvMicrokernelTester() diff --git a/test/f32-dwconv-minmax.yaml b/test/f32-dwconv-minmax.yaml index 2477eccb2..fdaccc7eb 100644 --- a/test/f32-dwconv-minmax.yaml +++ b/test/f32-dwconv-minmax.yaml @@ -79,18 +79,6 @@ - name: xnn_f32_dwconv_minmax_ukernel_up16x4__avx512f_acc2 - name: xnn_f32_dwconv_minmax_ukernel_up32x4__avx512f - name: xnn_f32_dwconv_minmax_ukernel_up32x4__avx512f_acc2 -- name: xnn_f32_dwconv_minmax_ukernel_up4x25__psimd -- name: xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2 -- name: xnn_f32_dwconv_minmax_ukernel_up8x25__psimd -- name: xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2 -- name: xnn_f32_dwconv_minmax_ukernel_up4x9__psimd -- name: xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2 -- name: xnn_f32_dwconv_minmax_ukernel_up8x9__psimd -- name: xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2 -- name: xnn_f32_dwconv_minmax_ukernel_up4x4__psimd -- name: xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2 -- name: xnn_f32_dwconv_minmax_ukernel_up8x4__psimd -- name: xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2 - name: xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm - name: xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm - name: xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm |