aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMarat Dukhan <maratek@google.com>2020-07-16 23:50:10 -0700
committerXNNPACK Team <xnnpack-github-robot@google.com>2020-07-16 23:50:49 -0700
commitbbf7f3f9be24b707392805c490a9b2dc884e9bb4 (patch)
tree5dba42cddc57ef8606910e76f78ff5bf1b564365
parent115d3e2f98d279e7c2f05aadfdf10f69a2d2875a (diff)
downloadXNNPACK-bbf7f3f9be24b707392805c490a9b2dc884e9bb4.tar.gz
Remove PSIMD variants of DWCONV microkernels
PiperOrigin-RevId: 321723576
-rw-r--r--BUILD.bazel12
-rwxr-xr-xCMakeLists.txt12
-rw-r--r--bench/f32-dwconv-e2e.cc31
-rw-r--r--bench/f32-dwconv.cc19
-rwxr-xr-xscripts/generate-f32-dwconv.sh16
-rw-r--r--src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c451
-rw-r--r--src/f32-dwconv/gen/up4x25-minmax-psimd.c447
-rw-r--r--src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c136
-rw-r--r--src/f32-dwconv/gen/up4x4-minmax-psimd.c132
-rw-r--r--src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c211
-rw-r--r--src/f32-dwconv/gen/up4x9-minmax-psimd.c207
-rw-r--r--src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c695
-rw-r--r--src/f32-dwconv/gen/up8x25-minmax-psimd.c688
-rw-r--r--src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c191
-rw-r--r--src/f32-dwconv/gen/up8x4-minmax-psimd.c184
-rw-r--r--src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c311
-rw-r--r--src/f32-dwconv/gen/up8x9-minmax-psimd.c304
-rw-r--r--src/f32-dwconv/up-psimd.c.in154
-rw-r--r--src/xnnpack/dwconv.h12
-rw-r--r--test/f32-dwconv-minmax.cc2232
-rw-r--r--test/f32-dwconv-minmax.yaml12
21 files changed, 0 insertions, 6457 deletions
diff --git a/BUILD.bazel b/BUILD.bazel
index 9956a8ebb..f816af28b 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -845,18 +845,6 @@ PSIMD_FASTMATH_UKERNELS = [
"src/f32-dwconv-chw/3x3s2p1-psimd.c",
"src/f32-dwconv-chw/5x5p2-psimd.c",
"src/f32-dwconv-chw/5x5s2p2-psimd.c",
- "src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c",
- "src/f32-dwconv/gen/up4x25-minmax-psimd.c",
- "src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c",
- "src/f32-dwconv/gen/up4x4-minmax-psimd.c",
- "src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c",
- "src/f32-dwconv/gen/up4x9-minmax-psimd.c",
- "src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c",
- "src/f32-dwconv/gen/up8x25-minmax-psimd.c",
- "src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c",
- "src/f32-dwconv/gen/up8x4-minmax-psimd.c",
- "src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c",
- "src/f32-dwconv/gen/up8x9-minmax-psimd.c",
"src/f32-gavgpool-cw/psimd-x4.c",
"src/f32-gavgpool/7p7x-minmax-psimd-c4.c",
"src/f32-gavgpool/7x-minmax-psimd-c4.c",
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c6899cd4b..3779507fe 100755
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -525,18 +525,6 @@ SET(XNNPACK_PSIMD_FASTMATH_MICROKERNEL_SRCS
src/f32-dwconv-chw/3x3s2p1-psimd.c
src/f32-dwconv-chw/5x5p2-psimd.c
src/f32-dwconv-chw/5x5s2p2-psimd.c
- src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c
- src/f32-dwconv/gen/up4x25-minmax-psimd.c
- src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c
- src/f32-dwconv/gen/up4x4-minmax-psimd.c
- src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c
- src/f32-dwconv/gen/up4x9-minmax-psimd.c
- src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c
- src/f32-dwconv/gen/up8x25-minmax-psimd.c
- src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c
- src/f32-dwconv/gen/up8x4-minmax-psimd.c
- src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c
- src/f32-dwconv/gen/up8x9-minmax-psimd.c
src/f32-gavgpool-cw/psimd-x4.c
src/f32-gavgpool/7p7x-minmax-psimd-c4.c
src/f32-gavgpool/7x-minmax-psimd-c4.c
diff --git a/bench/f32-dwconv-e2e.cc b/bench/f32-dwconv-e2e.cc
index 2d1e7d2ab..367f908a1 100644
--- a/bench/f32-dwconv-e2e.cc
+++ b/bench/f32-dwconv-e2e.cc
@@ -310,37 +310,6 @@ static void DWConvEnd2EndBenchmark(
BENCHMARK_FP32_END2END(f32_dwconv_up8x9__wasmsimd_acc2_x86);
#endif // XNN_ARCH_WASMSIMD
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- static void f32_dwconv_up4x9__psimd(benchmark::State& state, models::ExecutionPlanFactory model) {
- DWConvEnd2EndBenchmark(state, model,
- xnn_f32_dwconv_minmax_ukernel_up4x9__psimd,
- 4 /* cr */, 9 /* mr */);
- }
-
- static void f32_dwconv_up4x9__psimd_acc2(benchmark::State& state, models::ExecutionPlanFactory model) {
- DWConvEnd2EndBenchmark(state, model,
- xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2,
- 4 /* cr */, 9 /* mr */);
- }
-
- static void f32_dwconv_up8x9__psimd(benchmark::State& state, models::ExecutionPlanFactory model) {
- DWConvEnd2EndBenchmark(state, model,
- xnn_f32_dwconv_minmax_ukernel_up8x9__psimd,
- 8 /* cr */, 9 /* mr */);
- }
-
- static void f32_dwconv_up8x9__psimd_acc2(benchmark::State& state, models::ExecutionPlanFactory model) {
- DWConvEnd2EndBenchmark(state, model,
- xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2,
- 8 /* cr */, 9 /* mr */);
- }
-
- BENCHMARK_FP32_END2END(f32_dwconv_up4x9__psimd);
- BENCHMARK_FP32_END2END(f32_dwconv_up4x9__psimd_acc2);
- BENCHMARK_FP32_END2END(f32_dwconv_up8x9__psimd);
- BENCHMARK_FP32_END2END(f32_dwconv_up8x9__psimd_acc2);
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
static void f32_dwconv_up1x9__scalar(benchmark::State& state, models::ExecutionPlanFactory model) {
DWConvEnd2EndBenchmark(state, model,
xnn_f32_dwconv_minmax_ukernel_up1x9__scalar,
diff --git a/bench/f32-dwconv.cc b/bench/f32-dwconv.cc
index be22685f1..8121e5202 100644
--- a/bench/f32-dwconv.cc
+++ b/bench/f32-dwconv.cc
@@ -333,25 +333,6 @@ static void DWConvBenchmark(benchmark::State& state,
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- static void f32_dwconv_4x4__psimd(benchmark::State& state, const char* net) {
- DWConvBenchmark(state, xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, 4, 4);
- }
-
- static void f32_dwconv_4x9__psimd(benchmark::State& state, const char* net) {
- DWConvBenchmark(state, xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, 4, 9);
- }
-
- static void f32_dwconv_4x25__psimd(benchmark::State& state, const char* net) {
- DWConvBenchmark(state, xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, 4, 25);
- }
-
- BENCHMARK_DWCONV(f32_dwconv_4x4__psimd)
- BENCHMARK_DWCONV(f32_dwconv_4x9__psimd)
- BENCHMARK_DWCONV(f32_dwconv_4x25__psimd)
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
static void f32_dwconv_1x4__scalar(benchmark::State& state, const char* net) {
DWConvBenchmark(state, xnn_f32_dwconv_minmax_ukernel_up1x4__scalar, 1, 4);
}
diff --git a/scripts/generate-f32-dwconv.sh b/scripts/generate-f32-dwconv.sh
index 7abef21ef..d6b9c9e72 100755
--- a/scripts/generate-f32-dwconv.sh
+++ b/scripts/generate-f32-dwconv.sh
@@ -201,22 +201,6 @@ tools/xngen src/f32-dwconv/up-avx512.c.in -D CHANNEL_TILE=16 -D KERNEL_TILE=25 -
tools/xngen src/f32-dwconv/up-avx512.c.in -D CHANNEL_TILE=32 -D KERNEL_TILE=25 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up32x25-minmax-avx512f.c
tools/xngen src/f32-dwconv/up-avx512.c.in -D CHANNEL_TILE=32 -D KERNEL_TILE=25 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up32x25-minmax-avx512f-acc2.c
-#################################### PSIMD ####################################
-tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=4 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up4x4-minmax-psimd.c
-tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=4 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c
-tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=4 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up8x4-minmax-psimd.c
-tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=4 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c
-
-tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=9 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up4x9-minmax-psimd.c
-tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=9 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c
-tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up8x9-minmax-psimd.c
-tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c
-
-tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=25 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up4x25-minmax-psimd.c
-tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=25 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c
-tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D ACCUMULATORS=1 -o src/f32-dwconv/gen/up8x25-minmax-psimd.c
-tools/xngen src/f32-dwconv/up-psimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D ACCUMULATORS=2 -o src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c
-
################################## Unit tests #################################
tools/generate-dwconv-test.py --spec test/f32-dwconv.yaml --output test/f32-dwconv.cc
tools/generate-dwconv-test.py --spec test/f32-dwconv-minmax.yaml --output test/f32-dwconv-minmax.cc
diff --git a/src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c b/src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c
deleted file mode 100644
index 236cef2ad..000000000
--- a/src/f32-dwconv/gen/up4x25-minmax-psimd-acc2.c
+++ /dev/null
@@ -1,451 +0,0 @@
-// Auto-generated file. Do not edit!
-// Template: src/f32-dwconv/up-psimd.c.in
-// Generator: tools/xngen
-//
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- const float* i0 = input[0];
- assert(i0 != NULL);
- if XNN_UNPREDICTABLE(i0 != zero) {
- i0 = (const float*) ((uintptr_t) i0 + input_offset);
- }
- const float* i1 = input[1];
- assert(i1 != NULL);
- if XNN_UNPREDICTABLE(i1 != zero) {
- i1 = (const float*) ((uintptr_t) i1 + input_offset);
- }
- const float* i2 = input[2];
- assert(i2 != NULL);
- if XNN_UNPREDICTABLE(i2 != zero) {
- i2 = (const float*) ((uintptr_t) i2 + input_offset);
- }
- const float* i3 = input[3];
- assert(i3 != NULL);
- if XNN_UNPREDICTABLE(i3 != zero) {
- i3 = (const float*) ((uintptr_t) i3 + input_offset);
- }
- const float* i4 = input[4];
- assert(i4 != NULL);
- if XNN_UNPREDICTABLE(i4 != zero) {
- i4 = (const float*) ((uintptr_t) i4 + input_offset);
- }
- const float* i5 = input[5];
- assert(i5 != NULL);
- if XNN_UNPREDICTABLE(i5 != zero) {
- i5 = (const float*) ((uintptr_t) i5 + input_offset);
- }
- const float* i6 = input[6];
- assert(i6 != NULL);
- if XNN_UNPREDICTABLE(i6 != zero) {
- i6 = (const float*) ((uintptr_t) i6 + input_offset);
- }
- const float* i7 = input[7];
- assert(i7 != NULL);
- if XNN_UNPREDICTABLE(i7 != zero) {
- i7 = (const float*) ((uintptr_t) i7 + input_offset);
- }
- const float* i8 = input[8];
- assert(i8 != NULL);
- if XNN_UNPREDICTABLE(i8 != zero) {
- i8 = (const float*) ((uintptr_t) i8 + input_offset);
- }
- const float* i9 = input[9];
- assert(i9 != NULL);
- if XNN_UNPREDICTABLE(i9 != zero) {
- i9 = (const float*) ((uintptr_t) i9 + input_offset);
- }
- const float* i10 = input[10];
- assert(i10 != NULL);
- if XNN_UNPREDICTABLE(i10 != zero) {
- i10 = (const float*) ((uintptr_t) i10 + input_offset);
- }
- const float* i11 = input[11];
- assert(i11 != NULL);
- if XNN_UNPREDICTABLE(i11 != zero) {
- i11 = (const float*) ((uintptr_t) i11 + input_offset);
- }
- const float* i12 = input[12];
- assert(i12 != NULL);
- if XNN_UNPREDICTABLE(i12 != zero) {
- i12 = (const float*) ((uintptr_t) i12 + input_offset);
- }
- const float* i13 = input[13];
- assert(i13 != NULL);
- if XNN_UNPREDICTABLE(i13 != zero) {
- i13 = (const float*) ((uintptr_t) i13 + input_offset);
- }
- const float* i14 = input[14];
- assert(i14 != NULL);
- if XNN_UNPREDICTABLE(i14 != zero) {
- i14 = (const float*) ((uintptr_t) i14 + input_offset);
- }
- const float* i15 = input[15];
- assert(i15 != NULL);
- if XNN_UNPREDICTABLE(i15 != zero) {
- i15 = (const float*) ((uintptr_t) i15 + input_offset);
- }
- const float* i16 = input[16];
- assert(i16 != NULL);
- if XNN_UNPREDICTABLE(i16 != zero) {
- i16 = (const float*) ((uintptr_t) i16 + input_offset);
- }
- const float* i17 = input[17];
- assert(i17 != NULL);
- if XNN_UNPREDICTABLE(i17 != zero) {
- i17 = (const float*) ((uintptr_t) i17 + input_offset);
- }
- const float* i18 = input[18];
- assert(i18 != NULL);
- if XNN_UNPREDICTABLE(i18 != zero) {
- i18 = (const float*) ((uintptr_t) i18 + input_offset);
- }
- const float* i19 = input[19];
- assert(i19 != NULL);
- if XNN_UNPREDICTABLE(i19 != zero) {
- i19 = (const float*) ((uintptr_t) i19 + input_offset);
- }
- const float* i20 = input[20];
- assert(i20 != NULL);
- if XNN_UNPREDICTABLE(i20 != zero) {
- i20 = (const float*) ((uintptr_t) i20 + input_offset);
- }
- const float* i21 = input[21];
- assert(i21 != NULL);
- if XNN_UNPREDICTABLE(i21 != zero) {
- i21 = (const float*) ((uintptr_t) i21 + input_offset);
- }
- const float* i22 = input[22];
- assert(i22 != NULL);
- if XNN_UNPREDICTABLE(i22 != zero) {
- i22 = (const float*) ((uintptr_t) i22 + input_offset);
- }
- const float* i23 = input[23];
- assert(i23 != NULL);
- if XNN_UNPREDICTABLE(i23 != zero) {
- i23 = (const float*) ((uintptr_t) i23 + input_offset);
- }
- const float* i24 = input[24];
- assert(i24 != NULL);
- if XNN_UNPREDICTABLE(i24 != zero) {
- i24 = (const float*) ((uintptr_t) i24 + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- i0 += 4;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- i1 += 4;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- i2 += 4;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- i3 += 4;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- i4 += 4;
-
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 20);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- i5 += 4;
-
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 24);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- i6 += 4;
-
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- i7 += 4;
-
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 32);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- i8 += 4;
-
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 36);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- const psimd_f32 vi9x0123 = psimd_load_f32(i9);
- i9 += 4;
-
- const psimd_f32 vk9x0123 = psimd_load_f32(w + 40);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi9x0123, vk9x0123);
-
- const psimd_f32 vi10x0123 = psimd_load_f32(i10);
- i10 += 4;
-
- const psimd_f32 vk10x0123 = psimd_load_f32(w + 44);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123);
-
- const psimd_f32 vi11x0123 = psimd_load_f32(i11);
- i11 += 4;
-
- const psimd_f32 vk11x0123 = psimd_load_f32(w + 48);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi11x0123, vk11x0123);
-
- const psimd_f32 vi12x0123 = psimd_load_f32(i12);
- i12 += 4;
-
- const psimd_f32 vk12x0123 = psimd_load_f32(w + 52);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123);
-
- const psimd_f32 vi13x0123 = psimd_load_f32(i13);
- i13 += 4;
-
- const psimd_f32 vk13x0123 = psimd_load_f32(w + 56);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi13x0123, vk13x0123);
-
- const psimd_f32 vi14x0123 = psimd_load_f32(i14);
- i14 += 4;
-
- const psimd_f32 vk14x0123 = psimd_load_f32(w + 60);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123);
-
- const psimd_f32 vi15x0123 = psimd_load_f32(i15);
- i15 += 4;
-
- const psimd_f32 vk15x0123 = psimd_load_f32(w + 64);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi15x0123, vk15x0123);
-
- const psimd_f32 vi16x0123 = psimd_load_f32(i16);
- i16 += 4;
-
- const psimd_f32 vk16x0123 = psimd_load_f32(w + 68);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123);
-
- const psimd_f32 vi17x0123 = psimd_load_f32(i17);
- i17 += 4;
-
- const psimd_f32 vk17x0123 = psimd_load_f32(w + 72);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi17x0123, vk17x0123);
-
- const psimd_f32 vi18x0123 = psimd_load_f32(i18);
- i18 += 4;
-
- const psimd_f32 vk18x0123 = psimd_load_f32(w + 76);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123);
-
- const psimd_f32 vi19x0123 = psimd_load_f32(i19);
- i19 += 4;
-
- const psimd_f32 vk19x0123 = psimd_load_f32(w + 80);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi19x0123, vk19x0123);
-
- const psimd_f32 vi20x0123 = psimd_load_f32(i20);
- i20 += 4;
-
- const psimd_f32 vk20x0123 = psimd_load_f32(w + 84);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123);
-
- const psimd_f32 vi21x0123 = psimd_load_f32(i21);
- i21 += 4;
-
- const psimd_f32 vk21x0123 = psimd_load_f32(w + 88);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi21x0123, vk21x0123);
-
- const psimd_f32 vi22x0123 = psimd_load_f32(i22);
- i22 += 4;
-
- const psimd_f32 vk22x0123 = psimd_load_f32(w + 92);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123);
-
- const psimd_f32 vi23x0123 = psimd_load_f32(i23);
- i23 += 4;
-
- const psimd_f32 vk23x0123 = psimd_load_f32(w + 96);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi23x0123, vk23x0123);
-
- const psimd_f32 vi24x0123 = psimd_load_f32(i24);
- i24 += 4;
-
- const psimd_f32 vk24x0123 = psimd_load_f32(w + 100);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123);
-
- w += 104;
-
- // Add up all accumulators to vacc0123p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 20);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 24);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 32);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 36);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- const psimd_f32 vi9x0123 = psimd_load_f32(i9);
- const psimd_f32 vk9x0123 = psimd_load_f32(w + 40);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi9x0123, vk9x0123);
-
- const psimd_f32 vi10x0123 = psimd_load_f32(i10);
- const psimd_f32 vk10x0123 = psimd_load_f32(w + 44);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123);
-
- const psimd_f32 vi11x0123 = psimd_load_f32(i11);
- const psimd_f32 vk11x0123 = psimd_load_f32(w + 48);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi11x0123, vk11x0123);
-
- const psimd_f32 vi12x0123 = psimd_load_f32(i12);
- const psimd_f32 vk12x0123 = psimd_load_f32(w + 52);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123);
-
- const psimd_f32 vi13x0123 = psimd_load_f32(i13);
- const psimd_f32 vk13x0123 = psimd_load_f32(w + 56);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi13x0123, vk13x0123);
-
- const psimd_f32 vi14x0123 = psimd_load_f32(i14);
- const psimd_f32 vk14x0123 = psimd_load_f32(w + 60);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123);
-
- const psimd_f32 vi15x0123 = psimd_load_f32(i15);
- const psimd_f32 vk15x0123 = psimd_load_f32(w + 64);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi15x0123, vk15x0123);
-
- const psimd_f32 vi16x0123 = psimd_load_f32(i16);
- const psimd_f32 vk16x0123 = psimd_load_f32(w + 68);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123);
-
- const psimd_f32 vi17x0123 = psimd_load_f32(i17);
- const psimd_f32 vk17x0123 = psimd_load_f32(w + 72);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi17x0123, vk17x0123);
-
- const psimd_f32 vi18x0123 = psimd_load_f32(i18);
- const psimd_f32 vk18x0123 = psimd_load_f32(w + 76);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123);
-
- const psimd_f32 vi19x0123 = psimd_load_f32(i19);
- const psimd_f32 vk19x0123 = psimd_load_f32(w + 80);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi19x0123, vk19x0123);
-
- const psimd_f32 vi20x0123 = psimd_load_f32(i20);
- const psimd_f32 vk20x0123 = psimd_load_f32(w + 84);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123);
-
- const psimd_f32 vi21x0123 = psimd_load_f32(i21);
- const psimd_f32 vk21x0123 = psimd_load_f32(w + 88);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi21x0123, vk21x0123);
-
- const psimd_f32 vi22x0123 = psimd_load_f32(i22);
- const psimd_f32 vk22x0123 = psimd_load_f32(w + 92);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123);
-
- const psimd_f32 vi23x0123 = psimd_load_f32(i23);
- const psimd_f32 vk23x0123 = psimd_load_f32(w + 96);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi23x0123, vk23x0123);
-
- const psimd_f32 vi24x0123 = psimd_load_f32(i24);
- const psimd_f32 vk24x0123 = psimd_load_f32(w + 100);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123);
-
- // Add up all accumulators to vacc0123p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/f32-dwconv/gen/up4x25-minmax-psimd.c b/src/f32-dwconv/gen/up4x25-minmax-psimd.c
deleted file mode 100644
index a0c3adaf9..000000000
--- a/src/f32-dwconv/gen/up4x25-minmax-psimd.c
+++ /dev/null
@@ -1,447 +0,0 @@
-// Auto-generated file. Do not edit!
-// Template: src/f32-dwconv/up-psimd.c.in
-// Generator: tools/xngen
-//
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up4x25__psimd(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- const float* i0 = input[0];
- assert(i0 != NULL);
- if XNN_UNPREDICTABLE(i0 != zero) {
- i0 = (const float*) ((uintptr_t) i0 + input_offset);
- }
- const float* i1 = input[1];
- assert(i1 != NULL);
- if XNN_UNPREDICTABLE(i1 != zero) {
- i1 = (const float*) ((uintptr_t) i1 + input_offset);
- }
- const float* i2 = input[2];
- assert(i2 != NULL);
- if XNN_UNPREDICTABLE(i2 != zero) {
- i2 = (const float*) ((uintptr_t) i2 + input_offset);
- }
- const float* i3 = input[3];
- assert(i3 != NULL);
- if XNN_UNPREDICTABLE(i3 != zero) {
- i3 = (const float*) ((uintptr_t) i3 + input_offset);
- }
- const float* i4 = input[4];
- assert(i4 != NULL);
- if XNN_UNPREDICTABLE(i4 != zero) {
- i4 = (const float*) ((uintptr_t) i4 + input_offset);
- }
- const float* i5 = input[5];
- assert(i5 != NULL);
- if XNN_UNPREDICTABLE(i5 != zero) {
- i5 = (const float*) ((uintptr_t) i5 + input_offset);
- }
- const float* i6 = input[6];
- assert(i6 != NULL);
- if XNN_UNPREDICTABLE(i6 != zero) {
- i6 = (const float*) ((uintptr_t) i6 + input_offset);
- }
- const float* i7 = input[7];
- assert(i7 != NULL);
- if XNN_UNPREDICTABLE(i7 != zero) {
- i7 = (const float*) ((uintptr_t) i7 + input_offset);
- }
- const float* i8 = input[8];
- assert(i8 != NULL);
- if XNN_UNPREDICTABLE(i8 != zero) {
- i8 = (const float*) ((uintptr_t) i8 + input_offset);
- }
- const float* i9 = input[9];
- assert(i9 != NULL);
- if XNN_UNPREDICTABLE(i9 != zero) {
- i9 = (const float*) ((uintptr_t) i9 + input_offset);
- }
- const float* i10 = input[10];
- assert(i10 != NULL);
- if XNN_UNPREDICTABLE(i10 != zero) {
- i10 = (const float*) ((uintptr_t) i10 + input_offset);
- }
- const float* i11 = input[11];
- assert(i11 != NULL);
- if XNN_UNPREDICTABLE(i11 != zero) {
- i11 = (const float*) ((uintptr_t) i11 + input_offset);
- }
- const float* i12 = input[12];
- assert(i12 != NULL);
- if XNN_UNPREDICTABLE(i12 != zero) {
- i12 = (const float*) ((uintptr_t) i12 + input_offset);
- }
- const float* i13 = input[13];
- assert(i13 != NULL);
- if XNN_UNPREDICTABLE(i13 != zero) {
- i13 = (const float*) ((uintptr_t) i13 + input_offset);
- }
- const float* i14 = input[14];
- assert(i14 != NULL);
- if XNN_UNPREDICTABLE(i14 != zero) {
- i14 = (const float*) ((uintptr_t) i14 + input_offset);
- }
- const float* i15 = input[15];
- assert(i15 != NULL);
- if XNN_UNPREDICTABLE(i15 != zero) {
- i15 = (const float*) ((uintptr_t) i15 + input_offset);
- }
- const float* i16 = input[16];
- assert(i16 != NULL);
- if XNN_UNPREDICTABLE(i16 != zero) {
- i16 = (const float*) ((uintptr_t) i16 + input_offset);
- }
- const float* i17 = input[17];
- assert(i17 != NULL);
- if XNN_UNPREDICTABLE(i17 != zero) {
- i17 = (const float*) ((uintptr_t) i17 + input_offset);
- }
- const float* i18 = input[18];
- assert(i18 != NULL);
- if XNN_UNPREDICTABLE(i18 != zero) {
- i18 = (const float*) ((uintptr_t) i18 + input_offset);
- }
- const float* i19 = input[19];
- assert(i19 != NULL);
- if XNN_UNPREDICTABLE(i19 != zero) {
- i19 = (const float*) ((uintptr_t) i19 + input_offset);
- }
- const float* i20 = input[20];
- assert(i20 != NULL);
- if XNN_UNPREDICTABLE(i20 != zero) {
- i20 = (const float*) ((uintptr_t) i20 + input_offset);
- }
- const float* i21 = input[21];
- assert(i21 != NULL);
- if XNN_UNPREDICTABLE(i21 != zero) {
- i21 = (const float*) ((uintptr_t) i21 + input_offset);
- }
- const float* i22 = input[22];
- assert(i22 != NULL);
- if XNN_UNPREDICTABLE(i22 != zero) {
- i22 = (const float*) ((uintptr_t) i22 + input_offset);
- }
- const float* i23 = input[23];
- assert(i23 != NULL);
- if XNN_UNPREDICTABLE(i23 != zero) {
- i23 = (const float*) ((uintptr_t) i23 + input_offset);
- }
- const float* i24 = input[24];
- assert(i24 != NULL);
- if XNN_UNPREDICTABLE(i24 != zero) {
- i24 = (const float*) ((uintptr_t) i24 + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- i0 += 4;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- i1 += 4;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- i2 += 4;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- i3 += 4;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- i4 += 4;
-
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 20);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- i5 += 4;
-
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- i6 += 4;
-
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- i7 += 4;
-
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 32);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- i8 += 4;
-
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 36);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- const psimd_f32 vi9x0123 = psimd_load_f32(i9);
- i9 += 4;
-
- const psimd_f32 vk9x0123 = psimd_load_f32(w + 40);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi9x0123, vk9x0123);
-
- const psimd_f32 vi10x0123 = psimd_load_f32(i10);
- i10 += 4;
-
- const psimd_f32 vk10x0123 = psimd_load_f32(w + 44);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123);
-
- const psimd_f32 vi11x0123 = psimd_load_f32(i11);
- i11 += 4;
-
- const psimd_f32 vk11x0123 = psimd_load_f32(w + 48);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi11x0123, vk11x0123);
-
- const psimd_f32 vi12x0123 = psimd_load_f32(i12);
- i12 += 4;
-
- const psimd_f32 vk12x0123 = psimd_load_f32(w + 52);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123);
-
- const psimd_f32 vi13x0123 = psimd_load_f32(i13);
- i13 += 4;
-
- const psimd_f32 vk13x0123 = psimd_load_f32(w + 56);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi13x0123, vk13x0123);
-
- const psimd_f32 vi14x0123 = psimd_load_f32(i14);
- i14 += 4;
-
- const psimd_f32 vk14x0123 = psimd_load_f32(w + 60);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123);
-
- const psimd_f32 vi15x0123 = psimd_load_f32(i15);
- i15 += 4;
-
- const psimd_f32 vk15x0123 = psimd_load_f32(w + 64);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi15x0123, vk15x0123);
-
- const psimd_f32 vi16x0123 = psimd_load_f32(i16);
- i16 += 4;
-
- const psimd_f32 vk16x0123 = psimd_load_f32(w + 68);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123);
-
- const psimd_f32 vi17x0123 = psimd_load_f32(i17);
- i17 += 4;
-
- const psimd_f32 vk17x0123 = psimd_load_f32(w + 72);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi17x0123, vk17x0123);
-
- const psimd_f32 vi18x0123 = psimd_load_f32(i18);
- i18 += 4;
-
- const psimd_f32 vk18x0123 = psimd_load_f32(w + 76);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123);
-
- const psimd_f32 vi19x0123 = psimd_load_f32(i19);
- i19 += 4;
-
- const psimd_f32 vk19x0123 = psimd_load_f32(w + 80);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi19x0123, vk19x0123);
-
- const psimd_f32 vi20x0123 = psimd_load_f32(i20);
- i20 += 4;
-
- const psimd_f32 vk20x0123 = psimd_load_f32(w + 84);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123);
-
- const psimd_f32 vi21x0123 = psimd_load_f32(i21);
- i21 += 4;
-
- const psimd_f32 vk21x0123 = psimd_load_f32(w + 88);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi21x0123, vk21x0123);
-
- const psimd_f32 vi22x0123 = psimd_load_f32(i22);
- i22 += 4;
-
- const psimd_f32 vk22x0123 = psimd_load_f32(w + 92);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123);
-
- const psimd_f32 vi23x0123 = psimd_load_f32(i23);
- i23 += 4;
-
- const psimd_f32 vk23x0123 = psimd_load_f32(w + 96);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi23x0123, vk23x0123);
-
- const psimd_f32 vi24x0123 = psimd_load_f32(i24);
- i24 += 4;
-
- const psimd_f32 vk24x0123 = psimd_load_f32(w + 100);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123);
-
- w += 104;
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 20);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 32);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 36);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- const psimd_f32 vi9x0123 = psimd_load_f32(i9);
- const psimd_f32 vk9x0123 = psimd_load_f32(w + 40);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi9x0123, vk9x0123);
-
- const psimd_f32 vi10x0123 = psimd_load_f32(i10);
- const psimd_f32 vk10x0123 = psimd_load_f32(w + 44);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123);
-
- const psimd_f32 vi11x0123 = psimd_load_f32(i11);
- const psimd_f32 vk11x0123 = psimd_load_f32(w + 48);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi11x0123, vk11x0123);
-
- const psimd_f32 vi12x0123 = psimd_load_f32(i12);
- const psimd_f32 vk12x0123 = psimd_load_f32(w + 52);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123);
-
- const psimd_f32 vi13x0123 = psimd_load_f32(i13);
- const psimd_f32 vk13x0123 = psimd_load_f32(w + 56);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi13x0123, vk13x0123);
-
- const psimd_f32 vi14x0123 = psimd_load_f32(i14);
- const psimd_f32 vk14x0123 = psimd_load_f32(w + 60);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123);
-
- const psimd_f32 vi15x0123 = psimd_load_f32(i15);
- const psimd_f32 vk15x0123 = psimd_load_f32(w + 64);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi15x0123, vk15x0123);
-
- const psimd_f32 vi16x0123 = psimd_load_f32(i16);
- const psimd_f32 vk16x0123 = psimd_load_f32(w + 68);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123);
-
- const psimd_f32 vi17x0123 = psimd_load_f32(i17);
- const psimd_f32 vk17x0123 = psimd_load_f32(w + 72);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi17x0123, vk17x0123);
-
- const psimd_f32 vi18x0123 = psimd_load_f32(i18);
- const psimd_f32 vk18x0123 = psimd_load_f32(w + 76);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123);
-
- const psimd_f32 vi19x0123 = psimd_load_f32(i19);
- const psimd_f32 vk19x0123 = psimd_load_f32(w + 80);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi19x0123, vk19x0123);
-
- const psimd_f32 vi20x0123 = psimd_load_f32(i20);
- const psimd_f32 vk20x0123 = psimd_load_f32(w + 84);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123);
-
- const psimd_f32 vi21x0123 = psimd_load_f32(i21);
- const psimd_f32 vk21x0123 = psimd_load_f32(w + 88);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi21x0123, vk21x0123);
-
- const psimd_f32 vi22x0123 = psimd_load_f32(i22);
- const psimd_f32 vk22x0123 = psimd_load_f32(w + 92);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123);
-
- const psimd_f32 vi23x0123 = psimd_load_f32(i23);
- const psimd_f32 vk23x0123 = psimd_load_f32(w + 96);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi23x0123, vk23x0123);
-
- const psimd_f32 vi24x0123 = psimd_load_f32(i24);
- const psimd_f32 vk24x0123 = psimd_load_f32(w + 100);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123);
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c b/src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c
deleted file mode 100644
index 9c6a98389..000000000
--- a/src/f32-dwconv/gen/up4x4-minmax-psimd-acc2.c
+++ /dev/null
@@ -1,136 +0,0 @@
-// Auto-generated file. Do not edit!
-// Template: src/f32-dwconv/up-psimd.c.in
-// Generator: tools/xngen
-//
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- const float* i0 = input[0];
- assert(i0 != NULL);
- if XNN_UNPREDICTABLE(i0 != zero) {
- i0 = (const float*) ((uintptr_t) i0 + input_offset);
- }
- const float* i1 = input[1];
- assert(i1 != NULL);
- if XNN_UNPREDICTABLE(i1 != zero) {
- i1 = (const float*) ((uintptr_t) i1 + input_offset);
- }
- const float* i2 = input[2];
- assert(i2 != NULL);
- if XNN_UNPREDICTABLE(i2 != zero) {
- i2 = (const float*) ((uintptr_t) i2 + input_offset);
- }
- const float* i3 = input[3];
- assert(i3 != NULL);
- if XNN_UNPREDICTABLE(i3 != zero) {
- i3 = (const float*) ((uintptr_t) i3 + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- i0 += 4;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- i1 += 4;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- i2 += 4;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- i3 += 4;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
-
- w += 20;
-
- // Add up all accumulators to vacc0123p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
-
- // Add up all accumulators to vacc0123p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/f32-dwconv/gen/up4x4-minmax-psimd.c b/src/f32-dwconv/gen/up4x4-minmax-psimd.c
deleted file mode 100644
index 3814c025e..000000000
--- a/src/f32-dwconv/gen/up4x4-minmax-psimd.c
+++ /dev/null
@@ -1,132 +0,0 @@
-// Auto-generated file. Do not edit!
-// Template: src/f32-dwconv/up-psimd.c.in
-// Generator: tools/xngen
-//
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up4x4__psimd(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- const float* i0 = input[0];
- assert(i0 != NULL);
- if XNN_UNPREDICTABLE(i0 != zero) {
- i0 = (const float*) ((uintptr_t) i0 + input_offset);
- }
- const float* i1 = input[1];
- assert(i1 != NULL);
- if XNN_UNPREDICTABLE(i1 != zero) {
- i1 = (const float*) ((uintptr_t) i1 + input_offset);
- }
- const float* i2 = input[2];
- assert(i2 != NULL);
- if XNN_UNPREDICTABLE(i2 != zero) {
- i2 = (const float*) ((uintptr_t) i2 + input_offset);
- }
- const float* i3 = input[3];
- assert(i3 != NULL);
- if XNN_UNPREDICTABLE(i3 != zero) {
- i3 = (const float*) ((uintptr_t) i3 + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- i0 += 4;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- i1 += 4;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- i2 += 4;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- i3 += 4;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
-
- w += 20;
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c b/src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c
deleted file mode 100644
index e4071864f..000000000
--- a/src/f32-dwconv/gen/up4x9-minmax-psimd-acc2.c
+++ /dev/null
@@ -1,211 +0,0 @@
-// Auto-generated file. Do not edit!
-// Template: src/f32-dwconv/up-psimd.c.in
-// Generator: tools/xngen
-//
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- const float* i0 = input[0];
- assert(i0 != NULL);
- if XNN_UNPREDICTABLE(i0 != zero) {
- i0 = (const float*) ((uintptr_t) i0 + input_offset);
- }
- const float* i1 = input[1];
- assert(i1 != NULL);
- if XNN_UNPREDICTABLE(i1 != zero) {
- i1 = (const float*) ((uintptr_t) i1 + input_offset);
- }
- const float* i2 = input[2];
- assert(i2 != NULL);
- if XNN_UNPREDICTABLE(i2 != zero) {
- i2 = (const float*) ((uintptr_t) i2 + input_offset);
- }
- const float* i3 = input[3];
- assert(i3 != NULL);
- if XNN_UNPREDICTABLE(i3 != zero) {
- i3 = (const float*) ((uintptr_t) i3 + input_offset);
- }
- const float* i4 = input[4];
- assert(i4 != NULL);
- if XNN_UNPREDICTABLE(i4 != zero) {
- i4 = (const float*) ((uintptr_t) i4 + input_offset);
- }
- const float* i5 = input[5];
- assert(i5 != NULL);
- if XNN_UNPREDICTABLE(i5 != zero) {
- i5 = (const float*) ((uintptr_t) i5 + input_offset);
- }
- const float* i6 = input[6];
- assert(i6 != NULL);
- if XNN_UNPREDICTABLE(i6 != zero) {
- i6 = (const float*) ((uintptr_t) i6 + input_offset);
- }
- const float* i7 = input[7];
- assert(i7 != NULL);
- if XNN_UNPREDICTABLE(i7 != zero) {
- i7 = (const float*) ((uintptr_t) i7 + input_offset);
- }
- const float* i8 = input[8];
- assert(i8 != NULL);
- if XNN_UNPREDICTABLE(i8 != zero) {
- i8 = (const float*) ((uintptr_t) i8 + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- i0 += 4;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- i1 += 4;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- i2 += 4;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- i3 += 4;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- i4 += 4;
-
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 20);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- i5 += 4;
-
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 24);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- i6 += 4;
-
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- i7 += 4;
-
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 32);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- i8 += 4;
-
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 36);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- w += 40;
-
- // Add up all accumulators to vacc0123p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 20);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 24);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 32);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 36);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- // Add up all accumulators to vacc0123p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/f32-dwconv/gen/up4x9-minmax-psimd.c b/src/f32-dwconv/gen/up4x9-minmax-psimd.c
deleted file mode 100644
index 0bc1115ba..000000000
--- a/src/f32-dwconv/gen/up4x9-minmax-psimd.c
+++ /dev/null
@@ -1,207 +0,0 @@
-// Auto-generated file. Do not edit!
-// Template: src/f32-dwconv/up-psimd.c.in
-// Generator: tools/xngen
-//
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up4x9__psimd(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- const float* i0 = input[0];
- assert(i0 != NULL);
- if XNN_UNPREDICTABLE(i0 != zero) {
- i0 = (const float*) ((uintptr_t) i0 + input_offset);
- }
- const float* i1 = input[1];
- assert(i1 != NULL);
- if XNN_UNPREDICTABLE(i1 != zero) {
- i1 = (const float*) ((uintptr_t) i1 + input_offset);
- }
- const float* i2 = input[2];
- assert(i2 != NULL);
- if XNN_UNPREDICTABLE(i2 != zero) {
- i2 = (const float*) ((uintptr_t) i2 + input_offset);
- }
- const float* i3 = input[3];
- assert(i3 != NULL);
- if XNN_UNPREDICTABLE(i3 != zero) {
- i3 = (const float*) ((uintptr_t) i3 + input_offset);
- }
- const float* i4 = input[4];
- assert(i4 != NULL);
- if XNN_UNPREDICTABLE(i4 != zero) {
- i4 = (const float*) ((uintptr_t) i4 + input_offset);
- }
- const float* i5 = input[5];
- assert(i5 != NULL);
- if XNN_UNPREDICTABLE(i5 != zero) {
- i5 = (const float*) ((uintptr_t) i5 + input_offset);
- }
- const float* i6 = input[6];
- assert(i6 != NULL);
- if XNN_UNPREDICTABLE(i6 != zero) {
- i6 = (const float*) ((uintptr_t) i6 + input_offset);
- }
- const float* i7 = input[7];
- assert(i7 != NULL);
- if XNN_UNPREDICTABLE(i7 != zero) {
- i7 = (const float*) ((uintptr_t) i7 + input_offset);
- }
- const float* i8 = input[8];
- assert(i8 != NULL);
- if XNN_UNPREDICTABLE(i8 != zero) {
- i8 = (const float*) ((uintptr_t) i8 + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- i0 += 4;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- i1 += 4;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- i2 += 4;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- i3 += 4;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- i4 += 4;
-
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 20);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- i5 += 4;
-
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- i6 += 4;
-
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- i7 += 4;
-
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 32);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- i8 += 4;
-
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 36);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- w += 40;
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 4);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 16);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 20);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 32);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 36);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c b/src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c
deleted file mode 100644
index 2da6c1482..000000000
--- a/src/f32-dwconv/gen/up8x25-minmax-psimd-acc2.c
+++ /dev/null
@@ -1,695 +0,0 @@
-// Auto-generated file. Do not edit!
-// Template: src/f32-dwconv/up-psimd.c.in
-// Generator: tools/xngen
-//
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- const float* i0 = input[0];
- assert(i0 != NULL);
- if XNN_UNPREDICTABLE(i0 != zero) {
- i0 = (const float*) ((uintptr_t) i0 + input_offset);
- }
- const float* i1 = input[1];
- assert(i1 != NULL);
- if XNN_UNPREDICTABLE(i1 != zero) {
- i1 = (const float*) ((uintptr_t) i1 + input_offset);
- }
- const float* i2 = input[2];
- assert(i2 != NULL);
- if XNN_UNPREDICTABLE(i2 != zero) {
- i2 = (const float*) ((uintptr_t) i2 + input_offset);
- }
- const float* i3 = input[3];
- assert(i3 != NULL);
- if XNN_UNPREDICTABLE(i3 != zero) {
- i3 = (const float*) ((uintptr_t) i3 + input_offset);
- }
- const float* i4 = input[4];
- assert(i4 != NULL);
- if XNN_UNPREDICTABLE(i4 != zero) {
- i4 = (const float*) ((uintptr_t) i4 + input_offset);
- }
- const float* i5 = input[5];
- assert(i5 != NULL);
- if XNN_UNPREDICTABLE(i5 != zero) {
- i5 = (const float*) ((uintptr_t) i5 + input_offset);
- }
- const float* i6 = input[6];
- assert(i6 != NULL);
- if XNN_UNPREDICTABLE(i6 != zero) {
- i6 = (const float*) ((uintptr_t) i6 + input_offset);
- }
- const float* i7 = input[7];
- assert(i7 != NULL);
- if XNN_UNPREDICTABLE(i7 != zero) {
- i7 = (const float*) ((uintptr_t) i7 + input_offset);
- }
- const float* i8 = input[8];
- assert(i8 != NULL);
- if XNN_UNPREDICTABLE(i8 != zero) {
- i8 = (const float*) ((uintptr_t) i8 + input_offset);
- }
- const float* i9 = input[9];
- assert(i9 != NULL);
- if XNN_UNPREDICTABLE(i9 != zero) {
- i9 = (const float*) ((uintptr_t) i9 + input_offset);
- }
- const float* i10 = input[10];
- assert(i10 != NULL);
- if XNN_UNPREDICTABLE(i10 != zero) {
- i10 = (const float*) ((uintptr_t) i10 + input_offset);
- }
- const float* i11 = input[11];
- assert(i11 != NULL);
- if XNN_UNPREDICTABLE(i11 != zero) {
- i11 = (const float*) ((uintptr_t) i11 + input_offset);
- }
- const float* i12 = input[12];
- assert(i12 != NULL);
- if XNN_UNPREDICTABLE(i12 != zero) {
- i12 = (const float*) ((uintptr_t) i12 + input_offset);
- }
- const float* i13 = input[13];
- assert(i13 != NULL);
- if XNN_UNPREDICTABLE(i13 != zero) {
- i13 = (const float*) ((uintptr_t) i13 + input_offset);
- }
- const float* i14 = input[14];
- assert(i14 != NULL);
- if XNN_UNPREDICTABLE(i14 != zero) {
- i14 = (const float*) ((uintptr_t) i14 + input_offset);
- }
- const float* i15 = input[15];
- assert(i15 != NULL);
- if XNN_UNPREDICTABLE(i15 != zero) {
- i15 = (const float*) ((uintptr_t) i15 + input_offset);
- }
- const float* i16 = input[16];
- assert(i16 != NULL);
- if XNN_UNPREDICTABLE(i16 != zero) {
- i16 = (const float*) ((uintptr_t) i16 + input_offset);
- }
- const float* i17 = input[17];
- assert(i17 != NULL);
- if XNN_UNPREDICTABLE(i17 != zero) {
- i17 = (const float*) ((uintptr_t) i17 + input_offset);
- }
- const float* i18 = input[18];
- assert(i18 != NULL);
- if XNN_UNPREDICTABLE(i18 != zero) {
- i18 = (const float*) ((uintptr_t) i18 + input_offset);
- }
- const float* i19 = input[19];
- assert(i19 != NULL);
- if XNN_UNPREDICTABLE(i19 != zero) {
- i19 = (const float*) ((uintptr_t) i19 + input_offset);
- }
- const float* i20 = input[20];
- assert(i20 != NULL);
- if XNN_UNPREDICTABLE(i20 != zero) {
- i20 = (const float*) ((uintptr_t) i20 + input_offset);
- }
- const float* i21 = input[21];
- assert(i21 != NULL);
- if XNN_UNPREDICTABLE(i21 != zero) {
- i21 = (const float*) ((uintptr_t) i21 + input_offset);
- }
- const float* i22 = input[22];
- assert(i22 != NULL);
- if XNN_UNPREDICTABLE(i22 != zero) {
- i22 = (const float*) ((uintptr_t) i22 + input_offset);
- }
- const float* i23 = input[23];
- assert(i23 != NULL);
- if XNN_UNPREDICTABLE(i23 != zero) {
- i23 = (const float*) ((uintptr_t) i23 + input_offset);
- }
- const float* i24 = input[24];
- assert(i24 != NULL);
- if XNN_UNPREDICTABLE(i24 != zero) {
- i24 = (const float*) ((uintptr_t) i24 + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= 8; c -= 8) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
- psimd_f32 vacc4567p0 = psimd_load_f32(w + 4);
-
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vi0x4567 = psimd_load_f32(i0 + 4);
- i0 += 8;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- const psimd_f32 vk0x4567 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi0x4567, vk0x4567);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vi1x4567 = psimd_load_f32(i1 + 4);
- i1 += 8;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- const psimd_f32 vk1x4567 = psimd_load_f32(w + 20);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
- psimd_f32 vacc4567p1 = psimd_mul_f32(vi1x4567, vk1x4567);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vi2x4567 = psimd_load_f32(i2 + 4);
- i2 += 8;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- const psimd_f32 vk2x4567 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi2x4567, vk2x4567);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vi3x4567 = psimd_load_f32(i3 + 4);
- i3 += 8;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- const psimd_f32 vk3x4567 = psimd_load_f32(w + 36);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi3x4567, vk3x4567);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- const psimd_f32 vi4x4567 = psimd_load_f32(i4 + 4);
- i4 += 8;
-
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 40);
- const psimd_f32 vk4x4567 = psimd_load_f32(w + 44);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi4x4567, vk4x4567);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- const psimd_f32 vi5x4567 = psimd_load_f32(i5 + 4);
- i5 += 8;
-
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 48);
- const psimd_f32 vk5x4567 = psimd_load_f32(w + 52);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi5x4567, vk5x4567);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- const psimd_f32 vi6x4567 = psimd_load_f32(i6 + 4);
- i6 += 8;
-
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 56);
- const psimd_f32 vk6x4567 = psimd_load_f32(w + 60);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi6x4567, vk6x4567);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- const psimd_f32 vi7x4567 = psimd_load_f32(i7 + 4);
- i7 += 8;
-
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 64);
- const psimd_f32 vk7x4567 = psimd_load_f32(w + 68);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi7x4567, vk7x4567);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- const psimd_f32 vi8x4567 = psimd_load_f32(i8 + 4);
- i8 += 8;
-
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 72);
- const psimd_f32 vk8x4567 = psimd_load_f32(w + 76);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi8x4567, vk8x4567);
-
- const psimd_f32 vi9x0123 = psimd_load_f32(i9);
- const psimd_f32 vi9x4567 = psimd_load_f32(i9 + 4);
- i9 += 8;
-
- const psimd_f32 vk9x0123 = psimd_load_f32(w + 80);
- const psimd_f32 vk9x4567 = psimd_load_f32(w + 84);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi9x0123, vk9x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi9x4567, vk9x4567);
-
- const psimd_f32 vi10x0123 = psimd_load_f32(i10);
- const psimd_f32 vi10x4567 = psimd_load_f32(i10 + 4);
- i10 += 8;
-
- const psimd_f32 vk10x0123 = psimd_load_f32(w + 88);
- const psimd_f32 vk10x4567 = psimd_load_f32(w + 92);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi10x4567, vk10x4567);
-
- const psimd_f32 vi11x0123 = psimd_load_f32(i11);
- const psimd_f32 vi11x4567 = psimd_load_f32(i11 + 4);
- i11 += 8;
-
- const psimd_f32 vk11x0123 = psimd_load_f32(w + 96);
- const psimd_f32 vk11x4567 = psimd_load_f32(w + 100);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi11x0123, vk11x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi11x4567, vk11x4567);
-
- const psimd_f32 vi12x0123 = psimd_load_f32(i12);
- const psimd_f32 vi12x4567 = psimd_load_f32(i12 + 4);
- i12 += 8;
-
- const psimd_f32 vk12x0123 = psimd_load_f32(w + 104);
- const psimd_f32 vk12x4567 = psimd_load_f32(w + 108);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi12x4567, vk12x4567);
-
- const psimd_f32 vi13x0123 = psimd_load_f32(i13);
- const psimd_f32 vi13x4567 = psimd_load_f32(i13 + 4);
- i13 += 8;
-
- const psimd_f32 vk13x0123 = psimd_load_f32(w + 112);
- const psimd_f32 vk13x4567 = psimd_load_f32(w + 116);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi13x0123, vk13x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi13x4567, vk13x4567);
-
- const psimd_f32 vi14x0123 = psimd_load_f32(i14);
- const psimd_f32 vi14x4567 = psimd_load_f32(i14 + 4);
- i14 += 8;
-
- const psimd_f32 vk14x0123 = psimd_load_f32(w + 120);
- const psimd_f32 vk14x4567 = psimd_load_f32(w + 124);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi14x4567, vk14x4567);
-
- const psimd_f32 vi15x0123 = psimd_load_f32(i15);
- const psimd_f32 vi15x4567 = psimd_load_f32(i15 + 4);
- i15 += 8;
-
- const psimd_f32 vk15x0123 = psimd_load_f32(w + 128);
- const psimd_f32 vk15x4567 = psimd_load_f32(w + 132);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi15x0123, vk15x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi15x4567, vk15x4567);
-
- const psimd_f32 vi16x0123 = psimd_load_f32(i16);
- const psimd_f32 vi16x4567 = psimd_load_f32(i16 + 4);
- i16 += 8;
-
- const psimd_f32 vk16x0123 = psimd_load_f32(w + 136);
- const psimd_f32 vk16x4567 = psimd_load_f32(w + 140);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi16x4567, vk16x4567);
-
- const psimd_f32 vi17x0123 = psimd_load_f32(i17);
- const psimd_f32 vi17x4567 = psimd_load_f32(i17 + 4);
- i17 += 8;
-
- const psimd_f32 vk17x0123 = psimd_load_f32(w + 144);
- const psimd_f32 vk17x4567 = psimd_load_f32(w + 148);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi17x0123, vk17x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi17x4567, vk17x4567);
-
- const psimd_f32 vi18x0123 = psimd_load_f32(i18);
- const psimd_f32 vi18x4567 = psimd_load_f32(i18 + 4);
- i18 += 8;
-
- const psimd_f32 vk18x0123 = psimd_load_f32(w + 152);
- const psimd_f32 vk18x4567 = psimd_load_f32(w + 156);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi18x4567, vk18x4567);
-
- const psimd_f32 vi19x0123 = psimd_load_f32(i19);
- const psimd_f32 vi19x4567 = psimd_load_f32(i19 + 4);
- i19 += 8;
-
- const psimd_f32 vk19x0123 = psimd_load_f32(w + 160);
- const psimd_f32 vk19x4567 = psimd_load_f32(w + 164);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi19x0123, vk19x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi19x4567, vk19x4567);
-
- const psimd_f32 vi20x0123 = psimd_load_f32(i20);
- const psimd_f32 vi20x4567 = psimd_load_f32(i20 + 4);
- i20 += 8;
-
- const psimd_f32 vk20x0123 = psimd_load_f32(w + 168);
- const psimd_f32 vk20x4567 = psimd_load_f32(w + 172);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi20x4567, vk20x4567);
-
- const psimd_f32 vi21x0123 = psimd_load_f32(i21);
- const psimd_f32 vi21x4567 = psimd_load_f32(i21 + 4);
- i21 += 8;
-
- const psimd_f32 vk21x0123 = psimd_load_f32(w + 176);
- const psimd_f32 vk21x4567 = psimd_load_f32(w + 180);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi21x0123, vk21x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi21x4567, vk21x4567);
-
- const psimd_f32 vi22x0123 = psimd_load_f32(i22);
- const psimd_f32 vi22x4567 = psimd_load_f32(i22 + 4);
- i22 += 8;
-
- const psimd_f32 vk22x0123 = psimd_load_f32(w + 184);
- const psimd_f32 vk22x4567 = psimd_load_f32(w + 188);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi22x4567, vk22x4567);
-
- const psimd_f32 vi23x0123 = psimd_load_f32(i23);
- const psimd_f32 vi23x4567 = psimd_load_f32(i23 + 4);
- i23 += 8;
-
- const psimd_f32 vk23x0123 = psimd_load_f32(w + 192);
- const psimd_f32 vk23x4567 = psimd_load_f32(w + 196);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi23x0123, vk23x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi23x4567, vk23x4567);
-
- const psimd_f32 vi24x0123 = psimd_load_f32(i24);
- const psimd_f32 vi24x4567 = psimd_load_f32(i24 + 4);
- i24 += 8;
-
- const psimd_f32 vk24x0123 = psimd_load_f32(w + 200);
- const psimd_f32 vk24x4567 = psimd_load_f32(w + 204);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi24x4567, vk24x4567);
-
- w += 208;
-
- // Add up all accumulators to vacc01234567p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
- vacc4567p0 = psimd_add_f32(vacc4567p0, vacc4567p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- psimd_f32 vacc4567 = psimd_max_f32(vacc4567p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
- vacc4567 = psimd_min_f32(vacc4567, vmax);
-
- psimd_store_f32(output, vacc0123);
- psimd_store_f32(output + 4, vacc4567);
- output += 8;
- }
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- i0 += 4;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- i1 += 4;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- i2 += 4;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- i3 += 4;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- i4 += 4;
-
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 40);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- i5 += 4;
-
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 48);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- i6 += 4;
-
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 56);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- i7 += 4;
-
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 64);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- i8 += 4;
-
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 72);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- const psimd_f32 vi9x0123 = psimd_load_f32(i9);
- i9 += 4;
-
- const psimd_f32 vk9x0123 = psimd_load_f32(w + 80);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi9x0123, vk9x0123);
-
- const psimd_f32 vi10x0123 = psimd_load_f32(i10);
- i10 += 4;
-
- const psimd_f32 vk10x0123 = psimd_load_f32(w + 88);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123);
-
- const psimd_f32 vi11x0123 = psimd_load_f32(i11);
- i11 += 4;
-
- const psimd_f32 vk11x0123 = psimd_load_f32(w + 96);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi11x0123, vk11x0123);
-
- const psimd_f32 vi12x0123 = psimd_load_f32(i12);
- i12 += 4;
-
- const psimd_f32 vk12x0123 = psimd_load_f32(w + 104);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123);
-
- const psimd_f32 vi13x0123 = psimd_load_f32(i13);
- i13 += 4;
-
- const psimd_f32 vk13x0123 = psimd_load_f32(w + 112);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi13x0123, vk13x0123);
-
- const psimd_f32 vi14x0123 = psimd_load_f32(i14);
- i14 += 4;
-
- const psimd_f32 vk14x0123 = psimd_load_f32(w + 120);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123);
-
- const psimd_f32 vi15x0123 = psimd_load_f32(i15);
- i15 += 4;
-
- const psimd_f32 vk15x0123 = psimd_load_f32(w + 128);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi15x0123, vk15x0123);
-
- const psimd_f32 vi16x0123 = psimd_load_f32(i16);
- i16 += 4;
-
- const psimd_f32 vk16x0123 = psimd_load_f32(w + 136);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123);
-
- const psimd_f32 vi17x0123 = psimd_load_f32(i17);
- i17 += 4;
-
- const psimd_f32 vk17x0123 = psimd_load_f32(w + 144);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi17x0123, vk17x0123);
-
- const psimd_f32 vi18x0123 = psimd_load_f32(i18);
- i18 += 4;
-
- const psimd_f32 vk18x0123 = psimd_load_f32(w + 152);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123);
-
- const psimd_f32 vi19x0123 = psimd_load_f32(i19);
- i19 += 4;
-
- const psimd_f32 vk19x0123 = psimd_load_f32(w + 160);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi19x0123, vk19x0123);
-
- const psimd_f32 vi20x0123 = psimd_load_f32(i20);
- i20 += 4;
-
- const psimd_f32 vk20x0123 = psimd_load_f32(w + 168);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123);
-
- const psimd_f32 vi21x0123 = psimd_load_f32(i21);
- i21 += 4;
-
- const psimd_f32 vk21x0123 = psimd_load_f32(w + 176);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi21x0123, vk21x0123);
-
- const psimd_f32 vi22x0123 = psimd_load_f32(i22);
- i22 += 4;
-
- const psimd_f32 vk22x0123 = psimd_load_f32(w + 184);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123);
-
- const psimd_f32 vi23x0123 = psimd_load_f32(i23);
- i23 += 4;
-
- const psimd_f32 vk23x0123 = psimd_load_f32(w + 192);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi23x0123, vk23x0123);
-
- const psimd_f32 vi24x0123 = psimd_load_f32(i24);
- i24 += 4;
-
- const psimd_f32 vk24x0123 = psimd_load_f32(w + 200);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123);
-
- w += 4;
-
- // Add up all accumulators to vacc01234567p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 40);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 48);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 56);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 64);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 72);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- const psimd_f32 vi9x0123 = psimd_load_f32(i9);
- const psimd_f32 vk9x0123 = psimd_load_f32(w + 80);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi9x0123, vk9x0123);
-
- const psimd_f32 vi10x0123 = psimd_load_f32(i10);
- const psimd_f32 vk10x0123 = psimd_load_f32(w + 88);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123);
-
- const psimd_f32 vi11x0123 = psimd_load_f32(i11);
- const psimd_f32 vk11x0123 = psimd_load_f32(w + 96);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi11x0123, vk11x0123);
-
- const psimd_f32 vi12x0123 = psimd_load_f32(i12);
- const psimd_f32 vk12x0123 = psimd_load_f32(w + 104);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123);
-
- const psimd_f32 vi13x0123 = psimd_load_f32(i13);
- const psimd_f32 vk13x0123 = psimd_load_f32(w + 112);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi13x0123, vk13x0123);
-
- const psimd_f32 vi14x0123 = psimd_load_f32(i14);
- const psimd_f32 vk14x0123 = psimd_load_f32(w + 120);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123);
-
- const psimd_f32 vi15x0123 = psimd_load_f32(i15);
- const psimd_f32 vk15x0123 = psimd_load_f32(w + 128);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi15x0123, vk15x0123);
-
- const psimd_f32 vi16x0123 = psimd_load_f32(i16);
- const psimd_f32 vk16x0123 = psimd_load_f32(w + 136);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123);
-
- const psimd_f32 vi17x0123 = psimd_load_f32(i17);
- const psimd_f32 vk17x0123 = psimd_load_f32(w + 144);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi17x0123, vk17x0123);
-
- const psimd_f32 vi18x0123 = psimd_load_f32(i18);
- const psimd_f32 vk18x0123 = psimd_load_f32(w + 152);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123);
-
- const psimd_f32 vi19x0123 = psimd_load_f32(i19);
- const psimd_f32 vk19x0123 = psimd_load_f32(w + 160);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi19x0123, vk19x0123);
-
- const psimd_f32 vi20x0123 = psimd_load_f32(i20);
- const psimd_f32 vk20x0123 = psimd_load_f32(w + 168);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123);
-
- const psimd_f32 vi21x0123 = psimd_load_f32(i21);
- const psimd_f32 vk21x0123 = psimd_load_f32(w + 176);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi21x0123, vk21x0123);
-
- const psimd_f32 vi22x0123 = psimd_load_f32(i22);
- const psimd_f32 vk22x0123 = psimd_load_f32(w + 184);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123);
-
- const psimd_f32 vi23x0123 = psimd_load_f32(i23);
- const psimd_f32 vk23x0123 = psimd_load_f32(w + 192);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi23x0123, vk23x0123);
-
- const psimd_f32 vi24x0123 = psimd_load_f32(i24);
- const psimd_f32 vk24x0123 = psimd_load_f32(w + 200);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123);
-
- // Add up all accumulators to vacc01234567p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/f32-dwconv/gen/up8x25-minmax-psimd.c b/src/f32-dwconv/gen/up8x25-minmax-psimd.c
deleted file mode 100644
index 2126f5eff..000000000
--- a/src/f32-dwconv/gen/up8x25-minmax-psimd.c
+++ /dev/null
@@ -1,688 +0,0 @@
-// Auto-generated file. Do not edit!
-// Template: src/f32-dwconv/up-psimd.c.in
-// Generator: tools/xngen
-//
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up8x25__psimd(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- const float* i0 = input[0];
- assert(i0 != NULL);
- if XNN_UNPREDICTABLE(i0 != zero) {
- i0 = (const float*) ((uintptr_t) i0 + input_offset);
- }
- const float* i1 = input[1];
- assert(i1 != NULL);
- if XNN_UNPREDICTABLE(i1 != zero) {
- i1 = (const float*) ((uintptr_t) i1 + input_offset);
- }
- const float* i2 = input[2];
- assert(i2 != NULL);
- if XNN_UNPREDICTABLE(i2 != zero) {
- i2 = (const float*) ((uintptr_t) i2 + input_offset);
- }
- const float* i3 = input[3];
- assert(i3 != NULL);
- if XNN_UNPREDICTABLE(i3 != zero) {
- i3 = (const float*) ((uintptr_t) i3 + input_offset);
- }
- const float* i4 = input[4];
- assert(i4 != NULL);
- if XNN_UNPREDICTABLE(i4 != zero) {
- i4 = (const float*) ((uintptr_t) i4 + input_offset);
- }
- const float* i5 = input[5];
- assert(i5 != NULL);
- if XNN_UNPREDICTABLE(i5 != zero) {
- i5 = (const float*) ((uintptr_t) i5 + input_offset);
- }
- const float* i6 = input[6];
- assert(i6 != NULL);
- if XNN_UNPREDICTABLE(i6 != zero) {
- i6 = (const float*) ((uintptr_t) i6 + input_offset);
- }
- const float* i7 = input[7];
- assert(i7 != NULL);
- if XNN_UNPREDICTABLE(i7 != zero) {
- i7 = (const float*) ((uintptr_t) i7 + input_offset);
- }
- const float* i8 = input[8];
- assert(i8 != NULL);
- if XNN_UNPREDICTABLE(i8 != zero) {
- i8 = (const float*) ((uintptr_t) i8 + input_offset);
- }
- const float* i9 = input[9];
- assert(i9 != NULL);
- if XNN_UNPREDICTABLE(i9 != zero) {
- i9 = (const float*) ((uintptr_t) i9 + input_offset);
- }
- const float* i10 = input[10];
- assert(i10 != NULL);
- if XNN_UNPREDICTABLE(i10 != zero) {
- i10 = (const float*) ((uintptr_t) i10 + input_offset);
- }
- const float* i11 = input[11];
- assert(i11 != NULL);
- if XNN_UNPREDICTABLE(i11 != zero) {
- i11 = (const float*) ((uintptr_t) i11 + input_offset);
- }
- const float* i12 = input[12];
- assert(i12 != NULL);
- if XNN_UNPREDICTABLE(i12 != zero) {
- i12 = (const float*) ((uintptr_t) i12 + input_offset);
- }
- const float* i13 = input[13];
- assert(i13 != NULL);
- if XNN_UNPREDICTABLE(i13 != zero) {
- i13 = (const float*) ((uintptr_t) i13 + input_offset);
- }
- const float* i14 = input[14];
- assert(i14 != NULL);
- if XNN_UNPREDICTABLE(i14 != zero) {
- i14 = (const float*) ((uintptr_t) i14 + input_offset);
- }
- const float* i15 = input[15];
- assert(i15 != NULL);
- if XNN_UNPREDICTABLE(i15 != zero) {
- i15 = (const float*) ((uintptr_t) i15 + input_offset);
- }
- const float* i16 = input[16];
- assert(i16 != NULL);
- if XNN_UNPREDICTABLE(i16 != zero) {
- i16 = (const float*) ((uintptr_t) i16 + input_offset);
- }
- const float* i17 = input[17];
- assert(i17 != NULL);
- if XNN_UNPREDICTABLE(i17 != zero) {
- i17 = (const float*) ((uintptr_t) i17 + input_offset);
- }
- const float* i18 = input[18];
- assert(i18 != NULL);
- if XNN_UNPREDICTABLE(i18 != zero) {
- i18 = (const float*) ((uintptr_t) i18 + input_offset);
- }
- const float* i19 = input[19];
- assert(i19 != NULL);
- if XNN_UNPREDICTABLE(i19 != zero) {
- i19 = (const float*) ((uintptr_t) i19 + input_offset);
- }
- const float* i20 = input[20];
- assert(i20 != NULL);
- if XNN_UNPREDICTABLE(i20 != zero) {
- i20 = (const float*) ((uintptr_t) i20 + input_offset);
- }
- const float* i21 = input[21];
- assert(i21 != NULL);
- if XNN_UNPREDICTABLE(i21 != zero) {
- i21 = (const float*) ((uintptr_t) i21 + input_offset);
- }
- const float* i22 = input[22];
- assert(i22 != NULL);
- if XNN_UNPREDICTABLE(i22 != zero) {
- i22 = (const float*) ((uintptr_t) i22 + input_offset);
- }
- const float* i23 = input[23];
- assert(i23 != NULL);
- if XNN_UNPREDICTABLE(i23 != zero) {
- i23 = (const float*) ((uintptr_t) i23 + input_offset);
- }
- const float* i24 = input[24];
- assert(i24 != NULL);
- if XNN_UNPREDICTABLE(i24 != zero) {
- i24 = (const float*) ((uintptr_t) i24 + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= 8; c -= 8) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
- psimd_f32 vacc4567p0 = psimd_load_f32(w + 4);
-
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vi0x4567 = psimd_load_f32(i0 + 4);
- i0 += 8;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- const psimd_f32 vk0x4567 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi0x4567, vk0x4567);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vi1x4567 = psimd_load_f32(i1 + 4);
- i1 += 8;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- const psimd_f32 vk1x4567 = psimd_load_f32(w + 20);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi1x4567, vk1x4567);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vi2x4567 = psimd_load_f32(i2 + 4);
- i2 += 8;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- const psimd_f32 vk2x4567 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi2x4567, vk2x4567);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vi3x4567 = psimd_load_f32(i3 + 4);
- i3 += 8;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- const psimd_f32 vk3x4567 = psimd_load_f32(w + 36);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi3x4567, vk3x4567);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- const psimd_f32 vi4x4567 = psimd_load_f32(i4 + 4);
- i4 += 8;
-
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 40);
- const psimd_f32 vk4x4567 = psimd_load_f32(w + 44);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi4x4567, vk4x4567);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- const psimd_f32 vi5x4567 = psimd_load_f32(i5 + 4);
- i5 += 8;
-
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 48);
- const psimd_f32 vk5x4567 = psimd_load_f32(w + 52);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi5x4567, vk5x4567);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- const psimd_f32 vi6x4567 = psimd_load_f32(i6 + 4);
- i6 += 8;
-
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 56);
- const psimd_f32 vk6x4567 = psimd_load_f32(w + 60);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi6x4567, vk6x4567);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- const psimd_f32 vi7x4567 = psimd_load_f32(i7 + 4);
- i7 += 8;
-
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 64);
- const psimd_f32 vk7x4567 = psimd_load_f32(w + 68);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi7x4567, vk7x4567);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- const psimd_f32 vi8x4567 = psimd_load_f32(i8 + 4);
- i8 += 8;
-
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 72);
- const psimd_f32 vk8x4567 = psimd_load_f32(w + 76);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi8x4567, vk8x4567);
-
- const psimd_f32 vi9x0123 = psimd_load_f32(i9);
- const psimd_f32 vi9x4567 = psimd_load_f32(i9 + 4);
- i9 += 8;
-
- const psimd_f32 vk9x0123 = psimd_load_f32(w + 80);
- const psimd_f32 vk9x4567 = psimd_load_f32(w + 84);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi9x0123, vk9x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi9x4567, vk9x4567);
-
- const psimd_f32 vi10x0123 = psimd_load_f32(i10);
- const psimd_f32 vi10x4567 = psimd_load_f32(i10 + 4);
- i10 += 8;
-
- const psimd_f32 vk10x0123 = psimd_load_f32(w + 88);
- const psimd_f32 vk10x4567 = psimd_load_f32(w + 92);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi10x4567, vk10x4567);
-
- const psimd_f32 vi11x0123 = psimd_load_f32(i11);
- const psimd_f32 vi11x4567 = psimd_load_f32(i11 + 4);
- i11 += 8;
-
- const psimd_f32 vk11x0123 = psimd_load_f32(w + 96);
- const psimd_f32 vk11x4567 = psimd_load_f32(w + 100);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi11x0123, vk11x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi11x4567, vk11x4567);
-
- const psimd_f32 vi12x0123 = psimd_load_f32(i12);
- const psimd_f32 vi12x4567 = psimd_load_f32(i12 + 4);
- i12 += 8;
-
- const psimd_f32 vk12x0123 = psimd_load_f32(w + 104);
- const psimd_f32 vk12x4567 = psimd_load_f32(w + 108);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi12x4567, vk12x4567);
-
- const psimd_f32 vi13x0123 = psimd_load_f32(i13);
- const psimd_f32 vi13x4567 = psimd_load_f32(i13 + 4);
- i13 += 8;
-
- const psimd_f32 vk13x0123 = psimd_load_f32(w + 112);
- const psimd_f32 vk13x4567 = psimd_load_f32(w + 116);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi13x0123, vk13x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi13x4567, vk13x4567);
-
- const psimd_f32 vi14x0123 = psimd_load_f32(i14);
- const psimd_f32 vi14x4567 = psimd_load_f32(i14 + 4);
- i14 += 8;
-
- const psimd_f32 vk14x0123 = psimd_load_f32(w + 120);
- const psimd_f32 vk14x4567 = psimd_load_f32(w + 124);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi14x4567, vk14x4567);
-
- const psimd_f32 vi15x0123 = psimd_load_f32(i15);
- const psimd_f32 vi15x4567 = psimd_load_f32(i15 + 4);
- i15 += 8;
-
- const psimd_f32 vk15x0123 = psimd_load_f32(w + 128);
- const psimd_f32 vk15x4567 = psimd_load_f32(w + 132);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi15x0123, vk15x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi15x4567, vk15x4567);
-
- const psimd_f32 vi16x0123 = psimd_load_f32(i16);
- const psimd_f32 vi16x4567 = psimd_load_f32(i16 + 4);
- i16 += 8;
-
- const psimd_f32 vk16x0123 = psimd_load_f32(w + 136);
- const psimd_f32 vk16x4567 = psimd_load_f32(w + 140);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi16x4567, vk16x4567);
-
- const psimd_f32 vi17x0123 = psimd_load_f32(i17);
- const psimd_f32 vi17x4567 = psimd_load_f32(i17 + 4);
- i17 += 8;
-
- const psimd_f32 vk17x0123 = psimd_load_f32(w + 144);
- const psimd_f32 vk17x4567 = psimd_load_f32(w + 148);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi17x0123, vk17x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi17x4567, vk17x4567);
-
- const psimd_f32 vi18x0123 = psimd_load_f32(i18);
- const psimd_f32 vi18x4567 = psimd_load_f32(i18 + 4);
- i18 += 8;
-
- const psimd_f32 vk18x0123 = psimd_load_f32(w + 152);
- const psimd_f32 vk18x4567 = psimd_load_f32(w + 156);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi18x4567, vk18x4567);
-
- const psimd_f32 vi19x0123 = psimd_load_f32(i19);
- const psimd_f32 vi19x4567 = psimd_load_f32(i19 + 4);
- i19 += 8;
-
- const psimd_f32 vk19x0123 = psimd_load_f32(w + 160);
- const psimd_f32 vk19x4567 = psimd_load_f32(w + 164);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi19x0123, vk19x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi19x4567, vk19x4567);
-
- const psimd_f32 vi20x0123 = psimd_load_f32(i20);
- const psimd_f32 vi20x4567 = psimd_load_f32(i20 + 4);
- i20 += 8;
-
- const psimd_f32 vk20x0123 = psimd_load_f32(w + 168);
- const psimd_f32 vk20x4567 = psimd_load_f32(w + 172);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi20x4567, vk20x4567);
-
- const psimd_f32 vi21x0123 = psimd_load_f32(i21);
- const psimd_f32 vi21x4567 = psimd_load_f32(i21 + 4);
- i21 += 8;
-
- const psimd_f32 vk21x0123 = psimd_load_f32(w + 176);
- const psimd_f32 vk21x4567 = psimd_load_f32(w + 180);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi21x0123, vk21x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi21x4567, vk21x4567);
-
- const psimd_f32 vi22x0123 = psimd_load_f32(i22);
- const psimd_f32 vi22x4567 = psimd_load_f32(i22 + 4);
- i22 += 8;
-
- const psimd_f32 vk22x0123 = psimd_load_f32(w + 184);
- const psimd_f32 vk22x4567 = psimd_load_f32(w + 188);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi22x4567, vk22x4567);
-
- const psimd_f32 vi23x0123 = psimd_load_f32(i23);
- const psimd_f32 vi23x4567 = psimd_load_f32(i23 + 4);
- i23 += 8;
-
- const psimd_f32 vk23x0123 = psimd_load_f32(w + 192);
- const psimd_f32 vk23x4567 = psimd_load_f32(w + 196);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi23x0123, vk23x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi23x4567, vk23x4567);
-
- const psimd_f32 vi24x0123 = psimd_load_f32(i24);
- const psimd_f32 vi24x4567 = psimd_load_f32(i24 + 4);
- i24 += 8;
-
- const psimd_f32 vk24x0123 = psimd_load_f32(w + 200);
- const psimd_f32 vk24x4567 = psimd_load_f32(w + 204);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi24x4567, vk24x4567);
-
- w += 208;
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- psimd_f32 vacc4567 = psimd_max_f32(vacc4567p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
- vacc4567 = psimd_min_f32(vacc4567, vmax);
-
- psimd_store_f32(output, vacc0123);
- psimd_store_f32(output + 4, vacc4567);
- output += 8;
- }
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- i0 += 4;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- i1 += 4;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- i2 += 4;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- i3 += 4;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- i4 += 4;
-
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 40);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- i5 += 4;
-
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 48);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- i6 += 4;
-
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 56);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- i7 += 4;
-
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 64);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- i8 += 4;
-
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 72);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- const psimd_f32 vi9x0123 = psimd_load_f32(i9);
- i9 += 4;
-
- const psimd_f32 vk9x0123 = psimd_load_f32(w + 80);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi9x0123, vk9x0123);
-
- const psimd_f32 vi10x0123 = psimd_load_f32(i10);
- i10 += 4;
-
- const psimd_f32 vk10x0123 = psimd_load_f32(w + 88);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123);
-
- const psimd_f32 vi11x0123 = psimd_load_f32(i11);
- i11 += 4;
-
- const psimd_f32 vk11x0123 = psimd_load_f32(w + 96);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi11x0123, vk11x0123);
-
- const psimd_f32 vi12x0123 = psimd_load_f32(i12);
- i12 += 4;
-
- const psimd_f32 vk12x0123 = psimd_load_f32(w + 104);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123);
-
- const psimd_f32 vi13x0123 = psimd_load_f32(i13);
- i13 += 4;
-
- const psimd_f32 vk13x0123 = psimd_load_f32(w + 112);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi13x0123, vk13x0123);
-
- const psimd_f32 vi14x0123 = psimd_load_f32(i14);
- i14 += 4;
-
- const psimd_f32 vk14x0123 = psimd_load_f32(w + 120);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123);
-
- const psimd_f32 vi15x0123 = psimd_load_f32(i15);
- i15 += 4;
-
- const psimd_f32 vk15x0123 = psimd_load_f32(w + 128);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi15x0123, vk15x0123);
-
- const psimd_f32 vi16x0123 = psimd_load_f32(i16);
- i16 += 4;
-
- const psimd_f32 vk16x0123 = psimd_load_f32(w + 136);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123);
-
- const psimd_f32 vi17x0123 = psimd_load_f32(i17);
- i17 += 4;
-
- const psimd_f32 vk17x0123 = psimd_load_f32(w + 144);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi17x0123, vk17x0123);
-
- const psimd_f32 vi18x0123 = psimd_load_f32(i18);
- i18 += 4;
-
- const psimd_f32 vk18x0123 = psimd_load_f32(w + 152);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123);
-
- const psimd_f32 vi19x0123 = psimd_load_f32(i19);
- i19 += 4;
-
- const psimd_f32 vk19x0123 = psimd_load_f32(w + 160);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi19x0123, vk19x0123);
-
- const psimd_f32 vi20x0123 = psimd_load_f32(i20);
- i20 += 4;
-
- const psimd_f32 vk20x0123 = psimd_load_f32(w + 168);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123);
-
- const psimd_f32 vi21x0123 = psimd_load_f32(i21);
- i21 += 4;
-
- const psimd_f32 vk21x0123 = psimd_load_f32(w + 176);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi21x0123, vk21x0123);
-
- const psimd_f32 vi22x0123 = psimd_load_f32(i22);
- i22 += 4;
-
- const psimd_f32 vk22x0123 = psimd_load_f32(w + 184);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123);
-
- const psimd_f32 vi23x0123 = psimd_load_f32(i23);
- i23 += 4;
-
- const psimd_f32 vk23x0123 = psimd_load_f32(w + 192);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi23x0123, vk23x0123);
-
- const psimd_f32 vi24x0123 = psimd_load_f32(i24);
- i24 += 4;
-
- const psimd_f32 vk24x0123 = psimd_load_f32(w + 200);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123);
-
- w += 4;
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 40);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 48);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 56);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 64);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 72);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- const psimd_f32 vi9x0123 = psimd_load_f32(i9);
- const psimd_f32 vk9x0123 = psimd_load_f32(w + 80);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi9x0123, vk9x0123);
-
- const psimd_f32 vi10x0123 = psimd_load_f32(i10);
- const psimd_f32 vk10x0123 = psimd_load_f32(w + 88);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi10x0123, vk10x0123);
-
- const psimd_f32 vi11x0123 = psimd_load_f32(i11);
- const psimd_f32 vk11x0123 = psimd_load_f32(w + 96);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi11x0123, vk11x0123);
-
- const psimd_f32 vi12x0123 = psimd_load_f32(i12);
- const psimd_f32 vk12x0123 = psimd_load_f32(w + 104);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi12x0123, vk12x0123);
-
- const psimd_f32 vi13x0123 = psimd_load_f32(i13);
- const psimd_f32 vk13x0123 = psimd_load_f32(w + 112);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi13x0123, vk13x0123);
-
- const psimd_f32 vi14x0123 = psimd_load_f32(i14);
- const psimd_f32 vk14x0123 = psimd_load_f32(w + 120);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi14x0123, vk14x0123);
-
- const psimd_f32 vi15x0123 = psimd_load_f32(i15);
- const psimd_f32 vk15x0123 = psimd_load_f32(w + 128);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi15x0123, vk15x0123);
-
- const psimd_f32 vi16x0123 = psimd_load_f32(i16);
- const psimd_f32 vk16x0123 = psimd_load_f32(w + 136);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi16x0123, vk16x0123);
-
- const psimd_f32 vi17x0123 = psimd_load_f32(i17);
- const psimd_f32 vk17x0123 = psimd_load_f32(w + 144);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi17x0123, vk17x0123);
-
- const psimd_f32 vi18x0123 = psimd_load_f32(i18);
- const psimd_f32 vk18x0123 = psimd_load_f32(w + 152);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi18x0123, vk18x0123);
-
- const psimd_f32 vi19x0123 = psimd_load_f32(i19);
- const psimd_f32 vk19x0123 = psimd_load_f32(w + 160);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi19x0123, vk19x0123);
-
- const psimd_f32 vi20x0123 = psimd_load_f32(i20);
- const psimd_f32 vk20x0123 = psimd_load_f32(w + 168);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi20x0123, vk20x0123);
-
- const psimd_f32 vi21x0123 = psimd_load_f32(i21);
- const psimd_f32 vk21x0123 = psimd_load_f32(w + 176);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi21x0123, vk21x0123);
-
- const psimd_f32 vi22x0123 = psimd_load_f32(i22);
- const psimd_f32 vk22x0123 = psimd_load_f32(w + 184);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi22x0123, vk22x0123);
-
- const psimd_f32 vi23x0123 = psimd_load_f32(i23);
- const psimd_f32 vk23x0123 = psimd_load_f32(w + 192);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi23x0123, vk23x0123);
-
- const psimd_f32 vi24x0123 = psimd_load_f32(i24);
- const psimd_f32 vk24x0123 = psimd_load_f32(w + 200);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi24x0123, vk24x0123);
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c b/src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c
deleted file mode 100644
index 03faf9037..000000000
--- a/src/f32-dwconv/gen/up8x4-minmax-psimd-acc2.c
+++ /dev/null
@@ -1,191 +0,0 @@
-// Auto-generated file. Do not edit!
-// Template: src/f32-dwconv/up-psimd.c.in
-// Generator: tools/xngen
-//
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- const float* i0 = input[0];
- assert(i0 != NULL);
- if XNN_UNPREDICTABLE(i0 != zero) {
- i0 = (const float*) ((uintptr_t) i0 + input_offset);
- }
- const float* i1 = input[1];
- assert(i1 != NULL);
- if XNN_UNPREDICTABLE(i1 != zero) {
- i1 = (const float*) ((uintptr_t) i1 + input_offset);
- }
- const float* i2 = input[2];
- assert(i2 != NULL);
- if XNN_UNPREDICTABLE(i2 != zero) {
- i2 = (const float*) ((uintptr_t) i2 + input_offset);
- }
- const float* i3 = input[3];
- assert(i3 != NULL);
- if XNN_UNPREDICTABLE(i3 != zero) {
- i3 = (const float*) ((uintptr_t) i3 + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= 8; c -= 8) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
- psimd_f32 vacc4567p0 = psimd_load_f32(w + 4);
-
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vi0x4567 = psimd_load_f32(i0 + 4);
- i0 += 8;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- const psimd_f32 vk0x4567 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi0x4567, vk0x4567);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vi1x4567 = psimd_load_f32(i1 + 4);
- i1 += 8;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- const psimd_f32 vk1x4567 = psimd_load_f32(w + 20);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
- psimd_f32 vacc4567p1 = psimd_mul_f32(vi1x4567, vk1x4567);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vi2x4567 = psimd_load_f32(i2 + 4);
- i2 += 8;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- const psimd_f32 vk2x4567 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi2x4567, vk2x4567);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vi3x4567 = psimd_load_f32(i3 + 4);
- i3 += 8;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- const psimd_f32 vk3x4567 = psimd_load_f32(w + 36);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi3x4567, vk3x4567);
-
- w += 40;
-
- // Add up all accumulators to vacc01234567p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
- vacc4567p0 = psimd_add_f32(vacc4567p0, vacc4567p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- psimd_f32 vacc4567 = psimd_max_f32(vacc4567p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
- vacc4567 = psimd_min_f32(vacc4567, vmax);
-
- psimd_store_f32(output, vacc0123);
- psimd_store_f32(output + 4, vacc4567);
- output += 8;
- }
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- i0 += 4;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- i1 += 4;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- i2 += 4;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- i3 += 4;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
-
- w += 4;
-
- // Add up all accumulators to vacc01234567p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
-
- // Add up all accumulators to vacc01234567p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/f32-dwconv/gen/up8x4-minmax-psimd.c b/src/f32-dwconv/gen/up8x4-minmax-psimd.c
deleted file mode 100644
index 99c2f7468..000000000
--- a/src/f32-dwconv/gen/up8x4-minmax-psimd.c
+++ /dev/null
@@ -1,184 +0,0 @@
-// Auto-generated file. Do not edit!
-// Template: src/f32-dwconv/up-psimd.c.in
-// Generator: tools/xngen
-//
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up8x4__psimd(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- const float* i0 = input[0];
- assert(i0 != NULL);
- if XNN_UNPREDICTABLE(i0 != zero) {
- i0 = (const float*) ((uintptr_t) i0 + input_offset);
- }
- const float* i1 = input[1];
- assert(i1 != NULL);
- if XNN_UNPREDICTABLE(i1 != zero) {
- i1 = (const float*) ((uintptr_t) i1 + input_offset);
- }
- const float* i2 = input[2];
- assert(i2 != NULL);
- if XNN_UNPREDICTABLE(i2 != zero) {
- i2 = (const float*) ((uintptr_t) i2 + input_offset);
- }
- const float* i3 = input[3];
- assert(i3 != NULL);
- if XNN_UNPREDICTABLE(i3 != zero) {
- i3 = (const float*) ((uintptr_t) i3 + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= 8; c -= 8) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
- psimd_f32 vacc4567p0 = psimd_load_f32(w + 4);
-
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vi0x4567 = psimd_load_f32(i0 + 4);
- i0 += 8;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- const psimd_f32 vk0x4567 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi0x4567, vk0x4567);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vi1x4567 = psimd_load_f32(i1 + 4);
- i1 += 8;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- const psimd_f32 vk1x4567 = psimd_load_f32(w + 20);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi1x4567, vk1x4567);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vi2x4567 = psimd_load_f32(i2 + 4);
- i2 += 8;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- const psimd_f32 vk2x4567 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi2x4567, vk2x4567);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vi3x4567 = psimd_load_f32(i3 + 4);
- i3 += 8;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- const psimd_f32 vk3x4567 = psimd_load_f32(w + 36);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi3x4567, vk3x4567);
-
- w += 40;
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- psimd_f32 vacc4567 = psimd_max_f32(vacc4567p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
- vacc4567 = psimd_min_f32(vacc4567, vmax);
-
- psimd_store_f32(output, vacc0123);
- psimd_store_f32(output + 4, vacc4567);
- output += 8;
- }
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- i0 += 4;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- i1 += 4;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- i2 += 4;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- i3 += 4;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
-
- w += 4;
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c b/src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c
deleted file mode 100644
index cb5ee907e..000000000
--- a/src/f32-dwconv/gen/up8x9-minmax-psimd-acc2.c
+++ /dev/null
@@ -1,311 +0,0 @@
-// Auto-generated file. Do not edit!
-// Template: src/f32-dwconv/up-psimd.c.in
-// Generator: tools/xngen
-//
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- const float* i0 = input[0];
- assert(i0 != NULL);
- if XNN_UNPREDICTABLE(i0 != zero) {
- i0 = (const float*) ((uintptr_t) i0 + input_offset);
- }
- const float* i1 = input[1];
- assert(i1 != NULL);
- if XNN_UNPREDICTABLE(i1 != zero) {
- i1 = (const float*) ((uintptr_t) i1 + input_offset);
- }
- const float* i2 = input[2];
- assert(i2 != NULL);
- if XNN_UNPREDICTABLE(i2 != zero) {
- i2 = (const float*) ((uintptr_t) i2 + input_offset);
- }
- const float* i3 = input[3];
- assert(i3 != NULL);
- if XNN_UNPREDICTABLE(i3 != zero) {
- i3 = (const float*) ((uintptr_t) i3 + input_offset);
- }
- const float* i4 = input[4];
- assert(i4 != NULL);
- if XNN_UNPREDICTABLE(i4 != zero) {
- i4 = (const float*) ((uintptr_t) i4 + input_offset);
- }
- const float* i5 = input[5];
- assert(i5 != NULL);
- if XNN_UNPREDICTABLE(i5 != zero) {
- i5 = (const float*) ((uintptr_t) i5 + input_offset);
- }
- const float* i6 = input[6];
- assert(i6 != NULL);
- if XNN_UNPREDICTABLE(i6 != zero) {
- i6 = (const float*) ((uintptr_t) i6 + input_offset);
- }
- const float* i7 = input[7];
- assert(i7 != NULL);
- if XNN_UNPREDICTABLE(i7 != zero) {
- i7 = (const float*) ((uintptr_t) i7 + input_offset);
- }
- const float* i8 = input[8];
- assert(i8 != NULL);
- if XNN_UNPREDICTABLE(i8 != zero) {
- i8 = (const float*) ((uintptr_t) i8 + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= 8; c -= 8) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
- psimd_f32 vacc4567p0 = psimd_load_f32(w + 4);
-
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vi0x4567 = psimd_load_f32(i0 + 4);
- i0 += 8;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- const psimd_f32 vk0x4567 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi0x4567, vk0x4567);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vi1x4567 = psimd_load_f32(i1 + 4);
- i1 += 8;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- const psimd_f32 vk1x4567 = psimd_load_f32(w + 20);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
- psimd_f32 vacc4567p1 = psimd_mul_f32(vi1x4567, vk1x4567);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vi2x4567 = psimd_load_f32(i2 + 4);
- i2 += 8;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- const psimd_f32 vk2x4567 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi2x4567, vk2x4567);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vi3x4567 = psimd_load_f32(i3 + 4);
- i3 += 8;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- const psimd_f32 vk3x4567 = psimd_load_f32(w + 36);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi3x4567, vk3x4567);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- const psimd_f32 vi4x4567 = psimd_load_f32(i4 + 4);
- i4 += 8;
-
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 40);
- const psimd_f32 vk4x4567 = psimd_load_f32(w + 44);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi4x4567, vk4x4567);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- const psimd_f32 vi5x4567 = psimd_load_f32(i5 + 4);
- i5 += 8;
-
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 48);
- const psimd_f32 vk5x4567 = psimd_load_f32(w + 52);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi5x4567, vk5x4567);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- const psimd_f32 vi6x4567 = psimd_load_f32(i6 + 4);
- i6 += 8;
-
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 56);
- const psimd_f32 vk6x4567 = psimd_load_f32(w + 60);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi6x4567, vk6x4567);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- const psimd_f32 vi7x4567 = psimd_load_f32(i7 + 4);
- i7 += 8;
-
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 64);
- const psimd_f32 vk7x4567 = psimd_load_f32(w + 68);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123);
- vacc4567p1 = psimd_qfma_f32(vacc4567p1, vi7x4567, vk7x4567);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- const psimd_f32 vi8x4567 = psimd_load_f32(i8 + 4);
- i8 += 8;
-
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 72);
- const psimd_f32 vk8x4567 = psimd_load_f32(w + 76);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi8x4567, vk8x4567);
-
- w += 80;
-
- // Add up all accumulators to vacc01234567p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
- vacc4567p0 = psimd_add_f32(vacc4567p0, vacc4567p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- psimd_f32 vacc4567 = psimd_max_f32(vacc4567p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
- vacc4567 = psimd_min_f32(vacc4567, vmax);
-
- psimd_store_f32(output, vacc0123);
- psimd_store_f32(output + 4, vacc4567);
- output += 8;
- }
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- i0 += 4;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- i1 += 4;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- i2 += 4;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- i3 += 4;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- i4 += 4;
-
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 40);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- i5 += 4;
-
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 48);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- i6 += 4;
-
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 56);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- i7 += 4;
-
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 64);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- i8 += 4;
-
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 72);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- w += 4;
-
- // Add up all accumulators to vacc01234567p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- psimd_f32 vacc0123p1 = psimd_mul_f32(vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 40);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 48);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 56);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 64);
- vacc0123p1 = psimd_qfma_f32(vacc0123p1, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 72);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- // Add up all accumulators to vacc01234567p0
- vacc0123p0 = psimd_add_f32(vacc0123p0, vacc0123p1);
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/f32-dwconv/gen/up8x9-minmax-psimd.c b/src/f32-dwconv/gen/up8x9-minmax-psimd.c
deleted file mode 100644
index 13163a721..000000000
--- a/src/f32-dwconv/gen/up8x9-minmax-psimd.c
+++ /dev/null
@@ -1,304 +0,0 @@
-// Auto-generated file. Do not edit!
-// Template: src/f32-dwconv/up-psimd.c.in
-// Generator: tools/xngen
-//
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up8x9__psimd(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- const float* i0 = input[0];
- assert(i0 != NULL);
- if XNN_UNPREDICTABLE(i0 != zero) {
- i0 = (const float*) ((uintptr_t) i0 + input_offset);
- }
- const float* i1 = input[1];
- assert(i1 != NULL);
- if XNN_UNPREDICTABLE(i1 != zero) {
- i1 = (const float*) ((uintptr_t) i1 + input_offset);
- }
- const float* i2 = input[2];
- assert(i2 != NULL);
- if XNN_UNPREDICTABLE(i2 != zero) {
- i2 = (const float*) ((uintptr_t) i2 + input_offset);
- }
- const float* i3 = input[3];
- assert(i3 != NULL);
- if XNN_UNPREDICTABLE(i3 != zero) {
- i3 = (const float*) ((uintptr_t) i3 + input_offset);
- }
- const float* i4 = input[4];
- assert(i4 != NULL);
- if XNN_UNPREDICTABLE(i4 != zero) {
- i4 = (const float*) ((uintptr_t) i4 + input_offset);
- }
- const float* i5 = input[5];
- assert(i5 != NULL);
- if XNN_UNPREDICTABLE(i5 != zero) {
- i5 = (const float*) ((uintptr_t) i5 + input_offset);
- }
- const float* i6 = input[6];
- assert(i6 != NULL);
- if XNN_UNPREDICTABLE(i6 != zero) {
- i6 = (const float*) ((uintptr_t) i6 + input_offset);
- }
- const float* i7 = input[7];
- assert(i7 != NULL);
- if XNN_UNPREDICTABLE(i7 != zero) {
- i7 = (const float*) ((uintptr_t) i7 + input_offset);
- }
- const float* i8 = input[8];
- assert(i8 != NULL);
- if XNN_UNPREDICTABLE(i8 != zero) {
- i8 = (const float*) ((uintptr_t) i8 + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= 8; c -= 8) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
- psimd_f32 vacc4567p0 = psimd_load_f32(w + 4);
-
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vi0x4567 = psimd_load_f32(i0 + 4);
- i0 += 8;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- const psimd_f32 vk0x4567 = psimd_load_f32(w + 12);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi0x4567, vk0x4567);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vi1x4567 = psimd_load_f32(i1 + 4);
- i1 += 8;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- const psimd_f32 vk1x4567 = psimd_load_f32(w + 20);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi1x4567, vk1x4567);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vi2x4567 = psimd_load_f32(i2 + 4);
- i2 += 8;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- const psimd_f32 vk2x4567 = psimd_load_f32(w + 28);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi2x4567, vk2x4567);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vi3x4567 = psimd_load_f32(i3 + 4);
- i3 += 8;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- const psimd_f32 vk3x4567 = psimd_load_f32(w + 36);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi3x4567, vk3x4567);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- const psimd_f32 vi4x4567 = psimd_load_f32(i4 + 4);
- i4 += 8;
-
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 40);
- const psimd_f32 vk4x4567 = psimd_load_f32(w + 44);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi4x4567, vk4x4567);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- const psimd_f32 vi5x4567 = psimd_load_f32(i5 + 4);
- i5 += 8;
-
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 48);
- const psimd_f32 vk5x4567 = psimd_load_f32(w + 52);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi5x4567, vk5x4567);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- const psimd_f32 vi6x4567 = psimd_load_f32(i6 + 4);
- i6 += 8;
-
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 56);
- const psimd_f32 vk6x4567 = psimd_load_f32(w + 60);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi6x4567, vk6x4567);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- const psimd_f32 vi7x4567 = psimd_load_f32(i7 + 4);
- i7 += 8;
-
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 64);
- const psimd_f32 vk7x4567 = psimd_load_f32(w + 68);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi7x4567, vk7x4567);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- const psimd_f32 vi8x4567 = psimd_load_f32(i8 + 4);
- i8 += 8;
-
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 72);
- const psimd_f32 vk8x4567 = psimd_load_f32(w + 76);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
- vacc4567p0 = psimd_qfma_f32(vacc4567p0, vi8x4567, vk8x4567);
-
- w += 80;
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- psimd_f32 vacc4567 = psimd_max_f32(vacc4567p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
- vacc4567 = psimd_min_f32(vacc4567, vmax);
-
- psimd_store_f32(output, vacc0123);
- psimd_store_f32(output + 4, vacc4567);
- output += 8;
- }
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- i0 += 4;
-
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- i1 += 4;
-
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- i2 += 4;
-
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- i3 += 4;
-
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- i4 += 4;
-
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 40);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- i5 += 4;
-
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 48);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- i6 += 4;
-
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 56);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- i7 += 4;
-
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 64);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- i8 += 4;
-
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 72);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
- w += 4;
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
-
- const psimd_f32 vi0x0123 = psimd_load_f32(i0);
- const psimd_f32 vk0x0123 = psimd_load_f32(w + 8);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi0x0123, vk0x0123);
-
- const psimd_f32 vi1x0123 = psimd_load_f32(i1);
- const psimd_f32 vk1x0123 = psimd_load_f32(w + 16);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi1x0123, vk1x0123);
-
- const psimd_f32 vi2x0123 = psimd_load_f32(i2);
- const psimd_f32 vk2x0123 = psimd_load_f32(w + 24);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi2x0123, vk2x0123);
-
- const psimd_f32 vi3x0123 = psimd_load_f32(i3);
- const psimd_f32 vk3x0123 = psimd_load_f32(w + 32);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi3x0123, vk3x0123);
-
- const psimd_f32 vi4x0123 = psimd_load_f32(i4);
- const psimd_f32 vk4x0123 = psimd_load_f32(w + 40);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi4x0123, vk4x0123);
-
- const psimd_f32 vi5x0123 = psimd_load_f32(i5);
- const psimd_f32 vk5x0123 = psimd_load_f32(w + 48);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi5x0123, vk5x0123);
-
- const psimd_f32 vi6x0123 = psimd_load_f32(i6);
- const psimd_f32 vk6x0123 = psimd_load_f32(w + 56);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi6x0123, vk6x0123);
-
- const psimd_f32 vi7x0123 = psimd_load_f32(i7);
- const psimd_f32 vk7x0123 = psimd_load_f32(w + 64);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi7x0123, vk7x0123);
-
- const psimd_f32 vi8x0123 = psimd_load_f32(i8);
- const psimd_f32 vk8x0123 = psimd_load_f32(w + 72);
- vacc0123p0 = psimd_qfma_f32(vacc0123p0, vi8x0123, vk8x0123);
-
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/f32-dwconv/up-psimd.c.in b/src/f32-dwconv/up-psimd.c.in
deleted file mode 100644
index 0b7cee90c..000000000
--- a/src/f32-dwconv/up-psimd.c.in
+++ /dev/null
@@ -1,154 +0,0 @@
-// Copyright 2019 Google LLC
-//
-// This source code is licensed under the BSD-style license found in the
-// LICENSE file in the root directory of this source tree.
-
-$assert CHANNEL_TILE % 4 == 0
-$assert KERNEL_TILE >= 2
-$assert ACCUMULATORS >= 1
-$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-#include <assert.h>
-
-#include <psimd.h>
-
-#include <xnnpack/dwconv.h>
-
-
-void xnn_f32_dwconv_minmax_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__psimd${"" if ACCUMULATORS == 1 else "_acc%d" % ACCUMULATORS}(
- size_t channels,
- size_t output_width,
- const float** input,
- const float* weights,
- float* output,
- size_t input_stride,
- size_t output_increment,
- size_t input_offset,
- const float* zero,
- const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN
-{
- assert(channels != 0);
- assert(output_width != 0);
-
- const psimd_f32 vmax = psimd_load_splat_f32(&params->scalar.max);
- const psimd_f32 vmin = psimd_load_splat_f32(&params->scalar.min);
- do {
- $for K in range(KERNEL_TILE):
- const float* i${K} = input[${K}];
- assert(i${K} != NULL);
- if XNN_UNPREDICTABLE(i${K} != zero) {
- i${K} = (const float*) ((uintptr_t) i${K} + input_offset);
- }
- input = (const float**) ((uintptr_t) input + input_stride);
-
- size_t c = channels;
- const float* w = weights;
- for (; c >= ${CHANNEL_TILE}; c -= ${CHANNEL_TILE}) {
- psimd_f32 vacc${ABC[0:4]}p0 = psimd_load_f32(w);
- $for C in range(4, CHANNEL_TILE, 4):
- psimd_f32 vacc${ABC[C:C+4]}p0 = psimd_load_f32(w + ${C});
-
- $for K in range(KERNEL_TILE):
-
- const psimd_f32 vi${K}x${ABC[0:4]} = psimd_load_f32(i${K});
- $for C in range(4, CHANNEL_TILE, 4):
- const psimd_f32 vi${K}x${ABC[C:C+4]} = psimd_load_f32(i${K} + ${C});
- i${K} += ${CHANNEL_TILE};
-
- $for C in range(0, CHANNEL_TILE, 4):
- const psimd_f32 vk${K}x${ABC[C:C+4]} = psimd_load_f32(w + ${(K + 1) * CHANNEL_TILE + C});
- $for C in range(0, CHANNEL_TILE, 4):
- $if 1 <= K < ACCUMULATORS:
- psimd_f32 vacc${ABC[C:C+4]}p${K} = psimd_mul_f32(vi${K}x${ABC[C:C+4]}, vk${K}x${ABC[C:C+4]});
- $else:
- vacc${ABC[C:C+4]}p${K % ACCUMULATORS} = psimd_qfma_f32(vacc${ABC[C:C+4]}p${K % ACCUMULATORS}, vi${K}x${ABC[C:C+4]}, vk${K}x${ABC[C:C+4]});
-
- w += ${(KERNEL_TILE + 1) * CHANNEL_TILE};
-
- $if ACCUMULATORS > 1:
- // Add up all accumulators to vacc${ABC[0:CHANNEL_TILE]}p0
- $ACC_SLICE = 1
- $while ACC_SLICE < ACCUMULATORS:
- $for A in range(0, ACCUMULATORS, ACC_SLICE * 2):
- $if A + ACC_SLICE < ACCUMULATORS:
- $for C in range(0, CHANNEL_TILE, 4):
- vacc${ABC[C:C+4]}p${A} = psimd_add_f32(vacc${ABC[C:C+4]}p${A}, vacc${ABC[C:C+4]}p${A + ACC_SLICE});
- $ACC_SLICE *= 2
-
- $for C in range(0, CHANNEL_TILE, 4):
- psimd_f32 vacc${ABC[C:C+4]} = psimd_max_f32(vacc${ABC[C:C+4]}p0, vmin);
- $for C in range(0, CHANNEL_TILE, 4):
- vacc${ABC[C:C+4]} = psimd_min_f32(vacc${ABC[C:C+4]}, vmax);
-
- psimd_store_f32(output, vacc${ABC[0:4]});
- $for C in range(4, CHANNEL_TILE, 4):
- psimd_store_f32(output + ${C}, vacc${ABC[C:C+4]});
- output += ${CHANNEL_TILE};
- }
- $if CHANNEL_TILE > 4:
- for (; c >= 4; c -= 4) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
- $for K in range(KERNEL_TILE):
-
- const psimd_f32 vi${K}x0123 = psimd_load_f32(i${K});
- i${K} += 4;
-
- const psimd_f32 vk${K}x0123 = psimd_load_f32(w + ${(K + 1) * CHANNEL_TILE});
- $if 1 <= K < ACCUMULATORS:
- psimd_f32 vacc0123p${K} = psimd_mul_f32(vi${K}x0123, vk${K}x0123);
- $else:
- vacc0123p${K % ACCUMULATORS} = psimd_qfma_f32(vacc0123p${K % ACCUMULATORS}, vi${K}x0123, vk${K}x0123);
-
- w += 4;
-
- $if ACCUMULATORS > 1:
- // Add up all accumulators to vacc${ABC[0:CHANNEL_TILE]}p0
- $ACC_SLICE = 1
- $while ACC_SLICE < ACCUMULATORS:
- $for A in range(0, ACCUMULATORS, ACC_SLICE * 2):
- $if A + ACC_SLICE < ACCUMULATORS:
- vacc0123p${A} = psimd_add_f32(vacc0123p${A}, vacc0123p${A + ACC_SLICE});
- $ACC_SLICE *= 2
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- psimd_store_f32(output, vacc0123);
- output += 4;
- }
- if XNN_UNLIKELY(c != 0) {
- psimd_f32 vacc0123p0 = psimd_load_f32(w);
- $for K in range(KERNEL_TILE):
-
- const psimd_f32 vi${K}x0123 = psimd_load_f32(i${K});
- const psimd_f32 vk${K}x0123 = psimd_load_f32(w + ${(K+1) * CHANNEL_TILE});
- $if 1 <= K < ACCUMULATORS:
- psimd_f32 vacc0123p${K} = psimd_mul_f32(vi${K}x0123, vk${K}x0123);
- $else:
- vacc0123p${K % ACCUMULATORS} = psimd_qfma_f32(vacc0123p${K % ACCUMULATORS}, vi${K}x0123, vk${K}x0123);
-
- $if ACCUMULATORS > 1:
- // Add up all accumulators to vacc${ABC[0:CHANNEL_TILE]}p0
- $ACC_SLICE = 1
- $while ACC_SLICE < ACCUMULATORS:
- $for A in range(0, ACCUMULATORS, ACC_SLICE * 2):
- $if A + ACC_SLICE < ACCUMULATORS:
- vacc0123p${A} = psimd_add_f32(vacc0123p${A}, vacc0123p${A + ACC_SLICE});
- $ACC_SLICE *= 2
-
- psimd_f32 vacc0123 = psimd_max_f32(vacc0123p0, vmin);
- vacc0123 = psimd_min_f32(vacc0123, vmax);
-
- if (c & 2) {
- psimd_store2_f32(output, vacc0123);
- vacc0123 = psimd_concat_hi_f32(vacc0123, vacc0123);
- output += 2;
- }
- if (c & 1) {
- psimd_store1_f32(output, vacc0123);
- output += 1;
- }
- }
-
- output = (float*) ((uintptr_t) output + output_increment);
- } while (--output_width != 0);
-}
diff --git a/src/xnnpack/dwconv.h b/src/xnnpack/dwconv.h
index 0a2df8238..46d86097d 100644
--- a/src/xnnpack/dwconv.h
+++ b/src/xnnpack/dwconv.h
@@ -52,10 +52,6 @@ DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__neonfma_acc2)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__neonfma)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__neonfma_acc2)
-DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd)
-DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2)
-DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd)
-DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__sse)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__sse_acc2)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__sse)
@@ -83,10 +79,6 @@ DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__neonfma_acc2)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__aarch64_neonfma)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__aarch64_neonfma_cortex_a55)
-DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd)
-DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2)
-DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd)
-DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__sse)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__sse_acc2)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__sse)
@@ -112,10 +104,6 @@ DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__neonfma_acc2)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__neonfma)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__neonfma_acc2)
-DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd)
-DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2)
-DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd)
-DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__sse)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__sse_acc2)
DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__sse)
diff --git a/test/f32-dwconv-minmax.cc b/test/f32-dwconv-minmax.cc
index cb2418a93..d312c59e5 100644
--- a/test/f32-dwconv-minmax.cc
+++ b/test/f32-dwconv-minmax.cc
@@ -13793,2238 +13793,6 @@
#endif // XNN_ARCH_X86 || XNN_ARCH_X86_64
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_eq_4) {
- TEST_REQUIRES_PSIMD;
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(4)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_div_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_div_4_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_div_4_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_lt_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 1; channels < 4; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_gt_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_gt_4_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, c_gt_4_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, multipixel) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .width(3)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, multipixel_with_step) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- for (size_t step = 2; step <= 25; step++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .width(3)
- .step(step)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, multipixel_with_output_stride) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(4)
- .width(5)
- .output_stride(23)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, multipixel_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .width(3)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, multipixel_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .width(3)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, input_offset) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .input_offset(112)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD, zero) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t mz = 0; mz < 25; mz++) {
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .input_offset(112)
- .zero_index(mz)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_eq_4) {
- TEST_REQUIRES_PSIMD;
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(4)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_div_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_div_4_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_div_4_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_lt_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 1; channels < 4; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_gt_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_gt_4_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, c_gt_4_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, multipixel) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .width(3)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, multipixel_with_step) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- for (size_t step = 2; step <= 25; step++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .width(3)
- .step(step)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, multipixel_with_output_stride) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(4)
- .width(5)
- .output_stride(23)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, multipixel_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .width(3)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, multipixel_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .width(3)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, input_offset) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .input_offset(112)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X25__PSIMD_ACC2, zero) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t mz = 0; mz < 25; mz++) {
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(25)
- .channels(channels)
- .input_offset(112)
- .zero_index(mz)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_eq_8) {
- TEST_REQUIRES_PSIMD;
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(8)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_div_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_div_8_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_div_8_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_lt_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 1; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_gt_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_gt_8_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, c_gt_8_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, multipixel) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .width(3)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, multipixel_with_step) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- for (size_t step = 2; step <= 25; step++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .width(3)
- .step(step)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, multipixel_with_output_stride) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(8)
- .width(5)
- .output_stride(43)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, multipixel_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .width(3)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, multipixel_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .width(3)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, input_offset) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .input_offset(176)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD, zero) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t mz = 0; mz < 25; mz++) {
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .input_offset(176)
- .zero_index(mz)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_eq_8) {
- TEST_REQUIRES_PSIMD;
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(8)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_div_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_div_8_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_div_8_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_lt_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 1; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_gt_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_gt_8_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, c_gt_8_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, multipixel) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .width(3)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, multipixel_with_step) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- for (size_t step = 2; step <= 25; step++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .width(3)
- .step(step)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, multipixel_with_output_stride) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(8)
- .width(5)
- .output_stride(43)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, multipixel_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .width(3)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, multipixel_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .width(3)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, input_offset) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .input_offset(176)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X25__PSIMD_ACC2, zero) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t mz = 0; mz < 25; mz++) {
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(25)
- .channels(channels)
- .input_offset(176)
- .zero_index(mz)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_eq_4) {
- TEST_REQUIRES_PSIMD;
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(4)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_div_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_div_4_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_div_4_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_lt_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 1; channels < 4; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_gt_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_gt_4_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, c_gt_4_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, multipixel) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .width(3)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, multipixel_with_step) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- for (size_t step = 2; step <= 9; step++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .width(3)
- .step(step)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, multipixel_with_output_stride) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(4)
- .width(5)
- .output_stride(23)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, multipixel_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .width(3)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, multipixel_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .width(3)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, input_offset) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .input_offset(112)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD, zero) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t mz = 0; mz < 9; mz++) {
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .input_offset(112)
- .zero_index(mz)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_eq_4) {
- TEST_REQUIRES_PSIMD;
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(4)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_div_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_div_4_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_div_4_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_lt_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 1; channels < 4; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_gt_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_gt_4_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, c_gt_4_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, multipixel) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .width(3)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, multipixel_with_step) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- for (size_t step = 2; step <= 9; step++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .width(3)
- .step(step)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, multipixel_with_output_stride) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(4)
- .width(5)
- .output_stride(23)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, multipixel_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .width(3)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, multipixel_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .width(3)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, input_offset) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .input_offset(112)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X9__PSIMD_ACC2, zero) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t mz = 0; mz < 9; mz++) {
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(9)
- .channels(channels)
- .input_offset(112)
- .zero_index(mz)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_eq_8) {
- TEST_REQUIRES_PSIMD;
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(8)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_div_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_div_8_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_div_8_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_lt_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 1; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_gt_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_gt_8_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, c_gt_8_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, multipixel) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .width(3)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, multipixel_with_step) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- for (size_t step = 2; step <= 9; step++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .width(3)
- .step(step)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, multipixel_with_output_stride) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(8)
- .width(5)
- .output_stride(43)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, multipixel_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .width(3)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, multipixel_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .width(3)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, input_offset) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .input_offset(176)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD, zero) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t mz = 0; mz < 9; mz++) {
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .input_offset(176)
- .zero_index(mz)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_eq_8) {
- TEST_REQUIRES_PSIMD;
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(8)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_div_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_div_8_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_div_8_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_lt_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 1; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_gt_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_gt_8_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, c_gt_8_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, multipixel) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .width(3)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, multipixel_with_step) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- for (size_t step = 2; step <= 9; step++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .width(3)
- .step(step)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, multipixel_with_output_stride) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(8)
- .width(5)
- .output_stride(43)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, multipixel_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .width(3)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, multipixel_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .width(3)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, input_offset) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .input_offset(176)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X9__PSIMD_ACC2, zero) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t mz = 0; mz < 9; mz++) {
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(9)
- .channels(channels)
- .input_offset(176)
- .zero_index(mz)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_eq_4) {
- TEST_REQUIRES_PSIMD;
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(4)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_div_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_div_4_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_div_4_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_lt_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 1; channels < 4; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_gt_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_gt_4_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, c_gt_4_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, multipixel) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .width(3)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, multipixel_with_step) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- for (size_t step = 2; step <= 4; step++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .width(3)
- .step(step)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, multipixel_with_output_stride) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(4)
- .width(5)
- .output_stride(23)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, multipixel_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .width(3)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, multipixel_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .width(3)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, input_offset) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .input_offset(112)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD, zero) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t mz = 0; mz < 4; mz++) {
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .input_offset(112)
- .zero_index(mz)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_eq_4) {
- TEST_REQUIRES_PSIMD;
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(4)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_div_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_div_4_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_div_4_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_lt_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 1; channels < 4; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_gt_4) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_gt_4_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, c_gt_4_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 5; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, multipixel) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .width(3)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, multipixel_with_step) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- for (size_t step = 2; step <= 4; step++) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .width(3)
- .step(step)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, multipixel_with_output_stride) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(4)
- .width(5)
- .output_stride(23)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, multipixel_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .width(3)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, multipixel_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 20; channels += 3) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .width(3)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, input_offset) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .input_offset(112)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP4X4__PSIMD_ACC2, zero) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t mz = 0; mz < 4; mz++) {
- for (uint32_t channels = 8; channels < 64; channels += 12) {
- DWConvMicrokernelTester()
- .cr(4)
- .kr(4)
- .channels(channels)
- .input_offset(112)
- .zero_index(mz)
- .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_eq_8) {
- TEST_REQUIRES_PSIMD;
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(8)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_div_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_div_8_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_div_8_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_lt_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 1; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_gt_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_gt_8_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, c_gt_8_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, multipixel) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .width(3)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, multipixel_with_step) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- for (size_t step = 2; step <= 4; step++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .width(3)
- .step(step)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, multipixel_with_output_stride) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(8)
- .width(5)
- .output_stride(43)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, multipixel_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .width(3)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, multipixel_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .width(3)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, input_offset) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .input_offset(176)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD, zero) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t mz = 0; mz < 4; mz++) {
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .input_offset(176)
- .zero_index(mz)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
-#if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_eq_8) {
- TEST_REQUIRES_PSIMD;
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(8)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_div_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_div_8_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_div_8_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_lt_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 1; channels < 8; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_gt_8) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_gt_8_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, c_gt_8_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 9; channels < 16; channels++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, multipixel) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .width(3)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, multipixel_with_step) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- for (size_t step = 2; step <= 4; step++) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .width(3)
- .step(step)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, multipixel_with_output_stride) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(8)
- .width(5)
- .output_stride(43)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, multipixel_with_qmin) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .width(3)
- .qmin(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, multipixel_with_qmax) {
- TEST_REQUIRES_PSIMD;
- for (size_t channels = 1; channels <= 40; channels += 7) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .width(3)
- .qmax(128)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, input_offset) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .input_offset(176)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
-
- TEST(F32_DWCONV_MINMAX_UP8X4__PSIMD_ACC2, zero) {
- TEST_REQUIRES_PSIMD;
- for (uint32_t mz = 0; mz < 4; mz++) {
- for (uint32_t channels = 16; channels < 128; channels += 24) {
- DWConvMicrokernelTester()
- .cr(8)
- .kr(4)
- .channels(channels)
- .input_offset(176)
- .zero_index(mz)
- .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2, DWConvMicrokernelTester::Variant::Scalar);
- }
- }
- }
-#endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC
-
-
#if XNN_ARCH_WASMSIMD
TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, c_eq_4) {
DWConvMicrokernelTester()
diff --git a/test/f32-dwconv-minmax.yaml b/test/f32-dwconv-minmax.yaml
index 2477eccb2..fdaccc7eb 100644
--- a/test/f32-dwconv-minmax.yaml
+++ b/test/f32-dwconv-minmax.yaml
@@ -79,18 +79,6 @@
- name: xnn_f32_dwconv_minmax_ukernel_up16x4__avx512f_acc2
- name: xnn_f32_dwconv_minmax_ukernel_up32x4__avx512f
- name: xnn_f32_dwconv_minmax_ukernel_up32x4__avx512f_acc2
-- name: xnn_f32_dwconv_minmax_ukernel_up4x25__psimd
-- name: xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2
-- name: xnn_f32_dwconv_minmax_ukernel_up8x25__psimd
-- name: xnn_f32_dwconv_minmax_ukernel_up8x25__psimd_acc2
-- name: xnn_f32_dwconv_minmax_ukernel_up4x9__psimd
-- name: xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2
-- name: xnn_f32_dwconv_minmax_ukernel_up8x9__psimd
-- name: xnn_f32_dwconv_minmax_ukernel_up8x9__psimd_acc2
-- name: xnn_f32_dwconv_minmax_ukernel_up4x4__psimd
-- name: xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2
-- name: xnn_f32_dwconv_minmax_ukernel_up8x4__psimd
-- name: xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2
- name: xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm
- name: xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm
- name: xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm