diff options
author | Marat Dukhan <maratek@google.com> | 2020-06-16 08:36:47 -0700 |
---|---|---|
committer | XNNPACK Team <xnnpack-github-robot@google.com> | 2020-06-16 08:37:16 -0700 |
commit | ac014d79c35e9b4811ee835ebfdfdc2cbc5220d7 (patch) | |
tree | 3bab084a13c108d6ab704109f6821ee70868d4d4 | |
parent | 1bbf96bb92185ffe9e7cb7631f7d997d6b00ec88 (diff) | |
download | XNNPACK-ac014d79c35e9b4811ee835ebfdfdc2cbc5220d7.tar.gz |
DWCONV microkernels in WAsm SIMD intrinsics
PiperOrigin-RevId: 316687431
128 files changed, 13305 insertions, 907 deletions
diff --git a/BUILD.bazel b/BUILD.bazel index afb375739..a5694aa70 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -473,6 +473,30 @@ WASM_UKERNELS = [ ] WASMSIMD_UKERNELS = [ + "src/f32-dwconv/gen/up4x25-minmax-wasmsimd-acc2-arm.c", + "src/f32-dwconv/gen/up4x25-minmax-wasmsimd-arm.c", + "src/f32-dwconv/gen/up4x4-minmax-wasmsimd-acc2-arm.c", + "src/f32-dwconv/gen/up4x4-minmax-wasmsimd-arm.c", + "src/f32-dwconv/gen/up4x9-minmax-wasmsimd-acc2-arm.c", + "src/f32-dwconv/gen/up4x9-minmax-wasmsimd-arm.c", + "src/f32-dwconv/gen/up8x25-minmax-wasmsimd-acc2-arm.c", + "src/f32-dwconv/gen/up8x25-minmax-wasmsimd-arm.c", + "src/f32-dwconv/gen/up8x4-minmax-wasmsimd-acc2-arm.c", + "src/f32-dwconv/gen/up8x4-minmax-wasmsimd-arm.c", + "src/f32-dwconv/gen/up8x9-minmax-wasmsimd-acc2-arm.c", + "src/f32-dwconv/gen/up8x9-minmax-wasmsimd-arm.c", + "src/f32-dwconv/gen/up4x25-minmax-wasmsimd-acc2-x86.c", + "src/f32-dwconv/gen/up4x25-minmax-wasmsimd-x86.c", + "src/f32-dwconv/gen/up4x4-minmax-wasmsimd-acc2-x86.c", + "src/f32-dwconv/gen/up4x4-minmax-wasmsimd-x86.c", + "src/f32-dwconv/gen/up4x9-minmax-wasmsimd-acc2-x86.c", + "src/f32-dwconv/gen/up4x9-minmax-wasmsimd-x86.c", + "src/f32-dwconv/gen/up8x25-minmax-wasmsimd-acc2-x86.c", + "src/f32-dwconv/gen/up8x25-minmax-wasmsimd-x86.c", + "src/f32-dwconv/gen/up8x4-minmax-wasmsimd-acc2-x86.c", + "src/f32-dwconv/gen/up8x4-minmax-wasmsimd-x86.c", + "src/f32-dwconv/gen/up8x9-minmax-wasmsimd-acc2-x86.c", + "src/f32-dwconv/gen/up8x9-minmax-wasmsimd-x86.c", "src/f32-gemm/gen/1x8-minmax-wasmsimd-loadsplat-arm.c", "src/f32-gemm/gen/1x8-minmax-wasmsimd-splat-arm.c", "src/f32-gemm/gen/1x8s4-minmax-wasmsimd-arm.c", diff --git a/bench/f32-dwconv-e2e.cc b/bench/f32-dwconv-e2e.cc index 83334bae5..7c367e8f9 100644 --- a/bench/f32-dwconv-e2e.cc +++ b/bench/f32-dwconv-e2e.cc @@ -294,6 +294,80 @@ static void DWConvEnd2EndBenchmark( BENCHMARK_CAPTURE(f32_dwconv_up32x9__avx512f_acc2, mobilenet_v2, models::MobileNetV2)->Unit(benchmark::kMicrosecond)->UseRealTime(); #endif // XNN_ARCH_X86 || XNN_ARCH_X86_64 +#if XNN_ARCH_WASMSIMD + static void f32_dwconv_up4x9__wasmsimd_arm(benchmark::State& state, models::ExecutionPlanFactory model) { + DWConvEnd2EndBenchmark(state, model, + xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, + 4 /* cr */, 9 /* mr */); + } + + static void f32_dwconv_up4x9__wasmsimd_acc2_arm(benchmark::State& state, models::ExecutionPlanFactory model) { + DWConvEnd2EndBenchmark(state, model, + xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, + 4 /* cr */, 9 /* mr */); + } + + static void f32_dwconv_up8x9__wasmsimd_arm(benchmark::State& state, models::ExecutionPlanFactory model) { + DWConvEnd2EndBenchmark(state, model, + xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, + 8 /* cr */, 9 /* mr */); + } + + static void f32_dwconv_up8x9__wasmsimd_acc2_arm(benchmark::State& state, models::ExecutionPlanFactory model) { + DWConvEnd2EndBenchmark(state, model, + xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, + 8 /* cr */, 9 /* mr */); + } + + static void f32_dwconv_up4x9__wasmsimd_x86(benchmark::State& state, models::ExecutionPlanFactory model) { + DWConvEnd2EndBenchmark(state, model, + xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, + 4 /* cr */, 9 /* mr */); + } + + static void f32_dwconv_up4x9__wasmsimd_acc2_x86(benchmark::State& state, models::ExecutionPlanFactory model) { + DWConvEnd2EndBenchmark(state, model, + xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, + 4 /* cr */, 9 /* mr */); + } + + static void f32_dwconv_up8x9__wasmsimd_x86(benchmark::State& state, models::ExecutionPlanFactory model) { + DWConvEnd2EndBenchmark(state, model, + xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, + 8 /* cr */, 9 /* mr */); + } + + static void f32_dwconv_up8x9__wasmsimd_acc2_x86(benchmark::State& state, models::ExecutionPlanFactory model) { + DWConvEnd2EndBenchmark(state, model, + xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, + 8 /* cr */, 9 /* mr */); + } + + BENCHMARK_CAPTURE(f32_dwconv_up4x9__wasmsimd_arm, mobilenet_v1, models::MobileNetV1)->Unit(benchmark::kMicrosecond)->UseRealTime(); + BENCHMARK_CAPTURE(f32_dwconv_up4x9__wasmsimd_arm, mobilenet_v2, models::MobileNetV2)->Unit(benchmark::kMicrosecond)->UseRealTime(); + + BENCHMARK_CAPTURE(f32_dwconv_up4x9__wasmsimd_acc2_arm, mobilenet_v1, models::MobileNetV1)->Unit(benchmark::kMicrosecond)->UseRealTime(); + BENCHMARK_CAPTURE(f32_dwconv_up4x9__wasmsimd_acc2_arm, mobilenet_v2, models::MobileNetV2)->Unit(benchmark::kMicrosecond)->UseRealTime(); + + BENCHMARK_CAPTURE(f32_dwconv_up8x9__wasmsimd_arm, mobilenet_v1, models::MobileNetV1)->Unit(benchmark::kMicrosecond)->UseRealTime(); + BENCHMARK_CAPTURE(f32_dwconv_up8x9__wasmsimd_arm, mobilenet_v2, models::MobileNetV2)->Unit(benchmark::kMicrosecond)->UseRealTime(); + + BENCHMARK_CAPTURE(f32_dwconv_up8x9__wasmsimd_acc2_arm, mobilenet_v1, models::MobileNetV1)->Unit(benchmark::kMicrosecond)->UseRealTime(); + BENCHMARK_CAPTURE(f32_dwconv_up8x9__wasmsimd_acc2_arm, mobilenet_v2, models::MobileNetV2)->Unit(benchmark::kMicrosecond)->UseRealTime(); + + BENCHMARK_CAPTURE(f32_dwconv_up4x9__wasmsimd_x86, mobilenet_v1, models::MobileNetV1)->Unit(benchmark::kMicrosecond)->UseRealTime(); + BENCHMARK_CAPTURE(f32_dwconv_up4x9__wasmsimd_x86, mobilenet_v2, models::MobileNetV2)->Unit(benchmark::kMicrosecond)->UseRealTime(); + + BENCHMARK_CAPTURE(f32_dwconv_up4x9__wasmsimd_acc2_x86, mobilenet_v1, models::MobileNetV1)->Unit(benchmark::kMicrosecond)->UseRealTime(); + BENCHMARK_CAPTURE(f32_dwconv_up4x9__wasmsimd_acc2_x86, mobilenet_v2, models::MobileNetV2)->Unit(benchmark::kMicrosecond)->UseRealTime(); + + BENCHMARK_CAPTURE(f32_dwconv_up8x9__wasmsimd_x86, mobilenet_v1, models::MobileNetV1)->Unit(benchmark::kMicrosecond)->UseRealTime(); + BENCHMARK_CAPTURE(f32_dwconv_up8x9__wasmsimd_x86, mobilenet_v2, models::MobileNetV2)->Unit(benchmark::kMicrosecond)->UseRealTime(); + + BENCHMARK_CAPTURE(f32_dwconv_up8x9__wasmsimd_acc2_x86, mobilenet_v1, models::MobileNetV1)->Unit(benchmark::kMicrosecond)->UseRealTime(); + BENCHMARK_CAPTURE(f32_dwconv_up8x9__wasmsimd_acc2_x86, mobilenet_v2, models::MobileNetV2)->Unit(benchmark::kMicrosecond)->UseRealTime(); +#endif // XNN_ARCH_WASMSIMD + #if !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC static void f32_dwconv_up4x9__psimd(benchmark::State& state, models::ExecutionPlanFactory model) { DWConvEnd2EndBenchmark(state, model, diff --git a/scripts/generate-f32-dwconv.sh b/scripts/generate-f32-dwconv.sh index 2e1055c52..104a2ed72 100755 --- a/scripts/generate-f32-dwconv.sh +++ b/scripts/generate-f32-dwconv.sh @@ -67,6 +67,37 @@ tools/xngen src/f32-dwconv/up-scalar.c.in -D CHANNEL_TILE=1 -D KERNEL_TILE=25 -D tools/xngen src/f32-dwconv/up-scalar.c.in -D CHANNEL_TILE=2 -D KERNEL_TILE=25 -D ACCUMULATORS=1 -D WASM=1 -D ACTIVATION=MINMAX -o src/f32-dwconv/gen/up2x25-minmax-wasm.c tools/xngen src/f32-dwconv/up-scalar.c.in -D CHANNEL_TILE=2 -D KERNEL_TILE=25 -D ACCUMULATORS=2 -D WASM=1 -D ACTIVATION=MINMAX -o src/f32-dwconv/gen/up2x25-minmax-wasm-acc2.c +################################## WAsm SIMD ################################## +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=4 -D ACCUMULATORS=1 -D X86=0 -o src/f32-dwconv/gen/up4x4-minmax-wasmsimd-arm.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=4 -D ACCUMULATORS=2 -D X86=0 -o src/f32-dwconv/gen/up4x4-minmax-wasmsimd-acc2-arm.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=4 -D ACCUMULATORS=1 -D X86=0 -o src/f32-dwconv/gen/up8x4-minmax-wasmsimd-arm.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=4 -D ACCUMULATORS=2 -D X86=0 -o src/f32-dwconv/gen/up8x4-minmax-wasmsimd-acc2-arm.c + +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=4 -D ACCUMULATORS=1 -D X86=1 -o src/f32-dwconv/gen/up4x4-minmax-wasmsimd-x86.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=4 -D ACCUMULATORS=2 -D X86=1 -o src/f32-dwconv/gen/up4x4-minmax-wasmsimd-acc2-x86.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=4 -D ACCUMULATORS=1 -D X86=1 -o src/f32-dwconv/gen/up8x4-minmax-wasmsimd-x86.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=4 -D ACCUMULATORS=2 -D X86=1 -o src/f32-dwconv/gen/up8x4-minmax-wasmsimd-acc2-x86.c + +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=9 -D ACCUMULATORS=1 -D X86=0 -o src/f32-dwconv/gen/up4x9-minmax-wasmsimd-arm.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=9 -D ACCUMULATORS=2 -D X86=0 -o src/f32-dwconv/gen/up4x9-minmax-wasmsimd-acc2-arm.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D ACCUMULATORS=1 -D X86=0 -o src/f32-dwconv/gen/up8x9-minmax-wasmsimd-arm.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D ACCUMULATORS=2 -D X86=0 -o src/f32-dwconv/gen/up8x9-minmax-wasmsimd-acc2-arm.c + +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=9 -D ACCUMULATORS=1 -D X86=1 -o src/f32-dwconv/gen/up4x9-minmax-wasmsimd-x86.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=9 -D ACCUMULATORS=2 -D X86=1 -o src/f32-dwconv/gen/up4x9-minmax-wasmsimd-acc2-x86.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D ACCUMULATORS=1 -D X86=1 -o src/f32-dwconv/gen/up8x9-minmax-wasmsimd-x86.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=9 -D ACCUMULATORS=2 -D X86=1 -o src/f32-dwconv/gen/up8x9-minmax-wasmsimd-acc2-x86.c + +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=25 -D ACCUMULATORS=1 -D X86=0 -o src/f32-dwconv/gen/up4x25-minmax-wasmsimd-arm.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=25 -D ACCUMULATORS=2 -D X86=0 -o src/f32-dwconv/gen/up4x25-minmax-wasmsimd-acc2-arm.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D ACCUMULATORS=1 -D X86=0 -o src/f32-dwconv/gen/up8x25-minmax-wasmsimd-arm.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D ACCUMULATORS=2 -D X86=0 -o src/f32-dwconv/gen/up8x25-minmax-wasmsimd-acc2-arm.c + +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=25 -D ACCUMULATORS=1 -D X86=1 -o src/f32-dwconv/gen/up4x25-minmax-wasmsimd-x86.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=25 -D ACCUMULATORS=2 -D X86=1 -o src/f32-dwconv/gen/up4x25-minmax-wasmsimd-acc2-x86.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D ACCUMULATORS=1 -D X86=1 -o src/f32-dwconv/gen/up8x25-minmax-wasmsimd-x86.c +tools/xngen src/f32-dwconv/up-wasmsimd.c.in -D CHANNEL_TILE=8 -D KERNEL_TILE=25 -D ACCUMULATORS=2 -D X86=1 -o src/f32-dwconv/gen/up8x25-minmax-wasmsimd-acc2-x86.c + ################################### ARM NEON ################################## tools/xngen src/f32-dwconv/up-neon.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=4 -D ACCUMULATORS=1 -D FMA=0 -o src/f32-dwconv/gen/up4x4-minmax-neon.c tools/xngen src/f32-dwconv/up-neon.c.in -D CHANNEL_TILE=4 -D KERNEL_TILE=4 -D ACCUMULATORS=2 -D FMA=0 -o src/f32-dwconv/gen/up4x4-minmax-neon-acc2.c diff --git a/src/f32-dwconv/gen/up4x25-minmax-wasmsimd-acc2-arm.c b/src/f32-dwconv/gen/up4x25-minmax-wasmsimd-acc2-arm.c new file mode 100644 index 000000000..98532939d --- /dev/null +++ b/src/f32-dwconv/gen/up4x25-minmax-wasmsimd-acc2-arm.c @@ -0,0 +1,452 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + const float* i9 = input[9]; + assert(i9 != NULL); + if XNN_UNPREDICTABLE(i9 != zero) { + i9 = (const float*) ((uintptr_t) i9 + input_offset); + } + const float* i10 = input[10]; + assert(i10 != NULL); + if XNN_UNPREDICTABLE(i10 != zero) { + i10 = (const float*) ((uintptr_t) i10 + input_offset); + } + const float* i11 = input[11]; + assert(i11 != NULL); + if XNN_UNPREDICTABLE(i11 != zero) { + i11 = (const float*) ((uintptr_t) i11 + input_offset); + } + const float* i12 = input[12]; + assert(i12 != NULL); + if XNN_UNPREDICTABLE(i12 != zero) { + i12 = (const float*) ((uintptr_t) i12 + input_offset); + } + const float* i13 = input[13]; + assert(i13 != NULL); + if XNN_UNPREDICTABLE(i13 != zero) { + i13 = (const float*) ((uintptr_t) i13 + input_offset); + } + const float* i14 = input[14]; + assert(i14 != NULL); + if XNN_UNPREDICTABLE(i14 != zero) { + i14 = (const float*) ((uintptr_t) i14 + input_offset); + } + const float* i15 = input[15]; + assert(i15 != NULL); + if XNN_UNPREDICTABLE(i15 != zero) { + i15 = (const float*) ((uintptr_t) i15 + input_offset); + } + const float* i16 = input[16]; + assert(i16 != NULL); + if XNN_UNPREDICTABLE(i16 != zero) { + i16 = (const float*) ((uintptr_t) i16 + input_offset); + } + const float* i17 = input[17]; + assert(i17 != NULL); + if XNN_UNPREDICTABLE(i17 != zero) { + i17 = (const float*) ((uintptr_t) i17 + input_offset); + } + const float* i18 = input[18]; + assert(i18 != NULL); + if XNN_UNPREDICTABLE(i18 != zero) { + i18 = (const float*) ((uintptr_t) i18 + input_offset); + } + const float* i19 = input[19]; + assert(i19 != NULL); + if XNN_UNPREDICTABLE(i19 != zero) { + i19 = (const float*) ((uintptr_t) i19 + input_offset); + } + const float* i20 = input[20]; + assert(i20 != NULL); + if XNN_UNPREDICTABLE(i20 != zero) { + i20 = (const float*) ((uintptr_t) i20 + input_offset); + } + const float* i21 = input[21]; + assert(i21 != NULL); + if XNN_UNPREDICTABLE(i21 != zero) { + i21 = (const float*) ((uintptr_t) i21 + input_offset); + } + const float* i22 = input[22]; + assert(i22 != NULL); + if XNN_UNPREDICTABLE(i22 != zero) { + i22 = (const float*) ((uintptr_t) i22 + input_offset); + } + const float* i23 = input[23]; + assert(i23 != NULL); + if XNN_UNPREDICTABLE(i23 != zero) { + i23 = (const float*) ((uintptr_t) i23 + input_offset); + } + const float* i24 = input[24]; + assert(i24 != NULL); + if XNN_UNPREDICTABLE(i24 != zero) { + i24 = (const float*) ((uintptr_t) i24 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 8); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + i9 += 4; + + const v128_t vk9x0123 = wasm_v128_load(w + 40); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + i10 += 4; + + const v128_t vk10x0123 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + i11 += 4; + + const v128_t vk11x0123 = wasm_v128_load(w + 48); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + i12 += 4; + + const v128_t vk12x0123 = wasm_v128_load(w + 52); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + i13 += 4; + + const v128_t vk13x0123 = wasm_v128_load(w + 56); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + i14 += 4; + + const v128_t vk14x0123 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + i15 += 4; + + const v128_t vk15x0123 = wasm_v128_load(w + 64); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + i16 += 4; + + const v128_t vk16x0123 = wasm_v128_load(w + 68); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + i17 += 4; + + const v128_t vk17x0123 = wasm_v128_load(w + 72); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + i18 += 4; + + const v128_t vk18x0123 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + i19 += 4; + + const v128_t vk19x0123 = wasm_v128_load(w + 80); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + i20 += 4; + + const v128_t vk20x0123 = wasm_v128_load(w + 84); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + i21 += 4; + + const v128_t vk21x0123 = wasm_v128_load(w + 88); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + i22 += 4; + + const v128_t vk22x0123 = wasm_v128_load(w + 92); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + i23 += 4; + + const v128_t vk23x0123 = wasm_v128_load(w + 96); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + i24 += 4; + + const v128_t vk24x0123 = wasm_v128_load(w + 100); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + w += 104; + + // Add up all accumulators to vacc0123p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 8); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + const v128_t vk9x0123 = wasm_v128_load(w + 40); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + const v128_t vk10x0123 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + const v128_t vk11x0123 = wasm_v128_load(w + 48); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + const v128_t vk12x0123 = wasm_v128_load(w + 52); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + const v128_t vk13x0123 = wasm_v128_load(w + 56); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + const v128_t vk14x0123 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + const v128_t vk15x0123 = wasm_v128_load(w + 64); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + const v128_t vk16x0123 = wasm_v128_load(w + 68); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + const v128_t vk17x0123 = wasm_v128_load(w + 72); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + const v128_t vk18x0123 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + const v128_t vk19x0123 = wasm_v128_load(w + 80); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + const v128_t vk20x0123 = wasm_v128_load(w + 84); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + const v128_t vk21x0123 = wasm_v128_load(w + 88); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + const v128_t vk22x0123 = wasm_v128_load(w + 92); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + const v128_t vk23x0123 = wasm_v128_load(w + 96); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + const v128_t vk24x0123 = wasm_v128_load(w + 100); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + // Add up all accumulators to vacc0123p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up4x25-minmax-wasmsimd-acc2-x86.c b/src/f32-dwconv/gen/up4x25-minmax-wasmsimd-acc2-x86.c new file mode 100644 index 000000000..4f3a5e782 --- /dev/null +++ b/src/f32-dwconv/gen/up4x25-minmax-wasmsimd-acc2-x86.c @@ -0,0 +1,452 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + const float* i9 = input[9]; + assert(i9 != NULL); + if XNN_UNPREDICTABLE(i9 != zero) { + i9 = (const float*) ((uintptr_t) i9 + input_offset); + } + const float* i10 = input[10]; + assert(i10 != NULL); + if XNN_UNPREDICTABLE(i10 != zero) { + i10 = (const float*) ((uintptr_t) i10 + input_offset); + } + const float* i11 = input[11]; + assert(i11 != NULL); + if XNN_UNPREDICTABLE(i11 != zero) { + i11 = (const float*) ((uintptr_t) i11 + input_offset); + } + const float* i12 = input[12]; + assert(i12 != NULL); + if XNN_UNPREDICTABLE(i12 != zero) { + i12 = (const float*) ((uintptr_t) i12 + input_offset); + } + const float* i13 = input[13]; + assert(i13 != NULL); + if XNN_UNPREDICTABLE(i13 != zero) { + i13 = (const float*) ((uintptr_t) i13 + input_offset); + } + const float* i14 = input[14]; + assert(i14 != NULL); + if XNN_UNPREDICTABLE(i14 != zero) { + i14 = (const float*) ((uintptr_t) i14 + input_offset); + } + const float* i15 = input[15]; + assert(i15 != NULL); + if XNN_UNPREDICTABLE(i15 != zero) { + i15 = (const float*) ((uintptr_t) i15 + input_offset); + } + const float* i16 = input[16]; + assert(i16 != NULL); + if XNN_UNPREDICTABLE(i16 != zero) { + i16 = (const float*) ((uintptr_t) i16 + input_offset); + } + const float* i17 = input[17]; + assert(i17 != NULL); + if XNN_UNPREDICTABLE(i17 != zero) { + i17 = (const float*) ((uintptr_t) i17 + input_offset); + } + const float* i18 = input[18]; + assert(i18 != NULL); + if XNN_UNPREDICTABLE(i18 != zero) { + i18 = (const float*) ((uintptr_t) i18 + input_offset); + } + const float* i19 = input[19]; + assert(i19 != NULL); + if XNN_UNPREDICTABLE(i19 != zero) { + i19 = (const float*) ((uintptr_t) i19 + input_offset); + } + const float* i20 = input[20]; + assert(i20 != NULL); + if XNN_UNPREDICTABLE(i20 != zero) { + i20 = (const float*) ((uintptr_t) i20 + input_offset); + } + const float* i21 = input[21]; + assert(i21 != NULL); + if XNN_UNPREDICTABLE(i21 != zero) { + i21 = (const float*) ((uintptr_t) i21 + input_offset); + } + const float* i22 = input[22]; + assert(i22 != NULL); + if XNN_UNPREDICTABLE(i22 != zero) { + i22 = (const float*) ((uintptr_t) i22 + input_offset); + } + const float* i23 = input[23]; + assert(i23 != NULL); + if XNN_UNPREDICTABLE(i23 != zero) { + i23 = (const float*) ((uintptr_t) i23 + input_offset); + } + const float* i24 = input[24]; + assert(i24 != NULL); + if XNN_UNPREDICTABLE(i24 != zero) { + i24 = (const float*) ((uintptr_t) i24 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 8); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + i9 += 4; + + const v128_t vk9x0123 = wasm_v128_load(w + 40); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + i10 += 4; + + const v128_t vk10x0123 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + i11 += 4; + + const v128_t vk11x0123 = wasm_v128_load(w + 48); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + i12 += 4; + + const v128_t vk12x0123 = wasm_v128_load(w + 52); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + i13 += 4; + + const v128_t vk13x0123 = wasm_v128_load(w + 56); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + i14 += 4; + + const v128_t vk14x0123 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + i15 += 4; + + const v128_t vk15x0123 = wasm_v128_load(w + 64); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + i16 += 4; + + const v128_t vk16x0123 = wasm_v128_load(w + 68); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + i17 += 4; + + const v128_t vk17x0123 = wasm_v128_load(w + 72); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + i18 += 4; + + const v128_t vk18x0123 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + i19 += 4; + + const v128_t vk19x0123 = wasm_v128_load(w + 80); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + i20 += 4; + + const v128_t vk20x0123 = wasm_v128_load(w + 84); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + i21 += 4; + + const v128_t vk21x0123 = wasm_v128_load(w + 88); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + i22 += 4; + + const v128_t vk22x0123 = wasm_v128_load(w + 92); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + i23 += 4; + + const v128_t vk23x0123 = wasm_v128_load(w + 96); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + i24 += 4; + + const v128_t vk24x0123 = wasm_v128_load(w + 100); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + w += 104; + + // Add up all accumulators to vacc0123p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 8); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + const v128_t vk9x0123 = wasm_v128_load(w + 40); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + const v128_t vk10x0123 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + const v128_t vk11x0123 = wasm_v128_load(w + 48); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + const v128_t vk12x0123 = wasm_v128_load(w + 52); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + const v128_t vk13x0123 = wasm_v128_load(w + 56); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + const v128_t vk14x0123 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + const v128_t vk15x0123 = wasm_v128_load(w + 64); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + const v128_t vk16x0123 = wasm_v128_load(w + 68); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + const v128_t vk17x0123 = wasm_v128_load(w + 72); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + const v128_t vk18x0123 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + const v128_t vk19x0123 = wasm_v128_load(w + 80); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + const v128_t vk20x0123 = wasm_v128_load(w + 84); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + const v128_t vk21x0123 = wasm_v128_load(w + 88); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + const v128_t vk22x0123 = wasm_v128_load(w + 92); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + const v128_t vk23x0123 = wasm_v128_load(w + 96); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + const v128_t vk24x0123 = wasm_v128_load(w + 100); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + // Add up all accumulators to vacc0123p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up4x25-minmax-wasmsimd-arm.c b/src/f32-dwconv/gen/up4x25-minmax-wasmsimd-arm.c new file mode 100644 index 000000000..17be458d4 --- /dev/null +++ b/src/f32-dwconv/gen/up4x25-minmax-wasmsimd-arm.c @@ -0,0 +1,448 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + const float* i9 = input[9]; + assert(i9 != NULL); + if XNN_UNPREDICTABLE(i9 != zero) { + i9 = (const float*) ((uintptr_t) i9 + input_offset); + } + const float* i10 = input[10]; + assert(i10 != NULL); + if XNN_UNPREDICTABLE(i10 != zero) { + i10 = (const float*) ((uintptr_t) i10 + input_offset); + } + const float* i11 = input[11]; + assert(i11 != NULL); + if XNN_UNPREDICTABLE(i11 != zero) { + i11 = (const float*) ((uintptr_t) i11 + input_offset); + } + const float* i12 = input[12]; + assert(i12 != NULL); + if XNN_UNPREDICTABLE(i12 != zero) { + i12 = (const float*) ((uintptr_t) i12 + input_offset); + } + const float* i13 = input[13]; + assert(i13 != NULL); + if XNN_UNPREDICTABLE(i13 != zero) { + i13 = (const float*) ((uintptr_t) i13 + input_offset); + } + const float* i14 = input[14]; + assert(i14 != NULL); + if XNN_UNPREDICTABLE(i14 != zero) { + i14 = (const float*) ((uintptr_t) i14 + input_offset); + } + const float* i15 = input[15]; + assert(i15 != NULL); + if XNN_UNPREDICTABLE(i15 != zero) { + i15 = (const float*) ((uintptr_t) i15 + input_offset); + } + const float* i16 = input[16]; + assert(i16 != NULL); + if XNN_UNPREDICTABLE(i16 != zero) { + i16 = (const float*) ((uintptr_t) i16 + input_offset); + } + const float* i17 = input[17]; + assert(i17 != NULL); + if XNN_UNPREDICTABLE(i17 != zero) { + i17 = (const float*) ((uintptr_t) i17 + input_offset); + } + const float* i18 = input[18]; + assert(i18 != NULL); + if XNN_UNPREDICTABLE(i18 != zero) { + i18 = (const float*) ((uintptr_t) i18 + input_offset); + } + const float* i19 = input[19]; + assert(i19 != NULL); + if XNN_UNPREDICTABLE(i19 != zero) { + i19 = (const float*) ((uintptr_t) i19 + input_offset); + } + const float* i20 = input[20]; + assert(i20 != NULL); + if XNN_UNPREDICTABLE(i20 != zero) { + i20 = (const float*) ((uintptr_t) i20 + input_offset); + } + const float* i21 = input[21]; + assert(i21 != NULL); + if XNN_UNPREDICTABLE(i21 != zero) { + i21 = (const float*) ((uintptr_t) i21 + input_offset); + } + const float* i22 = input[22]; + assert(i22 != NULL); + if XNN_UNPREDICTABLE(i22 != zero) { + i22 = (const float*) ((uintptr_t) i22 + input_offset); + } + const float* i23 = input[23]; + assert(i23 != NULL); + if XNN_UNPREDICTABLE(i23 != zero) { + i23 = (const float*) ((uintptr_t) i23 + input_offset); + } + const float* i24 = input[24]; + assert(i24 != NULL); + if XNN_UNPREDICTABLE(i24 != zero) { + i24 = (const float*) ((uintptr_t) i24 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + i9 += 4; + + const v128_t vk9x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + i10 += 4; + + const v128_t vk10x0123 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + i11 += 4; + + const v128_t vk11x0123 = wasm_v128_load(w + 48); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + i12 += 4; + + const v128_t vk12x0123 = wasm_v128_load(w + 52); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + i13 += 4; + + const v128_t vk13x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + i14 += 4; + + const v128_t vk14x0123 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + i15 += 4; + + const v128_t vk15x0123 = wasm_v128_load(w + 64); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + i16 += 4; + + const v128_t vk16x0123 = wasm_v128_load(w + 68); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + i17 += 4; + + const v128_t vk17x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + i18 += 4; + + const v128_t vk18x0123 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + i19 += 4; + + const v128_t vk19x0123 = wasm_v128_load(w + 80); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + i20 += 4; + + const v128_t vk20x0123 = wasm_v128_load(w + 84); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + i21 += 4; + + const v128_t vk21x0123 = wasm_v128_load(w + 88); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + i22 += 4; + + const v128_t vk22x0123 = wasm_v128_load(w + 92); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + i23 += 4; + + const v128_t vk23x0123 = wasm_v128_load(w + 96); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + i24 += 4; + + const v128_t vk24x0123 = wasm_v128_load(w + 100); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + w += 104; + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + const v128_t vk9x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + const v128_t vk10x0123 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + const v128_t vk11x0123 = wasm_v128_load(w + 48); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + const v128_t vk12x0123 = wasm_v128_load(w + 52); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + const v128_t vk13x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + const v128_t vk14x0123 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + const v128_t vk15x0123 = wasm_v128_load(w + 64); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + const v128_t vk16x0123 = wasm_v128_load(w + 68); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + const v128_t vk17x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + const v128_t vk18x0123 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + const v128_t vk19x0123 = wasm_v128_load(w + 80); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + const v128_t vk20x0123 = wasm_v128_load(w + 84); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + const v128_t vk21x0123 = wasm_v128_load(w + 88); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + const v128_t vk22x0123 = wasm_v128_load(w + 92); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + const v128_t vk23x0123 = wasm_v128_load(w + 96); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + const v128_t vk24x0123 = wasm_v128_load(w + 100); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up4x25-minmax-wasmsimd-x86.c b/src/f32-dwconv/gen/up4x25-minmax-wasmsimd-x86.c new file mode 100644 index 000000000..1271fa69b --- /dev/null +++ b/src/f32-dwconv/gen/up4x25-minmax-wasmsimd-x86.c @@ -0,0 +1,448 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + const float* i9 = input[9]; + assert(i9 != NULL); + if XNN_UNPREDICTABLE(i9 != zero) { + i9 = (const float*) ((uintptr_t) i9 + input_offset); + } + const float* i10 = input[10]; + assert(i10 != NULL); + if XNN_UNPREDICTABLE(i10 != zero) { + i10 = (const float*) ((uintptr_t) i10 + input_offset); + } + const float* i11 = input[11]; + assert(i11 != NULL); + if XNN_UNPREDICTABLE(i11 != zero) { + i11 = (const float*) ((uintptr_t) i11 + input_offset); + } + const float* i12 = input[12]; + assert(i12 != NULL); + if XNN_UNPREDICTABLE(i12 != zero) { + i12 = (const float*) ((uintptr_t) i12 + input_offset); + } + const float* i13 = input[13]; + assert(i13 != NULL); + if XNN_UNPREDICTABLE(i13 != zero) { + i13 = (const float*) ((uintptr_t) i13 + input_offset); + } + const float* i14 = input[14]; + assert(i14 != NULL); + if XNN_UNPREDICTABLE(i14 != zero) { + i14 = (const float*) ((uintptr_t) i14 + input_offset); + } + const float* i15 = input[15]; + assert(i15 != NULL); + if XNN_UNPREDICTABLE(i15 != zero) { + i15 = (const float*) ((uintptr_t) i15 + input_offset); + } + const float* i16 = input[16]; + assert(i16 != NULL); + if XNN_UNPREDICTABLE(i16 != zero) { + i16 = (const float*) ((uintptr_t) i16 + input_offset); + } + const float* i17 = input[17]; + assert(i17 != NULL); + if XNN_UNPREDICTABLE(i17 != zero) { + i17 = (const float*) ((uintptr_t) i17 + input_offset); + } + const float* i18 = input[18]; + assert(i18 != NULL); + if XNN_UNPREDICTABLE(i18 != zero) { + i18 = (const float*) ((uintptr_t) i18 + input_offset); + } + const float* i19 = input[19]; + assert(i19 != NULL); + if XNN_UNPREDICTABLE(i19 != zero) { + i19 = (const float*) ((uintptr_t) i19 + input_offset); + } + const float* i20 = input[20]; + assert(i20 != NULL); + if XNN_UNPREDICTABLE(i20 != zero) { + i20 = (const float*) ((uintptr_t) i20 + input_offset); + } + const float* i21 = input[21]; + assert(i21 != NULL); + if XNN_UNPREDICTABLE(i21 != zero) { + i21 = (const float*) ((uintptr_t) i21 + input_offset); + } + const float* i22 = input[22]; + assert(i22 != NULL); + if XNN_UNPREDICTABLE(i22 != zero) { + i22 = (const float*) ((uintptr_t) i22 + input_offset); + } + const float* i23 = input[23]; + assert(i23 != NULL); + if XNN_UNPREDICTABLE(i23 != zero) { + i23 = (const float*) ((uintptr_t) i23 + input_offset); + } + const float* i24 = input[24]; + assert(i24 != NULL); + if XNN_UNPREDICTABLE(i24 != zero) { + i24 = (const float*) ((uintptr_t) i24 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + i9 += 4; + + const v128_t vk9x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + i10 += 4; + + const v128_t vk10x0123 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + i11 += 4; + + const v128_t vk11x0123 = wasm_v128_load(w + 48); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + i12 += 4; + + const v128_t vk12x0123 = wasm_v128_load(w + 52); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + i13 += 4; + + const v128_t vk13x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + i14 += 4; + + const v128_t vk14x0123 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + i15 += 4; + + const v128_t vk15x0123 = wasm_v128_load(w + 64); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + i16 += 4; + + const v128_t vk16x0123 = wasm_v128_load(w + 68); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + i17 += 4; + + const v128_t vk17x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + i18 += 4; + + const v128_t vk18x0123 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + i19 += 4; + + const v128_t vk19x0123 = wasm_v128_load(w + 80); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + i20 += 4; + + const v128_t vk20x0123 = wasm_v128_load(w + 84); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + i21 += 4; + + const v128_t vk21x0123 = wasm_v128_load(w + 88); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + i22 += 4; + + const v128_t vk22x0123 = wasm_v128_load(w + 92); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + i23 += 4; + + const v128_t vk23x0123 = wasm_v128_load(w + 96); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + i24 += 4; + + const v128_t vk24x0123 = wasm_v128_load(w + 100); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + w += 104; + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + const v128_t vk9x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + const v128_t vk10x0123 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + const v128_t vk11x0123 = wasm_v128_load(w + 48); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + const v128_t vk12x0123 = wasm_v128_load(w + 52); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + const v128_t vk13x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + const v128_t vk14x0123 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + const v128_t vk15x0123 = wasm_v128_load(w + 64); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + const v128_t vk16x0123 = wasm_v128_load(w + 68); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + const v128_t vk17x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + const v128_t vk18x0123 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + const v128_t vk19x0123 = wasm_v128_load(w + 80); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + const v128_t vk20x0123 = wasm_v128_load(w + 84); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + const v128_t vk21x0123 = wasm_v128_load(w + 88); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + const v128_t vk22x0123 = wasm_v128_load(w + 92); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + const v128_t vk23x0123 = wasm_v128_load(w + 96); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + const v128_t vk24x0123 = wasm_v128_load(w + 100); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up4x4-minmax-wasmsimd-acc2-arm.c b/src/f32-dwconv/gen/up4x4-minmax-wasmsimd-acc2-arm.c new file mode 100644 index 000000000..050b08948 --- /dev/null +++ b/src/f32-dwconv/gen/up4x4-minmax-wasmsimd-acc2-arm.c @@ -0,0 +1,137 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 8); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + w += 20; + + // Add up all accumulators to vacc0123p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 8); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + // Add up all accumulators to vacc0123p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up4x4-minmax-wasmsimd-acc2-x86.c b/src/f32-dwconv/gen/up4x4-minmax-wasmsimd-acc2-x86.c new file mode 100644 index 000000000..6d1e08ef0 --- /dev/null +++ b/src/f32-dwconv/gen/up4x4-minmax-wasmsimd-acc2-x86.c @@ -0,0 +1,137 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 8); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + w += 20; + + // Add up all accumulators to vacc0123p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 8); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + // Add up all accumulators to vacc0123p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up4x4-minmax-wasmsimd-arm.c b/src/f32-dwconv/gen/up4x4-minmax-wasmsimd-arm.c new file mode 100644 index 000000000..4b4c2ca5b --- /dev/null +++ b/src/f32-dwconv/gen/up4x4-minmax-wasmsimd-arm.c @@ -0,0 +1,133 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + w += 20; + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up4x4-minmax-wasmsimd-x86.c b/src/f32-dwconv/gen/up4x4-minmax-wasmsimd-x86.c new file mode 100644 index 000000000..a3df5cd07 --- /dev/null +++ b/src/f32-dwconv/gen/up4x4-minmax-wasmsimd-x86.c @@ -0,0 +1,133 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + w += 20; + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up4x9-minmax-wasmsimd-acc2-arm.c b/src/f32-dwconv/gen/up4x9-minmax-wasmsimd-acc2-arm.c new file mode 100644 index 000000000..04de92a84 --- /dev/null +++ b/src/f32-dwconv/gen/up4x9-minmax-wasmsimd-acc2-arm.c @@ -0,0 +1,212 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 8); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + w += 40; + + // Add up all accumulators to vacc0123p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 8); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + // Add up all accumulators to vacc0123p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up4x9-minmax-wasmsimd-acc2-x86.c b/src/f32-dwconv/gen/up4x9-minmax-wasmsimd-acc2-x86.c new file mode 100644 index 000000000..7ee841889 --- /dev/null +++ b/src/f32-dwconv/gen/up4x9-minmax-wasmsimd-acc2-x86.c @@ -0,0 +1,212 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 8); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + w += 40; + + // Add up all accumulators to vacc0123p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 8); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + // Add up all accumulators to vacc0123p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up4x9-minmax-wasmsimd-arm.c b/src/f32-dwconv/gen/up4x9-minmax-wasmsimd-arm.c new file mode 100644 index 000000000..d127591a8 --- /dev/null +++ b/src/f32-dwconv/gen/up4x9-minmax-wasmsimd-arm.c @@ -0,0 +1,208 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + w += 40; + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up4x9-minmax-wasmsimd-x86.c b/src/f32-dwconv/gen/up4x9-minmax-wasmsimd-x86.c new file mode 100644 index 000000000..5b53e5be8 --- /dev/null +++ b/src/f32-dwconv/gen/up4x9-minmax-wasmsimd-x86.c @@ -0,0 +1,208 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + w += 40; + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 4); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up8x25-minmax-wasmsimd-acc2-arm.c b/src/f32-dwconv/gen/up8x25-minmax-wasmsimd-acc2-arm.c new file mode 100644 index 000000000..d5eb7aea0 --- /dev/null +++ b/src/f32-dwconv/gen/up8x25-minmax-wasmsimd-acc2-arm.c @@ -0,0 +1,696 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + const float* i9 = input[9]; + assert(i9 != NULL); + if XNN_UNPREDICTABLE(i9 != zero) { + i9 = (const float*) ((uintptr_t) i9 + input_offset); + } + const float* i10 = input[10]; + assert(i10 != NULL); + if XNN_UNPREDICTABLE(i10 != zero) { + i10 = (const float*) ((uintptr_t) i10 + input_offset); + } + const float* i11 = input[11]; + assert(i11 != NULL); + if XNN_UNPREDICTABLE(i11 != zero) { + i11 = (const float*) ((uintptr_t) i11 + input_offset); + } + const float* i12 = input[12]; + assert(i12 != NULL); + if XNN_UNPREDICTABLE(i12 != zero) { + i12 = (const float*) ((uintptr_t) i12 + input_offset); + } + const float* i13 = input[13]; + assert(i13 != NULL); + if XNN_UNPREDICTABLE(i13 != zero) { + i13 = (const float*) ((uintptr_t) i13 + input_offset); + } + const float* i14 = input[14]; + assert(i14 != NULL); + if XNN_UNPREDICTABLE(i14 != zero) { + i14 = (const float*) ((uintptr_t) i14 + input_offset); + } + const float* i15 = input[15]; + assert(i15 != NULL); + if XNN_UNPREDICTABLE(i15 != zero) { + i15 = (const float*) ((uintptr_t) i15 + input_offset); + } + const float* i16 = input[16]; + assert(i16 != NULL); + if XNN_UNPREDICTABLE(i16 != zero) { + i16 = (const float*) ((uintptr_t) i16 + input_offset); + } + const float* i17 = input[17]; + assert(i17 != NULL); + if XNN_UNPREDICTABLE(i17 != zero) { + i17 = (const float*) ((uintptr_t) i17 + input_offset); + } + const float* i18 = input[18]; + assert(i18 != NULL); + if XNN_UNPREDICTABLE(i18 != zero) { + i18 = (const float*) ((uintptr_t) i18 + input_offset); + } + const float* i19 = input[19]; + assert(i19 != NULL); + if XNN_UNPREDICTABLE(i19 != zero) { + i19 = (const float*) ((uintptr_t) i19 + input_offset); + } + const float* i20 = input[20]; + assert(i20 != NULL); + if XNN_UNPREDICTABLE(i20 != zero) { + i20 = (const float*) ((uintptr_t) i20 + input_offset); + } + const float* i21 = input[21]; + assert(i21 != NULL); + if XNN_UNPREDICTABLE(i21 != zero) { + i21 = (const float*) ((uintptr_t) i21 + input_offset); + } + const float* i22 = input[22]; + assert(i22 != NULL); + if XNN_UNPREDICTABLE(i22 != zero) { + i22 = (const float*) ((uintptr_t) i22 + input_offset); + } + const float* i23 = input[23]; + assert(i23 != NULL); + if XNN_UNPREDICTABLE(i23 != zero) { + i23 = (const float*) ((uintptr_t) i23 + input_offset); + } + const float* i24 = input[24]; + assert(i24 != NULL); + if XNN_UNPREDICTABLE(i24 != zero) { + i24 = (const float*) ((uintptr_t) i24 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 8; c -= 8) { + v128_t vacc0123p0 = wasm_v128_load(w); + v128_t vacc4567p0 = wasm_v128_load(w + 4); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vi0x4567 = wasm_v128_load(i0 + 4); + i0 += 8; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + const v128_t vk0x4567 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi0x4567, vk0x4567)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vi1x4567 = wasm_v128_load(i1 + 4); + i1 += 8; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + const v128_t vk1x4567 = wasm_v128_load(w + 20); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + v128_t vacc4567p1 = wasm_f32x4_mul(vi1x4567, vk1x4567); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vi2x4567 = wasm_v128_load(i2 + 4); + i2 += 8; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + const v128_t vk2x4567 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi2x4567, vk2x4567)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vi3x4567 = wasm_v128_load(i3 + 4); + i3 += 8; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + const v128_t vk3x4567 = wasm_v128_load(w + 36); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi3x4567, vk3x4567)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vi4x4567 = wasm_v128_load(i4 + 4); + i4 += 8; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + const v128_t vk4x4567 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi4x4567, vk4x4567)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vi5x4567 = wasm_v128_load(i5 + 4); + i5 += 8; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + const v128_t vk5x4567 = wasm_v128_load(w + 52); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi5x4567, vk5x4567)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vi6x4567 = wasm_v128_load(i6 + 4); + i6 += 8; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + const v128_t vk6x4567 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi6x4567, vk6x4567)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vi7x4567 = wasm_v128_load(i7 + 4); + i7 += 8; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + const v128_t vk7x4567 = wasm_v128_load(w + 68); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi7x4567, vk7x4567)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vi8x4567 = wasm_v128_load(i8 + 4); + i8 += 8; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + const v128_t vk8x4567 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi8x4567, vk8x4567)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + const v128_t vi9x4567 = wasm_v128_load(i9 + 4); + i9 += 8; + + const v128_t vk9x0123 = wasm_v128_load(w + 80); + const v128_t vk9x4567 = wasm_v128_load(w + 84); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi9x0123, vk9x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi9x4567, vk9x4567)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + const v128_t vi10x4567 = wasm_v128_load(i10 + 4); + i10 += 8; + + const v128_t vk10x0123 = wasm_v128_load(w + 88); + const v128_t vk10x4567 = wasm_v128_load(w + 92); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi10x4567, vk10x4567)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + const v128_t vi11x4567 = wasm_v128_load(i11 + 4); + i11 += 8; + + const v128_t vk11x0123 = wasm_v128_load(w + 96); + const v128_t vk11x4567 = wasm_v128_load(w + 100); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi11x0123, vk11x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi11x4567, vk11x4567)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + const v128_t vi12x4567 = wasm_v128_load(i12 + 4); + i12 += 8; + + const v128_t vk12x0123 = wasm_v128_load(w + 104); + const v128_t vk12x4567 = wasm_v128_load(w + 108); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi12x4567, vk12x4567)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + const v128_t vi13x4567 = wasm_v128_load(i13 + 4); + i13 += 8; + + const v128_t vk13x0123 = wasm_v128_load(w + 112); + const v128_t vk13x4567 = wasm_v128_load(w + 116); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi13x0123, vk13x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi13x4567, vk13x4567)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + const v128_t vi14x4567 = wasm_v128_load(i14 + 4); + i14 += 8; + + const v128_t vk14x0123 = wasm_v128_load(w + 120); + const v128_t vk14x4567 = wasm_v128_load(w + 124); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi14x4567, vk14x4567)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + const v128_t vi15x4567 = wasm_v128_load(i15 + 4); + i15 += 8; + + const v128_t vk15x0123 = wasm_v128_load(w + 128); + const v128_t vk15x4567 = wasm_v128_load(w + 132); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi15x0123, vk15x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi15x4567, vk15x4567)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + const v128_t vi16x4567 = wasm_v128_load(i16 + 4); + i16 += 8; + + const v128_t vk16x0123 = wasm_v128_load(w + 136); + const v128_t vk16x4567 = wasm_v128_load(w + 140); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi16x4567, vk16x4567)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + const v128_t vi17x4567 = wasm_v128_load(i17 + 4); + i17 += 8; + + const v128_t vk17x0123 = wasm_v128_load(w + 144); + const v128_t vk17x4567 = wasm_v128_load(w + 148); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi17x0123, vk17x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi17x4567, vk17x4567)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + const v128_t vi18x4567 = wasm_v128_load(i18 + 4); + i18 += 8; + + const v128_t vk18x0123 = wasm_v128_load(w + 152); + const v128_t vk18x4567 = wasm_v128_load(w + 156); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi18x4567, vk18x4567)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + const v128_t vi19x4567 = wasm_v128_load(i19 + 4); + i19 += 8; + + const v128_t vk19x0123 = wasm_v128_load(w + 160); + const v128_t vk19x4567 = wasm_v128_load(w + 164); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi19x0123, vk19x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi19x4567, vk19x4567)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + const v128_t vi20x4567 = wasm_v128_load(i20 + 4); + i20 += 8; + + const v128_t vk20x0123 = wasm_v128_load(w + 168); + const v128_t vk20x4567 = wasm_v128_load(w + 172); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi20x4567, vk20x4567)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + const v128_t vi21x4567 = wasm_v128_load(i21 + 4); + i21 += 8; + + const v128_t vk21x0123 = wasm_v128_load(w + 176); + const v128_t vk21x4567 = wasm_v128_load(w + 180); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi21x0123, vk21x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi21x4567, vk21x4567)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + const v128_t vi22x4567 = wasm_v128_load(i22 + 4); + i22 += 8; + + const v128_t vk22x0123 = wasm_v128_load(w + 184); + const v128_t vk22x4567 = wasm_v128_load(w + 188); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi22x4567, vk22x4567)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + const v128_t vi23x4567 = wasm_v128_load(i23 + 4); + i23 += 8; + + const v128_t vk23x0123 = wasm_v128_load(w + 192); + const v128_t vk23x4567 = wasm_v128_load(w + 196); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi23x0123, vk23x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi23x4567, vk23x4567)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + const v128_t vi24x4567 = wasm_v128_load(i24 + 4); + i24 += 8; + + const v128_t vk24x0123 = wasm_v128_load(w + 200); + const v128_t vk24x4567 = wasm_v128_load(w + 204); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi24x4567, vk24x4567)); + + w += 208; + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, vacc4567p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + v128_t vacc4567 = wasm_f32x4_max(vacc4567p0, vmin); + + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + vacc4567 = wasm_f32x4_min(vacc4567, vmax); + + wasm_v128_store(output, vacc0123); + wasm_v128_store(output + 4, vacc4567); + output += 8; + } + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + i9 += 4; + + const v128_t vk9x0123 = wasm_v128_load(w + 80); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + i10 += 4; + + const v128_t vk10x0123 = wasm_v128_load(w + 88); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + i11 += 4; + + const v128_t vk11x0123 = wasm_v128_load(w + 96); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + i12 += 4; + + const v128_t vk12x0123 = wasm_v128_load(w + 104); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + i13 += 4; + + const v128_t vk13x0123 = wasm_v128_load(w + 112); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + i14 += 4; + + const v128_t vk14x0123 = wasm_v128_load(w + 120); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + i15 += 4; + + const v128_t vk15x0123 = wasm_v128_load(w + 128); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + i16 += 4; + + const v128_t vk16x0123 = wasm_v128_load(w + 136); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + i17 += 4; + + const v128_t vk17x0123 = wasm_v128_load(w + 144); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + i18 += 4; + + const v128_t vk18x0123 = wasm_v128_load(w + 152); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + i19 += 4; + + const v128_t vk19x0123 = wasm_v128_load(w + 160); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + i20 += 4; + + const v128_t vk20x0123 = wasm_v128_load(w + 168); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + i21 += 4; + + const v128_t vk21x0123 = wasm_v128_load(w + 176); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + i22 += 4; + + const v128_t vk22x0123 = wasm_v128_load(w + 184); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + i23 += 4; + + const v128_t vk23x0123 = wasm_v128_load(w + 192); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + i24 += 4; + + const v128_t vk24x0123 = wasm_v128_load(w + 200); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + w += 4; + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 16); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + const v128_t vk9x0123 = wasm_v128_load(w + 80); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + const v128_t vk10x0123 = wasm_v128_load(w + 88); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + const v128_t vk11x0123 = wasm_v128_load(w + 96); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + const v128_t vk12x0123 = wasm_v128_load(w + 104); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + const v128_t vk13x0123 = wasm_v128_load(w + 112); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + const v128_t vk14x0123 = wasm_v128_load(w + 120); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + const v128_t vk15x0123 = wasm_v128_load(w + 128); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + const v128_t vk16x0123 = wasm_v128_load(w + 136); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + const v128_t vk17x0123 = wasm_v128_load(w + 144); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + const v128_t vk18x0123 = wasm_v128_load(w + 152); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + const v128_t vk19x0123 = wasm_v128_load(w + 160); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + const v128_t vk20x0123 = wasm_v128_load(w + 168); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + const v128_t vk21x0123 = wasm_v128_load(w + 176); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + const v128_t vk22x0123 = wasm_v128_load(w + 184); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + const v128_t vk23x0123 = wasm_v128_load(w + 192); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + const v128_t vk24x0123 = wasm_v128_load(w + 200); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up8x25-minmax-wasmsimd-acc2-x86.c b/src/f32-dwconv/gen/up8x25-minmax-wasmsimd-acc2-x86.c new file mode 100644 index 000000000..8151aa722 --- /dev/null +++ b/src/f32-dwconv/gen/up8x25-minmax-wasmsimd-acc2-x86.c @@ -0,0 +1,696 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + const float* i9 = input[9]; + assert(i9 != NULL); + if XNN_UNPREDICTABLE(i9 != zero) { + i9 = (const float*) ((uintptr_t) i9 + input_offset); + } + const float* i10 = input[10]; + assert(i10 != NULL); + if XNN_UNPREDICTABLE(i10 != zero) { + i10 = (const float*) ((uintptr_t) i10 + input_offset); + } + const float* i11 = input[11]; + assert(i11 != NULL); + if XNN_UNPREDICTABLE(i11 != zero) { + i11 = (const float*) ((uintptr_t) i11 + input_offset); + } + const float* i12 = input[12]; + assert(i12 != NULL); + if XNN_UNPREDICTABLE(i12 != zero) { + i12 = (const float*) ((uintptr_t) i12 + input_offset); + } + const float* i13 = input[13]; + assert(i13 != NULL); + if XNN_UNPREDICTABLE(i13 != zero) { + i13 = (const float*) ((uintptr_t) i13 + input_offset); + } + const float* i14 = input[14]; + assert(i14 != NULL); + if XNN_UNPREDICTABLE(i14 != zero) { + i14 = (const float*) ((uintptr_t) i14 + input_offset); + } + const float* i15 = input[15]; + assert(i15 != NULL); + if XNN_UNPREDICTABLE(i15 != zero) { + i15 = (const float*) ((uintptr_t) i15 + input_offset); + } + const float* i16 = input[16]; + assert(i16 != NULL); + if XNN_UNPREDICTABLE(i16 != zero) { + i16 = (const float*) ((uintptr_t) i16 + input_offset); + } + const float* i17 = input[17]; + assert(i17 != NULL); + if XNN_UNPREDICTABLE(i17 != zero) { + i17 = (const float*) ((uintptr_t) i17 + input_offset); + } + const float* i18 = input[18]; + assert(i18 != NULL); + if XNN_UNPREDICTABLE(i18 != zero) { + i18 = (const float*) ((uintptr_t) i18 + input_offset); + } + const float* i19 = input[19]; + assert(i19 != NULL); + if XNN_UNPREDICTABLE(i19 != zero) { + i19 = (const float*) ((uintptr_t) i19 + input_offset); + } + const float* i20 = input[20]; + assert(i20 != NULL); + if XNN_UNPREDICTABLE(i20 != zero) { + i20 = (const float*) ((uintptr_t) i20 + input_offset); + } + const float* i21 = input[21]; + assert(i21 != NULL); + if XNN_UNPREDICTABLE(i21 != zero) { + i21 = (const float*) ((uintptr_t) i21 + input_offset); + } + const float* i22 = input[22]; + assert(i22 != NULL); + if XNN_UNPREDICTABLE(i22 != zero) { + i22 = (const float*) ((uintptr_t) i22 + input_offset); + } + const float* i23 = input[23]; + assert(i23 != NULL); + if XNN_UNPREDICTABLE(i23 != zero) { + i23 = (const float*) ((uintptr_t) i23 + input_offset); + } + const float* i24 = input[24]; + assert(i24 != NULL); + if XNN_UNPREDICTABLE(i24 != zero) { + i24 = (const float*) ((uintptr_t) i24 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 8; c -= 8) { + v128_t vacc0123p0 = wasm_v128_load(w); + v128_t vacc4567p0 = wasm_v128_load(w + 4); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vi0x4567 = wasm_v128_load(i0 + 4); + i0 += 8; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + const v128_t vk0x4567 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi0x4567, vk0x4567)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vi1x4567 = wasm_v128_load(i1 + 4); + i1 += 8; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + const v128_t vk1x4567 = wasm_v128_load(w + 20); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + v128_t vacc4567p1 = wasm_f32x4_mul(vi1x4567, vk1x4567); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vi2x4567 = wasm_v128_load(i2 + 4); + i2 += 8; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + const v128_t vk2x4567 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi2x4567, vk2x4567)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vi3x4567 = wasm_v128_load(i3 + 4); + i3 += 8; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + const v128_t vk3x4567 = wasm_v128_load(w + 36); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi3x4567, vk3x4567)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vi4x4567 = wasm_v128_load(i4 + 4); + i4 += 8; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + const v128_t vk4x4567 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi4x4567, vk4x4567)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vi5x4567 = wasm_v128_load(i5 + 4); + i5 += 8; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + const v128_t vk5x4567 = wasm_v128_load(w + 52); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi5x4567, vk5x4567)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vi6x4567 = wasm_v128_load(i6 + 4); + i6 += 8; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + const v128_t vk6x4567 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi6x4567, vk6x4567)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vi7x4567 = wasm_v128_load(i7 + 4); + i7 += 8; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + const v128_t vk7x4567 = wasm_v128_load(w + 68); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi7x4567, vk7x4567)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vi8x4567 = wasm_v128_load(i8 + 4); + i8 += 8; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + const v128_t vk8x4567 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi8x4567, vk8x4567)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + const v128_t vi9x4567 = wasm_v128_load(i9 + 4); + i9 += 8; + + const v128_t vk9x0123 = wasm_v128_load(w + 80); + const v128_t vk9x4567 = wasm_v128_load(w + 84); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi9x0123, vk9x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi9x4567, vk9x4567)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + const v128_t vi10x4567 = wasm_v128_load(i10 + 4); + i10 += 8; + + const v128_t vk10x0123 = wasm_v128_load(w + 88); + const v128_t vk10x4567 = wasm_v128_load(w + 92); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi10x4567, vk10x4567)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + const v128_t vi11x4567 = wasm_v128_load(i11 + 4); + i11 += 8; + + const v128_t vk11x0123 = wasm_v128_load(w + 96); + const v128_t vk11x4567 = wasm_v128_load(w + 100); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi11x0123, vk11x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi11x4567, vk11x4567)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + const v128_t vi12x4567 = wasm_v128_load(i12 + 4); + i12 += 8; + + const v128_t vk12x0123 = wasm_v128_load(w + 104); + const v128_t vk12x4567 = wasm_v128_load(w + 108); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi12x4567, vk12x4567)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + const v128_t vi13x4567 = wasm_v128_load(i13 + 4); + i13 += 8; + + const v128_t vk13x0123 = wasm_v128_load(w + 112); + const v128_t vk13x4567 = wasm_v128_load(w + 116); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi13x0123, vk13x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi13x4567, vk13x4567)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + const v128_t vi14x4567 = wasm_v128_load(i14 + 4); + i14 += 8; + + const v128_t vk14x0123 = wasm_v128_load(w + 120); + const v128_t vk14x4567 = wasm_v128_load(w + 124); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi14x4567, vk14x4567)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + const v128_t vi15x4567 = wasm_v128_load(i15 + 4); + i15 += 8; + + const v128_t vk15x0123 = wasm_v128_load(w + 128); + const v128_t vk15x4567 = wasm_v128_load(w + 132); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi15x0123, vk15x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi15x4567, vk15x4567)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + const v128_t vi16x4567 = wasm_v128_load(i16 + 4); + i16 += 8; + + const v128_t vk16x0123 = wasm_v128_load(w + 136); + const v128_t vk16x4567 = wasm_v128_load(w + 140); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi16x4567, vk16x4567)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + const v128_t vi17x4567 = wasm_v128_load(i17 + 4); + i17 += 8; + + const v128_t vk17x0123 = wasm_v128_load(w + 144); + const v128_t vk17x4567 = wasm_v128_load(w + 148); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi17x0123, vk17x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi17x4567, vk17x4567)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + const v128_t vi18x4567 = wasm_v128_load(i18 + 4); + i18 += 8; + + const v128_t vk18x0123 = wasm_v128_load(w + 152); + const v128_t vk18x4567 = wasm_v128_load(w + 156); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi18x4567, vk18x4567)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + const v128_t vi19x4567 = wasm_v128_load(i19 + 4); + i19 += 8; + + const v128_t vk19x0123 = wasm_v128_load(w + 160); + const v128_t vk19x4567 = wasm_v128_load(w + 164); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi19x0123, vk19x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi19x4567, vk19x4567)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + const v128_t vi20x4567 = wasm_v128_load(i20 + 4); + i20 += 8; + + const v128_t vk20x0123 = wasm_v128_load(w + 168); + const v128_t vk20x4567 = wasm_v128_load(w + 172); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi20x4567, vk20x4567)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + const v128_t vi21x4567 = wasm_v128_load(i21 + 4); + i21 += 8; + + const v128_t vk21x0123 = wasm_v128_load(w + 176); + const v128_t vk21x4567 = wasm_v128_load(w + 180); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi21x0123, vk21x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi21x4567, vk21x4567)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + const v128_t vi22x4567 = wasm_v128_load(i22 + 4); + i22 += 8; + + const v128_t vk22x0123 = wasm_v128_load(w + 184); + const v128_t vk22x4567 = wasm_v128_load(w + 188); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi22x4567, vk22x4567)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + const v128_t vi23x4567 = wasm_v128_load(i23 + 4); + i23 += 8; + + const v128_t vk23x0123 = wasm_v128_load(w + 192); + const v128_t vk23x4567 = wasm_v128_load(w + 196); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi23x0123, vk23x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi23x4567, vk23x4567)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + const v128_t vi24x4567 = wasm_v128_load(i24 + 4); + i24 += 8; + + const v128_t vk24x0123 = wasm_v128_load(w + 200); + const v128_t vk24x4567 = wasm_v128_load(w + 204); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi24x4567, vk24x4567)); + + w += 208; + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, vacc4567p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + v128_t vacc4567 = wasm_v128_bitselect(vmin, vacc4567p0, wasm_f32x4_lt(vacc4567p0, vmin)); + + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + vacc4567 = wasm_v128_bitselect(vacc4567, vmax, wasm_f32x4_le(vacc4567, vmax)); + + wasm_v128_store(output, vacc0123); + wasm_v128_store(output + 4, vacc4567); + output += 8; + } + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + i9 += 4; + + const v128_t vk9x0123 = wasm_v128_load(w + 80); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + i10 += 4; + + const v128_t vk10x0123 = wasm_v128_load(w + 88); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + i11 += 4; + + const v128_t vk11x0123 = wasm_v128_load(w + 96); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + i12 += 4; + + const v128_t vk12x0123 = wasm_v128_load(w + 104); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + i13 += 4; + + const v128_t vk13x0123 = wasm_v128_load(w + 112); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + i14 += 4; + + const v128_t vk14x0123 = wasm_v128_load(w + 120); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + i15 += 4; + + const v128_t vk15x0123 = wasm_v128_load(w + 128); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + i16 += 4; + + const v128_t vk16x0123 = wasm_v128_load(w + 136); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + i17 += 4; + + const v128_t vk17x0123 = wasm_v128_load(w + 144); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + i18 += 4; + + const v128_t vk18x0123 = wasm_v128_load(w + 152); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + i19 += 4; + + const v128_t vk19x0123 = wasm_v128_load(w + 160); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + i20 += 4; + + const v128_t vk20x0123 = wasm_v128_load(w + 168); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + i21 += 4; + + const v128_t vk21x0123 = wasm_v128_load(w + 176); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + i22 += 4; + + const v128_t vk22x0123 = wasm_v128_load(w + 184); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + i23 += 4; + + const v128_t vk23x0123 = wasm_v128_load(w + 192); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + i24 += 4; + + const v128_t vk24x0123 = wasm_v128_load(w + 200); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + w += 4; + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 16); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + const v128_t vk9x0123 = wasm_v128_load(w + 80); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + const v128_t vk10x0123 = wasm_v128_load(w + 88); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + const v128_t vk11x0123 = wasm_v128_load(w + 96); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + const v128_t vk12x0123 = wasm_v128_load(w + 104); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + const v128_t vk13x0123 = wasm_v128_load(w + 112); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + const v128_t vk14x0123 = wasm_v128_load(w + 120); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + const v128_t vk15x0123 = wasm_v128_load(w + 128); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + const v128_t vk16x0123 = wasm_v128_load(w + 136); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + const v128_t vk17x0123 = wasm_v128_load(w + 144); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + const v128_t vk18x0123 = wasm_v128_load(w + 152); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + const v128_t vk19x0123 = wasm_v128_load(w + 160); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + const v128_t vk20x0123 = wasm_v128_load(w + 168); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + const v128_t vk21x0123 = wasm_v128_load(w + 176); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + const v128_t vk22x0123 = wasm_v128_load(w + 184); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + const v128_t vk23x0123 = wasm_v128_load(w + 192); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + const v128_t vk24x0123 = wasm_v128_load(w + 200); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up8x25-minmax-wasmsimd-arm.c b/src/f32-dwconv/gen/up8x25-minmax-wasmsimd-arm.c new file mode 100644 index 000000000..1e16adfbe --- /dev/null +++ b/src/f32-dwconv/gen/up8x25-minmax-wasmsimd-arm.c @@ -0,0 +1,689 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + const float* i9 = input[9]; + assert(i9 != NULL); + if XNN_UNPREDICTABLE(i9 != zero) { + i9 = (const float*) ((uintptr_t) i9 + input_offset); + } + const float* i10 = input[10]; + assert(i10 != NULL); + if XNN_UNPREDICTABLE(i10 != zero) { + i10 = (const float*) ((uintptr_t) i10 + input_offset); + } + const float* i11 = input[11]; + assert(i11 != NULL); + if XNN_UNPREDICTABLE(i11 != zero) { + i11 = (const float*) ((uintptr_t) i11 + input_offset); + } + const float* i12 = input[12]; + assert(i12 != NULL); + if XNN_UNPREDICTABLE(i12 != zero) { + i12 = (const float*) ((uintptr_t) i12 + input_offset); + } + const float* i13 = input[13]; + assert(i13 != NULL); + if XNN_UNPREDICTABLE(i13 != zero) { + i13 = (const float*) ((uintptr_t) i13 + input_offset); + } + const float* i14 = input[14]; + assert(i14 != NULL); + if XNN_UNPREDICTABLE(i14 != zero) { + i14 = (const float*) ((uintptr_t) i14 + input_offset); + } + const float* i15 = input[15]; + assert(i15 != NULL); + if XNN_UNPREDICTABLE(i15 != zero) { + i15 = (const float*) ((uintptr_t) i15 + input_offset); + } + const float* i16 = input[16]; + assert(i16 != NULL); + if XNN_UNPREDICTABLE(i16 != zero) { + i16 = (const float*) ((uintptr_t) i16 + input_offset); + } + const float* i17 = input[17]; + assert(i17 != NULL); + if XNN_UNPREDICTABLE(i17 != zero) { + i17 = (const float*) ((uintptr_t) i17 + input_offset); + } + const float* i18 = input[18]; + assert(i18 != NULL); + if XNN_UNPREDICTABLE(i18 != zero) { + i18 = (const float*) ((uintptr_t) i18 + input_offset); + } + const float* i19 = input[19]; + assert(i19 != NULL); + if XNN_UNPREDICTABLE(i19 != zero) { + i19 = (const float*) ((uintptr_t) i19 + input_offset); + } + const float* i20 = input[20]; + assert(i20 != NULL); + if XNN_UNPREDICTABLE(i20 != zero) { + i20 = (const float*) ((uintptr_t) i20 + input_offset); + } + const float* i21 = input[21]; + assert(i21 != NULL); + if XNN_UNPREDICTABLE(i21 != zero) { + i21 = (const float*) ((uintptr_t) i21 + input_offset); + } + const float* i22 = input[22]; + assert(i22 != NULL); + if XNN_UNPREDICTABLE(i22 != zero) { + i22 = (const float*) ((uintptr_t) i22 + input_offset); + } + const float* i23 = input[23]; + assert(i23 != NULL); + if XNN_UNPREDICTABLE(i23 != zero) { + i23 = (const float*) ((uintptr_t) i23 + input_offset); + } + const float* i24 = input[24]; + assert(i24 != NULL); + if XNN_UNPREDICTABLE(i24 != zero) { + i24 = (const float*) ((uintptr_t) i24 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 8; c -= 8) { + v128_t vacc0123p0 = wasm_v128_load(w); + v128_t vacc4567p0 = wasm_v128_load(w + 4); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vi0x4567 = wasm_v128_load(i0 + 4); + i0 += 8; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + const v128_t vk0x4567 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi0x4567, vk0x4567)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vi1x4567 = wasm_v128_load(i1 + 4); + i1 += 8; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + const v128_t vk1x4567 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi1x4567, vk1x4567)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vi2x4567 = wasm_v128_load(i2 + 4); + i2 += 8; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + const v128_t vk2x4567 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi2x4567, vk2x4567)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vi3x4567 = wasm_v128_load(i3 + 4); + i3 += 8; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + const v128_t vk3x4567 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi3x4567, vk3x4567)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vi4x4567 = wasm_v128_load(i4 + 4); + i4 += 8; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + const v128_t vk4x4567 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi4x4567, vk4x4567)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vi5x4567 = wasm_v128_load(i5 + 4); + i5 += 8; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + const v128_t vk5x4567 = wasm_v128_load(w + 52); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi5x4567, vk5x4567)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vi6x4567 = wasm_v128_load(i6 + 4); + i6 += 8; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + const v128_t vk6x4567 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi6x4567, vk6x4567)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vi7x4567 = wasm_v128_load(i7 + 4); + i7 += 8; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + const v128_t vk7x4567 = wasm_v128_load(w + 68); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi7x4567, vk7x4567)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vi8x4567 = wasm_v128_load(i8 + 4); + i8 += 8; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + const v128_t vk8x4567 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi8x4567, vk8x4567)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + const v128_t vi9x4567 = wasm_v128_load(i9 + 4); + i9 += 8; + + const v128_t vk9x0123 = wasm_v128_load(w + 80); + const v128_t vk9x4567 = wasm_v128_load(w + 84); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi9x0123, vk9x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi9x4567, vk9x4567)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + const v128_t vi10x4567 = wasm_v128_load(i10 + 4); + i10 += 8; + + const v128_t vk10x0123 = wasm_v128_load(w + 88); + const v128_t vk10x4567 = wasm_v128_load(w + 92); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi10x4567, vk10x4567)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + const v128_t vi11x4567 = wasm_v128_load(i11 + 4); + i11 += 8; + + const v128_t vk11x0123 = wasm_v128_load(w + 96); + const v128_t vk11x4567 = wasm_v128_load(w + 100); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi11x0123, vk11x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi11x4567, vk11x4567)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + const v128_t vi12x4567 = wasm_v128_load(i12 + 4); + i12 += 8; + + const v128_t vk12x0123 = wasm_v128_load(w + 104); + const v128_t vk12x4567 = wasm_v128_load(w + 108); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi12x4567, vk12x4567)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + const v128_t vi13x4567 = wasm_v128_load(i13 + 4); + i13 += 8; + + const v128_t vk13x0123 = wasm_v128_load(w + 112); + const v128_t vk13x4567 = wasm_v128_load(w + 116); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi13x0123, vk13x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi13x4567, vk13x4567)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + const v128_t vi14x4567 = wasm_v128_load(i14 + 4); + i14 += 8; + + const v128_t vk14x0123 = wasm_v128_load(w + 120); + const v128_t vk14x4567 = wasm_v128_load(w + 124); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi14x4567, vk14x4567)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + const v128_t vi15x4567 = wasm_v128_load(i15 + 4); + i15 += 8; + + const v128_t vk15x0123 = wasm_v128_load(w + 128); + const v128_t vk15x4567 = wasm_v128_load(w + 132); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi15x0123, vk15x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi15x4567, vk15x4567)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + const v128_t vi16x4567 = wasm_v128_load(i16 + 4); + i16 += 8; + + const v128_t vk16x0123 = wasm_v128_load(w + 136); + const v128_t vk16x4567 = wasm_v128_load(w + 140); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi16x4567, vk16x4567)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + const v128_t vi17x4567 = wasm_v128_load(i17 + 4); + i17 += 8; + + const v128_t vk17x0123 = wasm_v128_load(w + 144); + const v128_t vk17x4567 = wasm_v128_load(w + 148); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi17x0123, vk17x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi17x4567, vk17x4567)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + const v128_t vi18x4567 = wasm_v128_load(i18 + 4); + i18 += 8; + + const v128_t vk18x0123 = wasm_v128_load(w + 152); + const v128_t vk18x4567 = wasm_v128_load(w + 156); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi18x4567, vk18x4567)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + const v128_t vi19x4567 = wasm_v128_load(i19 + 4); + i19 += 8; + + const v128_t vk19x0123 = wasm_v128_load(w + 160); + const v128_t vk19x4567 = wasm_v128_load(w + 164); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi19x0123, vk19x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi19x4567, vk19x4567)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + const v128_t vi20x4567 = wasm_v128_load(i20 + 4); + i20 += 8; + + const v128_t vk20x0123 = wasm_v128_load(w + 168); + const v128_t vk20x4567 = wasm_v128_load(w + 172); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi20x4567, vk20x4567)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + const v128_t vi21x4567 = wasm_v128_load(i21 + 4); + i21 += 8; + + const v128_t vk21x0123 = wasm_v128_load(w + 176); + const v128_t vk21x4567 = wasm_v128_load(w + 180); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi21x0123, vk21x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi21x4567, vk21x4567)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + const v128_t vi22x4567 = wasm_v128_load(i22 + 4); + i22 += 8; + + const v128_t vk22x0123 = wasm_v128_load(w + 184); + const v128_t vk22x4567 = wasm_v128_load(w + 188); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi22x4567, vk22x4567)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + const v128_t vi23x4567 = wasm_v128_load(i23 + 4); + i23 += 8; + + const v128_t vk23x0123 = wasm_v128_load(w + 192); + const v128_t vk23x4567 = wasm_v128_load(w + 196); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi23x0123, vk23x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi23x4567, vk23x4567)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + const v128_t vi24x4567 = wasm_v128_load(i24 + 4); + i24 += 8; + + const v128_t vk24x0123 = wasm_v128_load(w + 200); + const v128_t vk24x4567 = wasm_v128_load(w + 204); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi24x4567, vk24x4567)); + + w += 208; + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + v128_t vacc4567 = wasm_f32x4_max(vacc4567p0, vmin); + + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + vacc4567 = wasm_f32x4_min(vacc4567, vmax); + + wasm_v128_store(output, vacc0123); + wasm_v128_store(output + 4, vacc4567); + output += 8; + } + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + i9 += 4; + + const v128_t vk9x0123 = wasm_v128_load(w + 80); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + i10 += 4; + + const v128_t vk10x0123 = wasm_v128_load(w + 88); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + i11 += 4; + + const v128_t vk11x0123 = wasm_v128_load(w + 96); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + i12 += 4; + + const v128_t vk12x0123 = wasm_v128_load(w + 104); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + i13 += 4; + + const v128_t vk13x0123 = wasm_v128_load(w + 112); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + i14 += 4; + + const v128_t vk14x0123 = wasm_v128_load(w + 120); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + i15 += 4; + + const v128_t vk15x0123 = wasm_v128_load(w + 128); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + i16 += 4; + + const v128_t vk16x0123 = wasm_v128_load(w + 136); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + i17 += 4; + + const v128_t vk17x0123 = wasm_v128_load(w + 144); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + i18 += 4; + + const v128_t vk18x0123 = wasm_v128_load(w + 152); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + i19 += 4; + + const v128_t vk19x0123 = wasm_v128_load(w + 160); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + i20 += 4; + + const v128_t vk20x0123 = wasm_v128_load(w + 168); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + i21 += 4; + + const v128_t vk21x0123 = wasm_v128_load(w + 176); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + i22 += 4; + + const v128_t vk22x0123 = wasm_v128_load(w + 184); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + i23 += 4; + + const v128_t vk23x0123 = wasm_v128_load(w + 192); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + i24 += 4; + + const v128_t vk24x0123 = wasm_v128_load(w + 200); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + w += 4; + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + const v128_t vk9x0123 = wasm_v128_load(w + 80); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + const v128_t vk10x0123 = wasm_v128_load(w + 88); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + const v128_t vk11x0123 = wasm_v128_load(w + 96); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + const v128_t vk12x0123 = wasm_v128_load(w + 104); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + const v128_t vk13x0123 = wasm_v128_load(w + 112); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + const v128_t vk14x0123 = wasm_v128_load(w + 120); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + const v128_t vk15x0123 = wasm_v128_load(w + 128); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + const v128_t vk16x0123 = wasm_v128_load(w + 136); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + const v128_t vk17x0123 = wasm_v128_load(w + 144); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + const v128_t vk18x0123 = wasm_v128_load(w + 152); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + const v128_t vk19x0123 = wasm_v128_load(w + 160); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + const v128_t vk20x0123 = wasm_v128_load(w + 168); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + const v128_t vk21x0123 = wasm_v128_load(w + 176); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + const v128_t vk22x0123 = wasm_v128_load(w + 184); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + const v128_t vk23x0123 = wasm_v128_load(w + 192); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + const v128_t vk24x0123 = wasm_v128_load(w + 200); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up8x25-minmax-wasmsimd-x86.c b/src/f32-dwconv/gen/up8x25-minmax-wasmsimd-x86.c new file mode 100644 index 000000000..b01484a40 --- /dev/null +++ b/src/f32-dwconv/gen/up8x25-minmax-wasmsimd-x86.c @@ -0,0 +1,689 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + const float* i9 = input[9]; + assert(i9 != NULL); + if XNN_UNPREDICTABLE(i9 != zero) { + i9 = (const float*) ((uintptr_t) i9 + input_offset); + } + const float* i10 = input[10]; + assert(i10 != NULL); + if XNN_UNPREDICTABLE(i10 != zero) { + i10 = (const float*) ((uintptr_t) i10 + input_offset); + } + const float* i11 = input[11]; + assert(i11 != NULL); + if XNN_UNPREDICTABLE(i11 != zero) { + i11 = (const float*) ((uintptr_t) i11 + input_offset); + } + const float* i12 = input[12]; + assert(i12 != NULL); + if XNN_UNPREDICTABLE(i12 != zero) { + i12 = (const float*) ((uintptr_t) i12 + input_offset); + } + const float* i13 = input[13]; + assert(i13 != NULL); + if XNN_UNPREDICTABLE(i13 != zero) { + i13 = (const float*) ((uintptr_t) i13 + input_offset); + } + const float* i14 = input[14]; + assert(i14 != NULL); + if XNN_UNPREDICTABLE(i14 != zero) { + i14 = (const float*) ((uintptr_t) i14 + input_offset); + } + const float* i15 = input[15]; + assert(i15 != NULL); + if XNN_UNPREDICTABLE(i15 != zero) { + i15 = (const float*) ((uintptr_t) i15 + input_offset); + } + const float* i16 = input[16]; + assert(i16 != NULL); + if XNN_UNPREDICTABLE(i16 != zero) { + i16 = (const float*) ((uintptr_t) i16 + input_offset); + } + const float* i17 = input[17]; + assert(i17 != NULL); + if XNN_UNPREDICTABLE(i17 != zero) { + i17 = (const float*) ((uintptr_t) i17 + input_offset); + } + const float* i18 = input[18]; + assert(i18 != NULL); + if XNN_UNPREDICTABLE(i18 != zero) { + i18 = (const float*) ((uintptr_t) i18 + input_offset); + } + const float* i19 = input[19]; + assert(i19 != NULL); + if XNN_UNPREDICTABLE(i19 != zero) { + i19 = (const float*) ((uintptr_t) i19 + input_offset); + } + const float* i20 = input[20]; + assert(i20 != NULL); + if XNN_UNPREDICTABLE(i20 != zero) { + i20 = (const float*) ((uintptr_t) i20 + input_offset); + } + const float* i21 = input[21]; + assert(i21 != NULL); + if XNN_UNPREDICTABLE(i21 != zero) { + i21 = (const float*) ((uintptr_t) i21 + input_offset); + } + const float* i22 = input[22]; + assert(i22 != NULL); + if XNN_UNPREDICTABLE(i22 != zero) { + i22 = (const float*) ((uintptr_t) i22 + input_offset); + } + const float* i23 = input[23]; + assert(i23 != NULL); + if XNN_UNPREDICTABLE(i23 != zero) { + i23 = (const float*) ((uintptr_t) i23 + input_offset); + } + const float* i24 = input[24]; + assert(i24 != NULL); + if XNN_UNPREDICTABLE(i24 != zero) { + i24 = (const float*) ((uintptr_t) i24 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 8; c -= 8) { + v128_t vacc0123p0 = wasm_v128_load(w); + v128_t vacc4567p0 = wasm_v128_load(w + 4); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vi0x4567 = wasm_v128_load(i0 + 4); + i0 += 8; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + const v128_t vk0x4567 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi0x4567, vk0x4567)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vi1x4567 = wasm_v128_load(i1 + 4); + i1 += 8; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + const v128_t vk1x4567 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi1x4567, vk1x4567)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vi2x4567 = wasm_v128_load(i2 + 4); + i2 += 8; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + const v128_t vk2x4567 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi2x4567, vk2x4567)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vi3x4567 = wasm_v128_load(i3 + 4); + i3 += 8; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + const v128_t vk3x4567 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi3x4567, vk3x4567)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vi4x4567 = wasm_v128_load(i4 + 4); + i4 += 8; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + const v128_t vk4x4567 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi4x4567, vk4x4567)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vi5x4567 = wasm_v128_load(i5 + 4); + i5 += 8; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + const v128_t vk5x4567 = wasm_v128_load(w + 52); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi5x4567, vk5x4567)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vi6x4567 = wasm_v128_load(i6 + 4); + i6 += 8; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + const v128_t vk6x4567 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi6x4567, vk6x4567)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vi7x4567 = wasm_v128_load(i7 + 4); + i7 += 8; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + const v128_t vk7x4567 = wasm_v128_load(w + 68); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi7x4567, vk7x4567)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vi8x4567 = wasm_v128_load(i8 + 4); + i8 += 8; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + const v128_t vk8x4567 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi8x4567, vk8x4567)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + const v128_t vi9x4567 = wasm_v128_load(i9 + 4); + i9 += 8; + + const v128_t vk9x0123 = wasm_v128_load(w + 80); + const v128_t vk9x4567 = wasm_v128_load(w + 84); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi9x0123, vk9x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi9x4567, vk9x4567)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + const v128_t vi10x4567 = wasm_v128_load(i10 + 4); + i10 += 8; + + const v128_t vk10x0123 = wasm_v128_load(w + 88); + const v128_t vk10x4567 = wasm_v128_load(w + 92); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi10x4567, vk10x4567)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + const v128_t vi11x4567 = wasm_v128_load(i11 + 4); + i11 += 8; + + const v128_t vk11x0123 = wasm_v128_load(w + 96); + const v128_t vk11x4567 = wasm_v128_load(w + 100); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi11x0123, vk11x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi11x4567, vk11x4567)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + const v128_t vi12x4567 = wasm_v128_load(i12 + 4); + i12 += 8; + + const v128_t vk12x0123 = wasm_v128_load(w + 104); + const v128_t vk12x4567 = wasm_v128_load(w + 108); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi12x4567, vk12x4567)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + const v128_t vi13x4567 = wasm_v128_load(i13 + 4); + i13 += 8; + + const v128_t vk13x0123 = wasm_v128_load(w + 112); + const v128_t vk13x4567 = wasm_v128_load(w + 116); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi13x0123, vk13x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi13x4567, vk13x4567)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + const v128_t vi14x4567 = wasm_v128_load(i14 + 4); + i14 += 8; + + const v128_t vk14x0123 = wasm_v128_load(w + 120); + const v128_t vk14x4567 = wasm_v128_load(w + 124); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi14x4567, vk14x4567)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + const v128_t vi15x4567 = wasm_v128_load(i15 + 4); + i15 += 8; + + const v128_t vk15x0123 = wasm_v128_load(w + 128); + const v128_t vk15x4567 = wasm_v128_load(w + 132); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi15x0123, vk15x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi15x4567, vk15x4567)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + const v128_t vi16x4567 = wasm_v128_load(i16 + 4); + i16 += 8; + + const v128_t vk16x0123 = wasm_v128_load(w + 136); + const v128_t vk16x4567 = wasm_v128_load(w + 140); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi16x4567, vk16x4567)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + const v128_t vi17x4567 = wasm_v128_load(i17 + 4); + i17 += 8; + + const v128_t vk17x0123 = wasm_v128_load(w + 144); + const v128_t vk17x4567 = wasm_v128_load(w + 148); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi17x0123, vk17x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi17x4567, vk17x4567)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + const v128_t vi18x4567 = wasm_v128_load(i18 + 4); + i18 += 8; + + const v128_t vk18x0123 = wasm_v128_load(w + 152); + const v128_t vk18x4567 = wasm_v128_load(w + 156); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi18x4567, vk18x4567)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + const v128_t vi19x4567 = wasm_v128_load(i19 + 4); + i19 += 8; + + const v128_t vk19x0123 = wasm_v128_load(w + 160); + const v128_t vk19x4567 = wasm_v128_load(w + 164); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi19x0123, vk19x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi19x4567, vk19x4567)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + const v128_t vi20x4567 = wasm_v128_load(i20 + 4); + i20 += 8; + + const v128_t vk20x0123 = wasm_v128_load(w + 168); + const v128_t vk20x4567 = wasm_v128_load(w + 172); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi20x4567, vk20x4567)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + const v128_t vi21x4567 = wasm_v128_load(i21 + 4); + i21 += 8; + + const v128_t vk21x0123 = wasm_v128_load(w + 176); + const v128_t vk21x4567 = wasm_v128_load(w + 180); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi21x0123, vk21x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi21x4567, vk21x4567)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + const v128_t vi22x4567 = wasm_v128_load(i22 + 4); + i22 += 8; + + const v128_t vk22x0123 = wasm_v128_load(w + 184); + const v128_t vk22x4567 = wasm_v128_load(w + 188); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi22x4567, vk22x4567)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + const v128_t vi23x4567 = wasm_v128_load(i23 + 4); + i23 += 8; + + const v128_t vk23x0123 = wasm_v128_load(w + 192); + const v128_t vk23x4567 = wasm_v128_load(w + 196); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi23x0123, vk23x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi23x4567, vk23x4567)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + const v128_t vi24x4567 = wasm_v128_load(i24 + 4); + i24 += 8; + + const v128_t vk24x0123 = wasm_v128_load(w + 200); + const v128_t vk24x4567 = wasm_v128_load(w + 204); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi24x4567, vk24x4567)); + + w += 208; + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + v128_t vacc4567 = wasm_v128_bitselect(vmin, vacc4567p0, wasm_f32x4_lt(vacc4567p0, vmin)); + + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + vacc4567 = wasm_v128_bitselect(vacc4567, vmax, wasm_f32x4_le(vacc4567, vmax)); + + wasm_v128_store(output, vacc0123); + wasm_v128_store(output + 4, vacc4567); + output += 8; + } + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + i9 += 4; + + const v128_t vk9x0123 = wasm_v128_load(w + 80); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + i10 += 4; + + const v128_t vk10x0123 = wasm_v128_load(w + 88); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + i11 += 4; + + const v128_t vk11x0123 = wasm_v128_load(w + 96); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + i12 += 4; + + const v128_t vk12x0123 = wasm_v128_load(w + 104); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + i13 += 4; + + const v128_t vk13x0123 = wasm_v128_load(w + 112); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + i14 += 4; + + const v128_t vk14x0123 = wasm_v128_load(w + 120); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + i15 += 4; + + const v128_t vk15x0123 = wasm_v128_load(w + 128); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + i16 += 4; + + const v128_t vk16x0123 = wasm_v128_load(w + 136); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + i17 += 4; + + const v128_t vk17x0123 = wasm_v128_load(w + 144); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + i18 += 4; + + const v128_t vk18x0123 = wasm_v128_load(w + 152); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + i19 += 4; + + const v128_t vk19x0123 = wasm_v128_load(w + 160); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + i20 += 4; + + const v128_t vk20x0123 = wasm_v128_load(w + 168); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + i21 += 4; + + const v128_t vk21x0123 = wasm_v128_load(w + 176); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + i22 += 4; + + const v128_t vk22x0123 = wasm_v128_load(w + 184); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + i23 += 4; + + const v128_t vk23x0123 = wasm_v128_load(w + 192); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + i24 += 4; + + const v128_t vk24x0123 = wasm_v128_load(w + 200); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + w += 4; + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + const v128_t vi9x0123 = wasm_v128_load(i9); + const v128_t vk9x0123 = wasm_v128_load(w + 80); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi9x0123, vk9x0123)); + + const v128_t vi10x0123 = wasm_v128_load(i10); + const v128_t vk10x0123 = wasm_v128_load(w + 88); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi10x0123, vk10x0123)); + + const v128_t vi11x0123 = wasm_v128_load(i11); + const v128_t vk11x0123 = wasm_v128_load(w + 96); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi11x0123, vk11x0123)); + + const v128_t vi12x0123 = wasm_v128_load(i12); + const v128_t vk12x0123 = wasm_v128_load(w + 104); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi12x0123, vk12x0123)); + + const v128_t vi13x0123 = wasm_v128_load(i13); + const v128_t vk13x0123 = wasm_v128_load(w + 112); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi13x0123, vk13x0123)); + + const v128_t vi14x0123 = wasm_v128_load(i14); + const v128_t vk14x0123 = wasm_v128_load(w + 120); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi14x0123, vk14x0123)); + + const v128_t vi15x0123 = wasm_v128_load(i15); + const v128_t vk15x0123 = wasm_v128_load(w + 128); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi15x0123, vk15x0123)); + + const v128_t vi16x0123 = wasm_v128_load(i16); + const v128_t vk16x0123 = wasm_v128_load(w + 136); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi16x0123, vk16x0123)); + + const v128_t vi17x0123 = wasm_v128_load(i17); + const v128_t vk17x0123 = wasm_v128_load(w + 144); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi17x0123, vk17x0123)); + + const v128_t vi18x0123 = wasm_v128_load(i18); + const v128_t vk18x0123 = wasm_v128_load(w + 152); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi18x0123, vk18x0123)); + + const v128_t vi19x0123 = wasm_v128_load(i19); + const v128_t vk19x0123 = wasm_v128_load(w + 160); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi19x0123, vk19x0123)); + + const v128_t vi20x0123 = wasm_v128_load(i20); + const v128_t vk20x0123 = wasm_v128_load(w + 168); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi20x0123, vk20x0123)); + + const v128_t vi21x0123 = wasm_v128_load(i21); + const v128_t vk21x0123 = wasm_v128_load(w + 176); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi21x0123, vk21x0123)); + + const v128_t vi22x0123 = wasm_v128_load(i22); + const v128_t vk22x0123 = wasm_v128_load(w + 184); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi22x0123, vk22x0123)); + + const v128_t vi23x0123 = wasm_v128_load(i23); + const v128_t vk23x0123 = wasm_v128_load(w + 192); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi23x0123, vk23x0123)); + + const v128_t vi24x0123 = wasm_v128_load(i24); + const v128_t vk24x0123 = wasm_v128_load(w + 200); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi24x0123, vk24x0123)); + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up8x4-minmax-wasmsimd-acc2-arm.c b/src/f32-dwconv/gen/up8x4-minmax-wasmsimd-acc2-arm.c new file mode 100644 index 000000000..020cd8b8d --- /dev/null +++ b/src/f32-dwconv/gen/up8x4-minmax-wasmsimd-acc2-arm.c @@ -0,0 +1,192 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 8; c -= 8) { + v128_t vacc0123p0 = wasm_v128_load(w); + v128_t vacc4567p0 = wasm_v128_load(w + 4); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vi0x4567 = wasm_v128_load(i0 + 4); + i0 += 8; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + const v128_t vk0x4567 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi0x4567, vk0x4567)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vi1x4567 = wasm_v128_load(i1 + 4); + i1 += 8; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + const v128_t vk1x4567 = wasm_v128_load(w + 20); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + v128_t vacc4567p1 = wasm_f32x4_mul(vi1x4567, vk1x4567); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vi2x4567 = wasm_v128_load(i2 + 4); + i2 += 8; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + const v128_t vk2x4567 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi2x4567, vk2x4567)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vi3x4567 = wasm_v128_load(i3 + 4); + i3 += 8; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + const v128_t vk3x4567 = wasm_v128_load(w + 36); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi3x4567, vk3x4567)); + + w += 40; + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, vacc4567p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + v128_t vacc4567 = wasm_f32x4_max(vacc4567p0, vmin); + + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + vacc4567 = wasm_f32x4_min(vacc4567, vmax); + + wasm_v128_store(output, vacc0123); + wasm_v128_store(output + 4, vacc4567); + output += 8; + } + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + w += 4; + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 16); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up8x4-minmax-wasmsimd-acc2-x86.c b/src/f32-dwconv/gen/up8x4-minmax-wasmsimd-acc2-x86.c new file mode 100644 index 000000000..5e1ca91fb --- /dev/null +++ b/src/f32-dwconv/gen/up8x4-minmax-wasmsimd-acc2-x86.c @@ -0,0 +1,192 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 8; c -= 8) { + v128_t vacc0123p0 = wasm_v128_load(w); + v128_t vacc4567p0 = wasm_v128_load(w + 4); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vi0x4567 = wasm_v128_load(i0 + 4); + i0 += 8; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + const v128_t vk0x4567 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi0x4567, vk0x4567)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vi1x4567 = wasm_v128_load(i1 + 4); + i1 += 8; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + const v128_t vk1x4567 = wasm_v128_load(w + 20); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + v128_t vacc4567p1 = wasm_f32x4_mul(vi1x4567, vk1x4567); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vi2x4567 = wasm_v128_load(i2 + 4); + i2 += 8; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + const v128_t vk2x4567 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi2x4567, vk2x4567)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vi3x4567 = wasm_v128_load(i3 + 4); + i3 += 8; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + const v128_t vk3x4567 = wasm_v128_load(w + 36); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi3x4567, vk3x4567)); + + w += 40; + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, vacc4567p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + v128_t vacc4567 = wasm_v128_bitselect(vmin, vacc4567p0, wasm_f32x4_lt(vacc4567p0, vmin)); + + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + vacc4567 = wasm_v128_bitselect(vacc4567, vmax, wasm_f32x4_le(vacc4567, vmax)); + + wasm_v128_store(output, vacc0123); + wasm_v128_store(output + 4, vacc4567); + output += 8; + } + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + w += 4; + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 16); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up8x4-minmax-wasmsimd-arm.c b/src/f32-dwconv/gen/up8x4-minmax-wasmsimd-arm.c new file mode 100644 index 000000000..b2c4ae790 --- /dev/null +++ b/src/f32-dwconv/gen/up8x4-minmax-wasmsimd-arm.c @@ -0,0 +1,185 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 8; c -= 8) { + v128_t vacc0123p0 = wasm_v128_load(w); + v128_t vacc4567p0 = wasm_v128_load(w + 4); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vi0x4567 = wasm_v128_load(i0 + 4); + i0 += 8; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + const v128_t vk0x4567 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi0x4567, vk0x4567)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vi1x4567 = wasm_v128_load(i1 + 4); + i1 += 8; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + const v128_t vk1x4567 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi1x4567, vk1x4567)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vi2x4567 = wasm_v128_load(i2 + 4); + i2 += 8; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + const v128_t vk2x4567 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi2x4567, vk2x4567)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vi3x4567 = wasm_v128_load(i3 + 4); + i3 += 8; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + const v128_t vk3x4567 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi3x4567, vk3x4567)); + + w += 40; + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + v128_t vacc4567 = wasm_f32x4_max(vacc4567p0, vmin); + + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + vacc4567 = wasm_f32x4_min(vacc4567, vmax); + + wasm_v128_store(output, vacc0123); + wasm_v128_store(output + 4, vacc4567); + output += 8; + } + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + w += 4; + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up8x4-minmax-wasmsimd-x86.c b/src/f32-dwconv/gen/up8x4-minmax-wasmsimd-x86.c new file mode 100644 index 000000000..59f051300 --- /dev/null +++ b/src/f32-dwconv/gen/up8x4-minmax-wasmsimd-x86.c @@ -0,0 +1,185 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 8; c -= 8) { + v128_t vacc0123p0 = wasm_v128_load(w); + v128_t vacc4567p0 = wasm_v128_load(w + 4); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vi0x4567 = wasm_v128_load(i0 + 4); + i0 += 8; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + const v128_t vk0x4567 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi0x4567, vk0x4567)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vi1x4567 = wasm_v128_load(i1 + 4); + i1 += 8; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + const v128_t vk1x4567 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi1x4567, vk1x4567)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vi2x4567 = wasm_v128_load(i2 + 4); + i2 += 8; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + const v128_t vk2x4567 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi2x4567, vk2x4567)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vi3x4567 = wasm_v128_load(i3 + 4); + i3 += 8; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + const v128_t vk3x4567 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi3x4567, vk3x4567)); + + w += 40; + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + v128_t vacc4567 = wasm_v128_bitselect(vmin, vacc4567p0, wasm_f32x4_lt(vacc4567p0, vmin)); + + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + vacc4567 = wasm_v128_bitselect(vacc4567, vmax, wasm_f32x4_le(vacc4567, vmax)); + + wasm_v128_store(output, vacc0123); + wasm_v128_store(output + 4, vacc4567); + output += 8; + } + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + w += 4; + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up8x9-minmax-wasmsimd-acc2-arm.c b/src/f32-dwconv/gen/up8x9-minmax-wasmsimd-acc2-arm.c new file mode 100644 index 000000000..bcf763b40 --- /dev/null +++ b/src/f32-dwconv/gen/up8x9-minmax-wasmsimd-acc2-arm.c @@ -0,0 +1,312 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 8; c -= 8) { + v128_t vacc0123p0 = wasm_v128_load(w); + v128_t vacc4567p0 = wasm_v128_load(w + 4); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vi0x4567 = wasm_v128_load(i0 + 4); + i0 += 8; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + const v128_t vk0x4567 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi0x4567, vk0x4567)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vi1x4567 = wasm_v128_load(i1 + 4); + i1 += 8; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + const v128_t vk1x4567 = wasm_v128_load(w + 20); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + v128_t vacc4567p1 = wasm_f32x4_mul(vi1x4567, vk1x4567); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vi2x4567 = wasm_v128_load(i2 + 4); + i2 += 8; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + const v128_t vk2x4567 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi2x4567, vk2x4567)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vi3x4567 = wasm_v128_load(i3 + 4); + i3 += 8; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + const v128_t vk3x4567 = wasm_v128_load(w + 36); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi3x4567, vk3x4567)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vi4x4567 = wasm_v128_load(i4 + 4); + i4 += 8; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + const v128_t vk4x4567 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi4x4567, vk4x4567)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vi5x4567 = wasm_v128_load(i5 + 4); + i5 += 8; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + const v128_t vk5x4567 = wasm_v128_load(w + 52); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi5x4567, vk5x4567)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vi6x4567 = wasm_v128_load(i6 + 4); + i6 += 8; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + const v128_t vk6x4567 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi6x4567, vk6x4567)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vi7x4567 = wasm_v128_load(i7 + 4); + i7 += 8; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + const v128_t vk7x4567 = wasm_v128_load(w + 68); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi7x4567, vk7x4567)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vi8x4567 = wasm_v128_load(i8 + 4); + i8 += 8; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + const v128_t vk8x4567 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi8x4567, vk8x4567)); + + w += 80; + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, vacc4567p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + v128_t vacc4567 = wasm_f32x4_max(vacc4567p0, vmin); + + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + vacc4567 = wasm_f32x4_min(vacc4567, vmax); + + wasm_v128_store(output, vacc0123); + wasm_v128_store(output + 4, vacc4567); + output += 8; + } + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + w += 4; + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 16); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up8x9-minmax-wasmsimd-acc2-x86.c b/src/f32-dwconv/gen/up8x9-minmax-wasmsimd-acc2-x86.c new file mode 100644 index 000000000..41c1c4bc4 --- /dev/null +++ b/src/f32-dwconv/gen/up8x9-minmax-wasmsimd-acc2-x86.c @@ -0,0 +1,312 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 8; c -= 8) { + v128_t vacc0123p0 = wasm_v128_load(w); + v128_t vacc4567p0 = wasm_v128_load(w + 4); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vi0x4567 = wasm_v128_load(i0 + 4); + i0 += 8; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + const v128_t vk0x4567 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi0x4567, vk0x4567)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vi1x4567 = wasm_v128_load(i1 + 4); + i1 += 8; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + const v128_t vk1x4567 = wasm_v128_load(w + 20); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + v128_t vacc4567p1 = wasm_f32x4_mul(vi1x4567, vk1x4567); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vi2x4567 = wasm_v128_load(i2 + 4); + i2 += 8; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + const v128_t vk2x4567 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi2x4567, vk2x4567)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vi3x4567 = wasm_v128_load(i3 + 4); + i3 += 8; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + const v128_t vk3x4567 = wasm_v128_load(w + 36); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi3x4567, vk3x4567)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vi4x4567 = wasm_v128_load(i4 + 4); + i4 += 8; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + const v128_t vk4x4567 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi4x4567, vk4x4567)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vi5x4567 = wasm_v128_load(i5 + 4); + i5 += 8; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + const v128_t vk5x4567 = wasm_v128_load(w + 52); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi5x4567, vk5x4567)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vi6x4567 = wasm_v128_load(i6 + 4); + i6 += 8; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + const v128_t vk6x4567 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi6x4567, vk6x4567)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vi7x4567 = wasm_v128_load(i7 + 4); + i7 += 8; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + const v128_t vk7x4567 = wasm_v128_load(w + 68); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + vacc4567p1 = wasm_f32x4_add(vacc4567p1, wasm_f32x4_mul(vi7x4567, vk7x4567)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vi8x4567 = wasm_v128_load(i8 + 4); + i8 += 8; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + const v128_t vk8x4567 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi8x4567, vk8x4567)); + + w += 80; + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, vacc4567p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + v128_t vacc4567 = wasm_v128_bitselect(vmin, vacc4567p0, wasm_f32x4_lt(vacc4567p0, vmin)); + + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + vacc4567 = wasm_v128_bitselect(vacc4567, vmax, wasm_f32x4_le(vacc4567, vmax)); + + wasm_v128_store(output, vacc0123); + wasm_v128_store(output + 4, vacc4567); + output += 8; + } + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + w += 4; + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 16); + v128_t vacc0123p1 = wasm_f32x4_mul(vi1x0123, vk1x0123); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p1 = wasm_f32x4_add(vacc0123p1, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + // Add up all accumulators to vacc01234567p0 + vacc0123p0 = wasm_f32x4_add(vacc0123p0, vacc0123p1); + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up8x9-minmax-wasmsimd-arm.c b/src/f32-dwconv/gen/up8x9-minmax-wasmsimd-arm.c new file mode 100644 index 000000000..57f90fbb5 --- /dev/null +++ b/src/f32-dwconv/gen/up8x9-minmax-wasmsimd-arm.c @@ -0,0 +1,305 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 8; c -= 8) { + v128_t vacc0123p0 = wasm_v128_load(w); + v128_t vacc4567p0 = wasm_v128_load(w + 4); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vi0x4567 = wasm_v128_load(i0 + 4); + i0 += 8; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + const v128_t vk0x4567 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi0x4567, vk0x4567)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vi1x4567 = wasm_v128_load(i1 + 4); + i1 += 8; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + const v128_t vk1x4567 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi1x4567, vk1x4567)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vi2x4567 = wasm_v128_load(i2 + 4); + i2 += 8; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + const v128_t vk2x4567 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi2x4567, vk2x4567)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vi3x4567 = wasm_v128_load(i3 + 4); + i3 += 8; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + const v128_t vk3x4567 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi3x4567, vk3x4567)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vi4x4567 = wasm_v128_load(i4 + 4); + i4 += 8; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + const v128_t vk4x4567 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi4x4567, vk4x4567)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vi5x4567 = wasm_v128_load(i5 + 4); + i5 += 8; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + const v128_t vk5x4567 = wasm_v128_load(w + 52); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi5x4567, vk5x4567)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vi6x4567 = wasm_v128_load(i6 + 4); + i6 += 8; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + const v128_t vk6x4567 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi6x4567, vk6x4567)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vi7x4567 = wasm_v128_load(i7 + 4); + i7 += 8; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + const v128_t vk7x4567 = wasm_v128_load(w + 68); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi7x4567, vk7x4567)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vi8x4567 = wasm_v128_load(i8 + 4); + i8 += 8; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + const v128_t vk8x4567 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi8x4567, vk8x4567)); + + w += 80; + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + v128_t vacc4567 = wasm_f32x4_max(vacc4567p0, vmin); + + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + vacc4567 = wasm_f32x4_min(vacc4567, vmax); + + wasm_v128_store(output, vacc0123); + wasm_v128_store(output + 4, vacc4567); + output += 8; + } + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + w += 4; + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/gen/up8x9-minmax-wasmsimd-x86.c b/src/f32-dwconv/gen/up8x9-minmax-wasmsimd-x86.c new file mode 100644 index 000000000..4711c23cb --- /dev/null +++ b/src/f32-dwconv/gen/up8x9-minmax-wasmsimd-x86.c @@ -0,0 +1,305 @@ +// Auto-generated file. Do not edit! +// Template: src/f32-dwconv/up-wasmsimd.c.in +// Generator: tools/xngen +// +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + const float* i0 = input[0]; + assert(i0 != NULL); + if XNN_UNPREDICTABLE(i0 != zero) { + i0 = (const float*) ((uintptr_t) i0 + input_offset); + } + const float* i1 = input[1]; + assert(i1 != NULL); + if XNN_UNPREDICTABLE(i1 != zero) { + i1 = (const float*) ((uintptr_t) i1 + input_offset); + } + const float* i2 = input[2]; + assert(i2 != NULL); + if XNN_UNPREDICTABLE(i2 != zero) { + i2 = (const float*) ((uintptr_t) i2 + input_offset); + } + const float* i3 = input[3]; + assert(i3 != NULL); + if XNN_UNPREDICTABLE(i3 != zero) { + i3 = (const float*) ((uintptr_t) i3 + input_offset); + } + const float* i4 = input[4]; + assert(i4 != NULL); + if XNN_UNPREDICTABLE(i4 != zero) { + i4 = (const float*) ((uintptr_t) i4 + input_offset); + } + const float* i5 = input[5]; + assert(i5 != NULL); + if XNN_UNPREDICTABLE(i5 != zero) { + i5 = (const float*) ((uintptr_t) i5 + input_offset); + } + const float* i6 = input[6]; + assert(i6 != NULL); + if XNN_UNPREDICTABLE(i6 != zero) { + i6 = (const float*) ((uintptr_t) i6 + input_offset); + } + const float* i7 = input[7]; + assert(i7 != NULL); + if XNN_UNPREDICTABLE(i7 != zero) { + i7 = (const float*) ((uintptr_t) i7 + input_offset); + } + const float* i8 = input[8]; + assert(i8 != NULL); + if XNN_UNPREDICTABLE(i8 != zero) { + i8 = (const float*) ((uintptr_t) i8 + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= 8; c -= 8) { + v128_t vacc0123p0 = wasm_v128_load(w); + v128_t vacc4567p0 = wasm_v128_load(w + 4); + + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vi0x4567 = wasm_v128_load(i0 + 4); + i0 += 8; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + const v128_t vk0x4567 = wasm_v128_load(w + 12); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi0x4567, vk0x4567)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vi1x4567 = wasm_v128_load(i1 + 4); + i1 += 8; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + const v128_t vk1x4567 = wasm_v128_load(w + 20); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi1x4567, vk1x4567)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vi2x4567 = wasm_v128_load(i2 + 4); + i2 += 8; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + const v128_t vk2x4567 = wasm_v128_load(w + 28); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi2x4567, vk2x4567)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vi3x4567 = wasm_v128_load(i3 + 4); + i3 += 8; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + const v128_t vk3x4567 = wasm_v128_load(w + 36); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi3x4567, vk3x4567)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vi4x4567 = wasm_v128_load(i4 + 4); + i4 += 8; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + const v128_t vk4x4567 = wasm_v128_load(w + 44); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi4x4567, vk4x4567)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vi5x4567 = wasm_v128_load(i5 + 4); + i5 += 8; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + const v128_t vk5x4567 = wasm_v128_load(w + 52); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi5x4567, vk5x4567)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vi6x4567 = wasm_v128_load(i6 + 4); + i6 += 8; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + const v128_t vk6x4567 = wasm_v128_load(w + 60); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi6x4567, vk6x4567)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vi7x4567 = wasm_v128_load(i7 + 4); + i7 += 8; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + const v128_t vk7x4567 = wasm_v128_load(w + 68); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi7x4567, vk7x4567)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vi8x4567 = wasm_v128_load(i8 + 4); + i8 += 8; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + const v128_t vk8x4567 = wasm_v128_load(w + 76); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + vacc4567p0 = wasm_f32x4_add(vacc4567p0, wasm_f32x4_mul(vi8x4567, vk8x4567)); + + w += 80; + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + v128_t vacc4567 = wasm_v128_bitselect(vmin, vacc4567p0, wasm_f32x4_lt(vacc4567p0, vmin)); + + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + vacc4567 = wasm_v128_bitselect(vacc4567, vmax, wasm_f32x4_le(vacc4567, vmax)); + + wasm_v128_store(output, vacc0123); + wasm_v128_store(output + 4, vacc4567); + output += 8; + } + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + i0 += 4; + + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + i1 += 4; + + const v128_t vk1x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + i2 += 4; + + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + i3 += 4; + + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + i4 += 4; + + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + i5 += 4; + + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + i6 += 4; + + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + i7 += 4; + + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + w += 4; + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + + const v128_t vi0x0123 = wasm_v128_load(i0); + const v128_t vk0x0123 = wasm_v128_load(w + 8); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi0x0123, vk0x0123)); + + const v128_t vi1x0123 = wasm_v128_load(i1); + const v128_t vk1x0123 = wasm_v128_load(w + 16); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi1x0123, vk1x0123)); + + const v128_t vi2x0123 = wasm_v128_load(i2); + const v128_t vk2x0123 = wasm_v128_load(w + 24); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi2x0123, vk2x0123)); + + const v128_t vi3x0123 = wasm_v128_load(i3); + const v128_t vk3x0123 = wasm_v128_load(w + 32); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi3x0123, vk3x0123)); + + const v128_t vi4x0123 = wasm_v128_load(i4); + const v128_t vk4x0123 = wasm_v128_load(w + 40); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi4x0123, vk4x0123)); + + const v128_t vi5x0123 = wasm_v128_load(i5); + const v128_t vk5x0123 = wasm_v128_load(w + 48); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi5x0123, vk5x0123)); + + const v128_t vi6x0123 = wasm_v128_load(i6); + const v128_t vk6x0123 = wasm_v128_load(w + 56); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi6x0123, vk6x0123)); + + const v128_t vi7x0123 = wasm_v128_load(i7); + const v128_t vk7x0123 = wasm_v128_load(w + 64); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi7x0123, vk7x0123)); + + const v128_t vi8x0123 = wasm_v128_load(i8); + const v128_t vk8x0123 = wasm_v128_load(w + 72); + vacc0123p0 = wasm_f32x4_add(vacc0123p0, wasm_f32x4_mul(vi8x0123, vk8x0123)); + + + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-dwconv/up-wasmsimd.c.in b/src/f32-dwconv/up-wasmsimd.c.in new file mode 100644 index 000000000..80248e43d --- /dev/null +++ b/src/f32-dwconv/up-wasmsimd.c.in @@ -0,0 +1,169 @@ +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +$assert CHANNEL_TILE % 4 == 0 +$assert KERNEL_TILE >= 2 +$assert ACCUMULATORS >= 1 +$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/dwconv.h> + + +void xnn_f32_dwconv_minmax_ukernel_up${CHANNEL_TILE}x${KERNEL_TILE}__wasmsimd${"" if ACCUMULATORS == 1 else "_acc%d" % ACCUMULATORS}_${"x86" if X86 else "arm"}( + size_t channels, + size_t output_width, + const float** input, + const float* weights, + float* output, + size_t input_stride, + size_t output_increment, + size_t input_offset, + const float* zero, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(channels != 0); + assert(output_width != 0); + + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + do { + $for K in range(KERNEL_TILE): + const float* i${K} = input[${K}]; + assert(i${K} != NULL); + if XNN_UNPREDICTABLE(i${K} != zero) { + i${K} = (const float*) ((uintptr_t) i${K} + input_offset); + } + input = (const float**) ((uintptr_t) input + input_stride); + + size_t c = channels; + const float* w = weights; + for (; c >= ${CHANNEL_TILE}; c -= ${CHANNEL_TILE}) { + v128_t vacc${ABC[0:4]}p0 = wasm_v128_load(w); + $for C in range(4, CHANNEL_TILE, 4): + v128_t vacc${ABC[C:C+4]}p0 = wasm_v128_load(w + ${C}); + + $for K in range(KERNEL_TILE): + + const v128_t vi${K}x${ABC[0:4]} = wasm_v128_load(i${K}); + $for C in range(4, CHANNEL_TILE, 4): + const v128_t vi${K}x${ABC[C:C+4]} = wasm_v128_load(i${K} + ${C}); + i${K} += ${CHANNEL_TILE}; + + $for C in range(0, CHANNEL_TILE, 4): + const v128_t vk${K}x${ABC[C:C+4]} = wasm_v128_load(w + ${(K + 1) * CHANNEL_TILE + C}); + $for C in range(0, CHANNEL_TILE, 4): + $if 1 <= K < ACCUMULATORS: + v128_t vacc${ABC[C:C+4]}p${K} = wasm_f32x4_mul(vi${K}x${ABC[C:C+4]}, vk${K}x${ABC[C:C+4]}); + $else: + vacc${ABC[C:C+4]}p${K % ACCUMULATORS} = wasm_f32x4_add(vacc${ABC[C:C+4]}p${K % ACCUMULATORS}, wasm_f32x4_mul(vi${K}x${ABC[C:C+4]}, vk${K}x${ABC[C:C+4]})); + + w += ${(KERNEL_TILE + 1) * CHANNEL_TILE}; + + $if ACCUMULATORS > 1: + // Add up all accumulators to vacc${ABC[0:CHANNEL_TILE]}p0 + $ACC_SLICE = 1 + $while ACC_SLICE < ACCUMULATORS: + $for A in range(0, ACCUMULATORS, ACC_SLICE * 2): + $if A + ACC_SLICE < ACCUMULATORS: + $for C in range(0, CHANNEL_TILE, 4): + vacc${ABC[C:C+4]}p${A} = wasm_f32x4_add(vacc${ABC[C:C+4]}p${A}, vacc${ABC[C:C+4]}p${A + ACC_SLICE}); + $ACC_SLICE *= 2 + + $for C in range(0, CHANNEL_TILE, 4): + $if X86: + v128_t vacc${ABC[C:C+4]} = wasm_v128_bitselect(vmin, vacc${ABC[C:C+4]}p0, wasm_f32x4_lt(vacc${ABC[C:C+4]}p0, vmin)); + $else: + v128_t vacc${ABC[C:C+4]} = wasm_f32x4_max(vacc${ABC[C:C+4]}p0, vmin); + + $for C in range(0, CHANNEL_TILE, 4): + $if X86: + vacc${ABC[C:C+4]} = wasm_v128_bitselect(vacc${ABC[C:C+4]}, vmax, wasm_f32x4_le(vacc${ABC[C:C+4]}, vmax)); + $else: + vacc${ABC[C:C+4]} = wasm_f32x4_min(vacc${ABC[C:C+4]}, vmax); + + wasm_v128_store(output, vacc${ABC[0:4]}); + $for C in range(4, CHANNEL_TILE, 4): + wasm_v128_store(output + ${C}, vacc${ABC[C:C+4]}); + output += ${CHANNEL_TILE}; + } + $if CHANNEL_TILE > 4: + for (; c >= 4; c -= 4) { + v128_t vacc0123p0 = wasm_v128_load(w); + $for K in range(KERNEL_TILE): + + const v128_t vi${K}x0123 = wasm_v128_load(i${K}); + i${K} += 4; + + const v128_t vk${K}x0123 = wasm_v128_load(w + ${(K + 1) * CHANNEL_TILE}); + $if 1 <= K < ACCUMULATORS: + v128_t vacc0123p${K} = wasm_f32x4_mul(vi${K}x0123, vk${K}x0123); + $else: + vacc0123p${K % ACCUMULATORS} = wasm_f32x4_add(vacc0123p${K % ACCUMULATORS}, wasm_f32x4_mul(vi${K}x0123, vk${K}x0123)); + + w += 4; + + $if ACCUMULATORS > 1: + // Add up all accumulators to vacc${ABC[0:CHANNEL_TILE]}p0 + $ACC_SLICE = 1 + $while ACC_SLICE < ACCUMULATORS: + $for A in range(0, ACCUMULATORS, ACC_SLICE * 2): + $if A + ACC_SLICE < ACCUMULATORS: + vacc0123p${A} = wasm_f32x4_add(vacc0123p${A}, vacc0123p${A + ACC_SLICE}); + $ACC_SLICE *= 2 + + $if X86: + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + $else: + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + wasm_v128_store(output, vacc0123); + output += 4; + } + if XNN_UNLIKELY(c != 0) { + v128_t vacc0123p0 = wasm_v128_load(w); + $for K in range(KERNEL_TILE): + + const v128_t vi${K}x0123 = wasm_v128_load(i${K}); + const v128_t vk${K}x0123 = wasm_v128_load(w + ${(K+1) * CHANNEL_TILE}); + $if 1 <= K < ACCUMULATORS: + v128_t vacc0123p${K} = wasm_f32x4_mul(vi${K}x0123, vk${K}x0123); + $else: + vacc0123p${K % ACCUMULATORS} = wasm_f32x4_add(vacc0123p${K % ACCUMULATORS}, wasm_f32x4_mul(vi${K}x0123, vk${K}x0123)); + + $if ACCUMULATORS > 1: + // Add up all accumulators to vacc${ABC[0:CHANNEL_TILE]}p0 + $ACC_SLICE = 1 + $while ACC_SLICE < ACCUMULATORS: + $for A in range(0, ACCUMULATORS, ACC_SLICE * 2): + $if A + ACC_SLICE < ACCUMULATORS: + vacc0123p${A} = wasm_f32x4_add(vacc0123p${A}, vacc0123p${A + ACC_SLICE}); + $ACC_SLICE *= 2 + + $if X86: + v128_t vacc0123 = wasm_v128_bitselect(vmin, vacc0123p0, wasm_f32x4_lt(vacc0123p0, vmin)); + vacc0123 = wasm_v128_bitselect(vacc0123, vmax, wasm_f32x4_le(vacc0123, vmax)); + $else: + v128_t vacc0123 = wasm_f32x4_max(vacc0123p0, vmin); + vacc0123 = wasm_f32x4_min(vacc0123, vmax); + + if (c & 2) { + *((double*) output) = wasm_f64x2_extract_lane(vacc0123, 0); + vacc0123 = wasm_v32x4_shuffle(vacc0123, vacc0123, 2, 3, 2, 3); + output += 2; + } + if (c & 1) { + *output = wasm_f32x4_extract_lane(vacc0123, 0); + output += 1; + } + } + + output = (float*) ((uintptr_t) output + output_increment); + } while (--output_width != 0); +} diff --git a/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-loadsplat-arm.c b/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-loadsplat-arm.c index 2257383f0..68a15253d 100644 --- a/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-loadsplat-arm.c @@ -60,14 +60,14 @@ void xnn_f32_gemminc_minmax_ukernel_1x8__wasmsimd_loadsplat_arm( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc0x4567 = wasm_f32x4_max(vacc0x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-loadsplat-x86.c b/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-loadsplat-x86.c index 15032779c..2ea5c5a67 100644 --- a/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-loadsplat-x86.c @@ -60,14 +60,14 @@ void xnn_f32_gemminc_minmax_ukernel_1x8__wasmsimd_loadsplat_x86( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc0x4567 = wasm_v128_bitselect(vmin, vacc0x4567, wasm_f32x4_lt(vacc0x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-splat-arm.c b/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-splat-arm.c index 46694df14..186899774 100644 --- a/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-splat-arm.c +++ b/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-splat-arm.c @@ -98,14 +98,14 @@ void xnn_f32_gemminc_minmax_ukernel_1x8__wasmsimd_splat_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc0x4567 = wasm_f32x4_max(vacc0x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-splat-x86.c b/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-splat-x86.c index f97af392d..775b2a90b 100644 --- a/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-splat-x86.c +++ b/src/f32-gemm/gen-inc/1x8inc-minmax-wasmsimd-splat-x86.c @@ -98,14 +98,14 @@ void xnn_f32_gemminc_minmax_ukernel_1x8__wasmsimd_splat_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc0x4567 = wasm_v128_bitselect(vmin, vacc0x4567, wasm_f32x4_lt(vacc0x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-gemm/gen-inc/1x8s4inc-minmax-wasmsimd-arm.c b/src/f32-gemm/gen-inc/1x8s4inc-minmax-wasmsimd-arm.c index e9c86ae9b..67effbfaa 100644 --- a/src/f32-gemm/gen-inc/1x8s4inc-minmax-wasmsimd-arm.c +++ b/src/f32-gemm/gen-inc/1x8s4inc-minmax-wasmsimd-arm.c @@ -101,14 +101,14 @@ void xnn_f32_gemminc_minmax_ukernel_1x8s4__wasmsimd_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc0x4567 = wasm_f32x4_max(vacc0x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-gemm/gen-inc/1x8s4inc-minmax-wasmsimd-x86.c b/src/f32-gemm/gen-inc/1x8s4inc-minmax-wasmsimd-x86.c index a03aeb7db..f328cafaa 100644 --- a/src/f32-gemm/gen-inc/1x8s4inc-minmax-wasmsimd-x86.c +++ b/src/f32-gemm/gen-inc/1x8s4inc-minmax-wasmsimd-x86.c @@ -101,14 +101,14 @@ void xnn_f32_gemminc_minmax_ukernel_1x8s4__wasmsimd_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc0x4567 = wasm_v128_bitselect(vmin, vacc0x4567, wasm_f32x4_lt(vacc0x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-loadsplat-arm.c b/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-loadsplat-arm.c index e18e3b549..df6eadd81 100644 --- a/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-loadsplat-arm.c @@ -84,14 +84,6 @@ void xnn_f32_gemminc_minmax_ukernel_3x8__wasmsimd_loadsplat_arm( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -100,6 +92,14 @@ void xnn_f32_gemminc_minmax_ukernel_3x8__wasmsimd_loadsplat_arm( vacc1x4567 = wasm_f32x4_max(vacc1x4567, vmin); vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-loadsplat-x86.c b/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-loadsplat-x86.c index 16969d4e6..e723a4e6c 100644 --- a/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-loadsplat-x86.c @@ -84,14 +84,6 @@ void xnn_f32_gemminc_minmax_ukernel_3x8__wasmsimd_loadsplat_x86( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -100,6 +92,14 @@ void xnn_f32_gemminc_minmax_ukernel_3x8__wasmsimd_loadsplat_x86( vacc1x4567 = wasm_v128_bitselect(vmin, vacc1x4567, wasm_f32x4_lt(vacc1x4567, vmin)); vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-splat-arm.c b/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-splat-arm.c index cce92ff9f..6a48f44fd 100644 --- a/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-splat-arm.c +++ b/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-splat-arm.c @@ -150,14 +150,6 @@ void xnn_f32_gemminc_minmax_ukernel_3x8__wasmsimd_splat_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -166,6 +158,14 @@ void xnn_f32_gemminc_minmax_ukernel_3x8__wasmsimd_splat_arm( vacc1x4567 = wasm_f32x4_max(vacc1x4567, vmin); vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-splat-x86.c b/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-splat-x86.c index 94c4389ea..7ec96d08a 100644 --- a/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-splat-x86.c +++ b/src/f32-gemm/gen-inc/3x8inc-minmax-wasmsimd-splat-x86.c @@ -150,14 +150,6 @@ void xnn_f32_gemminc_minmax_ukernel_3x8__wasmsimd_splat_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -166,6 +158,14 @@ void xnn_f32_gemminc_minmax_ukernel_3x8__wasmsimd_splat_x86( vacc1x4567 = wasm_v128_bitselect(vmin, vacc1x4567, wasm_f32x4_lt(vacc1x4567, vmin)); vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-gemm/gen-inc/3x8s4inc-minmax-wasmsimd-arm.c b/src/f32-gemm/gen-inc/3x8s4inc-minmax-wasmsimd-arm.c index a2b554600..5d702340c 100644 --- a/src/f32-gemm/gen-inc/3x8s4inc-minmax-wasmsimd-arm.c +++ b/src/f32-gemm/gen-inc/3x8s4inc-minmax-wasmsimd-arm.c @@ -151,14 +151,6 @@ void xnn_f32_gemminc_minmax_ukernel_3x8s4__wasmsimd_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -167,6 +159,14 @@ void xnn_f32_gemminc_minmax_ukernel_3x8s4__wasmsimd_arm( vacc1x4567 = wasm_f32x4_max(vacc1x4567, vmin); vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-gemm/gen-inc/3x8s4inc-minmax-wasmsimd-x86.c b/src/f32-gemm/gen-inc/3x8s4inc-minmax-wasmsimd-x86.c index d55538994..bfb75210f 100644 --- a/src/f32-gemm/gen-inc/3x8s4inc-minmax-wasmsimd-x86.c +++ b/src/f32-gemm/gen-inc/3x8s4inc-minmax-wasmsimd-x86.c @@ -151,14 +151,6 @@ void xnn_f32_gemminc_minmax_ukernel_3x8s4__wasmsimd_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -167,6 +159,14 @@ void xnn_f32_gemminc_minmax_ukernel_3x8s4__wasmsimd_x86( vacc1x4567 = wasm_v128_bitselect(vmin, vacc1x4567, wasm_f32x4_lt(vacc1x4567, vmin)); vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-loadsplat-arm.c b/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-loadsplat-arm.c index a739b4c8d..077f236be 100644 --- a/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-loadsplat-arm.c @@ -96,16 +96,6 @@ void xnn_f32_gemminc_minmax_ukernel_4x8__wasmsimd_loadsplat_arm( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -116,6 +106,16 @@ void xnn_f32_gemminc_minmax_ukernel_4x8__wasmsimd_loadsplat_arm( vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-loadsplat-x86.c b/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-loadsplat-x86.c index 297f9d4d8..52f99615f 100644 --- a/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-loadsplat-x86.c @@ -96,16 +96,6 @@ void xnn_f32_gemminc_minmax_ukernel_4x8__wasmsimd_loadsplat_x86( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -116,6 +106,16 @@ void xnn_f32_gemminc_minmax_ukernel_4x8__wasmsimd_loadsplat_x86( vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-splat-arm.c b/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-splat-arm.c index ea0afc7fc..d00c9606a 100644 --- a/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-splat-arm.c +++ b/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-splat-arm.c @@ -176,16 +176,6 @@ void xnn_f32_gemminc_minmax_ukernel_4x8__wasmsimd_splat_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -196,6 +186,16 @@ void xnn_f32_gemminc_minmax_ukernel_4x8__wasmsimd_splat_arm( vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-splat-x86.c b/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-splat-x86.c index a46feb0a0..e589f1322 100644 --- a/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-splat-x86.c +++ b/src/f32-gemm/gen-inc/4x8inc-minmax-wasmsimd-splat-x86.c @@ -176,16 +176,6 @@ void xnn_f32_gemminc_minmax_ukernel_4x8__wasmsimd_splat_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -196,6 +186,16 @@ void xnn_f32_gemminc_minmax_ukernel_4x8__wasmsimd_splat_x86( vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-gemm/gen-inc/4x8s4inc-minmax-wasmsimd-arm.c b/src/f32-gemm/gen-inc/4x8s4inc-minmax-wasmsimd-arm.c index 8f5d8f22e..47f1a3333 100644 --- a/src/f32-gemm/gen-inc/4x8s4inc-minmax-wasmsimd-arm.c +++ b/src/f32-gemm/gen-inc/4x8s4inc-minmax-wasmsimd-arm.c @@ -176,16 +176,6 @@ void xnn_f32_gemminc_minmax_ukernel_4x8s4__wasmsimd_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -196,6 +186,16 @@ void xnn_f32_gemminc_minmax_ukernel_4x8s4__wasmsimd_arm( vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-gemm/gen-inc/4x8s4inc-minmax-wasmsimd-x86.c b/src/f32-gemm/gen-inc/4x8s4inc-minmax-wasmsimd-x86.c index 921ac56e8..be3caf49b 100644 --- a/src/f32-gemm/gen-inc/4x8s4inc-minmax-wasmsimd-x86.c +++ b/src/f32-gemm/gen-inc/4x8s4inc-minmax-wasmsimd-x86.c @@ -176,16 +176,6 @@ void xnn_f32_gemminc_minmax_ukernel_4x8s4__wasmsimd_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -196,6 +186,16 @@ void xnn_f32_gemminc_minmax_ukernel_4x8s4__wasmsimd_x86( vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-loadsplat-arm.c b/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-loadsplat-arm.c index e0a3bf82b..2e3608220 100644 --- a/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-loadsplat-arm.c @@ -108,18 +108,6 @@ void xnn_f32_gemminc_minmax_ukernel_5x8__wasmsimd_loadsplat_arm( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -132,6 +120,18 @@ void xnn_f32_gemminc_minmax_ukernel_5x8__wasmsimd_loadsplat_arm( vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-loadsplat-x86.c b/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-loadsplat-x86.c index 69fe74a23..a19939ddd 100644 --- a/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-loadsplat-x86.c @@ -108,18 +108,6 @@ void xnn_f32_gemminc_minmax_ukernel_5x8__wasmsimd_loadsplat_x86( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -132,6 +120,18 @@ void xnn_f32_gemminc_minmax_ukernel_5x8__wasmsimd_loadsplat_x86( vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-splat-arm.c b/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-splat-arm.c index 4b926cfc7..d0550a78f 100644 --- a/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-splat-arm.c +++ b/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-splat-arm.c @@ -202,18 +202,6 @@ void xnn_f32_gemminc_minmax_ukernel_5x8__wasmsimd_splat_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -226,6 +214,18 @@ void xnn_f32_gemminc_minmax_ukernel_5x8__wasmsimd_splat_arm( vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-splat-x86.c b/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-splat-x86.c index a7ab06efa..7d0d7784d 100644 --- a/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-splat-x86.c +++ b/src/f32-gemm/gen-inc/5x8inc-minmax-wasmsimd-splat-x86.c @@ -202,18 +202,6 @@ void xnn_f32_gemminc_minmax_ukernel_5x8__wasmsimd_splat_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -226,6 +214,18 @@ void xnn_f32_gemminc_minmax_ukernel_5x8__wasmsimd_splat_x86( vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-gemm/gen-inc/5x8s4inc-minmax-wasmsimd-arm.c b/src/f32-gemm/gen-inc/5x8s4inc-minmax-wasmsimd-arm.c index 50687cbc0..2dde359d6 100644 --- a/src/f32-gemm/gen-inc/5x8s4inc-minmax-wasmsimd-arm.c +++ b/src/f32-gemm/gen-inc/5x8s4inc-minmax-wasmsimd-arm.c @@ -201,18 +201,6 @@ void xnn_f32_gemminc_minmax_ukernel_5x8s4__wasmsimd_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -225,6 +213,18 @@ void xnn_f32_gemminc_minmax_ukernel_5x8s4__wasmsimd_arm( vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-gemm/gen-inc/5x8s4inc-minmax-wasmsimd-x86.c b/src/f32-gemm/gen-inc/5x8s4inc-minmax-wasmsimd-x86.c index 56ec724b0..74879645d 100644 --- a/src/f32-gemm/gen-inc/5x8s4inc-minmax-wasmsimd-x86.c +++ b/src/f32-gemm/gen-inc/5x8s4inc-minmax-wasmsimd-x86.c @@ -201,18 +201,6 @@ void xnn_f32_gemminc_minmax_ukernel_5x8s4__wasmsimd_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -225,6 +213,18 @@ void xnn_f32_gemminc_minmax_ukernel_5x8s4__wasmsimd_x86( vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-loadsplat-arm.c b/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-loadsplat-arm.c index a4e04cbcd..bb5f80b6b 100644 --- a/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-loadsplat-arm.c @@ -120,20 +120,6 @@ void xnn_f32_gemminc_minmax_ukernel_6x8__wasmsimd_loadsplat_arm( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -148,6 +134,20 @@ void xnn_f32_gemminc_minmax_ukernel_6x8__wasmsimd_loadsplat_arm( vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); vacc5x4567 = wasm_f32x4_max(vacc5x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-loadsplat-x86.c b/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-loadsplat-x86.c index ef6d75947..fbf70d3b4 100644 --- a/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-loadsplat-x86.c @@ -120,20 +120,6 @@ void xnn_f32_gemminc_minmax_ukernel_6x8__wasmsimd_loadsplat_x86( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -148,6 +134,20 @@ void xnn_f32_gemminc_minmax_ukernel_6x8__wasmsimd_loadsplat_x86( vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); vacc5x4567 = wasm_v128_bitselect(vmin, vacc5x4567, wasm_f32x4_lt(vacc5x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-splat-arm.c b/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-splat-arm.c index 1cba048ea..03523d892 100644 --- a/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-splat-arm.c +++ b/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-splat-arm.c @@ -228,20 +228,6 @@ void xnn_f32_gemminc_minmax_ukernel_6x8__wasmsimd_splat_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -256,6 +242,20 @@ void xnn_f32_gemminc_minmax_ukernel_6x8__wasmsimd_splat_arm( vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); vacc5x4567 = wasm_f32x4_max(vacc5x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-splat-x86.c b/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-splat-x86.c index fba162b1a..93522ce2e 100644 --- a/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-splat-x86.c +++ b/src/f32-gemm/gen-inc/6x8inc-minmax-wasmsimd-splat-x86.c @@ -228,20 +228,6 @@ void xnn_f32_gemminc_minmax_ukernel_6x8__wasmsimd_splat_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -256,6 +242,20 @@ void xnn_f32_gemminc_minmax_ukernel_6x8__wasmsimd_splat_x86( vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); vacc5x4567 = wasm_v128_bitselect(vmin, vacc5x4567, wasm_f32x4_lt(vacc5x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-gemm/gen-inc/6x8s4inc-minmax-wasmsimd-arm.c b/src/f32-gemm/gen-inc/6x8s4inc-minmax-wasmsimd-arm.c index 12624591c..5c890b348 100644 --- a/src/f32-gemm/gen-inc/6x8s4inc-minmax-wasmsimd-arm.c +++ b/src/f32-gemm/gen-inc/6x8s4inc-minmax-wasmsimd-arm.c @@ -226,20 +226,6 @@ void xnn_f32_gemminc_minmax_ukernel_6x8s4__wasmsimd_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -254,6 +240,20 @@ void xnn_f32_gemminc_minmax_ukernel_6x8s4__wasmsimd_arm( vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); vacc5x4567 = wasm_f32x4_max(vacc5x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-gemm/gen-inc/6x8s4inc-minmax-wasmsimd-x86.c b/src/f32-gemm/gen-inc/6x8s4inc-minmax-wasmsimd-x86.c index f5ff4ea86..4a1553972 100644 --- a/src/f32-gemm/gen-inc/6x8s4inc-minmax-wasmsimd-x86.c +++ b/src/f32-gemm/gen-inc/6x8s4inc-minmax-wasmsimd-x86.c @@ -226,20 +226,6 @@ void xnn_f32_gemminc_minmax_ukernel_6x8s4__wasmsimd_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -254,6 +240,20 @@ void xnn_f32_gemminc_minmax_ukernel_6x8s4__wasmsimd_x86( vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); vacc5x4567 = wasm_v128_bitselect(vmin, vacc5x4567, wasm_f32x4_lt(vacc5x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-gemm/gen/1x8-minmax-wasmsimd-loadsplat-arm.c b/src/f32-gemm/gen/1x8-minmax-wasmsimd-loadsplat-arm.c index 166e2f1eb..eb76a32eb 100644 --- a/src/f32-gemm/gen/1x8-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-gemm/gen/1x8-minmax-wasmsimd-loadsplat-arm.c @@ -58,14 +58,14 @@ void xnn_f32_gemm_minmax_ukernel_1x8__wasmsimd_loadsplat_arm( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc0x4567 = wasm_f32x4_max(vacc0x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-gemm/gen/1x8-minmax-wasmsimd-loadsplat-x86.c b/src/f32-gemm/gen/1x8-minmax-wasmsimd-loadsplat-x86.c index 3ffed28f9..2de3550ec 100644 --- a/src/f32-gemm/gen/1x8-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-gemm/gen/1x8-minmax-wasmsimd-loadsplat-x86.c @@ -58,14 +58,14 @@ void xnn_f32_gemm_minmax_ukernel_1x8__wasmsimd_loadsplat_x86( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc0x4567 = wasm_v128_bitselect(vmin, vacc0x4567, wasm_f32x4_lt(vacc0x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-gemm/gen/1x8-minmax-wasmsimd-splat-arm.c b/src/f32-gemm/gen/1x8-minmax-wasmsimd-splat-arm.c index 5d1172933..b73f7aaa4 100644 --- a/src/f32-gemm/gen/1x8-minmax-wasmsimd-splat-arm.c +++ b/src/f32-gemm/gen/1x8-minmax-wasmsimd-splat-arm.c @@ -96,14 +96,14 @@ void xnn_f32_gemm_minmax_ukernel_1x8__wasmsimd_splat_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc0x4567 = wasm_f32x4_max(vacc0x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-gemm/gen/1x8-minmax-wasmsimd-splat-x86.c b/src/f32-gemm/gen/1x8-minmax-wasmsimd-splat-x86.c index 16eb7657a..5b5f55266 100644 --- a/src/f32-gemm/gen/1x8-minmax-wasmsimd-splat-x86.c +++ b/src/f32-gemm/gen/1x8-minmax-wasmsimd-splat-x86.c @@ -96,14 +96,14 @@ void xnn_f32_gemm_minmax_ukernel_1x8__wasmsimd_splat_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc0x4567 = wasm_v128_bitselect(vmin, vacc0x4567, wasm_f32x4_lt(vacc0x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-gemm/gen/1x8s4-minmax-wasmsimd-arm.c b/src/f32-gemm/gen/1x8s4-minmax-wasmsimd-arm.c index 69b2d1af4..8de482d30 100644 --- a/src/f32-gemm/gen/1x8s4-minmax-wasmsimd-arm.c +++ b/src/f32-gemm/gen/1x8s4-minmax-wasmsimd-arm.c @@ -99,14 +99,14 @@ void xnn_f32_gemm_minmax_ukernel_1x8s4__wasmsimd_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc0x4567 = wasm_f32x4_max(vacc0x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-gemm/gen/1x8s4-minmax-wasmsimd-x86.c b/src/f32-gemm/gen/1x8s4-minmax-wasmsimd-x86.c index 4578dc6ad..d77e15e4f 100644 --- a/src/f32-gemm/gen/1x8s4-minmax-wasmsimd-x86.c +++ b/src/f32-gemm/gen/1x8s4-minmax-wasmsimd-x86.c @@ -99,14 +99,14 @@ void xnn_f32_gemm_minmax_ukernel_1x8s4__wasmsimd_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc0x4567 = wasm_v128_bitselect(vmin, vacc0x4567, wasm_f32x4_lt(vacc0x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-gemm/gen/3x8-minmax-wasmsimd-loadsplat-arm.c b/src/f32-gemm/gen/3x8-minmax-wasmsimd-loadsplat-arm.c index 13a34a6f0..66603db88 100644 --- a/src/f32-gemm/gen/3x8-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-gemm/gen/3x8-minmax-wasmsimd-loadsplat-arm.c @@ -82,14 +82,6 @@ void xnn_f32_gemm_minmax_ukernel_3x8__wasmsimd_loadsplat_arm( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -98,6 +90,14 @@ void xnn_f32_gemm_minmax_ukernel_3x8__wasmsimd_loadsplat_arm( vacc1x4567 = wasm_f32x4_max(vacc1x4567, vmin); vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-gemm/gen/3x8-minmax-wasmsimd-loadsplat-x86.c b/src/f32-gemm/gen/3x8-minmax-wasmsimd-loadsplat-x86.c index f88aeddb7..4ae21224b 100644 --- a/src/f32-gemm/gen/3x8-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-gemm/gen/3x8-minmax-wasmsimd-loadsplat-x86.c @@ -82,14 +82,6 @@ void xnn_f32_gemm_minmax_ukernel_3x8__wasmsimd_loadsplat_x86( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -98,6 +90,14 @@ void xnn_f32_gemm_minmax_ukernel_3x8__wasmsimd_loadsplat_x86( vacc1x4567 = wasm_v128_bitselect(vmin, vacc1x4567, wasm_f32x4_lt(vacc1x4567, vmin)); vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-gemm/gen/3x8-minmax-wasmsimd-splat-arm.c b/src/f32-gemm/gen/3x8-minmax-wasmsimd-splat-arm.c index 3eeeaeeec..da1806704 100644 --- a/src/f32-gemm/gen/3x8-minmax-wasmsimd-splat-arm.c +++ b/src/f32-gemm/gen/3x8-minmax-wasmsimd-splat-arm.c @@ -148,14 +148,6 @@ void xnn_f32_gemm_minmax_ukernel_3x8__wasmsimd_splat_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -164,6 +156,14 @@ void xnn_f32_gemm_minmax_ukernel_3x8__wasmsimd_splat_arm( vacc1x4567 = wasm_f32x4_max(vacc1x4567, vmin); vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-gemm/gen/3x8-minmax-wasmsimd-splat-x86.c b/src/f32-gemm/gen/3x8-minmax-wasmsimd-splat-x86.c index 5c92806a4..b51250f44 100644 --- a/src/f32-gemm/gen/3x8-minmax-wasmsimd-splat-x86.c +++ b/src/f32-gemm/gen/3x8-minmax-wasmsimd-splat-x86.c @@ -148,14 +148,6 @@ void xnn_f32_gemm_minmax_ukernel_3x8__wasmsimd_splat_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -164,6 +156,14 @@ void xnn_f32_gemm_minmax_ukernel_3x8__wasmsimd_splat_x86( vacc1x4567 = wasm_v128_bitselect(vmin, vacc1x4567, wasm_f32x4_lt(vacc1x4567, vmin)); vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-gemm/gen/3x8s4-minmax-wasmsimd-arm.c b/src/f32-gemm/gen/3x8s4-minmax-wasmsimd-arm.c index 03aeeb170..4d256f85a 100644 --- a/src/f32-gemm/gen/3x8s4-minmax-wasmsimd-arm.c +++ b/src/f32-gemm/gen/3x8s4-minmax-wasmsimd-arm.c @@ -149,14 +149,6 @@ void xnn_f32_gemm_minmax_ukernel_3x8s4__wasmsimd_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -165,6 +157,14 @@ void xnn_f32_gemm_minmax_ukernel_3x8s4__wasmsimd_arm( vacc1x4567 = wasm_f32x4_max(vacc1x4567, vmin); vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-gemm/gen/3x8s4-minmax-wasmsimd-x86.c b/src/f32-gemm/gen/3x8s4-minmax-wasmsimd-x86.c index 96ccfb19b..616656427 100644 --- a/src/f32-gemm/gen/3x8s4-minmax-wasmsimd-x86.c +++ b/src/f32-gemm/gen/3x8s4-minmax-wasmsimd-x86.c @@ -149,14 +149,6 @@ void xnn_f32_gemm_minmax_ukernel_3x8s4__wasmsimd_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -165,6 +157,14 @@ void xnn_f32_gemm_minmax_ukernel_3x8s4__wasmsimd_x86( vacc1x4567 = wasm_v128_bitselect(vmin, vacc1x4567, wasm_f32x4_lt(vacc1x4567, vmin)); vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-gemm/gen/4x8-minmax-wasmsimd-loadsplat-arm.c b/src/f32-gemm/gen/4x8-minmax-wasmsimd-loadsplat-arm.c index 8d4fa5863..1a8b9c86e 100644 --- a/src/f32-gemm/gen/4x8-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-gemm/gen/4x8-minmax-wasmsimd-loadsplat-arm.c @@ -94,16 +94,6 @@ void xnn_f32_gemm_minmax_ukernel_4x8__wasmsimd_loadsplat_arm( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -114,6 +104,16 @@ void xnn_f32_gemm_minmax_ukernel_4x8__wasmsimd_loadsplat_arm( vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-gemm/gen/4x8-minmax-wasmsimd-loadsplat-x86.c b/src/f32-gemm/gen/4x8-minmax-wasmsimd-loadsplat-x86.c index 3f18a912e..fd440c3b4 100644 --- a/src/f32-gemm/gen/4x8-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-gemm/gen/4x8-minmax-wasmsimd-loadsplat-x86.c @@ -94,16 +94,6 @@ void xnn_f32_gemm_minmax_ukernel_4x8__wasmsimd_loadsplat_x86( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -114,6 +104,16 @@ void xnn_f32_gemm_minmax_ukernel_4x8__wasmsimd_loadsplat_x86( vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-gemm/gen/4x8-minmax-wasmsimd-splat-arm.c b/src/f32-gemm/gen/4x8-minmax-wasmsimd-splat-arm.c index c746f905a..1b216c0da 100644 --- a/src/f32-gemm/gen/4x8-minmax-wasmsimd-splat-arm.c +++ b/src/f32-gemm/gen/4x8-minmax-wasmsimd-splat-arm.c @@ -174,16 +174,6 @@ void xnn_f32_gemm_minmax_ukernel_4x8__wasmsimd_splat_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -194,6 +184,16 @@ void xnn_f32_gemm_minmax_ukernel_4x8__wasmsimd_splat_arm( vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-gemm/gen/4x8-minmax-wasmsimd-splat-x86.c b/src/f32-gemm/gen/4x8-minmax-wasmsimd-splat-x86.c index 164ac58ac..0ab1857aa 100644 --- a/src/f32-gemm/gen/4x8-minmax-wasmsimd-splat-x86.c +++ b/src/f32-gemm/gen/4x8-minmax-wasmsimd-splat-x86.c @@ -174,16 +174,6 @@ void xnn_f32_gemm_minmax_ukernel_4x8__wasmsimd_splat_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -194,6 +184,16 @@ void xnn_f32_gemm_minmax_ukernel_4x8__wasmsimd_splat_x86( vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-gemm/gen/4x8s4-minmax-wasmsimd-arm.c b/src/f32-gemm/gen/4x8s4-minmax-wasmsimd-arm.c index 56f8797ec..95ff5540f 100644 --- a/src/f32-gemm/gen/4x8s4-minmax-wasmsimd-arm.c +++ b/src/f32-gemm/gen/4x8s4-minmax-wasmsimd-arm.c @@ -174,16 +174,6 @@ void xnn_f32_gemm_minmax_ukernel_4x8s4__wasmsimd_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -194,6 +184,16 @@ void xnn_f32_gemm_minmax_ukernel_4x8s4__wasmsimd_arm( vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-gemm/gen/4x8s4-minmax-wasmsimd-x86.c b/src/f32-gemm/gen/4x8s4-minmax-wasmsimd-x86.c index 2f98707a8..d005ec6a5 100644 --- a/src/f32-gemm/gen/4x8s4-minmax-wasmsimd-x86.c +++ b/src/f32-gemm/gen/4x8s4-minmax-wasmsimd-x86.c @@ -174,16 +174,6 @@ void xnn_f32_gemm_minmax_ukernel_4x8s4__wasmsimd_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -194,6 +184,16 @@ void xnn_f32_gemm_minmax_ukernel_4x8s4__wasmsimd_x86( vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-gemm/gen/5x8-minmax-wasmsimd-loadsplat-arm.c b/src/f32-gemm/gen/5x8-minmax-wasmsimd-loadsplat-arm.c index 98267c668..d367c7040 100644 --- a/src/f32-gemm/gen/5x8-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-gemm/gen/5x8-minmax-wasmsimd-loadsplat-arm.c @@ -106,18 +106,6 @@ void xnn_f32_gemm_minmax_ukernel_5x8__wasmsimd_loadsplat_arm( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -130,6 +118,18 @@ void xnn_f32_gemm_minmax_ukernel_5x8__wasmsimd_loadsplat_arm( vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-gemm/gen/5x8-minmax-wasmsimd-loadsplat-x86.c b/src/f32-gemm/gen/5x8-minmax-wasmsimd-loadsplat-x86.c index 5b3f22d6c..84f197fa3 100644 --- a/src/f32-gemm/gen/5x8-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-gemm/gen/5x8-minmax-wasmsimd-loadsplat-x86.c @@ -106,18 +106,6 @@ void xnn_f32_gemm_minmax_ukernel_5x8__wasmsimd_loadsplat_x86( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -130,6 +118,18 @@ void xnn_f32_gemm_minmax_ukernel_5x8__wasmsimd_loadsplat_x86( vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-gemm/gen/5x8-minmax-wasmsimd-splat-arm.c b/src/f32-gemm/gen/5x8-minmax-wasmsimd-splat-arm.c index c2fa57a76..c18010d46 100644 --- a/src/f32-gemm/gen/5x8-minmax-wasmsimd-splat-arm.c +++ b/src/f32-gemm/gen/5x8-minmax-wasmsimd-splat-arm.c @@ -200,18 +200,6 @@ void xnn_f32_gemm_minmax_ukernel_5x8__wasmsimd_splat_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -224,6 +212,18 @@ void xnn_f32_gemm_minmax_ukernel_5x8__wasmsimd_splat_arm( vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-gemm/gen/5x8-minmax-wasmsimd-splat-x86.c b/src/f32-gemm/gen/5x8-minmax-wasmsimd-splat-x86.c index f3da03db3..05136f147 100644 --- a/src/f32-gemm/gen/5x8-minmax-wasmsimd-splat-x86.c +++ b/src/f32-gemm/gen/5x8-minmax-wasmsimd-splat-x86.c @@ -200,18 +200,6 @@ void xnn_f32_gemm_minmax_ukernel_5x8__wasmsimd_splat_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -224,6 +212,18 @@ void xnn_f32_gemm_minmax_ukernel_5x8__wasmsimd_splat_x86( vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-gemm/gen/5x8s4-minmax-wasmsimd-arm.c b/src/f32-gemm/gen/5x8s4-minmax-wasmsimd-arm.c index fbc2fbd31..3220bbe52 100644 --- a/src/f32-gemm/gen/5x8s4-minmax-wasmsimd-arm.c +++ b/src/f32-gemm/gen/5x8s4-minmax-wasmsimd-arm.c @@ -199,18 +199,6 @@ void xnn_f32_gemm_minmax_ukernel_5x8s4__wasmsimd_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -223,6 +211,18 @@ void xnn_f32_gemm_minmax_ukernel_5x8s4__wasmsimd_arm( vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-gemm/gen/5x8s4-minmax-wasmsimd-x86.c b/src/f32-gemm/gen/5x8s4-minmax-wasmsimd-x86.c index 90b2ff96b..84621e025 100644 --- a/src/f32-gemm/gen/5x8s4-minmax-wasmsimd-x86.c +++ b/src/f32-gemm/gen/5x8s4-minmax-wasmsimd-x86.c @@ -199,18 +199,6 @@ void xnn_f32_gemm_minmax_ukernel_5x8s4__wasmsimd_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -223,6 +211,18 @@ void xnn_f32_gemm_minmax_ukernel_5x8s4__wasmsimd_x86( vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-gemm/gen/6x8-minmax-wasmsimd-loadsplat-arm.c b/src/f32-gemm/gen/6x8-minmax-wasmsimd-loadsplat-arm.c index 2b053efb4..8d4ab1d26 100644 --- a/src/f32-gemm/gen/6x8-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-gemm/gen/6x8-minmax-wasmsimd-loadsplat-arm.c @@ -118,20 +118,6 @@ void xnn_f32_gemm_minmax_ukernel_6x8__wasmsimd_loadsplat_arm( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -146,6 +132,20 @@ void xnn_f32_gemm_minmax_ukernel_6x8__wasmsimd_loadsplat_arm( vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); vacc5x4567 = wasm_f32x4_max(vacc5x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-gemm/gen/6x8-minmax-wasmsimd-loadsplat-x86.c b/src/f32-gemm/gen/6x8-minmax-wasmsimd-loadsplat-x86.c index 18b50464a..4795c093c 100644 --- a/src/f32-gemm/gen/6x8-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-gemm/gen/6x8-minmax-wasmsimd-loadsplat-x86.c @@ -118,20 +118,6 @@ void xnn_f32_gemm_minmax_ukernel_6x8__wasmsimd_loadsplat_x86( k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -146,6 +132,20 @@ void xnn_f32_gemm_minmax_ukernel_6x8__wasmsimd_loadsplat_x86( vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); vacc5x4567 = wasm_v128_bitselect(vmin, vacc5x4567, wasm_f32x4_lt(vacc5x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-gemm/gen/6x8-minmax-wasmsimd-splat-arm.c b/src/f32-gemm/gen/6x8-minmax-wasmsimd-splat-arm.c index 572c878c0..67e5916df 100644 --- a/src/f32-gemm/gen/6x8-minmax-wasmsimd-splat-arm.c +++ b/src/f32-gemm/gen/6x8-minmax-wasmsimd-splat-arm.c @@ -226,20 +226,6 @@ void xnn_f32_gemm_minmax_ukernel_6x8__wasmsimd_splat_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -254,6 +240,20 @@ void xnn_f32_gemm_minmax_ukernel_6x8__wasmsimd_splat_arm( vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); vacc5x4567 = wasm_f32x4_max(vacc5x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-gemm/gen/6x8-minmax-wasmsimd-splat-x86.c b/src/f32-gemm/gen/6x8-minmax-wasmsimd-splat-x86.c index 6029240ce..fa7e6174f 100644 --- a/src/f32-gemm/gen/6x8-minmax-wasmsimd-splat-x86.c +++ b/src/f32-gemm/gen/6x8-minmax-wasmsimd-splat-x86.c @@ -226,20 +226,6 @@ void xnn_f32_gemm_minmax_ukernel_6x8__wasmsimd_splat_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -254,6 +240,20 @@ void xnn_f32_gemm_minmax_ukernel_6x8__wasmsimd_splat_x86( vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); vacc5x4567 = wasm_v128_bitselect(vmin, vacc5x4567, wasm_f32x4_lt(vacc5x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-gemm/gen/6x8s4-minmax-wasmsimd-arm.c b/src/f32-gemm/gen/6x8s4-minmax-wasmsimd-arm.c index ebbb5c32f..32b83f0d0 100644 --- a/src/f32-gemm/gen/6x8s4-minmax-wasmsimd-arm.c +++ b/src/f32-gemm/gen/6x8s4-minmax-wasmsimd-arm.c @@ -224,20 +224,6 @@ void xnn_f32_gemm_minmax_ukernel_6x8s4__wasmsimd_arm( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -252,6 +238,20 @@ void xnn_f32_gemm_minmax_ukernel_6x8s4__wasmsimd_arm( vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); vacc5x4567 = wasm_f32x4_max(vacc5x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-gemm/gen/6x8s4-minmax-wasmsimd-x86.c b/src/f32-gemm/gen/6x8s4-minmax-wasmsimd-x86.c index 99c1fbdb6..1cd3c2568 100644 --- a/src/f32-gemm/gen/6x8s4-minmax-wasmsimd-x86.c +++ b/src/f32-gemm/gen/6x8s4-minmax-wasmsimd-x86.c @@ -224,20 +224,6 @@ void xnn_f32_gemm_minmax_ukernel_6x8s4__wasmsimd_x86( } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -252,6 +238,20 @@ void xnn_f32_gemm_minmax_ukernel_6x8s4__wasmsimd_x86( vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); vacc5x4567 = wasm_v128_bitselect(vmin, vacc5x4567, wasm_f32x4_lt(vacc5x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-gemm/wasmsimd-loadsplat.c.in b/src/f32-gemm/wasmsimd-loadsplat.c.in index 019aa77c3..7dd0e22e5 100644 --- a/src/f32-gemm/wasmsimd-loadsplat.c.in +++ b/src/f32-gemm/wasmsimd-loadsplat.c.in @@ -90,21 +90,21 @@ void xnn_f32_gemm${"inc" if INC else ""}_minmax_ukernel_${MR}x${NR}__wasmsimd_lo k -= sizeof(float); } while (k != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); $for N in range(0, NR, 4): $for M in range(MR): $if X86: - vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vacc${M}x${ABC[N:N+4]}, vmax, wasm_f32x4_le(vacc${M}x${ABC[N:N+4]}, vmax)); + vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vmin, vacc${M}x${ABC[N:N+4]}, wasm_f32x4_lt(vacc${M}x${ABC[N:N+4]}, vmin)); $else: - vacc${M}x${ABC[N:N+4]} = wasm_f32x4_min(vacc${M}x${ABC[N:N+4]}, vmax); + vacc${M}x${ABC[N:N+4]} = wasm_f32x4_max(vacc${M}x${ABC[N:N+4]}, vmin); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); $for N in range(0, NR, 4): $for M in range(MR): $if X86: - vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vmin, vacc${M}x${ABC[N:N+4]}, wasm_f32x4_lt(vacc${M}x${ABC[N:N+4]}, vmin)); + vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vacc${M}x${ABC[N:N+4]}, vmax, wasm_f32x4_le(vacc${M}x${ABC[N:N+4]}, vmax)); $else: - vacc${M}x${ABC[N:N+4]} = wasm_f32x4_max(vacc${M}x${ABC[N:N+4]}, vmin); + vacc${M}x${ABC[N:N+4]} = wasm_f32x4_min(vacc${M}x${ABC[N:N+4]}, vmax); if XNN_LIKELY(nc >= ${NR}) { $for M in reversed(range(MR)): diff --git a/src/f32-gemm/wasmsimd-s4.c.in b/src/f32-gemm/wasmsimd-s4.c.in index f92657799..a574f0b13 100644 --- a/src/f32-gemm/wasmsimd-s4.c.in +++ b/src/f32-gemm/wasmsimd-s4.c.in @@ -113,21 +113,21 @@ void xnn_f32_gemm${"inc" if INC else ""}_minmax_ukernel_${MR}x${NR}s4__wasmsimd_ } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); $for N in range(0, NR, 4): $for M in range(MR): $if X86: - vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vacc${M}x${ABC[N:N+4]}, vmax, wasm_f32x4_le(vacc${M}x${ABC[N:N+4]}, vmax)); + vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vmin, vacc${M}x${ABC[N:N+4]}, wasm_f32x4_lt(vacc${M}x${ABC[N:N+4]}, vmin)); $else: - vacc${M}x${ABC[N:N+4]} = wasm_f32x4_min(vacc${M}x${ABC[N:N+4]}, vmax); + vacc${M}x${ABC[N:N+4]} = wasm_f32x4_max(vacc${M}x${ABC[N:N+4]}, vmin); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); $for N in range(0, NR, 4): $for M in range(MR): $if X86: - vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vmin, vacc${M}x${ABC[N:N+4]}, wasm_f32x4_lt(vacc${M}x${ABC[N:N+4]}, vmin)); + vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vacc${M}x${ABC[N:N+4]}, vmax, wasm_f32x4_le(vacc${M}x${ABC[N:N+4]}, vmax)); $else: - vacc${M}x${ABC[N:N+4]} = wasm_f32x4_max(vacc${M}x${ABC[N:N+4]}, vmin); + vacc${M}x${ABC[N:N+4]} = wasm_f32x4_min(vacc${M}x${ABC[N:N+4]}, vmax); if XNN_LIKELY(nc >= ${NR}) { $for M in reversed(range(MR)): diff --git a/src/f32-gemm/wasmsimd-splat.c.in b/src/f32-gemm/wasmsimd-splat.c.in index b5dace621..a57e081d6 100644 --- a/src/f32-gemm/wasmsimd-splat.c.in +++ b/src/f32-gemm/wasmsimd-splat.c.in @@ -111,21 +111,21 @@ void xnn_f32_gemm${"inc" if INC else ""}_minmax_ukernel_${MR}x${NR}__wasmsimd_sp } while (k != 0); } - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); $for N in range(0, NR, 4): $for M in range(MR): $if X86: - vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vacc${M}x${ABC[N:N+4]}, vmax, wasm_f32x4_le(vacc${M}x${ABC[N:N+4]}, vmax)); + vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vmin, vacc${M}x${ABC[N:N+4]}, wasm_f32x4_lt(vacc${M}x${ABC[N:N+4]}, vmin)); $else: - vacc${M}x${ABC[N:N+4]} = wasm_f32x4_min(vacc${M}x${ABC[N:N+4]}, vmax); + vacc${M}x${ABC[N:N+4]} = wasm_f32x4_max(vacc${M}x${ABC[N:N+4]}, vmin); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); $for N in range(0, NR, 4): $for M in range(MR): $if X86: - vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vmin, vacc${M}x${ABC[N:N+4]}, wasm_f32x4_lt(vacc${M}x${ABC[N:N+4]}, vmin)); + vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vacc${M}x${ABC[N:N+4]}, vmax, wasm_f32x4_le(vacc${M}x${ABC[N:N+4]}, vmax)); $else: - vacc${M}x${ABC[N:N+4]} = wasm_f32x4_max(vacc${M}x${ABC[N:N+4]}, vmin); + vacc${M}x${ABC[N:N+4]} = wasm_f32x4_min(vacc${M}x${ABC[N:N+4]}, vmax); if XNN_LIKELY(nc >= ${NR}) { $for M in reversed(range(MR)): diff --git a/src/f32-igemm/gen/1x8-minmax-wasmsimd-loadsplat-arm.c b/src/f32-igemm/gen/1x8-minmax-wasmsimd-loadsplat-arm.c index 16752ad26..c41e2f467 100644 --- a/src/f32-igemm/gen/1x8-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-igemm/gen/1x8-minmax-wasmsimd-loadsplat-arm.c @@ -72,14 +72,14 @@ void xnn_f32_igemm_minmax_ukernel_1x8__wasmsimd_loadsplat_arm( p -= 1 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc0x4567 = wasm_f32x4_max(vacc0x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-igemm/gen/1x8-minmax-wasmsimd-loadsplat-x86.c b/src/f32-igemm/gen/1x8-minmax-wasmsimd-loadsplat-x86.c index d3732e7ff..bf1928b9d 100644 --- a/src/f32-igemm/gen/1x8-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-igemm/gen/1x8-minmax-wasmsimd-loadsplat-x86.c @@ -72,14 +72,14 @@ void xnn_f32_igemm_minmax_ukernel_1x8__wasmsimd_loadsplat_x86( p -= 1 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc0x4567 = wasm_v128_bitselect(vmin, vacc0x4567, wasm_f32x4_lt(vacc0x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-igemm/gen/1x8-minmax-wasmsimd-splat-arm.c b/src/f32-igemm/gen/1x8-minmax-wasmsimd-splat-arm.c index 4bf461564..d729fb7a5 100644 --- a/src/f32-igemm/gen/1x8-minmax-wasmsimd-splat-arm.c +++ b/src/f32-igemm/gen/1x8-minmax-wasmsimd-splat-arm.c @@ -110,14 +110,14 @@ void xnn_f32_igemm_minmax_ukernel_1x8__wasmsimd_splat_arm( p -= 1 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc0x4567 = wasm_f32x4_max(vacc0x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-igemm/gen/1x8-minmax-wasmsimd-splat-x86.c b/src/f32-igemm/gen/1x8-minmax-wasmsimd-splat-x86.c index 84c124bda..b83d2037b 100644 --- a/src/f32-igemm/gen/1x8-minmax-wasmsimd-splat-x86.c +++ b/src/f32-igemm/gen/1x8-minmax-wasmsimd-splat-x86.c @@ -110,14 +110,14 @@ void xnn_f32_igemm_minmax_ukernel_1x8__wasmsimd_splat_x86( p -= 1 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc0x4567 = wasm_v128_bitselect(vmin, vacc0x4567, wasm_f32x4_lt(vacc0x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-igemm/gen/1x8s4-minmax-wasmsimd-arm.c b/src/f32-igemm/gen/1x8s4-minmax-wasmsimd-arm.c index 030d8b468..9c10ca319 100644 --- a/src/f32-igemm/gen/1x8s4-minmax-wasmsimd-arm.c +++ b/src/f32-igemm/gen/1x8s4-minmax-wasmsimd-arm.c @@ -113,14 +113,14 @@ void xnn_f32_igemm_minmax_ukernel_1x8s4__wasmsimd_arm( p -= 1 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc0x4567 = wasm_f32x4_max(vacc0x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-igemm/gen/1x8s4-minmax-wasmsimd-x86.c b/src/f32-igemm/gen/1x8s4-minmax-wasmsimd-x86.c index 589de601e..11c4d7b25 100644 --- a/src/f32-igemm/gen/1x8s4-minmax-wasmsimd-x86.c +++ b/src/f32-igemm/gen/1x8s4-minmax-wasmsimd-x86.c @@ -113,14 +113,14 @@ void xnn_f32_igemm_minmax_ukernel_1x8s4__wasmsimd_x86( p -= 1 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc0x4567 = wasm_v128_bitselect(vmin, vacc0x4567, wasm_f32x4_lt(vacc0x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c0, vacc0x0123); wasm_v128_store(c0 + 4, vacc0x4567); diff --git a/src/f32-igemm/gen/3x8-minmax-wasmsimd-loadsplat-arm.c b/src/f32-igemm/gen/3x8-minmax-wasmsimd-loadsplat-arm.c index fb9ebcd6a..e25767d66 100644 --- a/src/f32-igemm/gen/3x8-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-igemm/gen/3x8-minmax-wasmsimd-loadsplat-arm.c @@ -102,14 +102,6 @@ void xnn_f32_igemm_minmax_ukernel_3x8__wasmsimd_loadsplat_arm( p -= 3 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -118,6 +110,14 @@ void xnn_f32_igemm_minmax_ukernel_3x8__wasmsimd_loadsplat_arm( vacc1x4567 = wasm_f32x4_max(vacc1x4567, vmin); vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-igemm/gen/3x8-minmax-wasmsimd-loadsplat-x86.c b/src/f32-igemm/gen/3x8-minmax-wasmsimd-loadsplat-x86.c index b8aae0e15..64d23dd0b 100644 --- a/src/f32-igemm/gen/3x8-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-igemm/gen/3x8-minmax-wasmsimd-loadsplat-x86.c @@ -102,14 +102,6 @@ void xnn_f32_igemm_minmax_ukernel_3x8__wasmsimd_loadsplat_x86( p -= 3 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -118,6 +110,14 @@ void xnn_f32_igemm_minmax_ukernel_3x8__wasmsimd_loadsplat_x86( vacc1x4567 = wasm_v128_bitselect(vmin, vacc1x4567, wasm_f32x4_lt(vacc1x4567, vmin)); vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-igemm/gen/3x8-minmax-wasmsimd-splat-arm.c b/src/f32-igemm/gen/3x8-minmax-wasmsimd-splat-arm.c index fde50264d..b6ecbef0c 100644 --- a/src/f32-igemm/gen/3x8-minmax-wasmsimd-splat-arm.c +++ b/src/f32-igemm/gen/3x8-minmax-wasmsimd-splat-arm.c @@ -168,14 +168,6 @@ void xnn_f32_igemm_minmax_ukernel_3x8__wasmsimd_splat_arm( p -= 3 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -184,6 +176,14 @@ void xnn_f32_igemm_minmax_ukernel_3x8__wasmsimd_splat_arm( vacc1x4567 = wasm_f32x4_max(vacc1x4567, vmin); vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-igemm/gen/3x8-minmax-wasmsimd-splat-x86.c b/src/f32-igemm/gen/3x8-minmax-wasmsimd-splat-x86.c index a12038b83..c0989c3f0 100644 --- a/src/f32-igemm/gen/3x8-minmax-wasmsimd-splat-x86.c +++ b/src/f32-igemm/gen/3x8-minmax-wasmsimd-splat-x86.c @@ -168,14 +168,6 @@ void xnn_f32_igemm_minmax_ukernel_3x8__wasmsimd_splat_x86( p -= 3 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -184,6 +176,14 @@ void xnn_f32_igemm_minmax_ukernel_3x8__wasmsimd_splat_x86( vacc1x4567 = wasm_v128_bitselect(vmin, vacc1x4567, wasm_f32x4_lt(vacc1x4567, vmin)); vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-igemm/gen/3x8s4-minmax-wasmsimd-arm.c b/src/f32-igemm/gen/3x8s4-minmax-wasmsimd-arm.c index ee2ef29cd..392192814 100644 --- a/src/f32-igemm/gen/3x8s4-minmax-wasmsimd-arm.c +++ b/src/f32-igemm/gen/3x8s4-minmax-wasmsimd-arm.c @@ -169,14 +169,6 @@ void xnn_f32_igemm_minmax_ukernel_3x8s4__wasmsimd_arm( p -= 3 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -185,6 +177,14 @@ void xnn_f32_igemm_minmax_ukernel_3x8s4__wasmsimd_arm( vacc1x4567 = wasm_f32x4_max(vacc1x4567, vmin); vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-igemm/gen/3x8s4-minmax-wasmsimd-x86.c b/src/f32-igemm/gen/3x8s4-minmax-wasmsimd-x86.c index 68461171a..98824b355 100644 --- a/src/f32-igemm/gen/3x8s4-minmax-wasmsimd-x86.c +++ b/src/f32-igemm/gen/3x8s4-minmax-wasmsimd-x86.c @@ -169,14 +169,6 @@ void xnn_f32_igemm_minmax_ukernel_3x8s4__wasmsimd_x86( p -= 3 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -185,6 +177,14 @@ void xnn_f32_igemm_minmax_ukernel_3x8s4__wasmsimd_x86( vacc1x4567 = wasm_v128_bitselect(vmin, vacc1x4567, wasm_f32x4_lt(vacc1x4567, vmin)); vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c2, vacc2x0123); wasm_v128_store(c2 + 4, vacc2x4567); diff --git a/src/f32-igemm/gen/4x8-minmax-wasmsimd-loadsplat-arm.c b/src/f32-igemm/gen/4x8-minmax-wasmsimd-loadsplat-arm.c index 32cf2e4e7..d0edc054d 100644 --- a/src/f32-igemm/gen/4x8-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-igemm/gen/4x8-minmax-wasmsimd-loadsplat-arm.c @@ -117,16 +117,6 @@ void xnn_f32_igemm_minmax_ukernel_4x8__wasmsimd_loadsplat_arm( p -= 4 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -137,6 +127,16 @@ void xnn_f32_igemm_minmax_ukernel_4x8__wasmsimd_loadsplat_arm( vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-igemm/gen/4x8-minmax-wasmsimd-loadsplat-x86.c b/src/f32-igemm/gen/4x8-minmax-wasmsimd-loadsplat-x86.c index 4e2ad3210..66e755e28 100644 --- a/src/f32-igemm/gen/4x8-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-igemm/gen/4x8-minmax-wasmsimd-loadsplat-x86.c @@ -117,16 +117,6 @@ void xnn_f32_igemm_minmax_ukernel_4x8__wasmsimd_loadsplat_x86( p -= 4 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -137,6 +127,16 @@ void xnn_f32_igemm_minmax_ukernel_4x8__wasmsimd_loadsplat_x86( vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-igemm/gen/4x8-minmax-wasmsimd-splat-arm.c b/src/f32-igemm/gen/4x8-minmax-wasmsimd-splat-arm.c index d50cca0e3..ce94c5187 100644 --- a/src/f32-igemm/gen/4x8-minmax-wasmsimd-splat-arm.c +++ b/src/f32-igemm/gen/4x8-minmax-wasmsimd-splat-arm.c @@ -197,16 +197,6 @@ void xnn_f32_igemm_minmax_ukernel_4x8__wasmsimd_splat_arm( p -= 4 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -217,6 +207,16 @@ void xnn_f32_igemm_minmax_ukernel_4x8__wasmsimd_splat_arm( vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-igemm/gen/4x8-minmax-wasmsimd-splat-x86.c b/src/f32-igemm/gen/4x8-minmax-wasmsimd-splat-x86.c index 02e55dd8e..50825d3ed 100644 --- a/src/f32-igemm/gen/4x8-minmax-wasmsimd-splat-x86.c +++ b/src/f32-igemm/gen/4x8-minmax-wasmsimd-splat-x86.c @@ -197,16 +197,6 @@ void xnn_f32_igemm_minmax_ukernel_4x8__wasmsimd_splat_x86( p -= 4 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -217,6 +207,16 @@ void xnn_f32_igemm_minmax_ukernel_4x8__wasmsimd_splat_x86( vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-igemm/gen/4x8s4-minmax-wasmsimd-arm.c b/src/f32-igemm/gen/4x8s4-minmax-wasmsimd-arm.c index 9d3bb6d7b..b114ef269 100644 --- a/src/f32-igemm/gen/4x8s4-minmax-wasmsimd-arm.c +++ b/src/f32-igemm/gen/4x8s4-minmax-wasmsimd-arm.c @@ -197,16 +197,6 @@ void xnn_f32_igemm_minmax_ukernel_4x8s4__wasmsimd_arm( p -= 4 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -217,6 +207,16 @@ void xnn_f32_igemm_minmax_ukernel_4x8s4__wasmsimd_arm( vacc2x4567 = wasm_f32x4_max(vacc2x4567, vmin); vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-igemm/gen/4x8s4-minmax-wasmsimd-x86.c b/src/f32-igemm/gen/4x8s4-minmax-wasmsimd-x86.c index bec0eadcd..b08fa49ce 100644 --- a/src/f32-igemm/gen/4x8s4-minmax-wasmsimd-x86.c +++ b/src/f32-igemm/gen/4x8s4-minmax-wasmsimd-x86.c @@ -197,16 +197,6 @@ void xnn_f32_igemm_minmax_ukernel_4x8s4__wasmsimd_x86( p -= 4 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -217,6 +207,16 @@ void xnn_f32_igemm_minmax_ukernel_4x8s4__wasmsimd_x86( vacc2x4567 = wasm_v128_bitselect(vmin, vacc2x4567, wasm_f32x4_lt(vacc2x4567, vmin)); vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c3, vacc3x0123); wasm_v128_store(c3 + 4, vacc3x4567); diff --git a/src/f32-igemm/gen/5x8-minmax-wasmsimd-loadsplat-arm.c b/src/f32-igemm/gen/5x8-minmax-wasmsimd-loadsplat-arm.c index 36eb956b9..40bd528dd 100644 --- a/src/f32-igemm/gen/5x8-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-igemm/gen/5x8-minmax-wasmsimd-loadsplat-arm.c @@ -132,18 +132,6 @@ void xnn_f32_igemm_minmax_ukernel_5x8__wasmsimd_loadsplat_arm( p -= 5 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -156,6 +144,18 @@ void xnn_f32_igemm_minmax_ukernel_5x8__wasmsimd_loadsplat_arm( vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-igemm/gen/5x8-minmax-wasmsimd-loadsplat-x86.c b/src/f32-igemm/gen/5x8-minmax-wasmsimd-loadsplat-x86.c index cf783a6b3..7ee07b74a 100644 --- a/src/f32-igemm/gen/5x8-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-igemm/gen/5x8-minmax-wasmsimd-loadsplat-x86.c @@ -132,18 +132,6 @@ void xnn_f32_igemm_minmax_ukernel_5x8__wasmsimd_loadsplat_x86( p -= 5 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -156,6 +144,18 @@ void xnn_f32_igemm_minmax_ukernel_5x8__wasmsimd_loadsplat_x86( vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-igemm/gen/5x8-minmax-wasmsimd-splat-arm.c b/src/f32-igemm/gen/5x8-minmax-wasmsimd-splat-arm.c index 12af706e4..450b1aae7 100644 --- a/src/f32-igemm/gen/5x8-minmax-wasmsimd-splat-arm.c +++ b/src/f32-igemm/gen/5x8-minmax-wasmsimd-splat-arm.c @@ -226,18 +226,6 @@ void xnn_f32_igemm_minmax_ukernel_5x8__wasmsimd_splat_arm( p -= 5 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -250,6 +238,18 @@ void xnn_f32_igemm_minmax_ukernel_5x8__wasmsimd_splat_arm( vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-igemm/gen/5x8-minmax-wasmsimd-splat-x86.c b/src/f32-igemm/gen/5x8-minmax-wasmsimd-splat-x86.c index cc8683b2e..8938b2f5c 100644 --- a/src/f32-igemm/gen/5x8-minmax-wasmsimd-splat-x86.c +++ b/src/f32-igemm/gen/5x8-minmax-wasmsimd-splat-x86.c @@ -226,18 +226,6 @@ void xnn_f32_igemm_minmax_ukernel_5x8__wasmsimd_splat_x86( p -= 5 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -250,6 +238,18 @@ void xnn_f32_igemm_minmax_ukernel_5x8__wasmsimd_splat_x86( vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-igemm/gen/5x8s4-minmax-wasmsimd-arm.c b/src/f32-igemm/gen/5x8s4-minmax-wasmsimd-arm.c index 979b9f266..804db49a9 100644 --- a/src/f32-igemm/gen/5x8s4-minmax-wasmsimd-arm.c +++ b/src/f32-igemm/gen/5x8s4-minmax-wasmsimd-arm.c @@ -225,18 +225,6 @@ void xnn_f32_igemm_minmax_ukernel_5x8s4__wasmsimd_arm( p -= 5 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -249,6 +237,18 @@ void xnn_f32_igemm_minmax_ukernel_5x8s4__wasmsimd_arm( vacc3x4567 = wasm_f32x4_max(vacc3x4567, vmin); vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-igemm/gen/5x8s4-minmax-wasmsimd-x86.c b/src/f32-igemm/gen/5x8s4-minmax-wasmsimd-x86.c index 3531539b2..61e3610be 100644 --- a/src/f32-igemm/gen/5x8s4-minmax-wasmsimd-x86.c +++ b/src/f32-igemm/gen/5x8s4-minmax-wasmsimd-x86.c @@ -225,18 +225,6 @@ void xnn_f32_igemm_minmax_ukernel_5x8s4__wasmsimd_x86( p -= 5 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -249,6 +237,18 @@ void xnn_f32_igemm_minmax_ukernel_5x8s4__wasmsimd_x86( vacc3x4567 = wasm_v128_bitselect(vmin, vacc3x4567, wasm_f32x4_lt(vacc3x4567, vmin)); vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c4, vacc4x0123); wasm_v128_store(c4 + 4, vacc4x4567); diff --git a/src/f32-igemm/gen/6x8-minmax-wasmsimd-loadsplat-arm.c b/src/f32-igemm/gen/6x8-minmax-wasmsimd-loadsplat-arm.c index 6aa43888b..a7958478a 100644 --- a/src/f32-igemm/gen/6x8-minmax-wasmsimd-loadsplat-arm.c +++ b/src/f32-igemm/gen/6x8-minmax-wasmsimd-loadsplat-arm.c @@ -147,20 +147,6 @@ void xnn_f32_igemm_minmax_ukernel_6x8__wasmsimd_loadsplat_arm( p -= 6 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -175,6 +161,20 @@ void xnn_f32_igemm_minmax_ukernel_6x8__wasmsimd_loadsplat_arm( vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); vacc5x4567 = wasm_f32x4_max(vacc5x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-igemm/gen/6x8-minmax-wasmsimd-loadsplat-x86.c b/src/f32-igemm/gen/6x8-minmax-wasmsimd-loadsplat-x86.c index e54bc1668..4590d6cf6 100644 --- a/src/f32-igemm/gen/6x8-minmax-wasmsimd-loadsplat-x86.c +++ b/src/f32-igemm/gen/6x8-minmax-wasmsimd-loadsplat-x86.c @@ -147,20 +147,6 @@ void xnn_f32_igemm_minmax_ukernel_6x8__wasmsimd_loadsplat_x86( p -= 6 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -175,6 +161,20 @@ void xnn_f32_igemm_minmax_ukernel_6x8__wasmsimd_loadsplat_x86( vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); vacc5x4567 = wasm_v128_bitselect(vmin, vacc5x4567, wasm_f32x4_lt(vacc5x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-igemm/gen/6x8-minmax-wasmsimd-splat-arm.c b/src/f32-igemm/gen/6x8-minmax-wasmsimd-splat-arm.c index 0ba5e02a5..822951df4 100644 --- a/src/f32-igemm/gen/6x8-minmax-wasmsimd-splat-arm.c +++ b/src/f32-igemm/gen/6x8-minmax-wasmsimd-splat-arm.c @@ -255,20 +255,6 @@ void xnn_f32_igemm_minmax_ukernel_6x8__wasmsimd_splat_arm( p -= 6 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -283,6 +269,20 @@ void xnn_f32_igemm_minmax_ukernel_6x8__wasmsimd_splat_arm( vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); vacc5x4567 = wasm_f32x4_max(vacc5x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-igemm/gen/6x8-minmax-wasmsimd-splat-x86.c b/src/f32-igemm/gen/6x8-minmax-wasmsimd-splat-x86.c index 40aeba4e7..825fd24b7 100644 --- a/src/f32-igemm/gen/6x8-minmax-wasmsimd-splat-x86.c +++ b/src/f32-igemm/gen/6x8-minmax-wasmsimd-splat-x86.c @@ -255,20 +255,6 @@ void xnn_f32_igemm_minmax_ukernel_6x8__wasmsimd_splat_x86( p -= 6 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -283,6 +269,20 @@ void xnn_f32_igemm_minmax_ukernel_6x8__wasmsimd_splat_x86( vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); vacc5x4567 = wasm_v128_bitselect(vmin, vacc5x4567, wasm_f32x4_lt(vacc5x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-igemm/gen/6x8s4-minmax-wasmsimd-arm.c b/src/f32-igemm/gen/6x8s4-minmax-wasmsimd-arm.c index 5677184bf..8d50f8ac9 100644 --- a/src/f32-igemm/gen/6x8s4-minmax-wasmsimd-arm.c +++ b/src/f32-igemm/gen/6x8s4-minmax-wasmsimd-arm.c @@ -253,20 +253,6 @@ void xnn_f32_igemm_minmax_ukernel_6x8s4__wasmsimd_arm( p -= 6 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); - vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); - vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); - vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); - vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); - vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); - vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); - vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); - vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); - vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); - vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); - vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_f32x4_max(vacc0x0123, vmin); vacc1x0123 = wasm_f32x4_max(vacc1x0123, vmin); @@ -281,6 +267,20 @@ void xnn_f32_igemm_minmax_ukernel_6x8s4__wasmsimd_arm( vacc4x4567 = wasm_f32x4_max(vacc4x4567, vmin); vacc5x4567 = wasm_f32x4_max(vacc5x4567, vmin); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_f32x4_min(vacc0x0123, vmax); + vacc1x0123 = wasm_f32x4_min(vacc1x0123, vmax); + vacc2x0123 = wasm_f32x4_min(vacc2x0123, vmax); + vacc3x0123 = wasm_f32x4_min(vacc3x0123, vmax); + vacc4x0123 = wasm_f32x4_min(vacc4x0123, vmax); + vacc5x0123 = wasm_f32x4_min(vacc5x0123, vmax); + vacc0x4567 = wasm_f32x4_min(vacc0x4567, vmax); + vacc1x4567 = wasm_f32x4_min(vacc1x4567, vmax); + vacc2x4567 = wasm_f32x4_min(vacc2x4567, vmax); + vacc3x4567 = wasm_f32x4_min(vacc3x4567, vmax); + vacc4x4567 = wasm_f32x4_min(vacc4x4567, vmax); + vacc5x4567 = wasm_f32x4_min(vacc5x4567, vmax); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-igemm/gen/6x8s4-minmax-wasmsimd-x86.c b/src/f32-igemm/gen/6x8s4-minmax-wasmsimd-x86.c index fcccac1ac..e16330711 100644 --- a/src/f32-igemm/gen/6x8s4-minmax-wasmsimd-x86.c +++ b/src/f32-igemm/gen/6x8s4-minmax-wasmsimd-x86.c @@ -253,20 +253,6 @@ void xnn_f32_igemm_minmax_ukernel_6x8s4__wasmsimd_x86( p -= 6 * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); - vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); - vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); - vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); - vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); - vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); - vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); - vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); - vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); - vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); - vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); - vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); - vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); vacc0x0123 = wasm_v128_bitselect(vmin, vacc0x0123, wasm_f32x4_lt(vacc0x0123, vmin)); vacc1x0123 = wasm_v128_bitselect(vmin, vacc1x0123, wasm_f32x4_lt(vacc1x0123, vmin)); @@ -281,6 +267,20 @@ void xnn_f32_igemm_minmax_ukernel_6x8s4__wasmsimd_x86( vacc4x4567 = wasm_v128_bitselect(vmin, vacc4x4567, wasm_f32x4_lt(vacc4x4567, vmin)); vacc5x4567 = wasm_v128_bitselect(vmin, vacc5x4567, wasm_f32x4_lt(vacc5x4567, vmin)); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + vacc0x0123 = wasm_v128_bitselect(vacc0x0123, vmax, wasm_f32x4_le(vacc0x0123, vmax)); + vacc1x0123 = wasm_v128_bitselect(vacc1x0123, vmax, wasm_f32x4_le(vacc1x0123, vmax)); + vacc2x0123 = wasm_v128_bitselect(vacc2x0123, vmax, wasm_f32x4_le(vacc2x0123, vmax)); + vacc3x0123 = wasm_v128_bitselect(vacc3x0123, vmax, wasm_f32x4_le(vacc3x0123, vmax)); + vacc4x0123 = wasm_v128_bitselect(vacc4x0123, vmax, wasm_f32x4_le(vacc4x0123, vmax)); + vacc5x0123 = wasm_v128_bitselect(vacc5x0123, vmax, wasm_f32x4_le(vacc5x0123, vmax)); + vacc0x4567 = wasm_v128_bitselect(vacc0x4567, vmax, wasm_f32x4_le(vacc0x4567, vmax)); + vacc1x4567 = wasm_v128_bitselect(vacc1x4567, vmax, wasm_f32x4_le(vacc1x4567, vmax)); + vacc2x4567 = wasm_v128_bitselect(vacc2x4567, vmax, wasm_f32x4_le(vacc2x4567, vmax)); + vacc3x4567 = wasm_v128_bitselect(vacc3x4567, vmax, wasm_f32x4_le(vacc3x4567, vmax)); + vacc4x4567 = wasm_v128_bitselect(vacc4x4567, vmax, wasm_f32x4_le(vacc4x4567, vmax)); + vacc5x4567 = wasm_v128_bitselect(vacc5x4567, vmax, wasm_f32x4_le(vacc5x4567, vmax)); + if XNN_LIKELY(nc >= 8) { wasm_v128_store(c5, vacc5x0123); wasm_v128_store(c5 + 4, vacc5x4567); diff --git a/src/f32-igemm/wasmsimd-loadsplat.c.in b/src/f32-igemm/wasmsimd-loadsplat.c.in index 9f547ddcb..bcf65f1ee 100644 --- a/src/f32-igemm/wasmsimd-loadsplat.c.in +++ b/src/f32-igemm/wasmsimd-loadsplat.c.in @@ -92,21 +92,21 @@ void xnn_f32_igemm_minmax_ukernel_${MR}x${NR}__wasmsimd_loadsplat_${"x86" if X86 p -= ${MR} * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); $for N in range(0, NR, 4): $for M in range(MR): $if X86: - vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vacc${M}x${ABC[N:N+4]}, vmax, wasm_f32x4_le(vacc${M}x${ABC[N:N+4]}, vmax)); + vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vmin, vacc${M}x${ABC[N:N+4]}, wasm_f32x4_lt(vacc${M}x${ABC[N:N+4]}, vmin)); $else: - vacc${M}x${ABC[N:N+4]} = wasm_f32x4_min(vacc${M}x${ABC[N:N+4]}, vmax); + vacc${M}x${ABC[N:N+4]} = wasm_f32x4_max(vacc${M}x${ABC[N:N+4]}, vmin); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); $for N in range(0, NR, 4): $for M in range(MR): $if X86: - vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vmin, vacc${M}x${ABC[N:N+4]}, wasm_f32x4_lt(vacc${M}x${ABC[N:N+4]}, vmin)); + vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vacc${M}x${ABC[N:N+4]}, vmax, wasm_f32x4_le(vacc${M}x${ABC[N:N+4]}, vmax)); $else: - vacc${M}x${ABC[N:N+4]} = wasm_f32x4_max(vacc${M}x${ABC[N:N+4]}, vmin); + vacc${M}x${ABC[N:N+4]} = wasm_f32x4_min(vacc${M}x${ABC[N:N+4]}, vmax); if XNN_LIKELY(nc >= ${NR}) { $for M in reversed(range(MR)): diff --git a/src/f32-igemm/wasmsimd-s4.c.in b/src/f32-igemm/wasmsimd-s4.c.in index adc501751..8c5da772c 100644 --- a/src/f32-igemm/wasmsimd-s4.c.in +++ b/src/f32-igemm/wasmsimd-s4.c.in @@ -115,21 +115,21 @@ void xnn_f32_igemm_minmax_ukernel_${MR}x${NR}s4__wasmsimd_${"x86" if X86 else "a p -= ${MR} * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); $for N in range(0, NR, 4): $for M in range(MR): $if X86: - vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vacc${M}x${ABC[N:N+4]}, vmax, wasm_f32x4_le(vacc${M}x${ABC[N:N+4]}, vmax)); + vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vmin, vacc${M}x${ABC[N:N+4]}, wasm_f32x4_lt(vacc${M}x${ABC[N:N+4]}, vmin)); $else: - vacc${M}x${ABC[N:N+4]} = wasm_f32x4_min(vacc${M}x${ABC[N:N+4]}, vmax); + vacc${M}x${ABC[N:N+4]} = wasm_f32x4_max(vacc${M}x${ABC[N:N+4]}, vmin); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); $for N in range(0, NR, 4): $for M in range(MR): $if X86: - vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vmin, vacc${M}x${ABC[N:N+4]}, wasm_f32x4_lt(vacc${M}x${ABC[N:N+4]}, vmin)); + vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vacc${M}x${ABC[N:N+4]}, vmax, wasm_f32x4_le(vacc${M}x${ABC[N:N+4]}, vmax)); $else: - vacc${M}x${ABC[N:N+4]} = wasm_f32x4_max(vacc${M}x${ABC[N:N+4]}, vmin); + vacc${M}x${ABC[N:N+4]} = wasm_f32x4_min(vacc${M}x${ABC[N:N+4]}, vmax); if XNN_LIKELY(nc >= ${NR}) { $for M in reversed(range(MR)): diff --git a/src/f32-igemm/wasmsimd-splat.c.in b/src/f32-igemm/wasmsimd-splat.c.in index a4edeaf49..56891d84c 100644 --- a/src/f32-igemm/wasmsimd-splat.c.in +++ b/src/f32-igemm/wasmsimd-splat.c.in @@ -113,21 +113,21 @@ void xnn_f32_igemm_minmax_ukernel_${MR}x${NR}__wasmsimd_splat_${"x86" if X86 els p -= ${MR} * sizeof(void*); } while (p != 0); - const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); + const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); $for N in range(0, NR, 4): $for M in range(MR): $if X86: - vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vacc${M}x${ABC[N:N+4]}, vmax, wasm_f32x4_le(vacc${M}x${ABC[N:N+4]}, vmax)); + vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vmin, vacc${M}x${ABC[N:N+4]}, wasm_f32x4_lt(vacc${M}x${ABC[N:N+4]}, vmin)); $else: - vacc${M}x${ABC[N:N+4]} = wasm_f32x4_min(vacc${M}x${ABC[N:N+4]}, vmax); + vacc${M}x${ABC[N:N+4]} = wasm_f32x4_max(vacc${M}x${ABC[N:N+4]}, vmin); - const v128_t vmin = wasm_v32x4_load_splat(¶ms->scalar.min); + const v128_t vmax = wasm_v32x4_load_splat(¶ms->scalar.max); $for N in range(0, NR, 4): $for M in range(MR): $if X86: - vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vmin, vacc${M}x${ABC[N:N+4]}, wasm_f32x4_lt(vacc${M}x${ABC[N:N+4]}, vmin)); + vacc${M}x${ABC[N:N+4]} = wasm_v128_bitselect(vacc${M}x${ABC[N:N+4]}, vmax, wasm_f32x4_le(vacc${M}x${ABC[N:N+4]}, vmax)); $else: - vacc${M}x${ABC[N:N+4]} = wasm_f32x4_max(vacc${M}x${ABC[N:N+4]}, vmin); + vacc${M}x${ABC[N:N+4]} = wasm_f32x4_min(vacc${M}x${ABC[N:N+4]}, vmax); if XNN_LIKELY(nc >= ${NR}) { $for M in reversed(range(MR)): diff --git a/src/init.c b/src/init.c index ed26a52bb..589d8947a 100644 --- a/src/init.c +++ b/src/init.c @@ -1627,15 +1627,25 @@ static void init(void) { xnn_params.f32.gemm2.nr = 2; xnn_params.f32.gemm2.log2_kr = 2; - xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2; - xnn_params.f32.dwconv[0].channel_tile = 4; - xnn_params.f32.dwconv[0].primary_tile = 4; + if (is_wasm_x86) { + xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86; + xnn_params.f32.dwconv[0].channel_tile = 8; + xnn_params.f32.dwconv[0].primary_tile = 4; - xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2; - xnn_params.f32.dwconv[1].channel_tile = 4; - xnn_params.f32.dwconv[1].primary_tile = 9; + xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86; + xnn_params.f32.dwconv[1].channel_tile = 8; + xnn_params.f32.dwconv[1].primary_tile = 9; + } else { + xnn_params.f32.dwconv[0].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm; + xnn_params.f32.dwconv[0].channel_tile = 4; + xnn_params.f32.dwconv[0].primary_tile = 4; + + xnn_params.f32.dwconv[1].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm; + xnn_params.f32.dwconv[1].channel_tile = 4; + xnn_params.f32.dwconv[1].primary_tile = 9; + } - xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2; + xnn_params.f32.dwconv[2].minmax.unipass = (xnn_dwconv_unipass_ukernel_function) xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm; xnn_params.f32.dwconv[2].channel_tile = 4; xnn_params.f32.dwconv[2].primary_tile = 25; diff --git a/src/xnnpack/dwconv.h b/src/xnnpack/dwconv.h index a3741f5df..7f5b8013d 100644 --- a/src/xnnpack/dwconv.h +++ b/src/xnnpack/dwconv.h @@ -52,6 +52,14 @@ DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__neonfma_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__neonfma) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__neonfma_acc2) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x4__psimd) @@ -83,6 +91,14 @@ DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__neonfma_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__aarch64_neonfma) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__aarch64_neonfma_cortex_a55) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x9__psimd_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x9__psimd) @@ -112,6 +128,14 @@ DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__neonfma_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__neonfma) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__neonfma_acc2) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86) +DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up4x25__psimd_acc2) DECLARE_F32_DWCONV_MINMAX_UNIPASS_UKERNEL_FUNCTION(xnn_f32_dwconv_minmax_ukernel_up8x25__psimd) diff --git a/test/f32-dwconv-minmax.cc b/test/f32-dwconv-minmax.cc index 6ed58f499..54b48f515 100644 --- a/test/f32-dwconv-minmax.cc +++ b/test/f32-dwconv-minmax.cc @@ -16025,6 +16025,4110 @@ #endif // !XNN_ARCH_ASMJS && !XNN_ARCH_WASM && !XNN_COMPILER_MSVC && !XNN_COMPILER_ICC +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, c_eq_4) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(4) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, c_div_4) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, c_div_4_with_qmin) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, c_div_4_with_qmax) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, c_lt_4) { + for (uint32_t channels = 1; channels < 4; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, c_gt_4) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, c_gt_4_with_qmin) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, c_gt_4_with_qmax) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, multipixel) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, multipixel_with_step) { + for (size_t channels = 1; channels <= 20; channels += 3) { + for (size_t step = 2; step <= 25; step++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(4) + .width(5) + .output_stride(23) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, input_offset) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .input_offset(112) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ARM, zero) { + for (uint32_t mz = 0; mz < 25; mz++) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .input_offset(112) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, c_eq_4) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(4) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, c_div_4) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, c_div_4_with_qmin) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, c_div_4_with_qmax) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, c_lt_4) { + for (uint32_t channels = 1; channels < 4; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, c_gt_4) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, c_gt_4_with_qmin) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, c_gt_4_with_qmax) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, multipixel) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, multipixel_with_step) { + for (size_t channels = 1; channels <= 20; channels += 3) { + for (size_t step = 2; step <= 25; step++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(4) + .width(5) + .output_stride(23) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, input_offset) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .input_offset(112) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_ARM, zero) { + for (uint32_t mz = 0; mz < 25; mz++) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .input_offset(112) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, c_eq_8) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(8) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, c_div_8) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, c_div_8_with_qmin) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, c_div_8_with_qmax) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, c_lt_8) { + for (uint32_t channels = 1; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, c_gt_8) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, c_gt_8_with_qmin) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, c_gt_8_with_qmax) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, multipixel) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, multipixel_with_step) { + for (size_t channels = 1; channels <= 40; channels += 7) { + for (size_t step = 2; step <= 25; step++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(8) + .width(5) + .output_stride(43) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, input_offset) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .input_offset(176) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ARM, zero) { + for (uint32_t mz = 0; mz < 25; mz++) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .input_offset(176) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, c_eq_8) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(8) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, c_div_8) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, c_div_8_with_qmin) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, c_div_8_with_qmax) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, c_lt_8) { + for (uint32_t channels = 1; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, c_gt_8) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, c_gt_8_with_qmin) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, c_gt_8_with_qmax) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, multipixel) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, multipixel_with_step) { + for (size_t channels = 1; channels <= 40; channels += 7) { + for (size_t step = 2; step <= 25; step++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(8) + .width(5) + .output_stride(43) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, input_offset) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .input_offset(176) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_ARM, zero) { + for (uint32_t mz = 0; mz < 25; mz++) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .input_offset(176) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, c_eq_4) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(4) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, c_div_4) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, c_div_4_with_qmin) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, c_div_4_with_qmax) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, c_lt_4) { + for (uint32_t channels = 1; channels < 4; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, c_gt_4) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, c_gt_4_with_qmin) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, c_gt_4_with_qmax) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, multipixel) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, multipixel_with_step) { + for (size_t channels = 1; channels <= 20; channels += 3) { + for (size_t step = 2; step <= 25; step++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(4) + .width(5) + .output_stride(23) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, input_offset) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .input_offset(112) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_X86, zero) { + for (uint32_t mz = 0; mz < 25; mz++) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .input_offset(112) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, c_eq_4) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(4) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, c_div_4) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, c_div_4_with_qmin) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, c_div_4_with_qmax) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, c_lt_4) { + for (uint32_t channels = 1; channels < 4; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, c_gt_4) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, c_gt_4_with_qmin) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, c_gt_4_with_qmax) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, multipixel) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, multipixel_with_step) { + for (size_t channels = 1; channels <= 20; channels += 3) { + for (size_t step = 2; step <= 25; step++) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(4) + .width(5) + .output_stride(23) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, input_offset) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .input_offset(112) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X25__WASMSIMD_ACC2_X86, zero) { + for (uint32_t mz = 0; mz < 25; mz++) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(25) + .channels(channels) + .input_offset(112) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, c_eq_8) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(8) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, c_div_8) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, c_div_8_with_qmin) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, c_div_8_with_qmax) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, c_lt_8) { + for (uint32_t channels = 1; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, c_gt_8) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, c_gt_8_with_qmin) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, c_gt_8_with_qmax) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, multipixel) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, multipixel_with_step) { + for (size_t channels = 1; channels <= 40; channels += 7) { + for (size_t step = 2; step <= 25; step++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(8) + .width(5) + .output_stride(43) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, input_offset) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .input_offset(176) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_X86, zero) { + for (uint32_t mz = 0; mz < 25; mz++) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .input_offset(176) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, c_eq_8) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(8) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, c_div_8) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, c_div_8_with_qmin) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, c_div_8_with_qmax) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, c_lt_8) { + for (uint32_t channels = 1; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, c_gt_8) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, c_gt_8_with_qmin) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, c_gt_8_with_qmax) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, multipixel) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, multipixel_with_step) { + for (size_t channels = 1; channels <= 40; channels += 7) { + for (size_t step = 2; step <= 25; step++) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(8) + .width(5) + .output_stride(43) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, input_offset) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .input_offset(176) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X25__WASMSIMD_ACC2_X86, zero) { + for (uint32_t mz = 0; mz < 25; mz++) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(25) + .channels(channels) + .input_offset(176) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, c_eq_4) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(4) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, c_div_4) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, c_div_4_with_qmin) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, c_div_4_with_qmax) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, c_lt_4) { + for (uint32_t channels = 1; channels < 4; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, c_gt_4) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, c_gt_4_with_qmin) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, c_gt_4_with_qmax) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, multipixel) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, multipixel_with_step) { + for (size_t channels = 1; channels <= 20; channels += 3) { + for (size_t step = 2; step <= 9; step++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(4) + .width(5) + .output_stride(23) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, input_offset) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .input_offset(112) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ARM, zero) { + for (uint32_t mz = 0; mz < 9; mz++) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .input_offset(112) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, c_eq_4) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(4) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, c_div_4) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, c_div_4_with_qmin) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, c_div_4_with_qmax) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, c_lt_4) { + for (uint32_t channels = 1; channels < 4; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, c_gt_4) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, c_gt_4_with_qmin) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, c_gt_4_with_qmax) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, multipixel) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, multipixel_with_step) { + for (size_t channels = 1; channels <= 20; channels += 3) { + for (size_t step = 2; step <= 9; step++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(4) + .width(5) + .output_stride(23) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, input_offset) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .input_offset(112) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_ARM, zero) { + for (uint32_t mz = 0; mz < 9; mz++) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .input_offset(112) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, c_eq_8) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(8) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, c_div_8) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, c_div_8_with_qmin) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, c_div_8_with_qmax) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, c_lt_8) { + for (uint32_t channels = 1; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, c_gt_8) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, c_gt_8_with_qmin) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, c_gt_8_with_qmax) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, multipixel) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, multipixel_with_step) { + for (size_t channels = 1; channels <= 40; channels += 7) { + for (size_t step = 2; step <= 9; step++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(8) + .width(5) + .output_stride(43) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, input_offset) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .input_offset(176) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ARM, zero) { + for (uint32_t mz = 0; mz < 9; mz++) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .input_offset(176) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, c_eq_8) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(8) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, c_div_8) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, c_div_8_with_qmin) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, c_div_8_with_qmax) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, c_lt_8) { + for (uint32_t channels = 1; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, c_gt_8) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, c_gt_8_with_qmin) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, c_gt_8_with_qmax) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, multipixel) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, multipixel_with_step) { + for (size_t channels = 1; channels <= 40; channels += 7) { + for (size_t step = 2; step <= 9; step++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(8) + .width(5) + .output_stride(43) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, input_offset) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .input_offset(176) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_ARM, zero) { + for (uint32_t mz = 0; mz < 9; mz++) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .input_offset(176) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, c_eq_4) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(4) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, c_div_4) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, c_div_4_with_qmin) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, c_div_4_with_qmax) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, c_lt_4) { + for (uint32_t channels = 1; channels < 4; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, c_gt_4) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, c_gt_4_with_qmin) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, c_gt_4_with_qmax) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, multipixel) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, multipixel_with_step) { + for (size_t channels = 1; channels <= 20; channels += 3) { + for (size_t step = 2; step <= 9; step++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(4) + .width(5) + .output_stride(23) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, input_offset) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .input_offset(112) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_X86, zero) { + for (uint32_t mz = 0; mz < 9; mz++) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .input_offset(112) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, c_eq_4) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(4) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, c_div_4) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, c_div_4_with_qmin) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, c_div_4_with_qmax) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, c_lt_4) { + for (uint32_t channels = 1; channels < 4; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, c_gt_4) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, c_gt_4_with_qmin) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, c_gt_4_with_qmax) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, multipixel) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, multipixel_with_step) { + for (size_t channels = 1; channels <= 20; channels += 3) { + for (size_t step = 2; step <= 9; step++) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(4) + .width(5) + .output_stride(23) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, input_offset) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .input_offset(112) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X9__WASMSIMD_ACC2_X86, zero) { + for (uint32_t mz = 0; mz < 9; mz++) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(9) + .channels(channels) + .input_offset(112) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, c_eq_8) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(8) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, c_div_8) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, c_div_8_with_qmin) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, c_div_8_with_qmax) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, c_lt_8) { + for (uint32_t channels = 1; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, c_gt_8) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, c_gt_8_with_qmin) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, c_gt_8_with_qmax) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, multipixel) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, multipixel_with_step) { + for (size_t channels = 1; channels <= 40; channels += 7) { + for (size_t step = 2; step <= 9; step++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(8) + .width(5) + .output_stride(43) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, input_offset) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .input_offset(176) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_X86, zero) { + for (uint32_t mz = 0; mz < 9; mz++) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .input_offset(176) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, c_eq_8) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(8) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, c_div_8) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, c_div_8_with_qmin) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, c_div_8_with_qmax) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, c_lt_8) { + for (uint32_t channels = 1; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, c_gt_8) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, c_gt_8_with_qmin) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, c_gt_8_with_qmax) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, multipixel) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, multipixel_with_step) { + for (size_t channels = 1; channels <= 40; channels += 7) { + for (size_t step = 2; step <= 9; step++) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(8) + .width(5) + .output_stride(43) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, input_offset) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .input_offset(176) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X9__WASMSIMD_ACC2_X86, zero) { + for (uint32_t mz = 0; mz < 9; mz++) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(9) + .channels(channels) + .input_offset(176) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, c_eq_4) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(4) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, c_div_4) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, c_div_4_with_qmin) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, c_div_4_with_qmax) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, c_lt_4) { + for (uint32_t channels = 1; channels < 4; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, c_gt_4) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, c_gt_4_with_qmin) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, c_gt_4_with_qmax) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, multipixel) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, multipixel_with_step) { + for (size_t channels = 1; channels <= 20; channels += 3) { + for (size_t step = 2; step <= 4; step++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(4) + .width(5) + .output_stride(23) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, input_offset) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .input_offset(112) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ARM, zero) { + for (uint32_t mz = 0; mz < 4; mz++) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .input_offset(112) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, c_eq_4) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(4) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, c_div_4) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, c_div_4_with_qmin) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, c_div_4_with_qmax) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, c_lt_4) { + for (uint32_t channels = 1; channels < 4; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, c_gt_4) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, c_gt_4_with_qmin) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, c_gt_4_with_qmax) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, multipixel) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, multipixel_with_step) { + for (size_t channels = 1; channels <= 20; channels += 3) { + for (size_t step = 2; step <= 4; step++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(4) + .width(5) + .output_stride(23) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, input_offset) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .input_offset(112) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_ARM, zero) { + for (uint32_t mz = 0; mz < 4; mz++) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .input_offset(112) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, c_eq_8) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(8) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, c_div_8) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, c_div_8_with_qmin) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, c_div_8_with_qmax) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, c_lt_8) { + for (uint32_t channels = 1; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, c_gt_8) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, c_gt_8_with_qmin) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, c_gt_8_with_qmax) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, multipixel) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, multipixel_with_step) { + for (size_t channels = 1; channels <= 40; channels += 7) { + for (size_t step = 2; step <= 4; step++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(8) + .width(5) + .output_stride(43) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, input_offset) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .input_offset(176) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ARM, zero) { + for (uint32_t mz = 0; mz < 4; mz++) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .input_offset(176) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, c_eq_8) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(8) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, c_div_8) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, c_div_8_with_qmin) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, c_div_8_with_qmax) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, c_lt_8) { + for (uint32_t channels = 1; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, c_gt_8) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, c_gt_8_with_qmin) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, c_gt_8_with_qmax) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, multipixel) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, multipixel_with_step) { + for (size_t channels = 1; channels <= 40; channels += 7) { + for (size_t step = 2; step <= 4; step++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(8) + .width(5) + .output_stride(43) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, input_offset) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .input_offset(176) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_ARM, zero) { + for (uint32_t mz = 0; mz < 4; mz++) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .input_offset(176) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, c_eq_4) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(4) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, c_div_4) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, c_div_4_with_qmin) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, c_div_4_with_qmax) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, c_lt_4) { + for (uint32_t channels = 1; channels < 4; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, c_gt_4) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, c_gt_4_with_qmin) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, c_gt_4_with_qmax) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, multipixel) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, multipixel_with_step) { + for (size_t channels = 1; channels <= 20; channels += 3) { + for (size_t step = 2; step <= 4; step++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(4) + .width(5) + .output_stride(23) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, input_offset) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .input_offset(112) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_X86, zero) { + for (uint32_t mz = 0; mz < 4; mz++) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .input_offset(112) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, c_eq_4) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(4) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, c_div_4) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, c_div_4_with_qmin) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, c_div_4_with_qmax) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, c_lt_4) { + for (uint32_t channels = 1; channels < 4; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, c_gt_4) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, c_gt_4_with_qmin) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, c_gt_4_with_qmax) { + for (uint32_t channels = 5; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, multipixel) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, multipixel_with_step) { + for (size_t channels = 1; channels <= 20; channels += 3) { + for (size_t step = 2; step <= 4; step++) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(4) + .width(5) + .output_stride(23) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 20; channels += 3) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, input_offset) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .input_offset(112) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP4X4__WASMSIMD_ACC2_X86, zero) { + for (uint32_t mz = 0; mz < 4; mz++) { + for (uint32_t channels = 8; channels < 64; channels += 12) { + DWConvMicrokernelTester() + .cr(4) + .kr(4) + .channels(channels) + .input_offset(112) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, c_eq_8) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(8) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, c_div_8) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, c_div_8_with_qmin) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, c_div_8_with_qmax) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, c_lt_8) { + for (uint32_t channels = 1; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, c_gt_8) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, c_gt_8_with_qmin) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, c_gt_8_with_qmax) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, multipixel) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, multipixel_with_step) { + for (size_t channels = 1; channels <= 40; channels += 7) { + for (size_t step = 2; step <= 4; step++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(8) + .width(5) + .output_stride(43) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, input_offset) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .input_offset(176) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_X86, zero) { + for (uint32_t mz = 0; mz < 4; mz++) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .input_offset(176) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + +#if XNN_ARCH_WASMSIMD + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, c_eq_8) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(8) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, c_div_8) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, c_div_8_with_qmin) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, c_div_8_with_qmax) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, c_lt_8) { + for (uint32_t channels = 1; channels < 8; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, c_gt_8) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, c_gt_8_with_qmin) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, c_gt_8_with_qmax) { + for (uint32_t channels = 9; channels < 16; channels++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, multipixel) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, multipixel_with_step) { + for (size_t channels = 1; channels <= 40; channels += 7) { + for (size_t step = 2; step <= 4; step++) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .step(step) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, multipixel_with_output_stride) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(8) + .width(5) + .output_stride(43) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, multipixel_with_qmin) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .qmin(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, multipixel_with_qmax) { + for (size_t channels = 1; channels <= 40; channels += 7) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .width(3) + .qmax(128) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, input_offset) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .input_offset(176) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + + TEST(F32_DWCONV_MINMAX_UP8X4__WASMSIMD_ACC2_X86, zero) { + for (uint32_t mz = 0; mz < 4; mz++) { + for (uint32_t channels = 16; channels < 128; channels += 24) { + DWConvMicrokernelTester() + .cr(8) + .kr(4) + .channels(channels) + .input_offset(176) + .zero_index(mz) + .Test(xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86, DWConvMicrokernelTester::Variant::Scalar); + } + } + } +#endif // XNN_ARCH_WASMSIMD + + #if XNN_ARCH_WASM TEST(F32_DWCONV_MINMAX_UP1X4__WASM, c_eq_1) { DWConvMicrokernelTester() diff --git a/test/f32-dwconv-minmax.yaml b/test/f32-dwconv-minmax.yaml index eff738c20..2477eccb2 100644 --- a/test/f32-dwconv-minmax.yaml +++ b/test/f32-dwconv-minmax.yaml @@ -91,6 +91,30 @@ - name: xnn_f32_dwconv_minmax_ukernel_up4x4__psimd_acc2 - name: xnn_f32_dwconv_minmax_ukernel_up8x4__psimd - name: xnn_f32_dwconv_minmax_ukernel_up8x4__psimd_acc2 +- name: xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_arm +- name: xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_arm +- name: xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_arm +- name: xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_arm +- name: xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_x86 +- name: xnn_f32_dwconv_minmax_ukernel_up4x25__wasmsimd_acc2_x86 +- name: xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_x86 +- name: xnn_f32_dwconv_minmax_ukernel_up8x25__wasmsimd_acc2_x86 +- name: xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_arm +- name: xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_arm +- name: xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_arm +- name: xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_arm +- name: xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_x86 +- name: xnn_f32_dwconv_minmax_ukernel_up4x9__wasmsimd_acc2_x86 +- name: xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_x86 +- name: xnn_f32_dwconv_minmax_ukernel_up8x9__wasmsimd_acc2_x86 +- name: xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_arm +- name: xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_arm +- name: xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_arm +- name: xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_arm +- name: xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_x86 +- name: xnn_f32_dwconv_minmax_ukernel_up4x4__wasmsimd_acc2_x86 +- name: xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_x86 +- name: xnn_f32_dwconv_minmax_ukernel_up8x4__wasmsimd_acc2_x86 - name: xnn_f32_dwconv_minmax_ukernel_up1x4__wasm - name: xnn_f32_dwconv_minmax_ukernel_up1x4__wasm_acc2 - name: xnn_f32_dwconv_minmax_ukernel_up2x4__wasm |