diff options
author | Marat Dukhan <maratek@google.com> | 2020-07-08 22:20:40 -0700 |
---|---|---|
committer | XNNPACK Team <xnnpack-github-robot@google.com> | 2020-07-08 22:21:26 -0700 |
commit | f6e2480bc00d3c0cda8d0c33875bcfdbfcc06d02 (patch) | |
tree | d6b3b71d051c1b8b327d22d8a77557a6d0eab278 /src/f32-maxpool/9p8x-minmax-wasmsimd-arm-c4.c | |
parent | 3fa52c8f975f181a938bcabbd05320b44bc993e0 (diff) | |
download | XNNPACK-f6e2480bc00d3c0cda8d0c33875bcfdbfcc06d02.tar.gz |
WAsm SIMD variants of F32 MAXPOOL microkernels
PiperOrigin-RevId: 320332301
Diffstat (limited to 'src/f32-maxpool/9p8x-minmax-wasmsimd-arm-c4.c')
-rw-r--r-- | src/f32-maxpool/9p8x-minmax-wasmsimd-arm-c4.c | 258 |
1 files changed, 258 insertions, 0 deletions
diff --git a/src/f32-maxpool/9p8x-minmax-wasmsimd-arm-c4.c b/src/f32-maxpool/9p8x-minmax-wasmsimd-arm-c4.c new file mode 100644 index 000000000..fef55b4a7 --- /dev/null +++ b/src/f32-maxpool/9p8x-minmax-wasmsimd-arm-c4.c @@ -0,0 +1,258 @@ +// Copyright 2020 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#include <assert.h> + +#include <wasm_simd128.h> + +#include <xnnpack/maxpool.h> + + +void xnn_f32_maxpool_minmax_ukernel_9p8x__wasmsimd_arm_c4( + size_t output_pixels, + size_t kernel_elements, + size_t channels, + const float** input, + size_t input_offset, + float* output, + size_t input_increment, + size_t output_increment, + const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN +{ + assert(output_pixels != 0); + assert(kernel_elements != 0); + assert(channels != 0); + + const v128_t voutput_max = wasm_v32x4_load_splat(¶ms->scalar.max); + const v128_t voutput_min = wasm_v32x4_load_splat(¶ms->scalar.min); + do { + float* o = output; + { + const float* i0 = *input++; + const float* i1 = *input++; + const float* i2 = *input++; + const float* i3 = *input++; + const float* i4 = *input++; + const float* i5 = *input++; + const float* i6 = *input++; + const float* i7 = *input++; + const float* i8 = *input++; + i0 = (const float*) ((uintptr_t) i0 + input_offset); + i1 = (const float*) ((uintptr_t) i1 + input_offset); + i2 = (const float*) ((uintptr_t) i2 + input_offset); + i3 = (const float*) ((uintptr_t) i3 + input_offset); + i4 = (const float*) ((uintptr_t) i4 + input_offset); + i5 = (const float*) ((uintptr_t) i5 + input_offset); + i6 = (const float*) ((uintptr_t) i6 + input_offset); + i7 = (const float*) ((uintptr_t) i7 + input_offset); + i8 = (const float*) ((uintptr_t) i8 + input_offset); + if (kernel_elements < 2) { + i1 = i0; + } + if (kernel_elements <= 2) { + i2 = i0; + } + if (kernel_elements < 4) { + i3 = i0; + } + if (kernel_elements <= 4) { + i4 = i0; + } + if (kernel_elements < 6) { + i5 = i0; + } + if (kernel_elements <= 6) { + i6 = i0; + } + if (kernel_elements < 8) { + i7 = i0; + } + if (kernel_elements <= 8) { + i8 = i0; + } + + size_t c = channels; + for (; c >= 4; c -= 4) { + const v128_t vi0 = wasm_v128_load(i0); + i0 += 4; + const v128_t vi1 = wasm_v128_load(i1); + i1 += 4; + const v128_t vi2 = wasm_v128_load(i2); + i2 += 4; + const v128_t vi3 = wasm_v128_load(i3); + i3 += 4; + const v128_t vi4 = wasm_v128_load(i4); + i4 += 4; + const v128_t vi5 = wasm_v128_load(i5); + i5 += 4; + const v128_t vi6 = wasm_v128_load(i6); + i6 += 4; + const v128_t vi7 = wasm_v128_load(i7); + i7 += 4; + const v128_t vi8 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vmax018 = wasm_f32x4_max(wasm_f32x4_max(vi0, vi1), vi8); + const v128_t vmax23 = wasm_f32x4_max(vi2, vi3); + const v128_t vmax45 = wasm_f32x4_max(vi4, vi5); + const v128_t vmax67 = wasm_f32x4_max(vi6, vi7); + + const v128_t vmax2345 = wasm_f32x4_max(vmax23, vmax45); + const v128_t vmax01678 = wasm_f32x4_max(vmax018, vmax67); + const v128_t vmax = wasm_f32x4_max(vmax2345, vmax01678); + const v128_t vout = wasm_f32x4_max(wasm_f32x4_min(vmax, voutput_max), voutput_min); + + wasm_v128_store(o, vout); + o += 4; + } + if (c != 0) { + const v128_t vi0 = wasm_v128_load(i0); + i0 += 4; + const v128_t vi1 = wasm_v128_load(i1); + i1 += 4; + const v128_t vi2 = wasm_v128_load(i2); + i2 += 4; + const v128_t vi3 = wasm_v128_load(i3); + i3 += 4; + const v128_t vi4 = wasm_v128_load(i4); + i4 += 4; + const v128_t vi5 = wasm_v128_load(i5); + i5 += 4; + const v128_t vi6 = wasm_v128_load(i6); + i6 += 4; + const v128_t vi7 = wasm_v128_load(i7); + i7 += 4; + const v128_t vi8 = wasm_v128_load(i8); + i8 += 4; + + const v128_t vmax018 = wasm_f32x4_max(wasm_f32x4_max(vi0, vi1), vi8); + const v128_t vmax23 = wasm_f32x4_max(vi2, vi3); + const v128_t vmax45 = wasm_f32x4_max(vi4, vi5); + const v128_t vmax67 = wasm_f32x4_max(vi6, vi7); + + const v128_t vmax2345 = wasm_f32x4_max(vmax23, vmax45); + const v128_t vmax01678 = wasm_f32x4_max(vmax018, vmax67); + const v128_t vmax = wasm_f32x4_max(vmax2345, vmax01678); + v128_t vout = wasm_f32x4_max(wasm_f32x4_min(vmax, voutput_max), voutput_min); + + if (c & 2) { + *((double*) o) = wasm_f64x2_extract_lane(vout, 0); + vout = wasm_v32x4_shuffle(vout, vout, 2, 3, 2, 3); + o += 2; + } + if (c & 1) { + *o++ = wasm_f32x4_extract_lane(vout, 0); + } + } + } + + for (ptrdiff_t k = (ptrdiff_t) kernel_elements - 9; k > 0; k -= 8) { + const float* i0 = *input++; + const float* i1 = *input++; + const float* i2 = *input++; + const float* i3 = *input++; + const float* i4 = *input++; + const float* i5 = *input++; + const float* i6 = *input++; + const float* i7 = *input++; + i0 = (const float*) ((uintptr_t) i0 + input_offset); + i1 = (const float*) ((uintptr_t) i1 + input_offset); + i2 = (const float*) ((uintptr_t) i2 + input_offset); + i3 = (const float*) ((uintptr_t) i3 + input_offset); + i4 = (const float*) ((uintptr_t) i4 + input_offset); + i5 = (const float*) ((uintptr_t) i5 + input_offset); + i6 = (const float*) ((uintptr_t) i6 + input_offset); + i7 = (const float*) ((uintptr_t) i7 + input_offset); + if (k < 2) { + i1 = i0; + } + if (k <= 2) { + i2 = i0; + } + if (k < 4) { + i3 = i0; + } + if (k <= 4) { + i4 = i0; + } + if (k < 6) { + i5 = i0; + } + if (k <= 6) { + i6 = i0; + } + if (k < 8) { + i7 = i0; + } + + o = output; + size_t c = channels; + for (; c >= 4; c -= 4) { + const v128_t vi0 = wasm_v128_load(i0); + i0 += 4; + const v128_t vi1 = wasm_v128_load(i1); + i1 += 4; + const v128_t vi2 = wasm_v128_load(i2); + i2 += 4; + const v128_t vi3 = wasm_v128_load(i3); + i3 += 4; + const v128_t vi4 = wasm_v128_load(i4); + i4 += 4; + const v128_t vi5 = wasm_v128_load(i5); + i5 += 4; + const v128_t vi6 = wasm_v128_load(i6); + i6 += 4; + const v128_t vi7 = wasm_v128_load(i7); + i7 += 4; + const v128_t vo = wasm_v128_load(o); + + const v128_t vmax01 = wasm_f32x4_max(wasm_f32x4_max(vi0, vi1), vo); + const v128_t vmax23 = wasm_f32x4_max(vi2, vi3); + const v128_t vmax45 = wasm_f32x4_max(vi4, vi5); + const v128_t vmax67 = wasm_f32x4_max(vi6, vi7); + + const v128_t vmax2345 = wasm_f32x4_max(vmax23, vmax45); + const v128_t vmax0167 = wasm_f32x4_max(vmax01, vmax67); + const v128_t vmax = wasm_f32x4_max(vmax2345, vmax0167); + const v128_t vout = wasm_f32x4_max(wasm_f32x4_min(vmax, voutput_max), voutput_min); + + wasm_v128_store(o, vout); + o += 4; + } + if (c != 0) { + const v128_t vi0 = wasm_v128_load(i0); + const v128_t vi1 = wasm_v128_load(i1); + const v128_t vi2 = wasm_v128_load(i2); + const v128_t vi3 = wasm_v128_load(i3); + const v128_t vi4 = wasm_v128_load(i4); + const v128_t vi5 = wasm_v128_load(i5); + const v128_t vi6 = wasm_v128_load(i6); + const v128_t vi7 = wasm_v128_load(i7); + const v128_t vo = wasm_v128_load(o); + + const v128_t vmax01 = wasm_f32x4_max(wasm_f32x4_max(vi0, vi1), vo); + const v128_t vmax23 = wasm_f32x4_max(vi2, vi3); + const v128_t vmax45 = wasm_f32x4_max(vi4, vi5); + const v128_t vmax67 = wasm_f32x4_max(vi6, vi7); + + const v128_t vmax2345 = wasm_f32x4_max(vmax23, vmax45); + const v128_t vmax0167 = wasm_f32x4_max(vmax01, vmax67); + const v128_t vmax = wasm_f32x4_max(vmax2345, vmax0167); + v128_t vout = wasm_f32x4_max(wasm_f32x4_min(vmax, voutput_max), voutput_min); + + if (c & 2) { + *((double*) o) = wasm_f64x2_extract_lane(vout, 0); + vout = wasm_v32x4_shuffle(vout, vout, 2, 3, 2, 3); + o += 2; + } + if (c & 1) { + *o++ = wasm_f32x4_extract_lane(vout, 0); + } + } + } + input = (const float**) ((uintptr_t) input + input_increment); + output = (float*) ((uintptr_t) o + output_increment); + } while (--output_pixels != 0); +} |