diff options
-rw-r--r-- | BUILD.bazel | 2 | ||||
-rwxr-xr-x | CMakeLists.txt | 4 | ||||
-rw-r--r-- | bench/qs8-gemm-e2e.cc | 11 | ||||
-rw-r--r-- | bench/qs8-gemm.cc | 4 | ||||
-rwxr-xr-x | scripts/generate-qs8-gemm.sh | 1 | ||||
-rwxr-xr-x | scripts/generate-qs8-igemm.sh | 1 | ||||
-rw-r--r-- | src/qs8-gemm/gen/12x8c4-minmax-neondot.c | 501 | ||||
-rw-r--r-- | src/qs8-igemm/gen/12x8c4-minmax-neondot.c | 519 | ||||
-rw-r--r-- | src/xnnpack/gemm.h | 1 | ||||
-rw-r--r-- | src/xnnpack/igemm.h | 1 | ||||
-rw-r--r-- | test/qs8-gemm-minmax.cc | 456 | ||||
-rw-r--r-- | test/qs8-gemm-minmax.yaml | 2 | ||||
-rw-r--r-- | test/qs8-igemm-minmax.cc | 468 | ||||
-rw-r--r-- | test/qs8-igemm-minmax.yaml | 2 |
14 files changed, 1 insertions, 1972 deletions
diff --git a/BUILD.bazel b/BUILD.bazel index fd27340ca..dc2287007 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -2276,7 +2276,6 @@ NEONDOT_UKERNELS = [ "src/qs8-gemm/gen/6x16c4-minmax-neondot.c", "src/qs8-gemm/gen/8x8c4-minmax-neondot.c", "src/qs8-gemm/gen/8x16c4-minmax-neondot.c", - "src/qs8-gemm/gen/12x8c4-minmax-neondot.c", "src/qs8-igemm/gen/1x8c4-minmax-neondot.c", "src/qs8-igemm/gen/1x16c4-minmax-neondot.c", "src/qs8-igemm/gen/4x8c4-minmax-neondot.c", @@ -2285,7 +2284,6 @@ NEONDOT_UKERNELS = [ "src/qs8-igemm/gen/6x16c4-minmax-neondot.c", "src/qs8-igemm/gen/8x8c4-minmax-neondot.c", "src/qs8-igemm/gen/8x16c4-minmax-neondot.c", - "src/qs8-igemm/gen/12x8c4-minmax-neondot.c", ] SSE_UKERNELS = [ diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e8f7742f..8d26e08c3 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1520,7 +1520,6 @@ SET(XNNPACK_NEONDOT_MICROKERNEL_SRCS src/qs8-gemm/gen/6x16c4-minmax-neondot.c src/qs8-gemm/gen/8x8c4-minmax-neondot.c src/qs8-gemm/gen/8x16c4-minmax-neondot.c - src/qs8-gemm/gen/12x8c4-minmax-neondot.c src/qs8-igemm/gen/1x8c4-minmax-neondot.c src/qs8-igemm/gen/1x16c4-minmax-neondot.c src/qs8-igemm/gen/4x8c4-minmax-neondot.c @@ -1528,8 +1527,7 @@ SET(XNNPACK_NEONDOT_MICROKERNEL_SRCS src/qs8-igemm/gen/6x8c4-minmax-neondot.c src/qs8-igemm/gen/6x16c4-minmax-neondot.c src/qs8-igemm/gen/8x8c4-minmax-neondot.c - src/qs8-igemm/gen/8x16c4-minmax-neondot.c - src/qs8-igemm/gen/12x8c4-minmax-neondot.c) + src/qs8-igemm/gen/8x16c4-minmax-neondot.c) SET(XNNPACK_SSE_MICROKERNEL_SRCS src/f32-avgpool/9p8x-minmax-sse-c4.c diff --git a/bench/qs8-gemm-e2e.cc b/bench/qs8-gemm-e2e.cc index 80d60bb4a..41f5c3da2 100644 --- a/bench/qs8-gemm-e2e.cc +++ b/bench/qs8-gemm-e2e.cc @@ -544,16 +544,6 @@ static void GEMMEnd2EndBenchmark( benchmark::utils::CheckNEONDOT); } - static void qs8_gemm_minmax_ukernel_12x8c4__neondot(benchmark::State& state, models::ExecutionPlanFactory model) { - GEMMEnd2EndBenchmark(state, model, - xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot, - xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot, - xnn_qs8_gemm_minmax_ukernel_1x8c4__neondot, - xnn_qs8_igemm_minmax_ukernel_1x8c4__neondot, - 12 /* mr */, 8 /* nr */, 2 /* log2_kr */, 0 /* log2_sr */, - benchmark::utils::CheckNEONDOT); - } - #if XNN_ENABLE_FULL_BENCHMARKS static void qs8_gemm_minmax_ukernel_1x8c8__neon_mull_padal(benchmark::State& state, models::ExecutionPlanFactory model) { GEMMEnd2EndBenchmark(state, model, @@ -810,7 +800,6 @@ static void GEMMEnd2EndBenchmark( BENCHMARK_QS8_END2END(qs8_gemm_minmax_ukernel_6x16c4__neondot); BENCHMARK_QS8_END2END(qs8_gemm_minmax_ukernel_8x8c4__neondot); BENCHMARK_QS8_END2END(qs8_gemm_minmax_ukernel_8x16c4__neondot); - BENCHMARK_QS8_END2END(qs8_gemm_minmax_ukernel_12x8c4__neondot); #if XNN_ENABLE_FULL_BENCHMARKS BENCHMARK_QS8_END2END(qs8_gemm_minmax_ukernel_1x8c8__neon_mlal_padal); diff --git a/bench/qs8-gemm.cc b/bench/qs8-gemm.cc index 524b784bf..cb1067b45 100644 --- a/bench/qs8-gemm.cc +++ b/bench/qs8-gemm.cc @@ -464,9 +464,6 @@ static void ruy_st(benchmark::State& state, const char* net) static void qs8_gemm_8x8c4__neondot(benchmark::State& state, const char* net) { GEMMBenchmark(state, xnn_qs8_gemm_minmax_ukernel_8x8c4__neondot, 8, 8, 4, 1, benchmark::utils::CheckNEONDOT); } - static void qs8_gemm_12x8c4__neondot(benchmark::State& state, const char* net) { - GEMMBenchmark(state, xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot, 12, 8, 4, 1, benchmark::utils::CheckNEONDOT); - } static void qs8_gemm_1x16c4__neondot(benchmark::State& state, const char* net) { GEMMBenchmark(state, xnn_qs8_gemm_minmax_ukernel_1x16c4__neondot, 1, 16, 4, 1, benchmark::utils::CheckNEONDOT); } @@ -540,7 +537,6 @@ static void ruy_st(benchmark::State& state, const char* net) BENCHMARK_GEMM(qs8_gemm_4x8c4__neondot) BENCHMARK_GEMM(qs8_gemm_6x8c4__neondot) BENCHMARK_GEMM(qs8_gemm_8x8c4__neondot) - BENCHMARK_GEMM(qs8_gemm_12x8c4__neondot) BENCHMARK_GEMM(qs8_gemm_1x16c4__neondot) BENCHMARK_GEMM(qs8_gemm_4x16c4__neondot) BENCHMARK_GEMM(qs8_gemm_6x16c4__neondot) diff --git a/scripts/generate-qs8-gemm.sh b/scripts/generate-qs8-gemm.sh index 25e5cf3bd..5f071acc1 100755 --- a/scripts/generate-qs8-gemm.sh +++ b/scripts/generate-qs8-gemm.sh @@ -92,7 +92,6 @@ tools/xngen src/qs8-gemm/MRxNRc4-neondot.c.in -D MR=1 -D NR=8 -o src/qs8-gemm/ tools/xngen src/qs8-gemm/MRxNRc4-neondot.c.in -D MR=4 -D NR=8 -o src/qs8-gemm/gen/4x8c4-minmax-neondot.c tools/xngen src/qs8-gemm/MRxNRc4-neondot.c.in -D MR=6 -D NR=8 -o src/qs8-gemm/gen/6x8c4-minmax-neondot.c tools/xngen src/qs8-gemm/MRxNRc4-neondot.c.in -D MR=8 -D NR=8 -o src/qs8-gemm/gen/8x8c4-minmax-neondot.c -tools/xngen src/qs8-gemm/MRxNRc4-neondot.c.in -D MR=12 -D NR=8 -o src/qs8-gemm/gen/12x8c4-minmax-neondot.c tools/xngen src/qs8-gemm/MRxNRc4-neondot.c.in -D MR=1 -D NR=16 -o src/qs8-gemm/gen/1x16c4-minmax-neondot.c tools/xngen src/qs8-gemm/MRxNRc4-neondot.c.in -D MR=4 -D NR=16 -o src/qs8-gemm/gen/4x16c4-minmax-neondot.c tools/xngen src/qs8-gemm/MRxNRc4-neondot.c.in -D MR=6 -D NR=16 -o src/qs8-gemm/gen/6x16c4-minmax-neondot.c diff --git a/scripts/generate-qs8-igemm.sh b/scripts/generate-qs8-igemm.sh index 8b93963a0..e2a1960bd 100755 --- a/scripts/generate-qs8-igemm.sh +++ b/scripts/generate-qs8-igemm.sh @@ -88,7 +88,6 @@ tools/xngen src/qs8-igemm/MRxNRc4-neondot.c.in -D MR=1 -D NR=8 -o src/qs8-igem tools/xngen src/qs8-igemm/MRxNRc4-neondot.c.in -D MR=4 -D NR=8 -o src/qs8-igemm/gen/4x8c4-minmax-neondot.c tools/xngen src/qs8-igemm/MRxNRc4-neondot.c.in -D MR=6 -D NR=8 -o src/qs8-igemm/gen/6x8c4-minmax-neondot.c tools/xngen src/qs8-igemm/MRxNRc4-neondot.c.in -D MR=8 -D NR=8 -o src/qs8-igemm/gen/8x8c4-minmax-neondot.c -tools/xngen src/qs8-igemm/MRxNRc4-neondot.c.in -D MR=12 -D NR=8 -o src/qs8-igemm/gen/12x8c4-minmax-neondot.c tools/xngen src/qs8-igemm/MRxNRc4-neondot.c.in -D MR=1 -D NR=16 -o src/qs8-igemm/gen/1x16c4-minmax-neondot.c tools/xngen src/qs8-igemm/MRxNRc4-neondot.c.in -D MR=4 -D NR=16 -o src/qs8-igemm/gen/4x16c4-minmax-neondot.c tools/xngen src/qs8-igemm/MRxNRc4-neondot.c.in -D MR=6 -D NR=16 -o src/qs8-igemm/gen/6x16c4-minmax-neondot.c diff --git a/src/qs8-gemm/gen/12x8c4-minmax-neondot.c b/src/qs8-gemm/gen/12x8c4-minmax-neondot.c deleted file mode 100644 index 71829b04b..000000000 --- a/src/qs8-gemm/gen/12x8c4-minmax-neondot.c +++ /dev/null @@ -1,501 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/qs8-gemm/MRxNRc4-neondot.c.in -// Generator: tools/xngen -// -// Copyright 2020 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <arm_neon.h> - -#include <xnnpack/gemm.h> -#include <xnnpack/math.h> - - -void xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot( - size_t mr, - size_t nc, - size_t kc, - const int8_t* restrict a, - size_t a_stride, - const void* restrict w, - int8_t* restrict c, - size_t cm_stride, - size_t cn_stride, - const union xnn_qs8_gemm_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN { - assert(mr != 0); - assert(mr <= 12); - assert(nc != 0); - assert(kc != 0); - assert(kc % sizeof(int8_t) == 0); - assert(a != NULL); - assert(w != NULL); - assert(c != NULL); - - kc = round_up_po2(kc, 4); - const int8_t* a0 = a; - int8_t* c0 = c; - const int8_t* a1 = (const int8_t*) ((uintptr_t) a0 + a_stride); - int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride); - if XNN_UNPREDICTABLE(mr < 2) { - a1 = a0; - c1 = c0; - } - const int8_t* a2 = (const int8_t*) ((uintptr_t) a1 + a_stride); - int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride); - if XNN_UNPREDICTABLE(mr <= 2) { - a2 = a1; - c2 = c1; - } - const int8_t* a3 = (const int8_t*) ((uintptr_t) a2 + a_stride); - int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride); - if XNN_UNPREDICTABLE(mr < 4) { - a3 = a2; - c3 = c2; - } - const int8_t* a4 = (const int8_t*) ((uintptr_t) a3 + a_stride); - int8_t* c4 = (int8_t*) ((uintptr_t) c3 + cm_stride); - if XNN_UNPREDICTABLE(mr <= 4) { - a4 = a3; - c4 = c3; - } - const int8_t* a5 = (const int8_t*) ((uintptr_t) a4 + a_stride); - int8_t* c5 = (int8_t*) ((uintptr_t) c4 + cm_stride); - if XNN_UNPREDICTABLE(mr < 6) { - a5 = a4; - c5 = c4; - } - const int8_t* a6 = (const int8_t*) ((uintptr_t) a5 + a_stride); - int8_t* c6 = (int8_t*) ((uintptr_t) c5 + cm_stride); - if XNN_UNPREDICTABLE(mr <= 6) { - a6 = a5; - c6 = c5; - } - const int8_t* a7 = (const int8_t*) ((uintptr_t) a6 + a_stride); - int8_t* c7 = (int8_t*) ((uintptr_t) c6 + cm_stride); - if XNN_UNPREDICTABLE(mr < 8) { - a7 = a6; - c7 = c6; - } - const int8_t* a8 = (const int8_t*) ((uintptr_t) a7 + a_stride); - int8_t* c8 = (int8_t*) ((uintptr_t) c7 + cm_stride); - if XNN_UNPREDICTABLE(mr <= 8) { - a8 = a7; - c8 = c7; - } - const int8_t* a9 = (const int8_t*) ((uintptr_t) a8 + a_stride); - int8_t* c9 = (int8_t*) ((uintptr_t) c8 + cm_stride); - if XNN_UNPREDICTABLE(mr < 10) { - a9 = a8; - c9 = c8; - } - const int8_t* a10 = (const int8_t*) ((uintptr_t) a9 + a_stride); - int8_t* c10 = (int8_t*) ((uintptr_t) c9 + cm_stride); - if XNN_UNPREDICTABLE(mr <= 10) { - a10 = a9; - c10 = c9; - } - const int8_t* a11 = (const int8_t*) ((uintptr_t) a10 + a_stride); - int8_t* c11 = (int8_t*) ((uintptr_t) c10 + cm_stride); - if XNN_UNPREDICTABLE(mr != 12) { - a11 = a10; - c11 = c10; - } - - // Loop over groups of 8 columns. - do { - // Initialize accumulators with bias. 8 bias values are loaded from the - // weight matrix, at the start of the group of 8 columns. - int32x4_t vacc0x0123 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)); - int32x4_t vacc0x4567 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)); - int32x4_t vacc1x0123 = vacc0x0123; - int32x4_t vacc1x4567 = vacc0x4567; - int32x4_t vacc2x0123 = vacc0x0123; - int32x4_t vacc2x4567 = vacc0x4567; - int32x4_t vacc3x0123 = vacc0x0123; - int32x4_t vacc3x4567 = vacc0x4567; - int32x4_t vacc4x0123 = vacc0x0123; - int32x4_t vacc4x4567 = vacc0x4567; - int32x4_t vacc5x0123 = vacc0x0123; - int32x4_t vacc5x4567 = vacc0x4567; - int32x4_t vacc6x0123 = vacc0x0123; - int32x4_t vacc6x4567 = vacc0x4567; - int32x4_t vacc7x0123 = vacc0x0123; - int32x4_t vacc7x4567 = vacc0x4567; - int32x4_t vacc8x0123 = vacc0x0123; - int32x4_t vacc8x4567 = vacc0x4567; - int32x4_t vacc9x0123 = vacc0x0123; - int32x4_t vacc9x4567 = vacc0x4567; - int32x4_t vacc10x0123 = vacc0x0123; - int32x4_t vacc10x4567 = vacc0x4567; - int32x4_t vacc11x0123 = vacc0x0123; - int32x4_t vacc11x4567 = vacc0x4567; - - // Inner accumulation loop along the 8 columns. - size_t k = kc; - // 2x partial unrolled loop to load 8 bytes at a time. - while (k >= 8 * sizeof(int8_t)) { - // Load a 12x8 block of activations. - const int8x8_t va0x01234567 = vld1_s8(a0); a0 += 8; - const int8x8_t va1x01234567 = vld1_s8(a1); a1 += 8; - const int8x8_t va2x01234567 = vld1_s8(a2); a2 += 8; - const int8x8_t va3x01234567 = vld1_s8(a3); a3 += 8; - const int8x8_t va4x01234567 = vld1_s8(a4); a4 += 8; - const int8x8_t va5x01234567 = vld1_s8(a5); a5 += 8; - const int8x8_t va6x01234567 = vld1_s8(a6); a6 += 8; - const int8x8_t va7x01234567 = vld1_s8(a7); a7 += 8; - const int8x8_t va8x01234567 = vld1_s8(a8); a8 += 8; - const int8x8_t va9x01234567 = vld1_s8(a9); a9 += 8; - const int8x8_t va10x01234567 = vld1_s8(a10); a10 += 8; - const int8x8_t va11x01234567 = vld1_s8(a11); a11 += 8; - - // Load a 8x8 block of weights. - const int8x16_t vb0123x0123 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); - const int8x16_t vb0123x4567 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); - const int8x16_t vb4567x0123 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); - const int8x16_t vb4567x4567 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); - - // Multiply-accumulate: 12x8 * 8x8 --> 12x8. - vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb0123x0123, va0x01234567, 0); - vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb0123x4567, va0x01234567, 0); - vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb0123x0123, va1x01234567, 0); - vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb0123x4567, va1x01234567, 0); - vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb0123x0123, va2x01234567, 0); - vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb0123x4567, va2x01234567, 0); - vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb0123x0123, va3x01234567, 0); - vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb0123x4567, va3x01234567, 0); - vacc4x0123 = vdotq_lane_s32(vacc4x0123, vb0123x0123, va4x01234567, 0); - vacc4x4567 = vdotq_lane_s32(vacc4x4567, vb0123x4567, va4x01234567, 0); - vacc5x0123 = vdotq_lane_s32(vacc5x0123, vb0123x0123, va5x01234567, 0); - vacc5x4567 = vdotq_lane_s32(vacc5x4567, vb0123x4567, va5x01234567, 0); - vacc6x0123 = vdotq_lane_s32(vacc6x0123, vb0123x0123, va6x01234567, 0); - vacc6x4567 = vdotq_lane_s32(vacc6x4567, vb0123x4567, va6x01234567, 0); - vacc7x0123 = vdotq_lane_s32(vacc7x0123, vb0123x0123, va7x01234567, 0); - vacc7x4567 = vdotq_lane_s32(vacc7x4567, vb0123x4567, va7x01234567, 0); - vacc8x0123 = vdotq_lane_s32(vacc8x0123, vb0123x0123, va8x01234567, 0); - vacc8x4567 = vdotq_lane_s32(vacc8x4567, vb0123x4567, va8x01234567, 0); - vacc9x0123 = vdotq_lane_s32(vacc9x0123, vb0123x0123, va9x01234567, 0); - vacc9x4567 = vdotq_lane_s32(vacc9x4567, vb0123x4567, va9x01234567, 0); - vacc10x0123 = vdotq_lane_s32(vacc10x0123, vb0123x0123, va10x01234567, 0); - vacc10x4567 = vdotq_lane_s32(vacc10x4567, vb0123x4567, va10x01234567, 0); - vacc11x0123 = vdotq_lane_s32(vacc11x0123, vb0123x0123, va11x01234567, 0); - vacc11x4567 = vdotq_lane_s32(vacc11x4567, vb0123x4567, va11x01234567, 0); - vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb4567x0123, va0x01234567, 1); - vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb4567x4567, va0x01234567, 1); - vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb4567x0123, va1x01234567, 1); - vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb4567x4567, va1x01234567, 1); - vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb4567x0123, va2x01234567, 1); - vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb4567x4567, va2x01234567, 1); - vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb4567x0123, va3x01234567, 1); - vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb4567x4567, va3x01234567, 1); - vacc4x0123 = vdotq_lane_s32(vacc4x0123, vb4567x0123, va4x01234567, 1); - vacc4x4567 = vdotq_lane_s32(vacc4x4567, vb4567x4567, va4x01234567, 1); - vacc5x0123 = vdotq_lane_s32(vacc5x0123, vb4567x0123, va5x01234567, 1); - vacc5x4567 = vdotq_lane_s32(vacc5x4567, vb4567x4567, va5x01234567, 1); - vacc6x0123 = vdotq_lane_s32(vacc6x0123, vb4567x0123, va6x01234567, 1); - vacc6x4567 = vdotq_lane_s32(vacc6x4567, vb4567x4567, va6x01234567, 1); - vacc7x0123 = vdotq_lane_s32(vacc7x0123, vb4567x0123, va7x01234567, 1); - vacc7x4567 = vdotq_lane_s32(vacc7x4567, vb4567x4567, va7x01234567, 1); - vacc8x0123 = vdotq_lane_s32(vacc8x0123, vb4567x0123, va8x01234567, 1); - vacc8x4567 = vdotq_lane_s32(vacc8x4567, vb4567x4567, va8x01234567, 1); - vacc9x0123 = vdotq_lane_s32(vacc9x0123, vb4567x0123, va9x01234567, 1); - vacc9x4567 = vdotq_lane_s32(vacc9x4567, vb4567x4567, va9x01234567, 1); - vacc10x0123 = vdotq_lane_s32(vacc10x0123, vb4567x0123, va10x01234567, 1); - vacc10x4567 = vdotq_lane_s32(vacc10x4567, vb4567x4567, va10x01234567, 1); - vacc11x0123 = vdotq_lane_s32(vacc11x0123, vb4567x0123, va11x01234567, 1); - vacc11x4567 = vdotq_lane_s32(vacc11x4567, vb4567x4567, va11x01234567, 1); - - k -= 8 * sizeof(int8_t); - } - // Handle up to 4 final positions of `k` - if XNN_UNLIKELY(k != 0) { - // Load a 12x4 block of activations. - const int8x8_t va0x01234567 = vld1_s8(a0); a0 += 4; - const int8x8_t va1x01234567 = vld1_s8(a1); a1 += 4; - const int8x8_t va2x01234567 = vld1_s8(a2); a2 += 4; - const int8x8_t va3x01234567 = vld1_s8(a3); a3 += 4; - const int8x8_t va4x01234567 = vld1_s8(a4); a4 += 4; - const int8x8_t va5x01234567 = vld1_s8(a5); a5 += 4; - const int8x8_t va6x01234567 = vld1_s8(a6); a6 += 4; - const int8x8_t va7x01234567 = vld1_s8(a7); a7 += 4; - const int8x8_t va8x01234567 = vld1_s8(a8); a8 += 4; - const int8x8_t va9x01234567 = vld1_s8(a9); a9 += 4; - const int8x8_t va10x01234567 = vld1_s8(a10); a10 += 4; - const int8x8_t va11x01234567 = vld1_s8(a11); a11 += 4; - - // Load a 4x8 block of weights. - const int8x16_t vb0123x0123 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); - const int8x16_t vb0123x4567 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); - - // Multiply-accumulate: 12x4 * 4x8 --> 12x8. - vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb0123x0123, va0x01234567, 0); - vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb0123x4567, va0x01234567, 0); - vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb0123x0123, va1x01234567, 0); - vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb0123x4567, va1x01234567, 0); - vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb0123x0123, va2x01234567, 0); - vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb0123x4567, va2x01234567, 0); - vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb0123x0123, va3x01234567, 0); - vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb0123x4567, va3x01234567, 0); - vacc4x0123 = vdotq_lane_s32(vacc4x0123, vb0123x0123, va4x01234567, 0); - vacc4x4567 = vdotq_lane_s32(vacc4x4567, vb0123x4567, va4x01234567, 0); - vacc5x0123 = vdotq_lane_s32(vacc5x0123, vb0123x0123, va5x01234567, 0); - vacc5x4567 = vdotq_lane_s32(vacc5x4567, vb0123x4567, va5x01234567, 0); - vacc6x0123 = vdotq_lane_s32(vacc6x0123, vb0123x0123, va6x01234567, 0); - vacc6x4567 = vdotq_lane_s32(vacc6x4567, vb0123x4567, va6x01234567, 0); - vacc7x0123 = vdotq_lane_s32(vacc7x0123, vb0123x0123, va7x01234567, 0); - vacc7x4567 = vdotq_lane_s32(vacc7x4567, vb0123x4567, va7x01234567, 0); - vacc8x0123 = vdotq_lane_s32(vacc8x0123, vb0123x0123, va8x01234567, 0); - vacc8x4567 = vdotq_lane_s32(vacc8x4567, vb0123x4567, va8x01234567, 0); - vacc9x0123 = vdotq_lane_s32(vacc9x0123, vb0123x0123, va9x01234567, 0); - vacc9x4567 = vdotq_lane_s32(vacc9x4567, vb0123x4567, va9x01234567, 0); - vacc10x0123 = vdotq_lane_s32(vacc10x0123, vb0123x0123, va10x01234567, 0); - vacc10x4567 = vdotq_lane_s32(vacc10x4567, vb0123x4567, va10x01234567, 0); - vacc11x0123 = vdotq_lane_s32(vacc11x0123, vb0123x0123, va11x01234567, 0); - vacc11x4567 = vdotq_lane_s32(vacc11x4567, vb0123x4567, va11x01234567, 0); - } - - // Post-accumulation work - const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift); - const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0))); - - const int32x4_t vproduct0x0123 = vqrdmulhq_n_s32(vacc0x0123, params->neon.multiplier); - const int32x4_t vproduct0x4567 = vqrdmulhq_n_s32(vacc0x4567, params->neon.multiplier); - const int32x4_t vproduct1x0123 = vqrdmulhq_n_s32(vacc1x0123, params->neon.multiplier); - const int32x4_t vproduct1x4567 = vqrdmulhq_n_s32(vacc1x4567, params->neon.multiplier); - const int32x4_t vproduct2x0123 = vqrdmulhq_n_s32(vacc2x0123, params->neon.multiplier); - const int32x4_t vproduct2x4567 = vqrdmulhq_n_s32(vacc2x4567, params->neon.multiplier); - const int32x4_t vproduct3x0123 = vqrdmulhq_n_s32(vacc3x0123, params->neon.multiplier); - const int32x4_t vproduct3x4567 = vqrdmulhq_n_s32(vacc3x4567, params->neon.multiplier); - const int32x4_t vproduct4x0123 = vqrdmulhq_n_s32(vacc4x0123, params->neon.multiplier); - const int32x4_t vproduct4x4567 = vqrdmulhq_n_s32(vacc4x4567, params->neon.multiplier); - const int32x4_t vproduct5x0123 = vqrdmulhq_n_s32(vacc5x0123, params->neon.multiplier); - const int32x4_t vproduct5x4567 = vqrdmulhq_n_s32(vacc5x4567, params->neon.multiplier); - const int32x4_t vproduct6x0123 = vqrdmulhq_n_s32(vacc6x0123, params->neon.multiplier); - const int32x4_t vproduct6x4567 = vqrdmulhq_n_s32(vacc6x4567, params->neon.multiplier); - const int32x4_t vproduct7x0123 = vqrdmulhq_n_s32(vacc7x0123, params->neon.multiplier); - const int32x4_t vproduct7x4567 = vqrdmulhq_n_s32(vacc7x4567, params->neon.multiplier); - const int32x4_t vproduct8x0123 = vqrdmulhq_n_s32(vacc8x0123, params->neon.multiplier); - const int32x4_t vproduct8x4567 = vqrdmulhq_n_s32(vacc8x4567, params->neon.multiplier); - const int32x4_t vproduct9x0123 = vqrdmulhq_n_s32(vacc9x0123, params->neon.multiplier); - const int32x4_t vproduct9x4567 = vqrdmulhq_n_s32(vacc9x4567, params->neon.multiplier); - const int32x4_t vproduct10x0123 = vqrdmulhq_n_s32(vacc10x0123, params->neon.multiplier); - const int32x4_t vproduct10x4567 = vqrdmulhq_n_s32(vacc10x4567, params->neon.multiplier); - const int32x4_t vproduct11x0123 = vqrdmulhq_n_s32(vacc11x0123, params->neon.multiplier); - const int32x4_t vproduct11x4567 = vqrdmulhq_n_s32(vacc11x4567, params->neon.multiplier); - - vacc0x0123 = vsraq_n_s32(vproduct0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31); - vacc0x4567 = vsraq_n_s32(vproduct0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31); - vacc1x0123 = vsraq_n_s32(vproduct1x0123, vbicq_s32(vacc1x0123, vzero_shift_mask), 31); - vacc1x4567 = vsraq_n_s32(vproduct1x4567, vbicq_s32(vacc1x4567, vzero_shift_mask), 31); - vacc2x0123 = vsraq_n_s32(vproduct2x0123, vbicq_s32(vacc2x0123, vzero_shift_mask), 31); - vacc2x4567 = vsraq_n_s32(vproduct2x4567, vbicq_s32(vacc2x4567, vzero_shift_mask), 31); - vacc3x0123 = vsraq_n_s32(vproduct3x0123, vbicq_s32(vacc3x0123, vzero_shift_mask), 31); - vacc3x4567 = vsraq_n_s32(vproduct3x4567, vbicq_s32(vacc3x4567, vzero_shift_mask), 31); - vacc4x0123 = vsraq_n_s32(vproduct4x0123, vbicq_s32(vacc4x0123, vzero_shift_mask), 31); - vacc4x4567 = vsraq_n_s32(vproduct4x4567, vbicq_s32(vacc4x4567, vzero_shift_mask), 31); - vacc5x0123 = vsraq_n_s32(vproduct5x0123, vbicq_s32(vacc5x0123, vzero_shift_mask), 31); - vacc5x4567 = vsraq_n_s32(vproduct5x4567, vbicq_s32(vacc5x4567, vzero_shift_mask), 31); - vacc6x0123 = vsraq_n_s32(vproduct6x0123, vbicq_s32(vacc6x0123, vzero_shift_mask), 31); - vacc6x4567 = vsraq_n_s32(vproduct6x4567, vbicq_s32(vacc6x4567, vzero_shift_mask), 31); - vacc7x0123 = vsraq_n_s32(vproduct7x0123, vbicq_s32(vacc7x0123, vzero_shift_mask), 31); - vacc7x4567 = vsraq_n_s32(vproduct7x4567, vbicq_s32(vacc7x4567, vzero_shift_mask), 31); - vacc8x0123 = vsraq_n_s32(vproduct8x0123, vbicq_s32(vacc8x0123, vzero_shift_mask), 31); - vacc8x4567 = vsraq_n_s32(vproduct8x4567, vbicq_s32(vacc8x4567, vzero_shift_mask), 31); - vacc9x0123 = vsraq_n_s32(vproduct9x0123, vbicq_s32(vacc9x0123, vzero_shift_mask), 31); - vacc9x4567 = vsraq_n_s32(vproduct9x4567, vbicq_s32(vacc9x4567, vzero_shift_mask), 31); - vacc10x0123 = vsraq_n_s32(vproduct10x0123, vbicq_s32(vacc10x0123, vzero_shift_mask), 31); - vacc10x4567 = vsraq_n_s32(vproduct10x4567, vbicq_s32(vacc10x4567, vzero_shift_mask), 31); - vacc11x0123 = vsraq_n_s32(vproduct11x0123, vbicq_s32(vacc11x0123, vzero_shift_mask), 31); - vacc11x4567 = vsraq_n_s32(vproduct11x4567, vbicq_s32(vacc11x4567, vzero_shift_mask), 31); - - vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift); - vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift); - vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift); - vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift); - vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift); - vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift); - vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift); - vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift); - vacc4x0123 = vrshlq_s32(vacc4x0123, vright_shift); - vacc4x4567 = vrshlq_s32(vacc4x4567, vright_shift); - vacc5x0123 = vrshlq_s32(vacc5x0123, vright_shift); - vacc5x4567 = vrshlq_s32(vacc5x4567, vright_shift); - vacc6x0123 = vrshlq_s32(vacc6x0123, vright_shift); - vacc6x4567 = vrshlq_s32(vacc6x4567, vright_shift); - vacc7x0123 = vrshlq_s32(vacc7x0123, vright_shift); - vacc7x4567 = vrshlq_s32(vacc7x4567, vright_shift); - vacc8x0123 = vrshlq_s32(vacc8x0123, vright_shift); - vacc8x4567 = vrshlq_s32(vacc8x4567, vright_shift); - vacc9x0123 = vrshlq_s32(vacc9x0123, vright_shift); - vacc9x4567 = vrshlq_s32(vacc9x4567, vright_shift); - vacc10x0123 = vrshlq_s32(vacc10x0123, vright_shift); - vacc10x4567 = vrshlq_s32(vacc10x4567, vright_shift); - vacc11x0123 = vrshlq_s32(vacc11x0123, vright_shift); - vacc11x4567 = vrshlq_s32(vacc11x4567, vright_shift); - - const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point); -#if XNN_ARCH_ARM64 - const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point); - const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point); - const int16x8_t vacc2x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc2x0123), vacc2x4567), voutput_zero_point); - const int16x8_t vacc3x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc3x0123), vacc3x4567), voutput_zero_point); - const int16x8_t vacc4x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc4x0123), vacc4x4567), voutput_zero_point); - const int16x8_t vacc5x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc5x0123), vacc5x4567), voutput_zero_point); - const int16x8_t vacc6x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc6x0123), vacc6x4567), voutput_zero_point); - const int16x8_t vacc7x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc7x0123), vacc7x4567), voutput_zero_point); - const int16x8_t vacc8x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc8x0123), vacc8x4567), voutput_zero_point); - const int16x8_t vacc9x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc9x0123), vacc9x4567), voutput_zero_point); - const int16x8_t vacc10x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc10x0123), vacc10x4567), voutput_zero_point); - const int16x8_t vacc11x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc11x0123), vacc11x4567), voutput_zero_point); - - int8x16_t vout0x01234567_1x01234567 = vqmovn_high_s16(vqmovn_s16(vacc0x01234567), vacc1x01234567); - int8x16_t vout2x01234567_3x01234567 = vqmovn_high_s16(vqmovn_s16(vacc2x01234567), vacc3x01234567); - int8x16_t vout4x01234567_5x01234567 = vqmovn_high_s16(vqmovn_s16(vacc4x01234567), vacc5x01234567); - int8x16_t vout6x01234567_7x01234567 = vqmovn_high_s16(vqmovn_s16(vacc6x01234567), vacc7x01234567); - int8x16_t vout8x01234567_9x01234567 = vqmovn_high_s16(vqmovn_s16(vacc8x01234567), vacc9x01234567); - int8x16_t vout10x01234567_11x01234567 = vqmovn_high_s16(vqmovn_s16(vacc10x01234567), vacc11x01234567); -#else - const int16x8_t vacc0x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0x0123), vqmovn_s32(vacc0x4567)), voutput_zero_point); - const int16x8_t vacc1x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc1x0123), vqmovn_s32(vacc1x4567)), voutput_zero_point); - const int16x8_t vacc2x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc2x0123), vqmovn_s32(vacc2x4567)), voutput_zero_point); - const int16x8_t vacc3x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc3x0123), vqmovn_s32(vacc3x4567)), voutput_zero_point); - const int16x8_t vacc4x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc4x0123), vqmovn_s32(vacc4x4567)), voutput_zero_point); - const int16x8_t vacc5x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc5x0123), vqmovn_s32(vacc5x4567)), voutput_zero_point); - const int16x8_t vacc6x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc6x0123), vqmovn_s32(vacc6x4567)), voutput_zero_point); - const int16x8_t vacc7x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc7x0123), vqmovn_s32(vacc7x4567)), voutput_zero_point); - const int16x8_t vacc8x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc8x0123), vqmovn_s32(vacc8x4567)), voutput_zero_point); - const int16x8_t vacc9x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc9x0123), vqmovn_s32(vacc9x4567)), voutput_zero_point); - const int16x8_t vacc10x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc10x0123), vqmovn_s32(vacc10x4567)), voutput_zero_point); - const int16x8_t vacc11x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc11x0123), vqmovn_s32(vacc11x4567)), voutput_zero_point); - - int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567)); - int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567)); - int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567)); - int8x16_t vout6x01234567_7x01234567 = vcombine_s8(vqmovn_s16(vacc6x01234567), vqmovn_s16(vacc7x01234567)); - int8x16_t vout8x01234567_9x01234567 = vcombine_s8(vqmovn_s16(vacc8x01234567), vqmovn_s16(vacc9x01234567)); - int8x16_t vout10x01234567_11x01234567 = vcombine_s8(vqmovn_s16(vacc10x01234567), vqmovn_s16(vacc11x01234567)); -#endif - const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min); - const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max); - - vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min); - vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min); - vout4x01234567_5x01234567 = vmaxq_s8(vout4x01234567_5x01234567, voutput_min); - vout6x01234567_7x01234567 = vmaxq_s8(vout6x01234567_7x01234567, voutput_min); - vout8x01234567_9x01234567 = vmaxq_s8(vout8x01234567_9x01234567, voutput_min); - vout10x01234567_11x01234567 = vmaxq_s8(vout10x01234567_11x01234567, voutput_min); - - vout0x01234567_1x01234567 = vminq_s8(vout0x01234567_1x01234567, voutput_max); - vout2x01234567_3x01234567 = vminq_s8(vout2x01234567_3x01234567, voutput_max); - vout4x01234567_5x01234567 = vminq_s8(vout4x01234567_5x01234567, voutput_max); - vout6x01234567_7x01234567 = vminq_s8(vout6x01234567_7x01234567, voutput_max); - vout8x01234567_9x01234567 = vminq_s8(vout8x01234567_9x01234567, voutput_max); - vout10x01234567_11x01234567 = vminq_s8(vout10x01234567_11x01234567, voutput_max); - - if (nc >= 8) { - // Main case where there the 8 columns fit in the destination. - vst1_s8(c0 + 0, vget_low_s8(vout0x01234567_1x01234567)); - vst1_s8(c1 + 0, vget_high_s8(vout0x01234567_1x01234567)); - vst1_s8(c2 + 0, vget_low_s8(vout2x01234567_3x01234567)); - vst1_s8(c3 + 0, vget_high_s8(vout2x01234567_3x01234567)); - vst1_s8(c4 + 0, vget_low_s8(vout4x01234567_5x01234567)); - vst1_s8(c5 + 0, vget_high_s8(vout4x01234567_5x01234567)); - vst1_s8(c6 + 0, vget_low_s8(vout6x01234567_7x01234567)); - vst1_s8(c7 + 0, vget_high_s8(vout6x01234567_7x01234567)); - vst1_s8(c8 + 0, vget_low_s8(vout8x01234567_9x01234567)); - vst1_s8(c9 + 0, vget_high_s8(vout8x01234567_9x01234567)); - vst1_s8(c10 + 0, vget_low_s8(vout10x01234567_11x01234567)); - vst1_s8(c11 + 0, vget_high_s8(vout10x01234567_11x01234567)); - - // Advance to the next 8 columns. - c0 = (int8_t*) ((uintptr_t) c0 + cn_stride); - c1 = (int8_t*) ((uintptr_t) c1 + cn_stride); - c2 = (int8_t*) ((uintptr_t) c2 + cn_stride); - c3 = (int8_t*) ((uintptr_t) c3 + cn_stride); - c4 = (int8_t*) ((uintptr_t) c4 + cn_stride); - c5 = (int8_t*) ((uintptr_t) c5 + cn_stride); - c6 = (int8_t*) ((uintptr_t) c6 + cn_stride); - c7 = (int8_t*) ((uintptr_t) c7 + cn_stride); - c8 = (int8_t*) ((uintptr_t) c8 + cn_stride); - c9 = (int8_t*) ((uintptr_t) c9 + cn_stride); - c10 = (int8_t*) ((uintptr_t) c10 + cn_stride); - c11 = (int8_t*) ((uintptr_t) c11 + cn_stride); - - a0 = (const int8_t*) ((uintptr_t) a0 - kc); - a1 = (const int8_t*) ((uintptr_t) a1 - kc); - a2 = (const int8_t*) ((uintptr_t) a2 - kc); - a3 = (const int8_t*) ((uintptr_t) a3 - kc); - a4 = (const int8_t*) ((uintptr_t) a4 - kc); - a5 = (const int8_t*) ((uintptr_t) a5 - kc); - a6 = (const int8_t*) ((uintptr_t) a6 - kc); - a7 = (const int8_t*) ((uintptr_t) a7 - kc); - a8 = (const int8_t*) ((uintptr_t) a8 - kc); - a9 = (const int8_t*) ((uintptr_t) a9 - kc); - a10 = (const int8_t*) ((uintptr_t) a10 - kc); - a11 = (const int8_t*) ((uintptr_t) a11 - kc); - - nc -= 8; - } else { - // Final case where not all of the 8 columns fit in the destination. - if (nc & 4) { - vst1q_lane_u32(__builtin_assume_aligned(c0, 1), vreinterpretq_u32_s8(vout0x01234567_1x01234567), 0); c0 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c1, 1), vreinterpretq_u32_s8(vout0x01234567_1x01234567), 2); c1 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c2, 1), vreinterpretq_u32_s8(vout2x01234567_3x01234567), 0); c2 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c3, 1), vreinterpretq_u32_s8(vout2x01234567_3x01234567), 2); c3 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c4, 1), vreinterpretq_u32_s8(vout4x01234567_5x01234567), 0); c4 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c5, 1), vreinterpretq_u32_s8(vout4x01234567_5x01234567), 2); c5 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c6, 1), vreinterpretq_u32_s8(vout6x01234567_7x01234567), 0); c6 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c7, 1), vreinterpretq_u32_s8(vout6x01234567_7x01234567), 2); c7 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c8, 1), vreinterpretq_u32_s8(vout8x01234567_9x01234567), 0); c8 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c9, 1), vreinterpretq_u32_s8(vout8x01234567_9x01234567), 2); c9 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c10, 1), vreinterpretq_u32_s8(vout10x01234567_11x01234567), 0); c10 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c11, 1), vreinterpretq_u32_s8(vout10x01234567_11x01234567), 2); c11 += 4; - vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 4); - vout2x01234567_3x01234567 = vextq_s8(vout2x01234567_3x01234567, vout2x01234567_3x01234567, 4); - vout4x01234567_5x01234567 = vextq_s8(vout4x01234567_5x01234567, vout4x01234567_5x01234567, 4); - vout6x01234567_7x01234567 = vextq_s8(vout6x01234567_7x01234567, vout6x01234567_7x01234567, 4); - vout8x01234567_9x01234567 = vextq_s8(vout8x01234567_9x01234567, vout8x01234567_9x01234567, 4); - vout10x01234567_11x01234567 = vextq_s8(vout10x01234567_11x01234567, vout10x01234567_11x01234567, 4); - } - if (nc & 2) { - vst1q_lane_u16(__builtin_assume_aligned(c0, 1), vreinterpretq_u16_s8(vout0x01234567_1x01234567), 0); c0 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c1, 1), vreinterpretq_u16_s8(vout0x01234567_1x01234567), 4); c1 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c2, 1), vreinterpretq_u16_s8(vout2x01234567_3x01234567), 0); c2 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c3, 1), vreinterpretq_u16_s8(vout2x01234567_3x01234567), 4); c3 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c4, 1), vreinterpretq_u16_s8(vout4x01234567_5x01234567), 0); c4 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c5, 1), vreinterpretq_u16_s8(vout4x01234567_5x01234567), 4); c5 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c6, 1), vreinterpretq_u16_s8(vout6x01234567_7x01234567), 0); c6 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c7, 1), vreinterpretq_u16_s8(vout6x01234567_7x01234567), 4); c7 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c8, 1), vreinterpretq_u16_s8(vout8x01234567_9x01234567), 0); c8 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c9, 1), vreinterpretq_u16_s8(vout8x01234567_9x01234567), 4); c9 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c10, 1), vreinterpretq_u16_s8(vout10x01234567_11x01234567), 0); c10 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c11, 1), vreinterpretq_u16_s8(vout10x01234567_11x01234567), 4); c11 += 2; - vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 2); - vout2x01234567_3x01234567 = vextq_s8(vout2x01234567_3x01234567, vout2x01234567_3x01234567, 2); - vout4x01234567_5x01234567 = vextq_s8(vout4x01234567_5x01234567, vout4x01234567_5x01234567, 2); - vout6x01234567_7x01234567 = vextq_s8(vout6x01234567_7x01234567, vout6x01234567_7x01234567, 2); - vout8x01234567_9x01234567 = vextq_s8(vout8x01234567_9x01234567, vout8x01234567_9x01234567, 2); - vout10x01234567_11x01234567 = vextq_s8(vout10x01234567_11x01234567, vout10x01234567_11x01234567, 2); - } - if (nc & 1) { - vst1q_lane_s8(c0, vout0x01234567_1x01234567, 0); - vst1q_lane_s8(c1, vout0x01234567_1x01234567, 8); - vst1q_lane_s8(c2, vout2x01234567_3x01234567, 0); - vst1q_lane_s8(c3, vout2x01234567_3x01234567, 8); - vst1q_lane_s8(c4, vout4x01234567_5x01234567, 0); - vst1q_lane_s8(c5, vout4x01234567_5x01234567, 8); - vst1q_lane_s8(c6, vout6x01234567_7x01234567, 0); - vst1q_lane_s8(c7, vout6x01234567_7x01234567, 8); - vst1q_lane_s8(c8, vout8x01234567_9x01234567, 0); - vst1q_lane_s8(c9, vout8x01234567_9x01234567, 8); - vst1q_lane_s8(c10, vout10x01234567_11x01234567, 0); - vst1q_lane_s8(c11, vout10x01234567_11x01234567, 8); - } - - nc = 0; - } - } while (nc != 0); -} diff --git a/src/qs8-igemm/gen/12x8c4-minmax-neondot.c b/src/qs8-igemm/gen/12x8c4-minmax-neondot.c deleted file mode 100644 index 0f0953b2e..000000000 --- a/src/qs8-igemm/gen/12x8c4-minmax-neondot.c +++ /dev/null @@ -1,519 +0,0 @@ -// Auto-generated file. Do not edit! -// Template: src/qs8-igemm/MRxNRc4-neondot.c.in -// Generator: tools/xngen -// -// Copyright 2020 Google LLC -// -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. - -#include <assert.h> - -#include <arm_neon.h> - -#include <xnnpack/igemm.h> -#include <xnnpack/math.h> - - -void xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot( - size_t mr, - size_t nc, - size_t kc, - size_t ks, - const int8_t** restrict a, - const void* restrict w, - int8_t* restrict c, - size_t cm_stride, - size_t cn_stride, - size_t a_offset, - const int8_t* zero, - const union xnn_qs8_gemm_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_DISABLE_TSAN -{ - assert(mr != 0); - assert(mr <= 12); - assert(nc != 0); - assert(kc != 0); - assert(ks != 0); - assert(ks % (12 * sizeof(void*)) == 0); - assert(a_offset % sizeof(int8_t) == 0); - assert(a != NULL); - assert(w != NULL); - assert(c != NULL); - - kc = round_up_po2(kc, 4); - int8_t* c0 = c; - int8_t* c1 = (int8_t*) ((uintptr_t) c0 + cm_stride); - if XNN_UNPREDICTABLE(mr < 2) { - c1 = c0; - } - int8_t* c2 = (int8_t*) ((uintptr_t) c1 + cm_stride); - if XNN_UNPREDICTABLE(mr <= 2) { - c2 = c1; - } - int8_t* c3 = (int8_t*) ((uintptr_t) c2 + cm_stride); - if XNN_UNPREDICTABLE(mr < 4) { - c3 = c2; - } - int8_t* c4 = (int8_t*) ((uintptr_t) c3 + cm_stride); - if XNN_UNPREDICTABLE(mr <= 4) { - c4 = c3; - } - int8_t* c5 = (int8_t*) ((uintptr_t) c4 + cm_stride); - if XNN_UNPREDICTABLE(mr < 6) { - c5 = c4; - } - int8_t* c6 = (int8_t*) ((uintptr_t) c5 + cm_stride); - if XNN_UNPREDICTABLE(mr <= 6) { - c6 = c5; - } - int8_t* c7 = (int8_t*) ((uintptr_t) c6 + cm_stride); - if XNN_UNPREDICTABLE(mr < 8) { - c7 = c6; - } - int8_t* c8 = (int8_t*) ((uintptr_t) c7 + cm_stride); - if XNN_UNPREDICTABLE(mr <= 8) { - c8 = c7; - } - int8_t* c9 = (int8_t*) ((uintptr_t) c8 + cm_stride); - if XNN_UNPREDICTABLE(mr < 10) { - c9 = c8; - } - int8_t* c10 = (int8_t*) ((uintptr_t) c9 + cm_stride); - if XNN_UNPREDICTABLE(mr <= 10) { - c10 = c9; - } - int8_t* c11 = (int8_t*) ((uintptr_t) c10 + cm_stride); - if XNN_UNPREDICTABLE(mr != 12) { - c11 = c10; - } - - do { - int32x4_t vacc0x0123 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)); - int32x4_t vacc0x4567 = vld1q_s32(w); w = (const void*) ((uintptr_t) w + 4 * sizeof(int32_t)); - int32x4_t vacc1x0123 = vacc0x0123; - int32x4_t vacc1x4567 = vacc0x4567; - int32x4_t vacc2x0123 = vacc0x0123; - int32x4_t vacc2x4567 = vacc0x4567; - int32x4_t vacc3x0123 = vacc0x0123; - int32x4_t vacc3x4567 = vacc0x4567; - int32x4_t vacc4x0123 = vacc0x0123; - int32x4_t vacc4x4567 = vacc0x4567; - int32x4_t vacc5x0123 = vacc0x0123; - int32x4_t vacc5x4567 = vacc0x4567; - int32x4_t vacc6x0123 = vacc0x0123; - int32x4_t vacc6x4567 = vacc0x4567; - int32x4_t vacc7x0123 = vacc0x0123; - int32x4_t vacc7x4567 = vacc0x4567; - int32x4_t vacc8x0123 = vacc0x0123; - int32x4_t vacc8x4567 = vacc0x4567; - int32x4_t vacc9x0123 = vacc0x0123; - int32x4_t vacc9x4567 = vacc0x4567; - int32x4_t vacc10x0123 = vacc0x0123; - int32x4_t vacc10x4567 = vacc0x4567; - int32x4_t vacc11x0123 = vacc0x0123; - int32x4_t vacc11x4567 = vacc0x4567; - - size_t p = ks; - do { - const int8_t* restrict a0 = a[0]; - if XNN_UNPREDICTABLE(a0 != zero) { - a0 = (const int8_t*) ((uintptr_t) a0 + a_offset); - } - const int8_t* restrict a1 = a[1]; - if XNN_UNPREDICTABLE(a1 != zero) { - a1 = (const int8_t*) ((uintptr_t) a1 + a_offset); - } - const int8_t* restrict a2 = a[2]; - if XNN_UNPREDICTABLE(a2 != zero) { - a2 = (const int8_t*) ((uintptr_t) a2 + a_offset); - } - const int8_t* restrict a3 = a[3]; - if XNN_UNPREDICTABLE(a3 != zero) { - a3 = (const int8_t*) ((uintptr_t) a3 + a_offset); - } - const int8_t* restrict a4 = a[4]; - if XNN_UNPREDICTABLE(a4 != zero) { - a4 = (const int8_t*) ((uintptr_t) a4 + a_offset); - } - const int8_t* restrict a5 = a[5]; - if XNN_UNPREDICTABLE(a5 != zero) { - a5 = (const int8_t*) ((uintptr_t) a5 + a_offset); - } - const int8_t* restrict a6 = a[6]; - if XNN_UNPREDICTABLE(a6 != zero) { - a6 = (const int8_t*) ((uintptr_t) a6 + a_offset); - } - const int8_t* restrict a7 = a[7]; - if XNN_UNPREDICTABLE(a7 != zero) { - a7 = (const int8_t*) ((uintptr_t) a7 + a_offset); - } - const int8_t* restrict a8 = a[8]; - if XNN_UNPREDICTABLE(a8 != zero) { - a8 = (const int8_t*) ((uintptr_t) a8 + a_offset); - } - const int8_t* restrict a9 = a[9]; - if XNN_UNPREDICTABLE(a9 != zero) { - a9 = (const int8_t*) ((uintptr_t) a9 + a_offset); - } - const int8_t* restrict a10 = a[10]; - if XNN_UNPREDICTABLE(a10 != zero) { - a10 = (const int8_t*) ((uintptr_t) a10 + a_offset); - } - const int8_t* restrict a11 = a[11]; - if XNN_UNPREDICTABLE(a11 != zero) { - a11 = (const int8_t*) ((uintptr_t) a11 + a_offset); - } - a += 12; - - // Inner accumulation loop along the 8 columns. - size_t k = kc; - // 2x partial unrolled loop to load 8 bytes at a time. - while (k >= 8 * sizeof(int8_t)) { - // Load a 12x8 block of activations. - const int8x8_t va0x01234567 = vld1_s8(a0); a0 += 8; - const int8x8_t va1x01234567 = vld1_s8(a1); a1 += 8; - const int8x8_t va2x01234567 = vld1_s8(a2); a2 += 8; - const int8x8_t va3x01234567 = vld1_s8(a3); a3 += 8; - const int8x8_t va4x01234567 = vld1_s8(a4); a4 += 8; - const int8x8_t va5x01234567 = vld1_s8(a5); a5 += 8; - const int8x8_t va6x01234567 = vld1_s8(a6); a6 += 8; - const int8x8_t va7x01234567 = vld1_s8(a7); a7 += 8; - const int8x8_t va8x01234567 = vld1_s8(a8); a8 += 8; - const int8x8_t va9x01234567 = vld1_s8(a9); a9 += 8; - const int8x8_t va10x01234567 = vld1_s8(a10); a10 += 8; - const int8x8_t va11x01234567 = vld1_s8(a11); a11 += 8; - - // Load a 8x8 block of weights. - const int8x16_t vb0123x0123 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); - const int8x16_t vb0123x4567 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); - const int8x16_t vb4567x0123 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); - const int8x16_t vb4567x4567 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); - - // Multiply-accumulate: 12x8 * 8x8 --> 12x8. - vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb0123x0123, va0x01234567, 0); - vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb0123x4567, va0x01234567, 0); - vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb0123x0123, va1x01234567, 0); - vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb0123x4567, va1x01234567, 0); - vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb0123x0123, va2x01234567, 0); - vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb0123x4567, va2x01234567, 0); - vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb0123x0123, va3x01234567, 0); - vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb0123x4567, va3x01234567, 0); - vacc4x0123 = vdotq_lane_s32(vacc4x0123, vb0123x0123, va4x01234567, 0); - vacc4x4567 = vdotq_lane_s32(vacc4x4567, vb0123x4567, va4x01234567, 0); - vacc5x0123 = vdotq_lane_s32(vacc5x0123, vb0123x0123, va5x01234567, 0); - vacc5x4567 = vdotq_lane_s32(vacc5x4567, vb0123x4567, va5x01234567, 0); - vacc6x0123 = vdotq_lane_s32(vacc6x0123, vb0123x0123, va6x01234567, 0); - vacc6x4567 = vdotq_lane_s32(vacc6x4567, vb0123x4567, va6x01234567, 0); - vacc7x0123 = vdotq_lane_s32(vacc7x0123, vb0123x0123, va7x01234567, 0); - vacc7x4567 = vdotq_lane_s32(vacc7x4567, vb0123x4567, va7x01234567, 0); - vacc8x0123 = vdotq_lane_s32(vacc8x0123, vb0123x0123, va8x01234567, 0); - vacc8x4567 = vdotq_lane_s32(vacc8x4567, vb0123x4567, va8x01234567, 0); - vacc9x0123 = vdotq_lane_s32(vacc9x0123, vb0123x0123, va9x01234567, 0); - vacc9x4567 = vdotq_lane_s32(vacc9x4567, vb0123x4567, va9x01234567, 0); - vacc10x0123 = vdotq_lane_s32(vacc10x0123, vb0123x0123, va10x01234567, 0); - vacc10x4567 = vdotq_lane_s32(vacc10x4567, vb0123x4567, va10x01234567, 0); - vacc11x0123 = vdotq_lane_s32(vacc11x0123, vb0123x0123, va11x01234567, 0); - vacc11x4567 = vdotq_lane_s32(vacc11x4567, vb0123x4567, va11x01234567, 0); - vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb4567x0123, va0x01234567, 1); - vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb4567x4567, va0x01234567, 1); - vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb4567x0123, va1x01234567, 1); - vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb4567x4567, va1x01234567, 1); - vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb4567x0123, va2x01234567, 1); - vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb4567x4567, va2x01234567, 1); - vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb4567x0123, va3x01234567, 1); - vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb4567x4567, va3x01234567, 1); - vacc4x0123 = vdotq_lane_s32(vacc4x0123, vb4567x0123, va4x01234567, 1); - vacc4x4567 = vdotq_lane_s32(vacc4x4567, vb4567x4567, va4x01234567, 1); - vacc5x0123 = vdotq_lane_s32(vacc5x0123, vb4567x0123, va5x01234567, 1); - vacc5x4567 = vdotq_lane_s32(vacc5x4567, vb4567x4567, va5x01234567, 1); - vacc6x0123 = vdotq_lane_s32(vacc6x0123, vb4567x0123, va6x01234567, 1); - vacc6x4567 = vdotq_lane_s32(vacc6x4567, vb4567x4567, va6x01234567, 1); - vacc7x0123 = vdotq_lane_s32(vacc7x0123, vb4567x0123, va7x01234567, 1); - vacc7x4567 = vdotq_lane_s32(vacc7x4567, vb4567x4567, va7x01234567, 1); - vacc8x0123 = vdotq_lane_s32(vacc8x0123, vb4567x0123, va8x01234567, 1); - vacc8x4567 = vdotq_lane_s32(vacc8x4567, vb4567x4567, va8x01234567, 1); - vacc9x0123 = vdotq_lane_s32(vacc9x0123, vb4567x0123, va9x01234567, 1); - vacc9x4567 = vdotq_lane_s32(vacc9x4567, vb4567x4567, va9x01234567, 1); - vacc10x0123 = vdotq_lane_s32(vacc10x0123, vb4567x0123, va10x01234567, 1); - vacc10x4567 = vdotq_lane_s32(vacc10x4567, vb4567x4567, va10x01234567, 1); - vacc11x0123 = vdotq_lane_s32(vacc11x0123, vb4567x0123, va11x01234567, 1); - vacc11x4567 = vdotq_lane_s32(vacc11x4567, vb4567x4567, va11x01234567, 1); - - k -= 8 * sizeof(int8_t); - } - // Handle up to 4 final positions of `k` - if XNN_UNLIKELY(k != 0) { - // Load a 12x4 block of activations. - const int8x8_t va0x01234567 = vld1_s8(a0); - const int8x8_t va1x01234567 = vld1_s8(a1); - const int8x8_t va2x01234567 = vld1_s8(a2); - const int8x8_t va3x01234567 = vld1_s8(a3); - const int8x8_t va4x01234567 = vld1_s8(a4); - const int8x8_t va5x01234567 = vld1_s8(a5); - const int8x8_t va6x01234567 = vld1_s8(a6); - const int8x8_t va7x01234567 = vld1_s8(a7); - const int8x8_t va8x01234567 = vld1_s8(a8); - const int8x8_t va9x01234567 = vld1_s8(a9); - const int8x8_t va10x01234567 = vld1_s8(a10); - const int8x8_t va11x01234567 = vld1_s8(a11); - - // Load a 4x8 block of weights. - const int8x16_t vb0123x0123 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); - const int8x16_t vb0123x4567 = vld1q_s8(w); w = (const void*) ((const int8_t*) w + 16); - - // Multiply-accumulate: 12x4 * 4x8 --> 12x8. - vacc0x0123 = vdotq_lane_s32(vacc0x0123, vb0123x0123, va0x01234567, 0); - vacc0x4567 = vdotq_lane_s32(vacc0x4567, vb0123x4567, va0x01234567, 0); - vacc1x0123 = vdotq_lane_s32(vacc1x0123, vb0123x0123, va1x01234567, 0); - vacc1x4567 = vdotq_lane_s32(vacc1x4567, vb0123x4567, va1x01234567, 0); - vacc2x0123 = vdotq_lane_s32(vacc2x0123, vb0123x0123, va2x01234567, 0); - vacc2x4567 = vdotq_lane_s32(vacc2x4567, vb0123x4567, va2x01234567, 0); - vacc3x0123 = vdotq_lane_s32(vacc3x0123, vb0123x0123, va3x01234567, 0); - vacc3x4567 = vdotq_lane_s32(vacc3x4567, vb0123x4567, va3x01234567, 0); - vacc4x0123 = vdotq_lane_s32(vacc4x0123, vb0123x0123, va4x01234567, 0); - vacc4x4567 = vdotq_lane_s32(vacc4x4567, vb0123x4567, va4x01234567, 0); - vacc5x0123 = vdotq_lane_s32(vacc5x0123, vb0123x0123, va5x01234567, 0); - vacc5x4567 = vdotq_lane_s32(vacc5x4567, vb0123x4567, va5x01234567, 0); - vacc6x0123 = vdotq_lane_s32(vacc6x0123, vb0123x0123, va6x01234567, 0); - vacc6x4567 = vdotq_lane_s32(vacc6x4567, vb0123x4567, va6x01234567, 0); - vacc7x0123 = vdotq_lane_s32(vacc7x0123, vb0123x0123, va7x01234567, 0); - vacc7x4567 = vdotq_lane_s32(vacc7x4567, vb0123x4567, va7x01234567, 0); - vacc8x0123 = vdotq_lane_s32(vacc8x0123, vb0123x0123, va8x01234567, 0); - vacc8x4567 = vdotq_lane_s32(vacc8x4567, vb0123x4567, va8x01234567, 0); - vacc9x0123 = vdotq_lane_s32(vacc9x0123, vb0123x0123, va9x01234567, 0); - vacc9x4567 = vdotq_lane_s32(vacc9x4567, vb0123x4567, va9x01234567, 0); - vacc10x0123 = vdotq_lane_s32(vacc10x0123, vb0123x0123, va10x01234567, 0); - vacc10x4567 = vdotq_lane_s32(vacc10x4567, vb0123x4567, va10x01234567, 0); - vacc11x0123 = vdotq_lane_s32(vacc11x0123, vb0123x0123, va11x01234567, 0); - vacc11x4567 = vdotq_lane_s32(vacc11x4567, vb0123x4567, va11x01234567, 0); - } - p -= 12 * sizeof(void*); - } while (p != 0); - - const int32x4_t vmultiplier = vld1q_dup_s32(¶ms->neon.multiplier); - vacc0x0123 = vqrdmulhq_s32(vacc0x0123, vmultiplier); - vacc0x4567 = vqrdmulhq_s32(vacc0x4567, vmultiplier); - vacc1x0123 = vqrdmulhq_s32(vacc1x0123, vmultiplier); - vacc1x4567 = vqrdmulhq_s32(vacc1x4567, vmultiplier); - vacc2x0123 = vqrdmulhq_s32(vacc2x0123, vmultiplier); - vacc2x4567 = vqrdmulhq_s32(vacc2x4567, vmultiplier); - vacc3x0123 = vqrdmulhq_s32(vacc3x0123, vmultiplier); - vacc3x4567 = vqrdmulhq_s32(vacc3x4567, vmultiplier); - vacc4x0123 = vqrdmulhq_s32(vacc4x0123, vmultiplier); - vacc4x4567 = vqrdmulhq_s32(vacc4x4567, vmultiplier); - vacc5x0123 = vqrdmulhq_s32(vacc5x0123, vmultiplier); - vacc5x4567 = vqrdmulhq_s32(vacc5x4567, vmultiplier); - vacc6x0123 = vqrdmulhq_s32(vacc6x0123, vmultiplier); - vacc6x4567 = vqrdmulhq_s32(vacc6x4567, vmultiplier); - vacc7x0123 = vqrdmulhq_s32(vacc7x0123, vmultiplier); - vacc7x4567 = vqrdmulhq_s32(vacc7x4567, vmultiplier); - vacc8x0123 = vqrdmulhq_s32(vacc8x0123, vmultiplier); - vacc8x4567 = vqrdmulhq_s32(vacc8x4567, vmultiplier); - vacc9x0123 = vqrdmulhq_s32(vacc9x0123, vmultiplier); - vacc9x4567 = vqrdmulhq_s32(vacc9x4567, vmultiplier); - vacc10x0123 = vqrdmulhq_s32(vacc10x0123, vmultiplier); - vacc10x4567 = vqrdmulhq_s32(vacc10x4567, vmultiplier); - vacc11x0123 = vqrdmulhq_s32(vacc11x0123, vmultiplier); - vacc11x4567 = vqrdmulhq_s32(vacc11x4567, vmultiplier); - - const int32x4_t vright_shift = vld1q_dup_s32(¶ms->neon.right_shift); - const int32x4_t vzero_shift_mask = vreinterpretq_s32_u32(vceqq_s32(vright_shift, vmovq_n_s32(0))); - vacc0x0123 = vsraq_n_s32(vacc0x0123, vbicq_s32(vacc0x0123, vzero_shift_mask), 31); - vacc0x4567 = vsraq_n_s32(vacc0x4567, vbicq_s32(vacc0x4567, vzero_shift_mask), 31); - vacc1x0123 = vsraq_n_s32(vacc1x0123, vbicq_s32(vacc1x0123, vzero_shift_mask), 31); - vacc1x4567 = vsraq_n_s32(vacc1x4567, vbicq_s32(vacc1x4567, vzero_shift_mask), 31); - vacc2x0123 = vsraq_n_s32(vacc2x0123, vbicq_s32(vacc2x0123, vzero_shift_mask), 31); - vacc2x4567 = vsraq_n_s32(vacc2x4567, vbicq_s32(vacc2x4567, vzero_shift_mask), 31); - vacc3x0123 = vsraq_n_s32(vacc3x0123, vbicq_s32(vacc3x0123, vzero_shift_mask), 31); - vacc3x4567 = vsraq_n_s32(vacc3x4567, vbicq_s32(vacc3x4567, vzero_shift_mask), 31); - vacc4x0123 = vsraq_n_s32(vacc4x0123, vbicq_s32(vacc4x0123, vzero_shift_mask), 31); - vacc4x4567 = vsraq_n_s32(vacc4x4567, vbicq_s32(vacc4x4567, vzero_shift_mask), 31); - vacc5x0123 = vsraq_n_s32(vacc5x0123, vbicq_s32(vacc5x0123, vzero_shift_mask), 31); - vacc5x4567 = vsraq_n_s32(vacc5x4567, vbicq_s32(vacc5x4567, vzero_shift_mask), 31); - vacc6x0123 = vsraq_n_s32(vacc6x0123, vbicq_s32(vacc6x0123, vzero_shift_mask), 31); - vacc6x4567 = vsraq_n_s32(vacc6x4567, vbicq_s32(vacc6x4567, vzero_shift_mask), 31); - vacc7x0123 = vsraq_n_s32(vacc7x0123, vbicq_s32(vacc7x0123, vzero_shift_mask), 31); - vacc7x4567 = vsraq_n_s32(vacc7x4567, vbicq_s32(vacc7x4567, vzero_shift_mask), 31); - vacc8x0123 = vsraq_n_s32(vacc8x0123, vbicq_s32(vacc8x0123, vzero_shift_mask), 31); - vacc8x4567 = vsraq_n_s32(vacc8x4567, vbicq_s32(vacc8x4567, vzero_shift_mask), 31); - vacc9x0123 = vsraq_n_s32(vacc9x0123, vbicq_s32(vacc9x0123, vzero_shift_mask), 31); - vacc9x4567 = vsraq_n_s32(vacc9x4567, vbicq_s32(vacc9x4567, vzero_shift_mask), 31); - vacc10x0123 = vsraq_n_s32(vacc10x0123, vbicq_s32(vacc10x0123, vzero_shift_mask), 31); - vacc10x4567 = vsraq_n_s32(vacc10x4567, vbicq_s32(vacc10x4567, vzero_shift_mask), 31); - vacc11x0123 = vsraq_n_s32(vacc11x0123, vbicq_s32(vacc11x0123, vzero_shift_mask), 31); - vacc11x4567 = vsraq_n_s32(vacc11x4567, vbicq_s32(vacc11x4567, vzero_shift_mask), 31); - - vacc0x0123 = vrshlq_s32(vacc0x0123, vright_shift); - vacc0x4567 = vrshlq_s32(vacc0x4567, vright_shift); - vacc1x0123 = vrshlq_s32(vacc1x0123, vright_shift); - vacc1x4567 = vrshlq_s32(vacc1x4567, vright_shift); - vacc2x0123 = vrshlq_s32(vacc2x0123, vright_shift); - vacc2x4567 = vrshlq_s32(vacc2x4567, vright_shift); - vacc3x0123 = vrshlq_s32(vacc3x0123, vright_shift); - vacc3x4567 = vrshlq_s32(vacc3x4567, vright_shift); - vacc4x0123 = vrshlq_s32(vacc4x0123, vright_shift); - vacc4x4567 = vrshlq_s32(vacc4x4567, vright_shift); - vacc5x0123 = vrshlq_s32(vacc5x0123, vright_shift); - vacc5x4567 = vrshlq_s32(vacc5x4567, vright_shift); - vacc6x0123 = vrshlq_s32(vacc6x0123, vright_shift); - vacc6x4567 = vrshlq_s32(vacc6x4567, vright_shift); - vacc7x0123 = vrshlq_s32(vacc7x0123, vright_shift); - vacc7x4567 = vrshlq_s32(vacc7x4567, vright_shift); - vacc8x0123 = vrshlq_s32(vacc8x0123, vright_shift); - vacc8x4567 = vrshlq_s32(vacc8x4567, vright_shift); - vacc9x0123 = vrshlq_s32(vacc9x0123, vright_shift); - vacc9x4567 = vrshlq_s32(vacc9x4567, vright_shift); - vacc10x0123 = vrshlq_s32(vacc10x0123, vright_shift); - vacc10x4567 = vrshlq_s32(vacc10x4567, vright_shift); - vacc11x0123 = vrshlq_s32(vacc11x0123, vright_shift); - vacc11x4567 = vrshlq_s32(vacc11x4567, vright_shift); - - const int16x8_t voutput_zero_point = vld1q_dup_s16(¶ms->neon.output_zero_point); -#if XNN_ARCH_ARM64 - const int16x8_t vacc0x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc0x0123), vacc0x4567), voutput_zero_point); - const int16x8_t vacc1x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc1x0123), vacc1x4567), voutput_zero_point); - const int16x8_t vacc2x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc2x0123), vacc2x4567), voutput_zero_point); - const int16x8_t vacc3x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc3x0123), vacc3x4567), voutput_zero_point); - const int16x8_t vacc4x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc4x0123), vacc4x4567), voutput_zero_point); - const int16x8_t vacc5x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc5x0123), vacc5x4567), voutput_zero_point); - const int16x8_t vacc6x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc6x0123), vacc6x4567), voutput_zero_point); - const int16x8_t vacc7x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc7x0123), vacc7x4567), voutput_zero_point); - const int16x8_t vacc8x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc8x0123), vacc8x4567), voutput_zero_point); - const int16x8_t vacc9x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc9x0123), vacc9x4567), voutput_zero_point); - const int16x8_t vacc10x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc10x0123), vacc10x4567), voutput_zero_point); - const int16x8_t vacc11x01234567 = vqaddq_s16(vqmovn_high_s32(vqmovn_s32(vacc11x0123), vacc11x4567), voutput_zero_point); - - int8x16_t vout0x01234567_1x01234567 = vqmovn_high_s16(vqmovn_s16(vacc0x01234567), vacc1x01234567); - int8x16_t vout2x01234567_3x01234567 = vqmovn_high_s16(vqmovn_s16(vacc2x01234567), vacc3x01234567); - int8x16_t vout4x01234567_5x01234567 = vqmovn_high_s16(vqmovn_s16(vacc4x01234567), vacc5x01234567); - int8x16_t vout6x01234567_7x01234567 = vqmovn_high_s16(vqmovn_s16(vacc6x01234567), vacc7x01234567); - int8x16_t vout8x01234567_9x01234567 = vqmovn_high_s16(vqmovn_s16(vacc8x01234567), vacc9x01234567); - int8x16_t vout10x01234567_11x01234567 = vqmovn_high_s16(vqmovn_s16(vacc10x01234567), vacc11x01234567); -#else - const int16x8_t vacc0x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc0x0123), vqmovn_s32(vacc0x4567)), voutput_zero_point); - const int16x8_t vacc1x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc1x0123), vqmovn_s32(vacc1x4567)), voutput_zero_point); - const int16x8_t vacc2x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc2x0123), vqmovn_s32(vacc2x4567)), voutput_zero_point); - const int16x8_t vacc3x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc3x0123), vqmovn_s32(vacc3x4567)), voutput_zero_point); - const int16x8_t vacc4x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc4x0123), vqmovn_s32(vacc4x4567)), voutput_zero_point); - const int16x8_t vacc5x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc5x0123), vqmovn_s32(vacc5x4567)), voutput_zero_point); - const int16x8_t vacc6x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc6x0123), vqmovn_s32(vacc6x4567)), voutput_zero_point); - const int16x8_t vacc7x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc7x0123), vqmovn_s32(vacc7x4567)), voutput_zero_point); - const int16x8_t vacc8x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc8x0123), vqmovn_s32(vacc8x4567)), voutput_zero_point); - const int16x8_t vacc9x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc9x0123), vqmovn_s32(vacc9x4567)), voutput_zero_point); - const int16x8_t vacc10x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc10x0123), vqmovn_s32(vacc10x4567)), voutput_zero_point); - const int16x8_t vacc11x01234567 = vqaddq_s16(vcombine_s16(vqmovn_s32(vacc11x0123), vqmovn_s32(vacc11x4567)), voutput_zero_point); - - int8x16_t vout0x01234567_1x01234567 = vcombine_s8(vqmovn_s16(vacc0x01234567), vqmovn_s16(vacc1x01234567)); - int8x16_t vout2x01234567_3x01234567 = vcombine_s8(vqmovn_s16(vacc2x01234567), vqmovn_s16(vacc3x01234567)); - int8x16_t vout4x01234567_5x01234567 = vcombine_s8(vqmovn_s16(vacc4x01234567), vqmovn_s16(vacc5x01234567)); - int8x16_t vout6x01234567_7x01234567 = vcombine_s8(vqmovn_s16(vacc6x01234567), vqmovn_s16(vacc7x01234567)); - int8x16_t vout8x01234567_9x01234567 = vcombine_s8(vqmovn_s16(vacc8x01234567), vqmovn_s16(vacc9x01234567)); - int8x16_t vout10x01234567_11x01234567 = vcombine_s8(vqmovn_s16(vacc10x01234567), vqmovn_s16(vacc11x01234567)); -#endif - const int8x16_t voutput_min = vld1q_dup_s8(¶ms->neon.output_min); - const int8x16_t voutput_max = vld1q_dup_s8(¶ms->neon.output_max); - - vout10x01234567_11x01234567 = vmaxq_s8(vout10x01234567_11x01234567, voutput_min); - vout8x01234567_9x01234567 = vmaxq_s8(vout8x01234567_9x01234567, voutput_min); - vout6x01234567_7x01234567 = vmaxq_s8(vout6x01234567_7x01234567, voutput_min); - vout4x01234567_5x01234567 = vmaxq_s8(vout4x01234567_5x01234567, voutput_min); - vout2x01234567_3x01234567 = vmaxq_s8(vout2x01234567_3x01234567, voutput_min); - vout0x01234567_1x01234567 = vmaxq_s8(vout0x01234567_1x01234567, voutput_min); - - vout10x01234567_11x01234567 = vminq_s8(vout10x01234567_11x01234567, voutput_max); - vout8x01234567_9x01234567 = vminq_s8(vout8x01234567_9x01234567, voutput_max); - vout6x01234567_7x01234567 = vminq_s8(vout6x01234567_7x01234567, voutput_max); - vout4x01234567_5x01234567 = vminq_s8(vout4x01234567_5x01234567, voutput_max); - vout2x01234567_3x01234567 = vminq_s8(vout2x01234567_3x01234567, voutput_max); - vout0x01234567_1x01234567 = vminq_s8(vout0x01234567_1x01234567, voutput_max); - - if (nc >= 8) { - vst1_s8(c11 + 0, vget_high_s8(vout10x01234567_11x01234567)); - vst1_s8(c10 + 0, vget_low_s8(vout10x01234567_11x01234567)); - vst1_s8(c9 + 0, vget_high_s8(vout8x01234567_9x01234567)); - vst1_s8(c8 + 0, vget_low_s8(vout8x01234567_9x01234567)); - vst1_s8(c7 + 0, vget_high_s8(vout6x01234567_7x01234567)); - vst1_s8(c6 + 0, vget_low_s8(vout6x01234567_7x01234567)); - vst1_s8(c5 + 0, vget_high_s8(vout4x01234567_5x01234567)); - vst1_s8(c4 + 0, vget_low_s8(vout4x01234567_5x01234567)); - vst1_s8(c3 + 0, vget_high_s8(vout2x01234567_3x01234567)); - vst1_s8(c2 + 0, vget_low_s8(vout2x01234567_3x01234567)); - vst1_s8(c1 + 0, vget_high_s8(vout0x01234567_1x01234567)); - vst1_s8(c0 + 0, vget_low_s8(vout0x01234567_1x01234567)); - - c11 = (int8_t*) ((uintptr_t) c11 + cn_stride); - c10 = (int8_t*) ((uintptr_t) c10 + cn_stride); - c9 = (int8_t*) ((uintptr_t) c9 + cn_stride); - c8 = (int8_t*) ((uintptr_t) c8 + cn_stride); - c7 = (int8_t*) ((uintptr_t) c7 + cn_stride); - c6 = (int8_t*) ((uintptr_t) c6 + cn_stride); - c5 = (int8_t*) ((uintptr_t) c5 + cn_stride); - c4 = (int8_t*) ((uintptr_t) c4 + cn_stride); - c3 = (int8_t*) ((uintptr_t) c3 + cn_stride); - c2 = (int8_t*) ((uintptr_t) c2 + cn_stride); - c1 = (int8_t*) ((uintptr_t) c1 + cn_stride); - c0 = (int8_t*) ((uintptr_t) c0 + cn_stride); - - a = (const int8_t**restrict) ((uintptr_t) a - ks); - - nc -= 8; - } else { - if (nc & 4) { - vst1q_lane_u32(__builtin_assume_aligned(c11, 1), vreinterpretq_u32_s8(vout10x01234567_11x01234567), 2); c11 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c10, 1), vreinterpretq_u32_s8(vout10x01234567_11x01234567), 0); c10 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c9, 1), vreinterpretq_u32_s8(vout8x01234567_9x01234567), 2); c9 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c8, 1), vreinterpretq_u32_s8(vout8x01234567_9x01234567), 0); c8 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c7, 1), vreinterpretq_u32_s8(vout6x01234567_7x01234567), 2); c7 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c6, 1), vreinterpretq_u32_s8(vout6x01234567_7x01234567), 0); c6 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c5, 1), vreinterpretq_u32_s8(vout4x01234567_5x01234567), 2); c5 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c4, 1), vreinterpretq_u32_s8(vout4x01234567_5x01234567), 0); c4 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c3, 1), vreinterpretq_u32_s8(vout2x01234567_3x01234567), 2); c3 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c2, 1), vreinterpretq_u32_s8(vout2x01234567_3x01234567), 0); c2 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c1, 1), vreinterpretq_u32_s8(vout0x01234567_1x01234567), 2); c1 += 4; - vst1q_lane_u32(__builtin_assume_aligned(c0, 1), vreinterpretq_u32_s8(vout0x01234567_1x01234567), 0); c0 += 4; - vout10x01234567_11x01234567 = vextq_s8(vout10x01234567_11x01234567, vout10x01234567_11x01234567, 4); - vout8x01234567_9x01234567 = vextq_s8(vout8x01234567_9x01234567, vout8x01234567_9x01234567, 4); - vout6x01234567_7x01234567 = vextq_s8(vout6x01234567_7x01234567, vout6x01234567_7x01234567, 4); - vout4x01234567_5x01234567 = vextq_s8(vout4x01234567_5x01234567, vout4x01234567_5x01234567, 4); - vout2x01234567_3x01234567 = vextq_s8(vout2x01234567_3x01234567, vout2x01234567_3x01234567, 4); - vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 4); - } - if (nc & 2) { - vst1q_lane_u16(__builtin_assume_aligned(c11, 1), vreinterpretq_u16_s8(vout10x01234567_11x01234567), 4); c11 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c10, 1), vreinterpretq_u16_s8(vout10x01234567_11x01234567), 0); c10 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c9, 1), vreinterpretq_u16_s8(vout8x01234567_9x01234567), 4); c9 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c8, 1), vreinterpretq_u16_s8(vout8x01234567_9x01234567), 0); c8 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c7, 1), vreinterpretq_u16_s8(vout6x01234567_7x01234567), 4); c7 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c6, 1), vreinterpretq_u16_s8(vout6x01234567_7x01234567), 0); c6 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c5, 1), vreinterpretq_u16_s8(vout4x01234567_5x01234567), 4); c5 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c4, 1), vreinterpretq_u16_s8(vout4x01234567_5x01234567), 0); c4 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c3, 1), vreinterpretq_u16_s8(vout2x01234567_3x01234567), 4); c3 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c2, 1), vreinterpretq_u16_s8(vout2x01234567_3x01234567), 0); c2 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c1, 1), vreinterpretq_u16_s8(vout0x01234567_1x01234567), 4); c1 += 2; - vst1q_lane_u16(__builtin_assume_aligned(c0, 1), vreinterpretq_u16_s8(vout0x01234567_1x01234567), 0); c0 += 2; - vout10x01234567_11x01234567 = vextq_s8(vout10x01234567_11x01234567, vout10x01234567_11x01234567, 2); - vout8x01234567_9x01234567 = vextq_s8(vout8x01234567_9x01234567, vout8x01234567_9x01234567, 2); - vout6x01234567_7x01234567 = vextq_s8(vout6x01234567_7x01234567, vout6x01234567_7x01234567, 2); - vout4x01234567_5x01234567 = vextq_s8(vout4x01234567_5x01234567, vout4x01234567_5x01234567, 2); - vout2x01234567_3x01234567 = vextq_s8(vout2x01234567_3x01234567, vout2x01234567_3x01234567, 2); - vout0x01234567_1x01234567 = vextq_s8(vout0x01234567_1x01234567, vout0x01234567_1x01234567, 2); - } - if (nc & 1) { - vst1q_lane_s8(c11, vout10x01234567_11x01234567, 8); - vst1q_lane_s8(c10, vout10x01234567_11x01234567, 0); - vst1q_lane_s8(c9, vout8x01234567_9x01234567, 8); - vst1q_lane_s8(c8, vout8x01234567_9x01234567, 0); - vst1q_lane_s8(c7, vout6x01234567_7x01234567, 8); - vst1q_lane_s8(c6, vout6x01234567_7x01234567, 0); - vst1q_lane_s8(c5, vout4x01234567_5x01234567, 8); - vst1q_lane_s8(c4, vout4x01234567_5x01234567, 0); - vst1q_lane_s8(c3, vout2x01234567_3x01234567, 8); - vst1q_lane_s8(c2, vout2x01234567_3x01234567, 0); - vst1q_lane_s8(c1, vout0x01234567_1x01234567, 8); - vst1q_lane_s8(c0, vout0x01234567_1x01234567, 0); - } - - nc = 0; - } - } while (nc != 0); -} diff --git a/src/xnnpack/gemm.h b/src/xnnpack/gemm.h index bb94b8a2e..f0da3b515 100644 --- a/src/xnnpack/gemm.h +++ b/src/xnnpack/gemm.h @@ -599,7 +599,6 @@ DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_ukernel_1x8c4__neon DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_ukernel_4x8c4__neondot) DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_ukernel_6x8c4__neondot) DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_ukernel_8x8c4__neondot) -DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot) DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_ukernel_1x16c4__neondot) DECLARE_QS8_GEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_gemm_minmax_ukernel_4x16c4__neondot) diff --git a/src/xnnpack/igemm.h b/src/xnnpack/igemm.h index fa418a511..219bd23fd 100644 --- a/src/xnnpack/igemm.h +++ b/src/xnnpack/igemm.h @@ -401,7 +401,6 @@ DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_ukernel_1x8c4__ne DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_ukernel_4x8c4__neondot) DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_ukernel_6x8c4__neondot) DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_ukernel_8x8c4__neondot) -DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot) DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_ukernel_1x16c4__neondot) DECLARE_QS8_IGEMM_MINMAX_UKERNEL_FUNCTION(xnn_qs8_igemm_minmax_ukernel_4x16c4__neondot) diff --git a/test/qs8-gemm-minmax.cc b/test/qs8-gemm-minmax.cc index 68cd1b17f..4b8a4912c 100644 --- a/test/qs8-gemm-minmax.cc +++ b/test/qs8-gemm-minmax.cc @@ -27383,462 +27383,6 @@ #if XNN_ARCH_ARM || XNN_ARCH_ARM64 - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_eq_8) { - TEST_REQUIRES_ARM_NEON_DOT; - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(8) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, strided_cn) { - TEST_REQUIRES_ARM_NEON_DOT; - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(8) - .cn_stride(11) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_eq_8_strided_a) { - TEST_REQUIRES_ARM_NEON_DOT; - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(8) - .a_stride(11) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_eq_8_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t m = 1; m <= 12; m++) { - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(8) - .iterations(1) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_eq_8_subtile_m) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t m = 1; m <= 12; m++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(8) - .k(8) - .iterations(1) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_eq_8_subtile_n) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(n) - .k(8) - .iterations(1) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_lt_8) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 1; k < 8; k++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_lt_8_strided_a) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 1; k < 8; k++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .a_stride(11) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_lt_8_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 1; k < 8; k++) { - for (uint32_t m = 1; m <= 12; m++) { - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .iterations(1) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_gt_8) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 9; k < 16; k++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_gt_8_strided_a) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 9; k < 16; k++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .a_stride(19) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_gt_8_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 9; k < 16; k++) { - for (uint32_t m = 1; m <= 12; m++) { - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .iterations(1) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_div_8) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 16; k <= 80; k += 8) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_div_8_strided_a) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 16; k <= 80; k += 8) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .a_stride(83) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, k_div_8_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 16; k <= 80; k += 8) { - for (uint32_t m = 1; m <= 12; m++) { - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .iterations(1) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, n_gt_8) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 9; n < 16; n++) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, n_gt_8_strided_cn) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 9; n < 16; n++) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .cn_stride(11) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, n_gt_8_strided_a) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 9; n < 16; n++) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(n) - .k(k) - .a_stride(43) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, n_gt_8_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 9; n < 16; n++) { - for (size_t k = 1; k <= 40; k += 9) { - for (uint32_t m = 1; m <= 12; m++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .iterations(1) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, n_div_8) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 16; n <= 24; n += 8) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, n_div_8_strided_cn) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 16; n <= 24; n += 8) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(n) - .k(k) - .cn_stride(11) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, n_div_8_strided_a) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 16; n <= 24; n += 8) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(n) - .k(k) - .a_stride(43) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, n_div_8_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 16; n <= 24; n += 8) { - for (size_t k = 1; k <= 40; k += 9) { - for (uint32_t m = 1; m <= 12; m++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .iterations(1) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, strided_cm_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 1; k <= 40; k += 9) { - for (uint32_t m = 1; m <= 12; m++) { - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .cm_stride(11) - .iterations(1) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, qmin) { - TEST_REQUIRES_ARM_NEON_DOT; - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(8) - .qmin(128) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, qmax) { - TEST_REQUIRES_ARM_NEON_DOT; - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(8) - .qmax(128) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } - - TEST(QS8_GEMM_MINMAX_12X8C4__NEONDOT, strided_cm) { - TEST_REQUIRES_ARM_NEON_DOT; - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(8) - .cm_stride(11) - .Test(xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot); - } -#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64 - - -#if XNN_ARCH_ARM || XNN_ARCH_ARM64 TEST(QS8_GEMM_MINMAX_1X16C4__NEONDOT, k_eq_8) { TEST_REQUIRES_ARM_NEON_DOT; GemmMicrokernelTester() diff --git a/test/qs8-gemm-minmax.yaml b/test/qs8-gemm-minmax.yaml index f2d98c154..e632257bc 100644 --- a/test/qs8-gemm-minmax.yaml +++ b/test/qs8-gemm-minmax.yaml @@ -122,8 +122,6 @@ k-block: 8 - name: xnn_qs8_gemm_minmax_ukernel_8x8c4__neondot k-block: 8 -- name: xnn_qs8_gemm_minmax_ukernel_12x8c4__neondot - k-block: 8 - name: xnn_qs8_gemm_minmax_ukernel_1x16c4__neondot k-block: 8 - name: xnn_qs8_gemm_minmax_ukernel_4x16c4__neondot diff --git a/test/qs8-igemm-minmax.cc b/test/qs8-igemm-minmax.cc index 325379e78..7314980a6 100644 --- a/test/qs8-igemm-minmax.cc +++ b/test/qs8-igemm-minmax.cc @@ -28103,474 +28103,6 @@ #if XNN_ARCH_ARM || XNN_ARCH_ARM64 - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, k_eq_8) { - TEST_REQUIRES_ARM_NEON_DOT; - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(8) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, strided_cn) { - TEST_REQUIRES_ARM_NEON_DOT; - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(8) - .cn_stride(11) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, k_eq_8_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t m = 1; m <= 12; m++) { - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(8) - .iterations(1) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, k_eq_8_subtile_m) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t m = 1; m <= 12; m++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(8) - .k(8) - .iterations(1) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, k_eq_8_subtile_n) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(n) - .k(8) - .iterations(1) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, k_lt_8) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 1; k < 8; k++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, k_lt_8_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 1; k < 8; k++) { - for (uint32_t m = 1; m <= 12; m++) { - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .iterations(1) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, k_gt_8) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 9; k < 16; k++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, k_gt_8_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 9; k < 16; k++) { - for (uint32_t m = 1; m <= 12; m++) { - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .iterations(1) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, k_div_8) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 16; k <= 80; k += 8) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, k_div_8_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 16; k <= 80; k += 8) { - for (uint32_t m = 1; m <= 12; m++) { - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .iterations(1) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, n_gt_8) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 9; n < 16; n++) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, n_gt_8_strided_cn) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 9; n < 16; n++) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .cn_stride(11) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, n_gt_8_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 9; n < 16; n++) { - for (size_t k = 1; k <= 40; k += 9) { - for (uint32_t m = 1; m <= 12; m++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .iterations(1) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, n_div_8) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 16; n <= 24; n += 8) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, n_div_8_strided_cn) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 16; n <= 24; n += 8) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(n) - .k(k) - .cn_stride(11) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, n_div_8_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 16; n <= 24; n += 8) { - for (size_t k = 1; k <= 40; k += 9) { - for (uint32_t m = 1; m <= 12; m++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .iterations(1) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, small_kernel) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .ks(3) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, small_kernel_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 1; k <= 40; k += 9) { - for (uint32_t m = 1; m <= 12; m++) { - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .ks(3) - .iterations(1) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, n_gt_8_small_kernel) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 9; n < 16; n++) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .ks(3) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, n_div_8_small_kernel) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t n = 16; n <= 24; n += 8) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .ks(3) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, strided_cm_subtile) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 1; k <= 40; k += 9) { - for (uint32_t m = 1; m <= 12; m++) { - for (uint32_t n = 1; n <= 8; n++) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(m) - .n(n) - .k(k) - .cm_stride(11) - .iterations(1) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, a_offset) { - TEST_REQUIRES_ARM_NEON_DOT; - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .ks(3) - .a_offset(487) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, zero) { - TEST_REQUIRES_ARM_NEON_DOT; - for (uint32_t mz = 0; mz < 12; mz++) { - for (size_t k = 1; k <= 40; k += 9) { - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(k) - .ks(3) - .a_offset(487) - .zero_index(mz) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - } - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, qmin) { - TEST_REQUIRES_ARM_NEON_DOT; - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(8) - .qmin(128) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, qmax) { - TEST_REQUIRES_ARM_NEON_DOT; - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(8) - .qmax(128) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } - - TEST(QS8_IGEMM_MINMAX_12X8C4__NEONDOT, strided_cm) { - TEST_REQUIRES_ARM_NEON_DOT; - GemmMicrokernelTester() - .mr(12) - .nr(8) - .kr(4) - .sr(1) - .m(12) - .n(8) - .k(8) - .cm_stride(11) - .Test(xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot); - } -#endif // XNN_ARCH_ARM || XNN_ARCH_ARM64 - - -#if XNN_ARCH_ARM || XNN_ARCH_ARM64 TEST(QS8_IGEMM_MINMAX_1X16C4__NEONDOT, k_eq_8) { TEST_REQUIRES_ARM_NEON_DOT; GemmMicrokernelTester() diff --git a/test/qs8-igemm-minmax.yaml b/test/qs8-igemm-minmax.yaml index b0e0f5799..26f13bb35 100644 --- a/test/qs8-igemm-minmax.yaml +++ b/test/qs8-igemm-minmax.yaml @@ -122,8 +122,6 @@ k-block: 8 - name: xnn_qs8_igemm_minmax_ukernel_8x8c4__neondot k-block: 8 -- name: xnn_qs8_igemm_minmax_ukernel_12x8c4__neondot - k-block: 8 - name: xnn_qs8_igemm_minmax_ukernel_1x16c4__neondot k-block: 8 - name: xnn_qs8_igemm_minmax_ukernel_4x16c4__neondot |