From 0672a7e1eb789b35709037ab5231d8d8a0f3bafc Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Sat, 2 May 2020 22:29:51 -0700 Subject: Fast path using atomic decrement instead of atomic compare-and-swap 50% higher throughput on x86 (disabled on other platforms) --- BUILD.bazel | 111 ++++++- CMakeLists.txt | 13 + src/fastpath.c | 793 ++++++++++++++++++++++++++++++++++++++++++++++++ src/portable-api.c | 190 +++++++++--- src/threadpool-object.h | 52 ++++ 5 files changed, 1117 insertions(+), 42 deletions(-) create mode 100644 src/fastpath.c diff --git a/BUILD.bazel b/BUILD.bazel index af1401b..fa00807 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -34,6 +34,10 @@ PORTABLE_SRCS = [ "src/portable-api.c", ] +ARCH_SPECIFIC_SRCS = [ + "src/fastpath.c", +] + PTHREADS_IMPL_SRCS = PORTABLE_SRCS + ["src/pthreads.c"] GCD_IMPL_SRCS = PORTABLE_SRCS + ["src/gcd.c"] @@ -54,9 +58,23 @@ cc_library( ":macos_x86": INTERNAL_HDRS + GCD_IMPL_SRCS, ":macos_x86_64": INTERNAL_HDRS + GCD_IMPL_SRCS, ":ios": INTERNAL_HDRS + GCD_IMPL_SRCS, + ":watchos": INTERNAL_HDRS + GCD_IMPL_SRCS, + ":tvos": INTERNAL_HDRS + GCD_IMPL_SRCS, ":windows_x86_64": INTERNAL_HDRS + WINDOWS_IMPL_SRCS, - ":windows_x86_64_msvc": INTERNAL_HDRS + WINDOWS_IMPL_SRCS, "//conditions:default": INTERNAL_HDRS + PTHREADS_IMPL_SRCS, + }) + select({ + ":linux_x86_64": ARCH_SPECIFIC_SRCS, + ":android_x86": ARCH_SPECIFIC_SRCS, + ":android_x86_64": ARCH_SPECIFIC_SRCS, + ":windows_x86_64": ARCH_SPECIFIC_SRCS, + ":macos_x86": ARCH_SPECIFIC_SRCS, + ":macos_x86_64": ARCH_SPECIFIC_SRCS, + ":ios_x86": ARCH_SPECIFIC_SRCS, + ":ios_x86_64": ARCH_SPECIFIC_SRCS, + ":watchos_x86": ARCH_SPECIFIC_SRCS, + ":watchos_x86_64": ARCH_SPECIFIC_SRCS, + ":tvos_x86_64": ARCH_SPECIFIC_SRCS, + "//conditions:default": [], }), copts = [ "-std=gnu11", @@ -96,6 +114,19 @@ cc_library( "-DPTHREADPOOL_USE_EVENT=1", ], "//conditions:default": [], + }) + select({ + ":linux_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"], + ":android_x86": ["-DPTHREADPOOL_USE_FASTPATH=1"], + ":android_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"], + ":windows_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"], + ":macos_x86": ["-DPTHREADPOOL_USE_FASTPATH=1"], + ":macos_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"], + ":ios_x86": ["-DPTHREADPOOL_USE_FASTPATH=1"], + ":ios_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"], + ":watchos_x86": ["-DPTHREADPOOL_USE_FASTPATH=1"], + ":watchos_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"], + ":tvos_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"], + "//conditions:default": ["-DPTHREADPOOL_USE_FASTPATH=0"], }), hdrs = [ "include/pthreadpool.h", @@ -219,6 +250,11 @@ config_setting( }, ) +config_setting( + name = "linux_x86_64", + values = {"cpu": "k8"}, +) + config_setting( name = "linux_arm", values = {"cpu": "arm"}, @@ -234,6 +270,22 @@ config_setting( values = {"cpu": "aarch64"}, ) +config_setting( + name = "android_x86", + values = { + "crosstool_top": "//external:android/crosstool", + "cpu": "x86", + }, +) + +config_setting( + name = "android_x86_64", + values = { + "crosstool_top": "//external:android/crosstool", + "cpu": "x86_64", + }, +) + config_setting( name = "android_armv7", values = { @@ -278,16 +330,65 @@ config_setting( ) config_setting( - name = "windows_x86_64", + name = "ios_x86", values = { - "cpu": "x64_windows", + "apple_platform_type": "ios", + "cpu": "ios_i386", + }, +) + +config_setting( + name = "ios_x86_64", + values = { + "apple_platform_type": "ios", + "cpu": "ios_x86_64", }, ) config_setting( - name = "windows_x86_64_msvc", + name = "watchos", values = { - "cpu": "x64_windows_msvc", + "crosstool_top": "@bazel_tools//tools/cpp:toolchain", + "apple_platform_type": "watchos", + }, +) + +config_setting( + name = "watchos_x86", + values = { + "apple_platform_type": "watchos", + "cpu": "watchos_i386", + }, +) + +config_setting( + name = "watchos_x86_64", + values = { + "apple_platform_type": "watchos", + "cpu": "watchos_x86_64", + }, +) + +config_setting( + name = "tvos", + values = { + "crosstool_top": "@bazel_tools//tools/cpp:toolchain", + "apple_platform_type": "tvos", + }, +) + +config_setting( + name = "tvos_x86_64", + values = { + "apple_platform_type": "tvos", + "cpu": "tvos_x86_64", + }, +) + +config_setting( + name = "windows_x86_64", + values = { + "cpu": "x64_windows", }, ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 51b0105..0db3264 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,6 +9,11 @@ SET_PROPERTY(CACHE PTHREADPOOL_LIBRARY_TYPE PROPERTY STRINGS default static shar OPTION(PTHREADPOOL_ALLOW_DEPRECATED_API "Enable deprecated API functions" ON) SET(PTHREADPOOL_SYNC_PRIMITIVE "default" CACHE STRING "Synchronization primitive (condvar, futex, gcd, event, or default) for worker threads") SET_PROPERTY(CACHE PTHREADPOOL_SYNC_PRIMITIVE PROPERTY STRINGS default condvar futex gcd event) +IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i[3-6]86|AMD64|x86(_64)?)$") + OPTION(PTHREADPOOL_ENABLE_FASTPATH "Enable fast path using atomic decrement instead of atomic compare-and-swap" ON) +ELSE() + OPTION(PTHREADPOOL_ENABLE_FASTPATH "Enable fast path using atomic decrement instead of atomic compare-and-swap" OFF) +ENDIF() IF("${CMAKE_SOURCE_DIR}" STREQUAL "${PROJECT_SOURCE_DIR}") OPTION(PTHREADPOOL_BUILD_TESTS "Build pthreadpool unit tests" ON) OPTION(PTHREADPOOL_BUILD_BENCHMARKS "Build pthreadpool micro-benchmarks" ON) @@ -76,6 +81,9 @@ ELSE() ELSE() LIST(APPEND PTHREADPOOL_SRCS src/pthreads.c) ENDIF() + IF(PTHREADPOOL_ENABLE_FASTPATH) + LIST(APPEND PTHREADPOOL_SRCS src/fastpath.c) + ENDIF() ENDIF() ADD_LIBRARY(pthreadpool_interface INTERFACE) @@ -114,6 +122,11 @@ ELSEIF(PTHREADPOOL_SYNC_PRIMITIVE STREQUAL "event") ELSEIF(NOT PTHREADPOOL_SYNC_PRIMITIVE STREQUAL "default") MESSAGE(FATAL_ERROR "Unsupported synchronization primitive ${PTHREADPOOL_SYNC_PRIMITIVE}") ENDIF() +IF(PTHREADPOOL_ENABLE_FASTPATH) + TARGET_COMPILE_DEFINITIONS(pthreadpool PRIVATE PTHREADPOOL_ENABLE_FASTPATH=1) +ELSE() + TARGET_COMPILE_DEFINITIONS(pthreadpool PRIVATE PTHREADPOOL_ENABLE_FASTPATH=0) +ENDIF() SET_TARGET_PROPERTIES(pthreadpool PROPERTIES C_STANDARD 11 diff --git a/src/fastpath.c b/src/fastpath.c new file mode 100644 index 0000000..1a5066a --- /dev/null +++ b/src/fastpath.c @@ -0,0 +1,793 @@ +/* Standard C headers */ +#include +#include +#include +#include +#include + +#if PTHREADPOOL_USE_CPUINFO + #include +#endif + +/* Dependencies */ +#include + +/* Public library header */ +#include + +/* Internal library headers */ +#include "threadpool-atomics.h" +#include "threadpool-common.h" +#include "threadpool-object.h" +#include "threadpool-utils.h" + + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_1d_t task = (pthreadpool_task_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, range_start++); + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + task(argument, index); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_with_uarch_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_1d_with_id_t task = (pthreadpool_task_1d_with_id_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const uint32_t default_uarch_index = threadpool->params.parallelize_1d_with_uarch.default_uarch_index; + uint32_t uarch_index = default_uarch_index; + #if PTHREADPOOL_USE_CPUINFO + uarch_index = cpuinfo_get_current_uarch_index(); + if (uarch_index > threadpool->params.parallelize_1d_with_uarch.max_uarch_index) { + uarch_index = default_uarch_index; + } + #endif + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, uarch_index, range_start++); + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + task(argument, uarch_index, index); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_tile_1d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_1d_tile_1d_t task = (pthreadpool_task_1d_tile_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const size_t tile = threadpool->params.parallelize_1d_tile_1d.tile; + size_t tile_start = range_start * tile; + + const size_t range = threadpool->params.parallelize_1d_tile_1d.range; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, tile_start, min(range - tile_start, tile)); + tile_start += tile; + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t tile_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const size_t tile_start = tile_index * tile; + task(argument, tile_start, min(range - tile_start, tile)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_2d_t task = (pthreadpool_task_2d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_2d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(range_start, range_j); + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, i, j); + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(linear_index, range_j); + task(argument, index_i_j.quotient, index_i_j.remainder); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_1d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_2d_tile_1d_t task = (pthreadpool_task_2d_tile_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_2d_tile_1d.tile_range_j; + const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(range_start, tile_range_j); + const size_t tile_j = threadpool->params.parallelize_2d_tile_1d.tile_j; + size_t i = tile_index_i_j.quotient; + size_t start_j = tile_index_i_j.remainder * tile_j; + + const size_t range_j = threadpool->params.parallelize_2d_tile_1d.range_j; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, i, start_j, min(range_j - start_j, tile_j)); + start_j += tile_j; + if (start_j >= range_j) { + start_j = 0; + i += 1; + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(linear_index, tile_range_j); + const size_t start_j = tile_index_i_j.remainder * tile_j; + task(argument, tile_index_i_j.quotient, start_j, min(range_j - start_j, tile_j)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_2d_tile_2d_t task = (pthreadpool_task_2d_tile_2d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_2d_tile_2d.tile_range_j; + const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(range_start, tile_range_j); + const size_t tile_i = threadpool->params.parallelize_2d_tile_2d.tile_i; + const size_t tile_j = threadpool->params.parallelize_2d_tile_2d.tile_j; + size_t start_i = tile_index_i_j.quotient * tile_i; + size_t start_j = tile_index_i_j.remainder * tile_j; + + const size_t range_i = threadpool->params.parallelize_2d_tile_2d.range_i; + const size_t range_j = threadpool->params.parallelize_2d_tile_2d.range_j; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, start_i, start_j, min(range_i - start_i, tile_i), min(range_j - start_j, tile_j)); + start_j += tile_j; + if (start_j >= range_j) { + start_j = 0; + start_i += tile_i; + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(linear_index, tile_range_j); + const size_t start_i = tile_index_i_j.quotient * tile_i; + const size_t start_j = tile_index_i_j.remainder * tile_j; + task(argument, start_i, start_j, min(range_i - start_i, tile_i), min(range_j - start_j, tile_j)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_with_uarch_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_2d_tile_2d_with_id_t task = (pthreadpool_task_2d_tile_2d_with_id_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const uint32_t default_uarch_index = threadpool->params.parallelize_2d_tile_2d_with_uarch.default_uarch_index; + uint32_t uarch_index = default_uarch_index; + #if PTHREADPOOL_USE_CPUINFO + uarch_index = cpuinfo_get_current_uarch_index(); + if (uarch_index > threadpool->params.parallelize_2d_tile_2d_with_uarch.max_uarch_index) { + uarch_index = default_uarch_index; + } + #endif + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_2d_tile_2d_with_uarch.tile_range_j; + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_result_size_t index = fxdiv_divide_size_t(range_start, tile_range_j); + const size_t range_i = threadpool->params.parallelize_2d_tile_2d_with_uarch.range_i; + const size_t tile_i = threadpool->params.parallelize_2d_tile_2d_with_uarch.tile_i; + const size_t range_j = threadpool->params.parallelize_2d_tile_2d_with_uarch.range_j; + const size_t tile_j = threadpool->params.parallelize_2d_tile_2d_with_uarch.tile_j; + size_t start_i = index.quotient * tile_i; + size_t start_j = index.remainder * tile_j; + + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, uarch_index, start_i, start_j, min(range_i - start_i, tile_i), min(range_j - start_j, tile_j)); + start_j += tile_j; + if (start_j >= range_j) { + start_j = 0; + start_i += tile_i; + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(linear_index, tile_range_j); + const size_t start_i = tile_index_i_j.quotient * tile_i; + const size_t start_j = tile_index_i_j.remainder * tile_j; + task(argument, uarch_index, start_i, start_j, min(range_i - start_i, tile_i), min(range_j - start_j, tile_j)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_3d_tile_2d_t task = (pthreadpool_task_3d_tile_2d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_k = threadpool->params.parallelize_3d_tile_2d.tile_range_k; + const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(range_start, tile_range_k); + const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_3d_tile_2d.tile_range_j; + const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, tile_range_j); + const size_t tile_j = threadpool->params.parallelize_3d_tile_2d.tile_j; + const size_t tile_k = threadpool->params.parallelize_3d_tile_2d.tile_k; + size_t i = tile_index_i_j.quotient; + size_t start_j = tile_index_i_j.remainder * tile_j; + size_t start_k = tile_index_ij_k.remainder * tile_k; + + const size_t range_k = threadpool->params.parallelize_3d_tile_2d.range_k; + const size_t range_j = threadpool->params.parallelize_3d_tile_2d.range_j; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, i, start_j, start_k, min(range_j - start_j, tile_j), min(range_k - start_k, tile_k)); + start_k += tile_k; + if (start_k >= range_k) { + start_k = 0; + start_j += tile_j; + if (start_j >= range_j) { + start_j = 0; + i += 1; + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(linear_index, tile_range_k); + const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, tile_range_j); + const size_t start_j = tile_index_i_j.remainder * tile_j; + const size_t start_k = tile_index_ij_k.remainder * tile_k; + task(argument, tile_index_i_j.quotient, start_j, start_k, min(range_j - start_j, tile_j), min(range_k - start_k, tile_k)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_with_uarch_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_3d_tile_2d_with_id_t task = (pthreadpool_task_3d_tile_2d_with_id_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const uint32_t default_uarch_index = threadpool->params.parallelize_3d_tile_2d_with_uarch.default_uarch_index; + uint32_t uarch_index = default_uarch_index; + #if PTHREADPOOL_USE_CPUINFO + uarch_index = cpuinfo_get_current_uarch_index(); + if (uarch_index > threadpool->params.parallelize_3d_tile_2d_with_uarch.max_uarch_index) { + uarch_index = default_uarch_index; + } + #endif + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_k = threadpool->params.parallelize_3d_tile_2d_with_uarch.tile_range_k; + const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(range_start, tile_range_k); + const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_3d_tile_2d_with_uarch.tile_range_j; + const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, tile_range_j); + const size_t tile_j = threadpool->params.parallelize_3d_tile_2d_with_uarch.tile_j; + const size_t tile_k = threadpool->params.parallelize_3d_tile_2d_with_uarch.tile_k; + size_t i = tile_index_i_j.quotient; + size_t start_j = tile_index_i_j.remainder * tile_j; + size_t start_k = tile_index_ij_k.remainder * tile_k; + + const size_t range_k = threadpool->params.parallelize_3d_tile_2d_with_uarch.range_k; + const size_t range_j = threadpool->params.parallelize_3d_tile_2d_with_uarch.range_j; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, uarch_index, i, start_j, start_k, min(range_j - start_j, tile_j), min(range_k - start_k, tile_k)); + start_k += tile_k; + if (start_k >= range_k) { + start_k = 0; + start_j += tile_j; + if (start_j >= range_j) { + start_j = 0; + i += 1; + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(linear_index, tile_range_k); + const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, tile_range_j); + const size_t start_j = tile_index_i_j.remainder * tile_j; + const size_t start_k = tile_index_ij_k.remainder * tile_k; + task(argument, uarch_index, tile_index_i_j.quotient, start_j, start_k, min(range_j - start_j, tile_j), min(range_k - start_k, tile_k)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_4d_tile_2d_t task = (pthreadpool_task_4d_tile_2d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_kl = threadpool->params.parallelize_4d_tile_2d.tile_range_kl; + const struct fxdiv_result_size_t tile_index_ij_kl = fxdiv_divide_size_t(range_start, tile_range_kl); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_4d_tile_2d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_kl.quotient, range_j); + const struct fxdiv_divisor_size_t tile_range_l = threadpool->params.parallelize_4d_tile_2d.tile_range_l; + const struct fxdiv_result_size_t tile_index_k_l = fxdiv_divide_size_t(tile_index_ij_kl.remainder, tile_range_l); + const size_t tile_k = threadpool->params.parallelize_4d_tile_2d.tile_k; + const size_t tile_l = threadpool->params.parallelize_4d_tile_2d.tile_l; + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t start_k = tile_index_k_l.quotient * tile_k; + size_t start_l = tile_index_k_l.remainder * tile_l; + + const size_t range_l = threadpool->params.parallelize_4d_tile_2d.range_l; + const size_t range_k = threadpool->params.parallelize_4d_tile_2d.range_k; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, i, j, start_k, start_l, min(range_k - start_k, tile_k), min(range_l - start_l, tile_l)); + start_l += tile_l; + if (start_l >= range_l) { + start_l = 0; + start_k += tile_k; + if (start_k >= range_k) { + start_k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_ij_kl = fxdiv_divide_size_t(linear_index, tile_range_kl); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_kl.quotient, range_j); + const struct fxdiv_result_size_t tile_index_k_l = fxdiv_divide_size_t(tile_index_ij_kl.remainder, tile_range_l); + const size_t start_k = tile_index_k_l.quotient * tile_k; + const size_t start_l = tile_index_k_l.remainder * tile_l; + task(argument, index_i_j.quotient, index_i_j.remainder, start_k, start_l, min(range_k - start_k, tile_k), min(range_l - start_l, tile_l)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_with_uarch_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_4d_tile_2d_with_id_t task = (pthreadpool_task_4d_tile_2d_with_id_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const uint32_t default_uarch_index = threadpool->params.parallelize_4d_tile_2d_with_uarch.default_uarch_index; + uint32_t uarch_index = default_uarch_index; + #if PTHREADPOOL_USE_CPUINFO + uarch_index = cpuinfo_get_current_uarch_index(); + if (uarch_index > threadpool->params.parallelize_4d_tile_2d_with_uarch.max_uarch_index) { + uarch_index = default_uarch_index; + } + #endif + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_kl = threadpool->params.parallelize_4d_tile_2d_with_uarch.tile_range_kl; + const struct fxdiv_result_size_t tile_index_ij_kl = fxdiv_divide_size_t(range_start, tile_range_kl); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_4d_tile_2d_with_uarch.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_kl.quotient, range_j); + const struct fxdiv_divisor_size_t tile_range_l = threadpool->params.parallelize_4d_tile_2d_with_uarch.tile_range_l; + const struct fxdiv_result_size_t tile_index_k_l = fxdiv_divide_size_t(tile_index_ij_kl.remainder, tile_range_l); + const size_t tile_k = threadpool->params.parallelize_4d_tile_2d_with_uarch.tile_k; + const size_t tile_l = threadpool->params.parallelize_4d_tile_2d_with_uarch.tile_l; + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t start_k = tile_index_k_l.quotient * tile_k; + size_t start_l = tile_index_k_l.remainder * tile_l; + + const size_t range_l = threadpool->params.parallelize_4d_tile_2d_with_uarch.range_l; + const size_t range_k = threadpool->params.parallelize_4d_tile_2d_with_uarch.range_k; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, uarch_index, i, j, start_k, start_l, min(range_k - start_k, tile_k), min(range_l - start_l, tile_l)); + start_l += tile_l; + if (start_l >= range_l) { + start_l = 0; + start_k += tile_k; + if (start_k >= range_k) { + start_k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_ij_kl = fxdiv_divide_size_t(linear_index, tile_range_kl); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_kl.quotient, range_j); + const struct fxdiv_result_size_t tile_index_k_l = fxdiv_divide_size_t(tile_index_ij_kl.remainder, tile_range_l); + const size_t start_k = tile_index_k_l.quotient * tile_k; + const size_t start_l = tile_index_k_l.remainder * tile_l; + task(argument, uarch_index, index_i_j.quotient, index_i_j.remainder, start_k, start_l, min(range_k - start_k, tile_k), min(range_l - start_l, tile_l)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_5d_tile_2d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_5d_tile_2d_t task = (pthreadpool_task_5d_tile_2d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_lm = threadpool->params.parallelize_5d_tile_2d.tile_range_lm; + const struct fxdiv_result_size_t tile_index_ijk_lm = fxdiv_divide_size_t(range_start, tile_range_lm); + const struct fxdiv_divisor_size_t range_k = threadpool->params.parallelize_5d_tile_2d.range_k; + const struct fxdiv_result_size_t index_ij_k = fxdiv_divide_size_t(tile_index_ijk_lm.quotient, range_k); + const struct fxdiv_divisor_size_t tile_range_m = threadpool->params.parallelize_5d_tile_2d.tile_range_m; + const struct fxdiv_result_size_t tile_index_l_m = fxdiv_divide_size_t(tile_index_ijk_lm.remainder, tile_range_m); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_5d_tile_2d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_k.quotient, range_j); + const size_t tile_l = threadpool->params.parallelize_5d_tile_2d.tile_l; + const size_t tile_m = threadpool->params.parallelize_5d_tile_2d.tile_m; + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t k = index_ij_k.remainder; + size_t start_l = tile_index_l_m.quotient * tile_l; + size_t start_m = tile_index_l_m.remainder * tile_m; + + const size_t range_m = threadpool->params.parallelize_5d_tile_2d.range_m; + const size_t range_l = threadpool->params.parallelize_5d_tile_2d.range_l; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, i, j, k, start_l, start_m, min(range_l - start_l, tile_l), min(range_m - start_m, tile_m)); + start_m += tile_m; + if (start_m >= range_m) { + start_m = 0; + start_l += tile_l; + if (start_l >= range_l) { + start_l = 0; + if (++k == range_k.value) { + k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_ijk_lm = fxdiv_divide_size_t(linear_index, tile_range_lm); + const struct fxdiv_result_size_t index_ij_k = fxdiv_divide_size_t(tile_index_ijk_lm.quotient, range_k); + const struct fxdiv_result_size_t tile_index_l_m = fxdiv_divide_size_t(tile_index_ijk_lm.remainder, tile_range_m); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_k.quotient, range_j); + const size_t start_l = tile_index_l_m.quotient * tile_l; + const size_t start_m = tile_index_l_m.remainder * tile_m; + task(argument, index_i_j.quotient, index_i_j.remainder, index_ij_k.remainder, + start_l, start_m, min(range_l - start_l, tile_l), min(range_m - start_m, tile_m)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_6d_tile_2d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_6d_tile_2d_t task = (pthreadpool_task_6d_tile_2d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_mn = threadpool->params.parallelize_6d_tile_2d.tile_range_mn; + const struct fxdiv_result_size_t tile_index_ijkl_mn = fxdiv_divide_size_t(range_start, tile_range_mn); + const struct fxdiv_divisor_size_t range_kl = threadpool->params.parallelize_6d_tile_2d.range_kl; + const struct fxdiv_result_size_t index_ij_kl = fxdiv_divide_size_t(tile_index_ijkl_mn.quotient, range_kl); + const struct fxdiv_divisor_size_t tile_range_n = threadpool->params.parallelize_6d_tile_2d.tile_range_n; + const struct fxdiv_result_size_t tile_index_m_n = fxdiv_divide_size_t(tile_index_ijkl_mn.remainder, tile_range_n); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_6d_tile_2d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_kl.quotient, range_j); + const struct fxdiv_divisor_size_t range_l = threadpool->params.parallelize_6d_tile_2d.range_l; + const struct fxdiv_result_size_t index_k_l = fxdiv_divide_size_t(index_ij_kl.remainder, range_l); + const size_t tile_m = threadpool->params.parallelize_6d_tile_2d.tile_m; + const size_t tile_n = threadpool->params.parallelize_6d_tile_2d.tile_n; + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t k = index_k_l.quotient; + size_t l = index_k_l.remainder; + size_t start_m = tile_index_m_n.quotient * tile_m; + size_t start_n = tile_index_m_n.remainder * tile_n; + + const size_t range_n = threadpool->params.parallelize_6d_tile_2d.range_n; + const size_t range_m = threadpool->params.parallelize_6d_tile_2d.range_m; + const size_t range_k = threadpool->params.parallelize_6d_tile_2d.range_k; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, i, j, k, l, start_m, start_n, min(range_m - start_m, tile_m), min(range_n - start_n, tile_n)); + start_n += tile_n; + if (start_n >= range_n) { + start_n = 0; + start_m += tile_m; + if (start_m >= range_m) { + start_m = 0; + if (++l == range_l.value) { + l = 0; + if (++k == range_k) { + k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_ijkl_mn = fxdiv_divide_size_t(linear_index, tile_range_mn); + const struct fxdiv_result_size_t index_ij_kl = fxdiv_divide_size_t(tile_index_ijkl_mn.quotient, range_kl); + const struct fxdiv_result_size_t tile_index_m_n = fxdiv_divide_size_t(tile_index_ijkl_mn.remainder, tile_range_n); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_kl.quotient, range_j); + const struct fxdiv_result_size_t index_k_l = fxdiv_divide_size_t(index_ij_kl.remainder, range_l); + const size_t start_m = tile_index_m_n.quotient * tile_m; + const size_t start_n = tile_index_m_n.remainder * tile_n; + task(argument, index_i_j.quotient, index_i_j.remainder, index_k_l.quotient, index_k_l.remainder, + start_m, start_n, min(range_m - start_m, tile_m), min(range_n - start_n, tile_n)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} diff --git a/src/portable-api.c b/src/portable-api.c index 84d6eda..6b16674 100644 --- a/src/portable-api.c +++ b/src/portable-api.c @@ -35,6 +35,7 @@ static void thread_parallelize_1d(struct pthreadpool* threadpool, struct thread_ const pthreadpool_task_1d_t task = (pthreadpool_task_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + /* Process thread's own range of items */ size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) { @@ -740,7 +741,8 @@ void pthreadpool_parallelize_1d( size_t range, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || range <= 1) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || range <= 1) { /* No thread pool used: execute task sequentially on the calling thread */ struct fpu_state saved_fpu_state = { 0 }; if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { @@ -754,8 +756,15 @@ void pthreadpool_parallelize_1d( set_fpu_state(saved_fpu_state); } } else { + thread_function_t parallelize_1d = &thread_parallelize_1d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (range < range_threshold) { + parallelize_1d = &pthreadpool_thread_parallelize_1d_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_1d, NULL, 0, + threadpool, parallelize_1d, NULL, 0, (void*) task, argument, range, flags); } } @@ -769,7 +778,8 @@ void pthreadpool_parallelize_1d_with_uarch( size_t range, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || range <= 1) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || range <= 1) { /* No thread pool used: execute task sequentially on the calling thread */ uint32_t uarch_index = default_uarch_index; @@ -796,8 +806,15 @@ void pthreadpool_parallelize_1d_with_uarch( .default_uarch_index = default_uarch_index, .max_uarch_index = max_uarch_index, }; + thread_function_t parallelize_1d_with_uarch = &thread_parallelize_1d_with_uarch; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (range < range_threshold) { + parallelize_1d_with_uarch = &pthreadpool_thread_parallelize_1d_with_uarch_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_1d_with_uarch, ¶ms, sizeof(params), + threadpool, parallelize_1d_with_uarch, ¶ms, sizeof(params), task, argument, range, flags); } } @@ -810,7 +827,8 @@ void pthreadpool_parallelize_1d_tile_1d( size_t tile, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || range <= tile) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || range <= tile) { /* No thread pool used: execute task sequentially on the calling thread */ struct fpu_state saved_fpu_state = { 0 }; if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { @@ -824,13 +842,21 @@ void pthreadpool_parallelize_1d_tile_1d( set_fpu_state(saved_fpu_state); } } else { + const size_t tile_range = divide_round_up(range, tile); const struct pthreadpool_1d_tile_1d_params params = { .range = range, .tile = tile, }; + thread_function_t parallelize_1d_tile_1d = &thread_parallelize_1d_tile_1d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (range < range_threshold) { + parallelize_1d_tile_1d = &pthreadpool_thread_parallelize_1d_tile_1d_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_1d_tile_1d, ¶ms, sizeof(params), - task, argument, divide_round_up(range, tile), flags); + threadpool, parallelize_1d_tile_1d, ¶ms, sizeof(params), + task, argument, tile_range, flags); } } @@ -842,7 +868,8 @@ void pthreadpool_parallelize_2d( size_t range_j, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || (range_i | range_j) <= 1) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i | range_j) <= 1) { /* No thread pool used: execute task sequentially on the calling thread */ struct fpu_state saved_fpu_state = { 0 }; if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { @@ -858,12 +885,20 @@ void pthreadpool_parallelize_2d( set_fpu_state(saved_fpu_state); } } else { + const size_t range = range_i * range_j; const struct pthreadpool_2d_params params = { .range_j = fxdiv_init_size_t(range_j), }; + thread_function_t parallelize_2d = &thread_parallelize_2d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (range < range_threshold) { + parallelize_2d = &pthreadpool_thread_parallelize_2d_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_2d, ¶ms, sizeof(params), - task, argument, range_i * range_j, flags); + threadpool, parallelize_2d, ¶ms, sizeof(params), + task, argument, range, flags); } } @@ -876,7 +911,8 @@ void pthreadpool_parallelize_2d_tile_1d( size_t tile_j, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || (range_i <= 1 && range_j <= tile_j)) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i <= 1 && range_j <= tile_j)) { /* No thread pool used: execute task sequentially on the calling thread */ struct fpu_state saved_fpu_state = { 0 }; if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { @@ -893,14 +929,22 @@ void pthreadpool_parallelize_2d_tile_1d( } } else { const size_t tile_range_j = divide_round_up(range_j, tile_j); + const size_t tile_range = range_i * tile_range_j; const struct pthreadpool_2d_tile_1d_params params = { .range_j = range_j, .tile_j = tile_j, .tile_range_j = fxdiv_init_size_t(tile_range_j), }; + thread_function_t parallelize_2d_tile_1d = &thread_parallelize_2d_tile_1d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (tile_range < range_threshold) { + parallelize_2d_tile_1d = &pthreadpool_thread_parallelize_2d_tile_1d_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_2d_tile_1d, ¶ms, sizeof(params), - task, argument, range_i * tile_range_j, flags); + threadpool, parallelize_2d_tile_1d, ¶ms, sizeof(params), + task, argument, tile_range, flags); } } @@ -914,7 +958,8 @@ void pthreadpool_parallelize_2d_tile_2d( size_t tile_j, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || (range_i <= tile_i && range_j <= tile_j)) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i <= tile_i && range_j <= tile_j)) { /* No thread pool used: execute task sequentially on the calling thread */ struct fpu_state saved_fpu_state = { 0 }; if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { @@ -932,6 +977,7 @@ void pthreadpool_parallelize_2d_tile_2d( } else { const size_t tile_range_i = divide_round_up(range_i, tile_i); const size_t tile_range_j = divide_round_up(range_j, tile_j); + const size_t tile_range = tile_range_i * tile_range_j; const struct pthreadpool_2d_tile_2d_params params = { .range_i = range_i, .tile_i = tile_i, @@ -939,9 +985,16 @@ void pthreadpool_parallelize_2d_tile_2d( .tile_j = tile_j, .tile_range_j = fxdiv_init_size_t(tile_range_j), }; + thread_function_t parallelize_2d_tile_2d = &thread_parallelize_2d_tile_2d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (tile_range < range_threshold) { + parallelize_2d_tile_2d = &pthreadpool_thread_parallelize_2d_tile_2d_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_2d_tile_2d, ¶ms, sizeof(params), - task, argument, tile_range_i * tile_range_j, flags); + threadpool, parallelize_2d_tile_2d, ¶ms, sizeof(params), + task, argument, tile_range, flags); } } @@ -957,7 +1010,8 @@ void pthreadpool_parallelize_2d_tile_2d_with_uarch( size_t tile_j, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || (range_i <= tile_i && range_j <= tile_j)) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i <= tile_i && range_j <= tile_j)) { /* No thread pool used: execute task sequentially on the calling thread */ uint32_t uarch_index = default_uarch_index; @@ -984,6 +1038,7 @@ void pthreadpool_parallelize_2d_tile_2d_with_uarch( } else { const size_t tile_range_i = divide_round_up(range_i, tile_i); const size_t tile_range_j = divide_round_up(range_j, tile_j); + const size_t tile_range = tile_range_i * tile_range_j; const struct pthreadpool_2d_tile_2d_with_uarch_params params = { .default_uarch_index = default_uarch_index, .max_uarch_index = max_uarch_index, @@ -993,9 +1048,16 @@ void pthreadpool_parallelize_2d_tile_2d_with_uarch( .tile_j = tile_j, .tile_range_j = fxdiv_init_size_t(tile_range_j), }; + thread_function_t parallelize_2d_tile_2d_with_uarch = &thread_parallelize_2d_tile_2d_with_uarch; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (tile_range < range_threshold) { + parallelize_2d_tile_2d_with_uarch = &pthreadpool_thread_parallelize_2d_tile_2d_with_uarch_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_2d_tile_2d_with_uarch, ¶ms, sizeof(params), - task, argument, tile_range_i * tile_range_j, flags); + threadpool, parallelize_2d_tile_2d_with_uarch, ¶ms, sizeof(params), + task, argument, tile_range, flags); } } @@ -1010,7 +1072,8 @@ void pthreadpool_parallelize_3d_tile_2d( size_t tile_k, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || (range_i <= 1 && range_j <= tile_j && range_k <= tile_k)) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i <= 1 && range_j <= tile_j && range_k <= tile_k)) { /* No thread pool used: execute task sequentially on the calling thread */ struct fpu_state saved_fpu_state = { 0 }; if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { @@ -1030,6 +1093,7 @@ void pthreadpool_parallelize_3d_tile_2d( } else { const size_t tile_range_j = divide_round_up(range_j, tile_j); const size_t tile_range_k = divide_round_up(range_k, tile_k); + const size_t tile_range = range_i * tile_range_j * tile_range_k; const struct pthreadpool_3d_tile_2d_params params = { .range_j = range_j, .tile_j = tile_j, @@ -1038,9 +1102,16 @@ void pthreadpool_parallelize_3d_tile_2d( .tile_range_j = fxdiv_init_size_t(tile_range_j), .tile_range_k = fxdiv_init_size_t(tile_range_k), }; + thread_function_t parallelize_3d_tile_2d = &thread_parallelize_3d_tile_2d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (tile_range < range_threshold) { + parallelize_3d_tile_2d = &pthreadpool_thread_parallelize_3d_tile_2d_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_3d_tile_2d, ¶ms, sizeof(params), - task, argument, range_i * tile_range_j * tile_range_k, flags); + threadpool, parallelize_3d_tile_2d, ¶ms, sizeof(params), + task, argument, tile_range, flags); } } @@ -1057,7 +1128,8 @@ void pthreadpool_parallelize_3d_tile_2d_with_uarch( size_t tile_k, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || (range_i <= 1 && range_j <= tile_j && range_k <= tile_k)) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i <= 1 && range_j <= tile_j && range_k <= tile_k)) { /* No thread pool used: execute task sequentially on the calling thread */ uint32_t uarch_index = default_uarch_index; @@ -1086,6 +1158,7 @@ void pthreadpool_parallelize_3d_tile_2d_with_uarch( } else { const size_t tile_range_j = divide_round_up(range_j, tile_j); const size_t tile_range_k = divide_round_up(range_k, tile_k); + const size_t tile_range = range_i * tile_range_j * tile_range_k; const struct pthreadpool_3d_tile_2d_with_uarch_params params = { .default_uarch_index = default_uarch_index, .max_uarch_index = max_uarch_index, @@ -1096,9 +1169,16 @@ void pthreadpool_parallelize_3d_tile_2d_with_uarch( .tile_range_j = fxdiv_init_size_t(tile_range_j), .tile_range_k = fxdiv_init_size_t(tile_range_k), }; + thread_function_t parallelize_3d_tile_2d_with_uarch = &thread_parallelize_3d_tile_2d_with_uarch; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (tile_range < range_threshold) { + parallelize_3d_tile_2d_with_uarch = &pthreadpool_thread_parallelize_3d_tile_2d_with_uarch_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_3d_tile_2d_with_uarch, ¶ms, sizeof(params), - task, argument, range_i * tile_range_j * tile_range_k, flags); + threadpool, parallelize_3d_tile_2d_with_uarch, ¶ms, sizeof(params), + task, argument, tile_range, flags); } } @@ -1114,7 +1194,8 @@ void pthreadpool_parallelize_4d_tile_2d( size_t tile_l, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || ((range_i | range_j) <= 1 && range_k <= tile_k && range_l <= tile_l)) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j) <= 1 && range_k <= tile_k && range_l <= tile_l)) { /* No thread pool used: execute task sequentially on the calling thread */ struct fpu_state saved_fpu_state = { 0 }; if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { @@ -1137,6 +1218,7 @@ void pthreadpool_parallelize_4d_tile_2d( } else { const size_t tile_range_l = divide_round_up(range_l, tile_l); const size_t tile_range_kl = divide_round_up(range_k, tile_k) * tile_range_l; + const size_t tile_range = range_i * range_j * tile_range_kl; const struct pthreadpool_4d_tile_2d_params params = { .range_k = range_k, .tile_k = tile_k, @@ -1146,9 +1228,16 @@ void pthreadpool_parallelize_4d_tile_2d( .tile_range_kl = fxdiv_init_size_t(tile_range_kl), .tile_range_l = fxdiv_init_size_t(tile_range_l), }; + thread_function_t parallelize_4d_tile_2d = &thread_parallelize_4d_tile_2d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (tile_range < range_threshold) { + parallelize_4d_tile_2d = &pthreadpool_thread_parallelize_4d_tile_2d_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_4d_tile_2d, ¶ms, sizeof(params), - task, argument, range_i * range_j * tile_range_kl, flags); + threadpool, parallelize_4d_tile_2d, ¶ms, sizeof(params), + task, argument, tile_range, flags); } } @@ -1166,7 +1255,8 @@ void pthreadpool_parallelize_4d_tile_2d_with_uarch( size_t tile_l, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || ((range_i | range_j) <= 1 && range_k <= tile_k && range_l <= tile_l)) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j) <= 1 && range_k <= tile_k && range_l <= tile_l)) { /* No thread pool used: execute task sequentially on the calling thread */ uint32_t uarch_index = default_uarch_index; @@ -1198,6 +1288,7 @@ void pthreadpool_parallelize_4d_tile_2d_with_uarch( } else { const size_t tile_range_l = divide_round_up(range_l, tile_l); const size_t tile_range_kl = divide_round_up(range_k, tile_k) * tile_range_l; + const size_t tile_range = range_i * range_j * tile_range_kl; const struct pthreadpool_4d_tile_2d_with_uarch_params params = { .default_uarch_index = default_uarch_index, .max_uarch_index = max_uarch_index, @@ -1209,9 +1300,16 @@ void pthreadpool_parallelize_4d_tile_2d_with_uarch( .tile_range_kl = fxdiv_init_size_t(tile_range_kl), .tile_range_l = fxdiv_init_size_t(tile_range_l), }; + thread_function_t parallelize_4d_tile_2d_with_uarch = &thread_parallelize_4d_tile_2d_with_uarch; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (tile_range < range_threshold) { + parallelize_4d_tile_2d_with_uarch = &pthreadpool_thread_parallelize_4d_tile_2d_with_uarch_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_4d_tile_2d_with_uarch, ¶ms, sizeof(params), - task, argument, range_i * range_j * tile_range_kl, flags); + threadpool, parallelize_4d_tile_2d_with_uarch, ¶ms, sizeof(params), + task, argument, tile_range, flags); } } @@ -1228,7 +1326,8 @@ void pthreadpool_parallelize_5d_tile_2d( size_t tile_m, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || ((range_i | range_j | range_k) <= 1 && range_l <= tile_l && range_m <= tile_m)) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j | range_k) <= 1 && range_l <= tile_l && range_m <= tile_m)) { /* No thread pool used: execute task sequentially on the calling thread */ struct fpu_state saved_fpu_state = { 0 }; if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { @@ -1253,6 +1352,7 @@ void pthreadpool_parallelize_5d_tile_2d( } else { const size_t tile_range_m = divide_round_up(range_m, tile_m); const size_t tile_range_lm = divide_round_up(range_l, tile_l) * tile_range_m; + const size_t tile_range = range_i * range_j * range_k * tile_range_lm; const struct pthreadpool_5d_tile_2d_params params = { .range_l = range_l, .tile_l = tile_l, @@ -1263,9 +1363,16 @@ void pthreadpool_parallelize_5d_tile_2d( .tile_range_lm = fxdiv_init_size_t(tile_range_lm), .tile_range_m = fxdiv_init_size_t(tile_range_m), }; + thread_function_t parallelize_5d_tile_2d = &thread_parallelize_5d_tile_2d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (tile_range < range_threshold) { + parallelize_5d_tile_2d = &pthreadpool_thread_parallelize_5d_tile_2d_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_5d_tile_2d, ¶ms, sizeof(params), - task, argument, range_i * range_j * range_k * tile_range_lm, flags); + threadpool, parallelize_5d_tile_2d, ¶ms, sizeof(params), + task, argument, tile_range, flags); } } @@ -1283,7 +1390,8 @@ void pthreadpool_parallelize_6d_tile_2d( size_t tile_n, uint32_t flags) { - if (threadpool == NULL || threadpool->threads_count.value <= 1 || ((range_i | range_j | range_k | range_l) <= 1 && range_m <= tile_m && range_n <= tile_n)) { + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j | range_k | range_l) <= 1 && range_m <= tile_m && range_n <= tile_n)) { /* No thread pool used: execute task sequentially on the calling thread */ struct fpu_state saved_fpu_state = { 0 }; if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { @@ -1311,6 +1419,7 @@ void pthreadpool_parallelize_6d_tile_2d( const size_t range_kl = range_k * range_l; const size_t tile_range_n = divide_round_up(range_n, tile_n); const size_t tile_range_mn = divide_round_up(range_m, tile_m) * tile_range_n; + const size_t tile_range = range_i * range_j * range_kl * tile_range_mn; const struct pthreadpool_6d_tile_2d_params params = { .range_k = range_k, .range_m = range_m, @@ -1323,8 +1432,15 @@ void pthreadpool_parallelize_6d_tile_2d( .tile_range_mn = fxdiv_init_size_t(tile_range_mn), .tile_range_n = fxdiv_init_size_t(tile_range_n), }; + thread_function_t parallelize_6d_tile_2d = &thread_parallelize_6d_tile_2d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (tile_range < range_threshold) { + parallelize_6d_tile_2d = &pthreadpool_thread_parallelize_6d_tile_2d_fastpath; + } + #endif pthreadpool_parallelize( - threadpool, &thread_parallelize_6d_tile_2d, ¶ms, sizeof(params), - task, argument, range_i * range_j * range_kl * tile_range_mn, flags); + threadpool, parallelize_6d_tile_2d, ¶ms, sizeof(params), + task, argument, tile_range, flags); } } diff --git a/src/threadpool-object.h b/src/threadpool-object.h index 239d116..7b643c6 100644 --- a/src/threadpool-object.h +++ b/src/threadpool-object.h @@ -526,3 +526,55 @@ PTHREADPOOL_INTERNAL void pthreadpool_parallelize( void* context, size_t linear_range, uint32_t flags); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_with_uarch_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_tile_1d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_1d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_with_uarch_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_with_uarch_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_with_uarch_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_5d_tile_2d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_6d_tile_2d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); -- cgit v1.2.3 From 28f251ed286e4b9d1f8e4d2912ff95943cc06aaf Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Mon, 4 May 2020 15:00:58 -0700 Subject: Fix Emscripten build with Bazel --- BUILD.bazel | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/BUILD.bazel b/BUILD.bazel index fa00807..0bad9a7 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -16,24 +16,6 @@ PORTABLE_SRCS = [ "src/portable-api.c", ] -PTHREADS_IMPL_SRCS = PORTABLE_SRCS + ["src/pthreads.c"] - -GCD_IMPL_SRCS = PORTABLE_SRCS + ["src/gcd.c"] - -SHIM_IMPL_SRCS = ["src/shim.c"] - -INTERNAL_HDRS = [ - "src/threadpool-atomics.h", - "src/threadpool-common.h", - "src/threadpool-object.h", - "src/threadpool-utils.h", -] - -PORTABLE_SRCS = [ - "src/memory.c", - "src/portable-api.c", -] - ARCH_SPECIFIC_SRCS = [ "src/fastpath.c", ] @@ -54,7 +36,7 @@ cc_library( ":pthreadpool_sync_primitive_explicit_gcd": INTERNAL_HDRS + GCD_IMPL_SRCS, ":pthreadpool_sync_primitive_explicit_event": INTERNAL_HDRS + WINDOWS_IMPL_SRCS, ":emscripten_with_threads": INTERNAL_HDRS + PTHREADS_IMPL_SRCS, - ":emscripten": SHIM_IMPL_SRCS, + ":emscripten": INTERNAL_HDRS + SHIM_IMPL_SRCS, ":macos_x86": INTERNAL_HDRS + GCD_IMPL_SRCS, ":macos_x86_64": INTERNAL_HDRS + GCD_IMPL_SRCS, ":ios": INTERNAL_HDRS + GCD_IMPL_SRCS, -- cgit v1.2.3 From 6525d8bb736b323eb4df9e4f3afdd3a8458d1a20 Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Mon, 4 May 2020 15:02:25 -0700 Subject: Avoid including stdatomic.h in any WAsm builds --- src/threadpool-atomics.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/threadpool-atomics.h b/src/threadpool-atomics.h index 474d12b..135166f 100644 --- a/src/threadpool-atomics.h +++ b/src/threadpool-atomics.h @@ -13,7 +13,7 @@ #endif -#if defined(__wasm__) && defined(__EMSCRIPTEN_PTHREADS__) && defined(__clang__) +#if defined(__wasm__) && defined(__clang__) /* * Clang for WebAssembly target lacks stdatomic.h header, * even though it supports the necessary low-level intrinsics. -- cgit v1.2.3 From ada4eedd7113b82e7d1ab35394b78eaf03a60906 Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Thu, 7 May 2020 15:57:07 -0700 Subject: Thumb-1 compatible assembly for disable_fpu_denormals --- src/threadpool-utils.h | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/threadpool-utils.h b/src/threadpool-utils.h index 24fee43..5443c6e 100644 --- a/src/threadpool-utils.h +++ b/src/threadpool-utils.h @@ -55,12 +55,21 @@ static inline void disable_fpu_denormals() { #if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) _mm_setcsr(_mm_getcsr() | 0x8040); #elif defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0) - uint32_t fpscr; - __asm__ __volatile__( - "VMRS %[fpscr], fpscr\n" - "ORR %[fpscr], #0x1000000\n" - "VMSR fpscr, %[fpscr]\n" - : [fpscr] "=r" (fpscr)); + #if defined(__thumb__) && !defined(__thumb2__) + __asm__ __volatile__( + "VMRS %[fpscr], fpscr\n" + "ORRS %[fpscr], %[bitmask]\n" + "VMSR fpscr, %[fpscr]\n" + : [fpscr] "=l" (fpscr) + : [bitmask] "l" (0x1000000) + : "cc"); + #else + __asm__ __volatile__( + "VMRS %[fpscr], fpscr\n" + "ORR %[fpscr], #0x1000000\n" + "VMSR fpscr, %[fpscr]\n" + : [fpscr] "=r" (fpscr)); + #endif #elif defined(__aarch64__) uint64_t fpcr; __asm__ __volatile__( -- cgit v1.2.3 From 5690b5ceada160444a916d31ef72e381f5e52d67 Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Thu, 7 May 2020 17:23:12 -0700 Subject: MSVC-compatible FPU state functions --- src/threadpool-utils.h | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/threadpool-utils.h b/src/threadpool-utils.h index 5443c6e..d95e3a5 100644 --- a/src/threadpool-utils.h +++ b/src/threadpool-utils.h @@ -9,7 +9,7 @@ #endif /* MSVC-specific headers */ -#if defined(_MSC_VER) && _MSC_VER >= 1920 +#if defined(_MSC_VER) #include #if defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) #include @@ -20,9 +20,9 @@ struct fpu_state { #if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) uint32_t mxcsr; -#elif defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0) +#elif defined(__GNUC__) && defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0) || defined(_MSC_VER) && defined(_M_ARM) uint32_t fpscr; -#elif defined(__aarch64__) +#elif defined(__GNUC__) && defined(__aarch64__) || defined(_MSC_VER) && defined(_M_ARM64) uint64_t fpcr; #else char unused; @@ -33,9 +33,13 @@ static inline struct fpu_state get_fpu_state() { struct fpu_state state = { 0 }; #if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) state.mxcsr = (uint32_t) _mm_getcsr(); -#elif defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0) +#elif defined(_MSC_VER) && defined(_M_ARM) + state.fpscr = (uint32_t) _MoveFromCoprocessor(10, 7, 1, 0, 0); +#elif defined(_MSC_VER) && defined(_M_ARM64) + state.fpcr = (uint64_t) _ReadStatusReg(0x5A20); +#elif defined(__GNUC__) && defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0) __asm__ __volatile__("VMRS %[fpscr], fpscr" : [fpscr] "=r" (state.fpscr)); -#elif defined(__aarch64__) +#elif defined(__GNUC__) && defined(__aarch64__) __asm__ __volatile__("MRS %[fpcr], fpcr" : [fpcr] "=r" (state.fpcr)); #endif return state; @@ -44,9 +48,13 @@ static inline struct fpu_state get_fpu_state() { static inline void set_fpu_state(const struct fpu_state state) { #if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) _mm_setcsr((unsigned int) state.mxcsr); -#elif defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0) +#elif defined(_MSC_VER) && defined(_M_ARM) + _MoveToCoprocessor((int) state.fpscr, 10, 7, 1, 0, 0); +#elif defined(_MSC_VER) && defined(_M_ARM64) + _WriteStatusReg(0x5A20, (__int64) state.fpcr); +#elif defined(__GNUC__) && defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0) __asm__ __volatile__("VMSR fpscr, %[fpscr]" : : [fpscr] "r" (state.fpscr)); -#elif defined(__aarch64__) +#elif defined(__GNUC__) && defined(__aarch64__) __asm__ __volatile__("MSR fpcr, %[fpcr]" : : [fpcr] "r" (state.fpcr)); #endif } @@ -54,7 +62,16 @@ static inline void set_fpu_state(const struct fpu_state state) { static inline void disable_fpu_denormals() { #if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) _mm_setcsr(_mm_getcsr() | 0x8040); -#elif defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0) +#elif defined(_MSC_VER) && defined(_M_ARM) + int fpscr = _MoveFromCoprocessor(10, 7, 1, 0, 0); + fpscr |= 0x1000000; + _MoveToCoprocessor(fpscr, 10, 7, 1, 0, 0); +#elif defined(_MSC_VER) && defined(_M_ARM64) + __int64 fpcr = _ReadStatusReg(0x5A20); + fpcr |= 0x1080000; + _WriteStatusReg(0x5A20, fpcr); +#elif defined(__GNUC__) && defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0) + uint32_t fpscr; #if defined(__thumb__) && !defined(__thumb2__) __asm__ __volatile__( "VMRS %[fpscr], fpscr\n" @@ -70,7 +87,7 @@ static inline void disable_fpu_denormals() { "VMSR fpscr, %[fpscr]\n" : [fpscr] "=r" (fpscr)); #endif -#elif defined(__aarch64__) +#elif defined(__GNUC__) && defined(__aarch64__) uint64_t fpcr; __asm__ __volatile__( "MRS %[fpcr], fpcr\n" -- cgit v1.2.3 From 6ae95d3d7b407ac310e59958a92bc8be4583340e Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Fri, 8 May 2020 20:45:20 -0700 Subject: Use platform-specific yield/pause instructions --- src/pthreads.c | 6 ++---- src/threadpool-atomics.h | 33 +++++++++++++++++++++++++++++---- src/threadpool-utils.h | 13 +++++-------- src/windows.c | 6 ++---- 4 files changed, 38 insertions(+), 20 deletions(-) diff --git a/src/pthreads.c b/src/pthreads.c index 2d945a0..430ca79 100644 --- a/src/pthreads.c +++ b/src/pthreads.c @@ -108,8 +108,7 @@ static void wait_worker_threads(struct pthreadpool* threadpool) { /* Spin-wait */ for (uint32_t i = PTHREADPOOL_SPIN_WAIT_ITERATIONS; i != 0; i--) { - /* This fence serves as a sleep instruction */ - pthreadpool_fence_acquire(); + pthreadpool_yield(); #if PTHREADPOOL_USE_FUTEX has_active_threads = pthreadpool_load_acquire_uint32_t(&threadpool->has_active_threads); @@ -151,8 +150,7 @@ static uint32_t wait_for_new_command( if ((last_flags & PTHREADPOOL_FLAG_YIELD_WORKERS) == 0) { /* Spin-wait loop */ for (uint32_t i = PTHREADPOOL_SPIN_WAIT_ITERATIONS; i != 0; i--) { - /* This fence serves as a sleep instruction */ - pthreadpool_fence_acquire(); + pthreadpool_yield(); command = pthreadpool_load_acquire_uint32_t(&threadpool->command); if (command != last_command) { diff --git a/src/threadpool-atomics.h b/src/threadpool-atomics.h index 135166f..a8491e4 100644 --- a/src/threadpool-atomics.h +++ b/src/threadpool-atomics.h @@ -4,12 +4,19 @@ #include #include +/* SSE-specific headers */ +#if defined(__i386__) || defined(__i686__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64) + #include +#endif + +/* ARM-specific headers */ +#if defined(__ARM_ACLE) + #include +#endif + /* MSVC-specific headers */ #ifdef _MSC_VER #include - #if defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) - #include - #endif #endif @@ -123,7 +130,7 @@ static inline void pthreadpool_fence_release() { __c11_atomic_thread_fence(__ATOMIC_RELEASE); } -#elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_AMD64)) +#elif defined(_MSC_VER) && defined(_M_X64) typedef volatile uint32_t pthreadpool_atomic_uint32_t; typedef volatile size_t pthreadpool_atomic_size_t; typedef void *volatile pthreadpool_atomic_void_p; @@ -701,3 +708,21 @@ #else #error "Platform-specific implementation of threadpool-atomics.h required" #endif + +#if defined(__i386__) || defined(__i686__) || defined(__x86_64__) || defined(_M_IX86) || defined(_M_X64) + static inline void pthreadpool_yield() { + _mm_pause(); + } +#elif defined(__ARM_ACLE) || defined(_MSC_VER) && (defined(_M_ARM) || defined(_M_ARM64)) + static inline void pthreadpool_yield() { + __yield(); + } +#elif defined(__GNUC__) && (defined(__arm__) || defined(__aarch64__)) + static inline void pthreadpool_yield() { + __asm__ __volatile__("yield"); + } +#else + static inline void pthreadpool_yield() { + pthreadpool_fence_acquire(); + } +#endif diff --git a/src/threadpool-utils.h b/src/threadpool-utils.h index d95e3a5..91e2445 100644 --- a/src/threadpool-utils.h +++ b/src/threadpool-utils.h @@ -4,21 +4,18 @@ #include /* SSE-specific headers */ -#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) +#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) #include #endif /* MSVC-specific headers */ #if defined(_MSC_VER) #include - #if defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) - #include - #endif #endif struct fpu_state { -#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) +#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) uint32_t mxcsr; #elif defined(__GNUC__) && defined(__arm__) && defined(__ARM_FP) && (__ARM_FP != 0) || defined(_MSC_VER) && defined(_M_ARM) uint32_t fpscr; @@ -31,7 +28,7 @@ struct fpu_state { static inline struct fpu_state get_fpu_state() { struct fpu_state state = { 0 }; -#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) +#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) state.mxcsr = (uint32_t) _mm_getcsr(); #elif defined(_MSC_VER) && defined(_M_ARM) state.fpscr = (uint32_t) _MoveFromCoprocessor(10, 7, 1, 0, 0); @@ -46,7 +43,7 @@ static inline struct fpu_state get_fpu_state() { } static inline void set_fpu_state(const struct fpu_state state) { -#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) +#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) _mm_setcsr((unsigned int) state.mxcsr); #elif defined(_MSC_VER) && defined(_M_ARM) _MoveToCoprocessor((int) state.fpscr, 10, 7, 1, 0, 0); @@ -60,7 +57,7 @@ static inline void set_fpu_state(const struct fpu_state state) { } static inline void disable_fpu_denormals() { -#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || defined(_M_AMD64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) +#if defined(__SSE__) || defined(__x86_64__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) _mm_setcsr(_mm_getcsr() | 0x8040); #elif defined(_MSC_VER) && defined(_M_ARM) int fpscr = _MoveFromCoprocessor(10, 7, 1, 0, 0); diff --git a/src/windows.c b/src/windows.c index 19e534f..c9b88f7 100644 --- a/src/windows.c +++ b/src/windows.c @@ -35,8 +35,7 @@ static void wait_worker_threads(struct pthreadpool* threadpool, uint32_t event_i /* Spin-wait */ for (uint32_t i = PTHREADPOOL_SPIN_WAIT_ITERATIONS; i != 0; i--) { - /* This fence serves as a sleep instruction */ - pthreadpool_fence_acquire(); + pthreadpool_yield(); active_threads = pthreadpool_load_acquire_size_t(&threadpool->active_threads); if (active_threads == 0) { @@ -63,8 +62,7 @@ static uint32_t wait_for_new_command( if ((last_flags & PTHREADPOOL_FLAG_YIELD_WORKERS) == 0) { /* Spin-wait loop */ for (uint32_t i = PTHREADPOOL_SPIN_WAIT_ITERATIONS; i != 0; i--) { - /* This fence serves as a sleep instruction */ - pthreadpool_fence_acquire(); + pthreadpool_yield(); command = pthreadpool_load_acquire_uint32_t(&threadpool->command); if (command != last_command) { -- cgit v1.2.3 From 5d088b1f1436e060cf2e2cd25c8c7003544eb266 Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Fri, 8 May 2020 20:54:20 -0700 Subject: Reorder C11 atomics before MSVC x64 atomics clang-cl, which supports both, should prefer C11 atomics --- src/threadpool-atomics.h | 160 +++++++++++++++++++++++------------------------ 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/src/threadpool-atomics.h b/src/threadpool-atomics.h index a8491e4..cb193c6 100644 --- a/src/threadpool-atomics.h +++ b/src/threadpool-atomics.h @@ -130,239 +130,239 @@ static inline void pthreadpool_fence_release() { __c11_atomic_thread_fence(__ATOMIC_RELEASE); } -#elif defined(_MSC_VER) && defined(_M_X64) - typedef volatile uint32_t pthreadpool_atomic_uint32_t; - typedef volatile size_t pthreadpool_atomic_size_t; - typedef void *volatile pthreadpool_atomic_void_p; +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) + #include + + typedef _Atomic(uint32_t) pthreadpool_atomic_uint32_t; + typedef _Atomic(size_t) pthreadpool_atomic_size_t; + typedef _Atomic(void*) pthreadpool_atomic_void_p; static inline uint32_t pthreadpool_load_relaxed_uint32_t( pthreadpool_atomic_uint32_t* address) { - return *address; + return atomic_load_explicit(address, memory_order_relaxed); } static inline size_t pthreadpool_load_relaxed_size_t( pthreadpool_atomic_size_t* address) { - return *address; + return atomic_load_explicit(address, memory_order_relaxed); } static inline void* pthreadpool_load_relaxed_void_p( pthreadpool_atomic_void_p* address) { - return *address; + return atomic_load_explicit(address, memory_order_relaxed); } static inline uint32_t pthreadpool_load_acquire_uint32_t( pthreadpool_atomic_uint32_t* address) { - /* x86-64 loads always have acquire semantics; use only a compiler barrier */ - const uint32_t value = *address; - _ReadBarrier(); - return value; + return atomic_load_explicit(address, memory_order_acquire); } static inline size_t pthreadpool_load_acquire_size_t( pthreadpool_atomic_size_t* address) { - /* x86-64 loads always have acquire semantics; use only a compiler barrier */ - const size_t value = *address; - _ReadBarrier(); - return value; + return atomic_load_explicit(address, memory_order_acquire); } static inline void pthreadpool_store_relaxed_uint32_t( pthreadpool_atomic_uint32_t* address, uint32_t value) { - *address = value; + atomic_store_explicit(address, value, memory_order_relaxed); } static inline void pthreadpool_store_relaxed_size_t( pthreadpool_atomic_size_t* address, size_t value) { - *address = value; + atomic_store_explicit(address, value, memory_order_relaxed); } static inline void pthreadpool_store_relaxed_void_p( pthreadpool_atomic_void_p* address, void* value) { - *address = value; + atomic_store_explicit(address, value, memory_order_relaxed); } static inline void pthreadpool_store_release_uint32_t( pthreadpool_atomic_uint32_t* address, uint32_t value) { - /* x86-64 stores always have release semantics; use only a compiler barrier */ - _WriteBarrier(); - *address = value; + atomic_store_explicit(address, value, memory_order_release); } static inline void pthreadpool_store_release_size_t( pthreadpool_atomic_size_t* address, size_t value) { - /* x86-64 stores always have release semantics; use only a compiler barrier */ - _WriteBarrier(); - *address = value; + atomic_store_explicit(address, value, memory_order_release); } static inline size_t pthreadpool_decrement_fetch_relaxed_size_t( pthreadpool_atomic_size_t* address) { - return (size_t) _InterlockedDecrement64((volatile __int64*) address); + return atomic_fetch_sub_explicit(address, 1, memory_order_relaxed) - 1; } static inline size_t pthreadpool_decrement_fetch_release_size_t( pthreadpool_atomic_size_t* address) { - return (size_t) _InterlockedDecrement64((volatile __int64*) address); + return atomic_fetch_sub_explicit(address, 1, memory_order_release) - 1; } static inline bool pthreadpool_try_decrement_relaxed_size_t( pthreadpool_atomic_size_t* value) { - size_t actual_value = *value; - while (actual_value != 0) { - const size_t new_value = actual_value - 1; - const size_t expected_value = actual_value; - actual_value = _InterlockedCompareExchange64( - (volatile __int64*) value, (__int64) new_value, (__int64) expected_value); - if (actual_value == expected_value) { - return true; + #if defined(__clang__) && (defined(__arm__) || defined(__aarch64__)) + size_t actual_value; + do { + actual_value = __builtin_arm_ldrex((const volatile size_t*) value); + if (actual_value == 0) { + __builtin_arm_clrex(); + return false; + } + } while (__builtin_arm_strex(actual_value - 1, (volatile size_t*) value) != 0); + return true; + #else + size_t actual_value = pthreadpool_load_relaxed_size_t(value); + while (actual_value != 0) { + if (atomic_compare_exchange_weak_explicit( + value, &actual_value, actual_value - 1, memory_order_relaxed, memory_order_relaxed)) + { + return true; + } } - } - return false; + return false; + #endif } static inline void pthreadpool_fence_acquire() { - _mm_lfence(); - _ReadBarrier(); + atomic_thread_fence(memory_order_acquire); } static inline void pthreadpool_fence_release() { - _WriteBarrier(); - _mm_sfence(); + atomic_thread_fence(memory_order_release); } -#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) - #include - - typedef _Atomic(uint32_t) pthreadpool_atomic_uint32_t; - typedef _Atomic(size_t) pthreadpool_atomic_size_t; - typedef _Atomic(void*) pthreadpool_atomic_void_p; +#elif defined(_MSC_VER) && defined(_M_X64) + typedef volatile uint32_t pthreadpool_atomic_uint32_t; + typedef volatile size_t pthreadpool_atomic_size_t; + typedef void *volatile pthreadpool_atomic_void_p; static inline uint32_t pthreadpool_load_relaxed_uint32_t( pthreadpool_atomic_uint32_t* address) { - return atomic_load_explicit(address, memory_order_relaxed); + return *address; } static inline size_t pthreadpool_load_relaxed_size_t( pthreadpool_atomic_size_t* address) { - return atomic_load_explicit(address, memory_order_relaxed); + return *address; } static inline void* pthreadpool_load_relaxed_void_p( pthreadpool_atomic_void_p* address) { - return atomic_load_explicit(address, memory_order_relaxed); + return *address; } static inline uint32_t pthreadpool_load_acquire_uint32_t( pthreadpool_atomic_uint32_t* address) { - return atomic_load_explicit(address, memory_order_acquire); + /* x86-64 loads always have acquire semantics; use only a compiler barrier */ + const uint32_t value = *address; + _ReadBarrier(); + return value; } static inline size_t pthreadpool_load_acquire_size_t( pthreadpool_atomic_size_t* address) { - return atomic_load_explicit(address, memory_order_acquire); + /* x86-64 loads always have acquire semantics; use only a compiler barrier */ + const size_t value = *address; + _ReadBarrier(); + return value; } static inline void pthreadpool_store_relaxed_uint32_t( pthreadpool_atomic_uint32_t* address, uint32_t value) { - atomic_store_explicit(address, value, memory_order_relaxed); + *address = value; } static inline void pthreadpool_store_relaxed_size_t( pthreadpool_atomic_size_t* address, size_t value) { - atomic_store_explicit(address, value, memory_order_relaxed); + *address = value; } static inline void pthreadpool_store_relaxed_void_p( pthreadpool_atomic_void_p* address, void* value) { - atomic_store_explicit(address, value, memory_order_relaxed); + *address = value; } static inline void pthreadpool_store_release_uint32_t( pthreadpool_atomic_uint32_t* address, uint32_t value) { - atomic_store_explicit(address, value, memory_order_release); + /* x86-64 stores always have release semantics; use only a compiler barrier */ + _WriteBarrier(); + *address = value; } static inline void pthreadpool_store_release_size_t( pthreadpool_atomic_size_t* address, size_t value) { - atomic_store_explicit(address, value, memory_order_release); + /* x86-64 stores always have release semantics; use only a compiler barrier */ + _WriteBarrier(); + *address = value; } static inline size_t pthreadpool_decrement_fetch_relaxed_size_t( pthreadpool_atomic_size_t* address) { - return atomic_fetch_sub_explicit(address, 1, memory_order_relaxed) - 1; + return (size_t) _InterlockedDecrement64((volatile __int64*) address); } static inline size_t pthreadpool_decrement_fetch_release_size_t( pthreadpool_atomic_size_t* address) { - return atomic_fetch_sub_explicit(address, 1, memory_order_release) - 1; + return (size_t) _InterlockedDecrement64((volatile __int64*) address); } static inline bool pthreadpool_try_decrement_relaxed_size_t( pthreadpool_atomic_size_t* value) { - #if defined(__clang__) && (defined(__arm__) || defined(__aarch64__)) - size_t actual_value; - do { - actual_value = __builtin_arm_ldrex((const volatile size_t*) value); - if (actual_value == 0) { - __builtin_arm_clrex(); - return false; - } - } while (__builtin_arm_strex(actual_value - 1, (volatile size_t*) value) != 0); - return true; - #else - size_t actual_value = pthreadpool_load_relaxed_size_t(value); - while (actual_value != 0) { - if (atomic_compare_exchange_weak_explicit( - value, &actual_value, actual_value - 1, memory_order_relaxed, memory_order_relaxed)) - { - return true; - } + size_t actual_value = *value; + while (actual_value != 0) { + const size_t new_value = actual_value - 1; + const size_t expected_value = actual_value; + actual_value = _InterlockedCompareExchange64( + (volatile __int64*) value, (__int64) new_value, (__int64) expected_value); + if (actual_value == expected_value) { + return true; } - return false; - #endif + } + return false; } static inline void pthreadpool_fence_acquire() { - atomic_thread_fence(memory_order_acquire); + _mm_lfence(); + _ReadBarrier(); } static inline void pthreadpool_fence_release() { - atomic_thread_fence(memory_order_release); + _WriteBarrier(); + _mm_sfence(); } #elif defined(_MSC_VER) && defined(_M_IX86) typedef volatile uint32_t pthreadpool_atomic_uint32_t; -- cgit v1.2.3 From afb880df0639945c854d70269f6b403cb81518b5 Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Sat, 16 May 2020 09:57:53 +0100 Subject: Guard against generating ARM yield instruction for unsupporting processors --- src/threadpool-atomics.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/threadpool-atomics.h b/src/threadpool-atomics.h index cb193c6..f0ddd89 100644 --- a/src/threadpool-atomics.h +++ b/src/threadpool-atomics.h @@ -717,7 +717,7 @@ static inline void pthreadpool_yield() { __yield(); } -#elif defined(__GNUC__) && (defined(__arm__) || defined(__aarch64__)) +#elif defined(__GNUC__) && (defined(__ARM_ARCH) && (__ARM_ARCH >= 7) || (defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6KZ__)) && !defined(__thumb__)) static inline void pthreadpool_yield() { __asm__ __volatile__("yield"); } -- cgit v1.2.3 From bfe07ff3d9ed6eb5e7803b9761c85b254a417742 Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Tue, 26 May 2020 09:41:08 -0700 Subject: 3D/4D/5D parallelization functions with 1D or no tiling --- include/pthreadpool.h | 270 ++++ src/fastpath.c | 377 ++++++ src/portable-api.c | 671 +++++++++- src/shim.c | 129 ++ src/threadpool-object.h | 160 +++ test/pthreadpool.cc | 3197 ++++++++++++++++++++++++++++++++++++++--------- 6 files changed, 4216 insertions(+), 588 deletions(-) diff --git a/include/pthreadpool.h b/include/pthreadpool.h index de4016b..6a7d61f 100644 --- a/include/pthreadpool.h +++ b/include/pthreadpool.h @@ -11,8 +11,14 @@ typedef void (*pthreadpool_task_1d_tile_1d_t)(void*, size_t, size_t); typedef void (*pthreadpool_task_2d_t)(void*, size_t, size_t); typedef void (*pthreadpool_task_2d_tile_1d_t)(void*, size_t, size_t, size_t); typedef void (*pthreadpool_task_2d_tile_2d_t)(void*, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_t)(void*, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_1d_t)(void*, size_t, size_t, size_t, size_t); typedef void (*pthreadpool_task_3d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_4d_t)(void*, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_4d_tile_1d_t)(void*, size_t, size_t, size_t, size_t, size_t); typedef void (*pthreadpool_task_4d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_5d_t)(void*, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_5d_tile_1d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t); typedef void (*pthreadpool_task_5d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t); typedef void (*pthreadpool_task_6d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t); @@ -359,6 +365,86 @@ void pthreadpool_parallelize_2d_tile_2d_with_uarch( size_t tile_j, uint32_t flags); +/** + * Process items on a 3D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * function(context, i, j, k); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d( + pthreadpool_t threadpool, + pthreadpool_task_3d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, i, j, k, min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param tile_k the maximum number of items along the third dimension of + * the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags); + /** * Process items on a 3D grid with the specified maximum tile size along the * last two grid dimensions. @@ -467,6 +553,94 @@ void pthreadpool_parallelize_3d_tile_2d_with_uarch( size_t tile_k, uint32_t flags); +/** + * Process items on a 4D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * function(context, i, j, k, l); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_4d( + pthreadpool_t threadpool, + pthreadpool_task_4d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + uint32_t flags); + +/** + * Process items on a 4D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l += tile_l) + * function(context, i, j, k, l, min(range_l - l, tile_l)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param tile_l the maximum number of items along the fourth dimension of + * the 4D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_4d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_4d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_l, + uint32_t flags); + /** * Process items on a 4D grid with the specified maximum tile size along the * last two grid dimensions. @@ -583,6 +757,102 @@ void pthreadpool_parallelize_4d_tile_2d_with_uarch( size_t tile_l, uint32_t flags); +/** + * Process items on a 5D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * function(context, i, j, k, l, m); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_5d( + pthreadpool_t threadpool, + pthreadpool_task_5d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + uint32_t flags); + +/** + * Process items on a 5D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m += tile_m) + * function(context, i, j, k, l, m, min(range_m - m, tile_m)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param tile_m the maximum number of items along the fifth dimension of + * the 5D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_5d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_5d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t tile_m, + uint32_t flags); + /** * Process items on a 5D grid with the specified maximum tile size along the * last two grid dimensions. diff --git a/src/fastpath.c b/src/fastpath.c index 1a5066a..b4e40c5 100644 --- a/src/fastpath.c +++ b/src/fastpath.c @@ -356,6 +356,116 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_with_uarch_f pthreadpool_fence_release(); } +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_3d_t task = (pthreadpool_task_3d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t range_k = threadpool->params.parallelize_3d.range_k; + const struct fxdiv_result_size_t index_ij_k = fxdiv_divide_size_t(range_start, range_k); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_3d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_k.quotient, range_j); + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t k = index_ij_k.remainder; + + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, i, j, k); + if (++k == range_k.value) { + k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t index_ij_k = fxdiv_divide_size_t(linear_index, range_k); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_k.quotient, range_j); + task(argument, index_i_j.quotient, index_i_j.remainder, index_ij_k.remainder); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_1d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_3d_tile_1d_t task = (pthreadpool_task_3d_tile_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_k = threadpool->params.parallelize_3d_tile_1d.tile_range_k; + const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(range_start, tile_range_k); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_3d_tile_1d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j); + const size_t tile_k = threadpool->params.parallelize_3d_tile_1d.tile_k; + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t start_k = tile_index_ij_k.remainder * tile_k; + + const size_t range_k = threadpool->params.parallelize_3d_tile_1d.range_k; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, i, j, start_k, min(range_k - start_k, tile_k)); + start_k += tile_k; + if (start_k >= range_k) { + start_k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(linear_index, tile_range_k); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j); + const size_t start_k = tile_index_ij_k.remainder * tile_k; + task(argument, index_i_j.quotient, index_i_j.remainder, start_k, min(range_k - start_k, tile_k)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_fastpath( struct pthreadpool* threadpool, struct thread_info* thread) @@ -487,6 +597,132 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_with_uarch_f pthreadpool_fence_release(); } +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_4d_t task = (pthreadpool_task_4d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t range_kl = threadpool->params.parallelize_4d.range_kl; + const struct fxdiv_result_size_t index_ij_kl = fxdiv_divide_size_t(range_start, range_kl); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_4d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_kl.quotient, range_j); + const struct fxdiv_divisor_size_t range_l = threadpool->params.parallelize_4d.range_l; + const struct fxdiv_result_size_t index_k_l = fxdiv_divide_size_t(index_ij_kl.remainder, range_l); + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t k = index_k_l.quotient; + size_t l = index_k_l.remainder; + + const size_t range_k = threadpool->params.parallelize_4d.range_k; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, i, j, k, l); + if (++l == range_l.value) { + l = 0; + if (++k == range_k) { + k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t index_ij_kl = fxdiv_divide_size_t(linear_index, range_kl); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_kl.quotient, range_j); + const struct fxdiv_result_size_t index_k_l = fxdiv_divide_size_t(index_ij_kl.remainder, range_l); + task(argument, index_i_j.quotient, index_i_j.remainder, index_k_l.quotient, index_k_l.remainder); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_1d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_4d_tile_1d_t task = (pthreadpool_task_4d_tile_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_kl = threadpool->params.parallelize_4d_tile_1d.tile_range_kl; + const struct fxdiv_result_size_t tile_index_ij_kl = fxdiv_divide_size_t(range_start, tile_range_kl); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_4d_tile_1d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_kl.quotient, range_j); + const struct fxdiv_divisor_size_t tile_range_l = threadpool->params.parallelize_4d_tile_1d.tile_range_l; + const struct fxdiv_result_size_t tile_index_k_l = fxdiv_divide_size_t(tile_index_ij_kl.remainder, tile_range_l); + const size_t tile_l = threadpool->params.parallelize_4d_tile_1d.tile_l; + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t k = tile_index_k_l.quotient; + size_t start_l = tile_index_k_l.remainder * tile_l; + + const size_t range_l = threadpool->params.parallelize_4d_tile_1d.range_l; + const size_t range_k = threadpool->params.parallelize_4d_tile_1d.range_k; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, i, j, k, start_l, min(range_l - start_l, tile_l)); + start_l += tile_l; + if (start_l >= range_l) { + start_l = 0; + if (++k == range_k) { + k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_ij_kl = fxdiv_divide_size_t(linear_index, tile_range_kl); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_kl.quotient, range_j); + const struct fxdiv_result_size_t tile_index_k_l = fxdiv_divide_size_t(tile_index_ij_kl.remainder, tile_range_l); + const size_t start_l = tile_index_k_l.remainder * tile_l; + task(argument, index_i_j.quotient, index_i_j.remainder, tile_index_k_l.quotient, start_l, min(range_l - start_l, tile_l)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_fastpath( struct pthreadpool* threadpool, struct thread_info* thread) @@ -632,6 +868,147 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_with_uarch_f pthreadpool_fence_release(); } +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_5d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_5d_t task = (pthreadpool_task_5d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t range_lm = threadpool->params.parallelize_5d.range_lm; + const struct fxdiv_result_size_t index_ijk_lm = fxdiv_divide_size_t(range_start, range_lm); + const struct fxdiv_divisor_size_t range_k = threadpool->params.parallelize_5d.range_k; + const struct fxdiv_result_size_t index_ij_k = fxdiv_divide_size_t(index_ijk_lm.quotient, range_k); + const struct fxdiv_divisor_size_t range_m = threadpool->params.parallelize_5d.range_m; + const struct fxdiv_result_size_t index_l_m = fxdiv_divide_size_t(index_ijk_lm.remainder, range_m); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_5d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_k.quotient, range_j); + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t k = index_ij_k.remainder; + size_t l = index_l_m.quotient; + size_t m = index_l_m.remainder; + + const size_t range_l = threadpool->params.parallelize_5d.range_l; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, i, j, k, l, m); + if (++m == range_m.value) { + m = 0; + if (++l == range_l) { + l = 0; + if (++k == range_k.value) { + k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t index_ijk_lm = fxdiv_divide_size_t(linear_index, range_lm); + const struct fxdiv_result_size_t index_ij_k = fxdiv_divide_size_t(index_ijk_lm.quotient, range_k); + const struct fxdiv_result_size_t index_l_m = fxdiv_divide_size_t(index_ijk_lm.remainder, range_m); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_k.quotient, range_j); + task(argument, index_i_j.quotient, index_i_j.remainder, index_ij_k.remainder, index_l_m.quotient, index_l_m.remainder); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_5d_tile_1d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread) +{ + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_5d_tile_1d_t task = (pthreadpool_task_5d_tile_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + const size_t threads_count = threadpool->threads_count.value; + const size_t range_threshold = -threads_count; + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_m = threadpool->params.parallelize_5d_tile_1d.tile_range_m; + const struct fxdiv_result_size_t tile_index_ijkl_m = fxdiv_divide_size_t(range_start, tile_range_m); + const struct fxdiv_divisor_size_t range_kl = threadpool->params.parallelize_5d_tile_1d.range_kl; + const struct fxdiv_result_size_t index_ij_kl = fxdiv_divide_size_t(tile_index_ijkl_m.quotient, range_kl); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_5d_tile_1d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_kl.quotient, range_j); + const struct fxdiv_divisor_size_t range_l = threadpool->params.parallelize_5d_tile_1d.range_l; + const struct fxdiv_result_size_t index_k_l = fxdiv_divide_size_t(index_ij_kl.remainder, range_l); + const size_t tile_m = threadpool->params.parallelize_5d_tile_1d.tile_m; + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t k = index_k_l.quotient; + size_t l = index_k_l.remainder; + size_t start_m = tile_index_ijkl_m.remainder * tile_m; + + const size_t range_m = threadpool->params.parallelize_5d_tile_1d.range_m; + const size_t range_k = threadpool->params.parallelize_5d_tile_1d.range_k; + while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) { + task(argument, i, j, k, l, start_m, min(range_m - start_m, tile_m)); + start_m += tile_m; + if (start_m >= range_m) { + start_m = 0; + if (++l == range_l.value) { + l = 0; + if (++k == range_k) { + k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_ijkl_m = fxdiv_divide_size_t(linear_index, tile_range_m); + const struct fxdiv_result_size_t index_ij_kl = fxdiv_divide_size_t(tile_index_ijkl_m.quotient, range_kl); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_kl.quotient, range_j); + const struct fxdiv_result_size_t index_k_l = fxdiv_divide_size_t(index_ij_kl.remainder, range_l); + size_t start_m = tile_index_ijkl_m.remainder * tile_m; + task(argument, index_i_j.quotient, index_i_j.remainder, index_k_l.quotient, index_k_l.remainder, start_m, + min(range_m - start_m, tile_m)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_5d_tile_2d_fastpath( struct pthreadpool* threadpool, struct thread_info* thread) diff --git a/src/portable-api.c b/src/portable-api.c index 6b16674..6a8ccf2 100644 --- a/src/portable-api.c +++ b/src/portable-api.c @@ -328,6 +328,106 @@ static void thread_parallelize_2d_tile_2d_with_uarch(struct pthreadpool* threadp pthreadpool_fence_release(); } +static void thread_parallelize_3d(struct pthreadpool* threadpool, struct thread_info* thread) { + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_3d_t task = (pthreadpool_task_3d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t range_k = threadpool->params.parallelize_3d.range_k; + const struct fxdiv_result_size_t index_ij_k = fxdiv_divide_size_t(range_start, range_k); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_3d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_k.quotient, range_j); + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t k = index_ij_k.remainder; + + while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) { + task(argument, i, j, k); + if (++k == range_k.value) { + k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + const size_t threads_count = threadpool->threads_count.value; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_try_decrement_relaxed_size_t(&other_thread->range_length)) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t index_ij_k = fxdiv_divide_size_t(linear_index, range_k); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_k.quotient, range_j); + task(argument, index_i_j.quotient, index_i_j.remainder, index_ij_k.remainder); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +static void thread_parallelize_3d_tile_1d(struct pthreadpool* threadpool, struct thread_info* thread) { + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_3d_tile_1d_t task = (pthreadpool_task_3d_tile_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_k = threadpool->params.parallelize_3d_tile_1d.tile_range_k; + const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(range_start, tile_range_k); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_3d_tile_1d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j); + const size_t tile_k = threadpool->params.parallelize_3d_tile_1d.tile_k; + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t start_k = tile_index_ij_k.remainder * tile_k; + + const size_t range_k = threadpool->params.parallelize_3d_tile_1d.range_k; + while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) { + task(argument, i, j, start_k, min(range_k - start_k, tile_k)); + start_k += tile_k; + if (start_k >= range_k) { + start_k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + const size_t threads_count = threadpool->threads_count.value; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_try_decrement_relaxed_size_t(&other_thread->range_length)) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(linear_index, tile_range_k); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, range_j); + const size_t start_k = tile_index_ij_k.remainder * tile_k; + task(argument, index_i_j.quotient, index_i_j.remainder, start_k, min(range_k - start_k, tile_k)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + static void thread_parallelize_3d_tile_2d(struct pthreadpool* threadpool, struct thread_info* thread) { assert(threadpool != NULL); assert(thread != NULL); @@ -449,6 +549,122 @@ static void thread_parallelize_3d_tile_2d_with_uarch(struct pthreadpool* threadp pthreadpool_fence_release(); } +static void thread_parallelize_4d(struct pthreadpool* threadpool, struct thread_info* thread) { + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_4d_t task = (pthreadpool_task_4d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t range_kl = threadpool->params.parallelize_4d.range_kl; + const struct fxdiv_result_size_t index_ij_kl = fxdiv_divide_size_t(range_start, range_kl); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_4d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_kl.quotient, range_j); + const struct fxdiv_divisor_size_t range_l = threadpool->params.parallelize_4d.range_l; + const struct fxdiv_result_size_t index_k_l = fxdiv_divide_size_t(index_ij_kl.remainder, range_l); + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t k = index_k_l.quotient; + size_t l = index_k_l.remainder; + + const size_t range_k = threadpool->params.parallelize_4d.range_k; + while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) { + task(argument, i, j, k, l); + if (++l == range_l.value) { + l = 0; + if (++k == range_k) { + k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + const size_t threads_count = threadpool->threads_count.value; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_try_decrement_relaxed_size_t(&other_thread->range_length)) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t index_ij_kl = fxdiv_divide_size_t(linear_index, range_kl); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_kl.quotient, range_j); + const struct fxdiv_result_size_t index_k_l = fxdiv_divide_size_t(index_ij_kl.remainder, range_l); + task(argument, index_i_j.quotient, index_i_j.remainder, index_k_l.quotient, index_k_l.remainder); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +static void thread_parallelize_4d_tile_1d(struct pthreadpool* threadpool, struct thread_info* thread) { + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_4d_tile_1d_t task = (pthreadpool_task_4d_tile_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_kl = threadpool->params.parallelize_4d_tile_1d.tile_range_kl; + const struct fxdiv_result_size_t tile_index_ij_kl = fxdiv_divide_size_t(range_start, tile_range_kl); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_4d_tile_1d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_kl.quotient, range_j); + const struct fxdiv_divisor_size_t tile_range_l = threadpool->params.parallelize_4d_tile_1d.tile_range_l; + const struct fxdiv_result_size_t tile_index_k_l = fxdiv_divide_size_t(tile_index_ij_kl.remainder, tile_range_l); + const size_t tile_l = threadpool->params.parallelize_4d_tile_1d.tile_l; + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t k = tile_index_k_l.quotient; + size_t start_l = tile_index_k_l.remainder * tile_l; + + const size_t range_k = threadpool->params.parallelize_4d_tile_1d.range_k; + const size_t range_l = threadpool->params.parallelize_4d_tile_1d.range_l; + while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) { + task(argument, i, j, k, start_l, min(range_l - start_l, tile_l)); + start_l += tile_l; + if (start_l >= range_l) { + start_l = 0; + if (++k == range_k) { + k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + const size_t threads_count = threadpool->threads_count.value; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_try_decrement_relaxed_size_t(&other_thread->range_length)) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_ij_kl = fxdiv_divide_size_t(linear_index, tile_range_kl); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_kl.quotient, range_j); + const struct fxdiv_result_size_t tile_index_k_l = fxdiv_divide_size_t(tile_index_ij_kl.remainder, tile_range_l); + const size_t start_l = tile_index_k_l.remainder * tile_l; + task(argument, index_i_j.quotient, index_i_j.remainder, tile_index_k_l.quotient, start_l, min(range_l - start_l, tile_l)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + static void thread_parallelize_4d_tile_2d(struct pthreadpool* threadpool, struct thread_info* thread) { assert(threadpool != NULL); assert(thread != NULL); @@ -584,6 +800,137 @@ static void thread_parallelize_4d_tile_2d_with_uarch(struct pthreadpool* threadp pthreadpool_fence_release(); } +static void thread_parallelize_5d(struct pthreadpool* threadpool, struct thread_info* thread) { + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_5d_t task = (pthreadpool_task_5d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t range_lm = threadpool->params.parallelize_5d.range_lm; + const struct fxdiv_result_size_t index_ijk_lm = fxdiv_divide_size_t(range_start, range_lm); + const struct fxdiv_divisor_size_t range_k = threadpool->params.parallelize_5d.range_k; + const struct fxdiv_result_size_t index_ij_k = fxdiv_divide_size_t(index_ijk_lm.quotient, range_k); + const struct fxdiv_divisor_size_t range_m = threadpool->params.parallelize_5d.range_m; + const struct fxdiv_result_size_t index_l_m = fxdiv_divide_size_t(index_ijk_lm.remainder, range_m); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_5d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_k.quotient, range_j); + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t k = index_ij_k.remainder; + size_t l = index_l_m.quotient; + size_t m = index_l_m.remainder; + + const size_t range_l = threadpool->params.parallelize_5d.range_l; + while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) { + task(argument, i, j, k, l, m); + if (++m == range_m.value) { + m = 0; + if (++l == range_l) { + l = 0; + if (++k == range_k.value) { + k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + const size_t threads_count = threadpool->threads_count.value; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_try_decrement_relaxed_size_t(&other_thread->range_length)) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t index_ijk_lm = fxdiv_divide_size_t(linear_index, range_lm); + const struct fxdiv_result_size_t index_ij_k = fxdiv_divide_size_t(index_ijk_lm.quotient, range_k); + const struct fxdiv_result_size_t index_l_m = fxdiv_divide_size_t(index_ijk_lm.remainder, range_m); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_k.quotient, range_j); + task(argument, index_i_j.quotient, index_i_j.remainder, index_ij_k.remainder, index_l_m.quotient, index_l_m.remainder); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + +static void thread_parallelize_5d_tile_1d(struct pthreadpool* threadpool, struct thread_info* thread) { + assert(threadpool != NULL); + assert(thread != NULL); + + const pthreadpool_task_5d_tile_1d_t task = (pthreadpool_task_5d_tile_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task); + void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument); + + /* Process thread's own range of items */ + const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start); + const struct fxdiv_divisor_size_t tile_range_m = threadpool->params.parallelize_5d_tile_1d.tile_range_m; + const struct fxdiv_result_size_t tile_index_ijkl_m = fxdiv_divide_size_t(range_start, tile_range_m); + const struct fxdiv_divisor_size_t range_kl = threadpool->params.parallelize_5d_tile_1d.range_kl; + const struct fxdiv_result_size_t index_ij_kl = fxdiv_divide_size_t(tile_index_ijkl_m.quotient, range_kl); + const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_5d_tile_1d.range_j; + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_kl.quotient, range_j); + const struct fxdiv_divisor_size_t range_l = threadpool->params.parallelize_5d_tile_1d.range_l; + const struct fxdiv_result_size_t index_k_l = fxdiv_divide_size_t(index_ij_kl.remainder, range_l); + const size_t tile_m = threadpool->params.parallelize_5d_tile_1d.tile_m; + size_t i = index_i_j.quotient; + size_t j = index_i_j.remainder; + size_t k = index_k_l.quotient; + size_t l = index_k_l.remainder; + size_t start_m = tile_index_ijkl_m.remainder * tile_m; + + const size_t range_m = threadpool->params.parallelize_5d_tile_1d.range_m; + const size_t range_k = threadpool->params.parallelize_5d_tile_1d.range_k; + while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) { + task(argument, i, j, k, l, start_m, min(range_m - start_m, tile_m)); + start_m += tile_m; + if (start_m >= range_m) { + start_m = 0; + if (++l == range_l.value) { + l = 0; + if (++k == range_k) { + k = 0; + if (++j == range_j.value) { + j = 0; + i += 1; + } + } + } + } + } + + /* There still may be other threads with work */ + const size_t thread_number = thread->thread_number; + const size_t threads_count = threadpool->threads_count.value; + for (size_t tid = modulo_decrement(thread_number, threads_count); + tid != thread_number; + tid = modulo_decrement(tid, threads_count)) + { + struct thread_info* other_thread = &threadpool->threads[tid]; + while (pthreadpool_try_decrement_relaxed_size_t(&other_thread->range_length)) { + const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end); + const struct fxdiv_result_size_t tile_index_ijkl_m = fxdiv_divide_size_t(linear_index, tile_range_m); + const struct fxdiv_result_size_t index_ij_kl = fxdiv_divide_size_t(tile_index_ijkl_m.quotient, range_kl); + const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_kl.quotient, range_j); + const struct fxdiv_result_size_t index_k_l = fxdiv_divide_size_t(index_ij_kl.remainder, range_l); + size_t start_m = tile_index_ijkl_m.remainder * tile_m; + task(argument, index_i_j.quotient, index_i_j.remainder, index_k_l.quotient, index_k_l.remainder, start_m, + min(range_m - start_m, tile_m)); + } + } + + /* Make changes by this thread visible to other threads */ + pthreadpool_fence_release(); +} + static void thread_parallelize_5d_tile_2d(struct pthreadpool* threadpool, struct thread_info* thread) { assert(threadpool != NULL); assert(thread != NULL); @@ -1061,9 +1408,105 @@ void pthreadpool_parallelize_2d_tile_2d_with_uarch( } } -void pthreadpool_parallelize_3d_tile_2d( +void pthreadpool_parallelize_3d( pthreadpool_t threadpool, - pthreadpool_task_3d_tile_2d_t task, + pthreadpool_task_3d_t task, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + uint32_t flags) +{ + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i | range_j | range_k) <= 1) { + /* No thread pool used: execute task sequentially on the calling thread */ + struct fpu_state saved_fpu_state = { 0 }; + if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { + saved_fpu_state = get_fpu_state(); + disable_fpu_denormals(); + } + for (size_t i = 0; i < range_i; i++) { + for (size_t j = 0; j < range_j; j++) { + for (size_t k = 0; k < range_k; k++) { + task(argument, i, j, k); + } + } + } + if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { + set_fpu_state(saved_fpu_state); + } + } else { + const size_t range = range_i * range_j * range_k; + const struct pthreadpool_3d_params params = { + .range_j = fxdiv_init_size_t(range_j), + .range_k = fxdiv_init_size_t(range_k), + }; + thread_function_t parallelize_3d = &thread_parallelize_3d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (range < range_threshold) { + parallelize_3d = &pthreadpool_thread_parallelize_3d_fastpath; + } + #endif + pthreadpool_parallelize( + threadpool, parallelize_3d, ¶ms, sizeof(params), + task, argument, range, flags); + } +} + +void pthreadpool_parallelize_3d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_1d_t task, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags) +{ + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j) <= 1 && range_k <= tile_k)) { + /* No thread pool used: execute task sequentially on the calling thread */ + struct fpu_state saved_fpu_state = { 0 }; + if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { + saved_fpu_state = get_fpu_state(); + disable_fpu_denormals(); + } + for (size_t i = 0; i < range_i; i++) { + for (size_t j = 0; j < range_j; j++) { + for (size_t k = 0; k < range_k; k += tile_k) { + task(argument, i, j, k, min(range_k - k, tile_k)); + } + } + } + if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { + set_fpu_state(saved_fpu_state); + } + } else { + const size_t tile_range_k = divide_round_up(range_k, tile_k); + const size_t tile_range = range_i * range_j * tile_range_k; + const struct pthreadpool_3d_tile_1d_params params = { + .range_k = range_k, + .tile_k = tile_k, + .range_j = fxdiv_init_size_t(range_j), + .tile_range_k = fxdiv_init_size_t(tile_range_k), + }; + thread_function_t parallelize_3d_tile_1d = &thread_parallelize_3d_tile_1d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (tile_range < range_threshold) { + parallelize_3d_tile_1d = &pthreadpool_thread_parallelize_3d_tile_1d_fastpath; + } + #endif + pthreadpool_parallelize( + threadpool, parallelize_3d_tile_1d, ¶ms, sizeof(params), + task, argument, tile_range, flags); + } +} + +void pthreadpool_parallelize_3d_tile_2d( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_2d_t task, void* argument, size_t range_i, size_t range_j, @@ -1182,6 +1625,114 @@ void pthreadpool_parallelize_3d_tile_2d_with_uarch( } } +void pthreadpool_parallelize_4d( + pthreadpool_t threadpool, + pthreadpool_task_4d_t task, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + uint32_t flags) +{ + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i | range_j | range_k | range_l) <= 1) { + /* No thread pool used: execute task sequentially on the calling thread */ + struct fpu_state saved_fpu_state = { 0 }; + if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { + saved_fpu_state = get_fpu_state(); + disable_fpu_denormals(); + } + for (size_t i = 0; i < range_i; i++) { + for (size_t j = 0; j < range_j; j++) { + for (size_t k = 0; k < range_k; k++) { + for (size_t l = 0; l < range_l; l++) { + task(argument, i, j, k, l); + } + } + } + } + if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { + set_fpu_state(saved_fpu_state); + } + } else { + const size_t range_kl = range_k * range_l; + const size_t range = range_i * range_j * range_kl; + const struct pthreadpool_4d_params params = { + .range_k = range_k, + .range_j = fxdiv_init_size_t(range_j), + .range_kl = fxdiv_init_size_t(range_kl), + .range_l = fxdiv_init_size_t(range_l), + }; + thread_function_t parallelize_4d = &thread_parallelize_4d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (range < range_threshold) { + parallelize_4d = &pthreadpool_thread_parallelize_4d_fastpath; + } + #endif + pthreadpool_parallelize( + threadpool, parallelize_4d, ¶ms, sizeof(params), + task, argument, range, flags); + } +} + +void pthreadpool_parallelize_4d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_4d_tile_1d_t task, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_l, + uint32_t flags) +{ + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j | range_k) <= 1 && range_l <= tile_l)) { + /* No thread pool used: execute task sequentially on the calling thread */ + struct fpu_state saved_fpu_state = { 0 }; + if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { + saved_fpu_state = get_fpu_state(); + disable_fpu_denormals(); + } + for (size_t i = 0; i < range_i; i++) { + for (size_t j = 0; j < range_j; j++) { + for (size_t k = 0; k < range_k; k++) { + for (size_t l = 0; l < range_l; l += tile_l) { + task(argument, i, j, k, l, min(range_l - l, tile_l)); + } + } + } + } + if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { + set_fpu_state(saved_fpu_state); + } + } else { + const size_t tile_range_l = divide_round_up(range_l, tile_l); + const size_t tile_range_kl = range_k * tile_range_l; + const size_t tile_range = range_i * range_j * tile_range_kl; + const struct pthreadpool_4d_tile_1d_params params = { + .range_k = range_k, + .range_l = range_l, + .tile_l = tile_l, + .range_j = fxdiv_init_size_t(range_j), + .tile_range_kl = fxdiv_init_size_t(tile_range_kl), + .tile_range_l = fxdiv_init_size_t(tile_range_l), + }; + thread_function_t parallelize_4d_tile_1d = &thread_parallelize_4d_tile_1d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (tile_range < range_threshold) { + parallelize_4d_tile_1d = &pthreadpool_thread_parallelize_4d_tile_1d_fastpath; + } + #endif + pthreadpool_parallelize( + threadpool, parallelize_4d_tile_1d, ¶ms, sizeof(params), + task, argument, tile_range, flags); + } +} + void pthreadpool_parallelize_4d_tile_2d( pthreadpool_t threadpool, pthreadpool_task_4d_tile_2d_t task, @@ -1313,6 +1864,122 @@ void pthreadpool_parallelize_4d_tile_2d_with_uarch( } } +void pthreadpool_parallelize_5d( + pthreadpool_t threadpool, + pthreadpool_task_5d_t task, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + uint32_t flags) +{ + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i | range_j | range_k | range_l | range_m) <= 1) { + /* No thread pool used: execute task sequentially on the calling thread */ + struct fpu_state saved_fpu_state = { 0 }; + if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { + saved_fpu_state = get_fpu_state(); + disable_fpu_denormals(); + } + for (size_t i = 0; i < range_i; i++) { + for (size_t j = 0; j < range_j; j++) { + for (size_t k = 0; k < range_k; k++) { + for (size_t l = 0; l < range_l; l++) { + for (size_t m = 0; m < range_m; m++) { + task(argument, i, j, k, l, m); + } + } + } + } + } + if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { + set_fpu_state(saved_fpu_state); + } + } else { + const size_t range_lm = range_l * range_m; + const size_t range = range_i * range_j * range_k * range_lm; + const struct pthreadpool_5d_params params = { + .range_l = range_l, + .range_j = fxdiv_init_size_t(range_j), + .range_k = fxdiv_init_size_t(range_k), + .range_lm = fxdiv_init_size_t(range_lm), + .range_m = fxdiv_init_size_t(range_m), + }; + thread_function_t parallelize_5d = &thread_parallelize_5d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (range < range_threshold) { + parallelize_5d = &pthreadpool_thread_parallelize_5d_fastpath; + } + #endif + pthreadpool_parallelize( + threadpool, parallelize_5d, ¶ms, sizeof(params), + task, argument, range, flags); + } +} + +void pthreadpool_parallelize_5d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_5d_tile_1d_t task, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t tile_m, + uint32_t flags) +{ + size_t threads_count; + if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j | range_k | range_l) <= 1 && range_m <= tile_m)) { + /* No thread pool used: execute task sequentially on the calling thread */ + struct fpu_state saved_fpu_state = { 0 }; + if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { + saved_fpu_state = get_fpu_state(); + disable_fpu_denormals(); + } + for (size_t i = 0; i < range_i; i++) { + for (size_t j = 0; j < range_j; j++) { + for (size_t k = 0; k < range_k; k++) { + for (size_t l = 0; l < range_l; l++) { + for (size_t m = 0; m < range_m; m += tile_m) { + task(argument, i, j, k, l, m, min(range_m - m, tile_m)); + } + } + } + } + } + if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) { + set_fpu_state(saved_fpu_state); + } + } else { + const size_t tile_range_m = divide_round_up(range_m, tile_m); + const size_t range_kl = range_k * range_l; + const size_t tile_range = range_i * range_j * range_kl * tile_range_m; + const struct pthreadpool_5d_tile_1d_params params = { + .range_k = range_k, + .range_m = range_m, + .tile_m = tile_m, + .range_j = fxdiv_init_size_t(range_j), + .range_kl = fxdiv_init_size_t(range_kl), + .range_l = fxdiv_init_size_t(range_l), + .tile_range_m = fxdiv_init_size_t(tile_range_m), + }; + thread_function_t parallelize_5d_tile_1d = &thread_parallelize_5d_tile_1d; + #if PTHREADPOOL_USE_FASTPATH + const size_t range_threshold = -threads_count; + if (tile_range < range_threshold) { + parallelize_5d_tile_1d = &pthreadpool_thread_parallelize_5d_tile_1d_fastpath; + } + #endif + pthreadpool_parallelize( + threadpool, parallelize_5d_tile_1d, ¶ms, sizeof(params), + task, argument, tile_range, flags); + } +} + void pthreadpool_parallelize_5d_tile_2d( pthreadpool_t threadpool, pthreadpool_task_5d_tile_2d_t task, diff --git a/src/shim.c b/src/shim.c index 7bf378c..e90ac45 100644 --- a/src/shim.c +++ b/src/shim.c @@ -133,6 +133,43 @@ void pthreadpool_parallelize_2d_tile_2d_with_uarch( } } +void pthreadpool_parallelize_3d( + pthreadpool_t threadpool, + pthreadpool_task_3d_t task, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + uint32_t flags) +{ + for (size_t i = 0; i < range_i; i++) { + for (size_t j = 0; j < range_j; j++) { + for (size_t k = 0; k < range_k; k++) { + task(argument, i, j, k); + } + } + } +} + +void pthreadpool_parallelize_3d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_1d_t task, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags) +{ + for (size_t i = 0; i < range_i; i++) { + for (size_t j = 0; j < range_j; j++) { + for (size_t k = 0; k < range_k; k += tile_k) { + task(argument, i, j, k, min(range_k - k, tile_k)); + } + } + } +} + void pthreadpool_parallelize_3d_tile_2d( pthreadpool_t threadpool, pthreadpool_task_3d_tile_2d_t task, @@ -177,6 +214,49 @@ void pthreadpool_parallelize_3d_tile_2d_with_uarch( } } +void pthreadpool_parallelize_4d( + pthreadpool_t threadpool, + pthreadpool_task_4d_t task, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + uint32_t flags) +{ + for (size_t i = 0; i < range_i; i++) { + for (size_t j = 0; j < range_j; j++) { + for (size_t k = 0; k < range_k; k++) { + for (size_t l = 0; l < range_l; l++) { + task(argument, i, j, k, l); + } + } + } + } +} + +void pthreadpool_parallelize_4d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_4d_tile_1d_t task, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_l, + uint32_t flags) +{ + for (size_t i = 0; i < range_i; i++) { + for (size_t j = 0; j < range_j; j++) { + for (size_t k = 0; k < range_k; k++) { + for (size_t l = 0; l < range_l; l += tile_l) { + task(argument, i, j, k, l, min(range_l - l, tile_l)); + } + } + } + } +} + void pthreadpool_parallelize_4d_tile_2d( pthreadpool_t threadpool, pthreadpool_task_4d_tile_2d_t task, @@ -227,6 +307,55 @@ void pthreadpool_parallelize_4d_tile_2d_with_uarch( } } +void pthreadpool_parallelize_5d( + pthreadpool_t threadpool, + pthreadpool_task_5d_t task, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + uint32_t flags) +{ + for (size_t i = 0; i < range_i; i++) { + for (size_t j = 0; j < range_j; j++) { + for (size_t k = 0; k < range_k; k++) { + for (size_t l = 0; l < range_l; l++) { + for (size_t m = 0; m < range_m; m++) { + task(argument, i, j, k, l, m); + } + } + } + } + } +} + +void pthreadpool_parallelize_5d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_5d_tile_1d_t task, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t tile_m, + uint32_t flags) +{ + for (size_t i = 0; i < range_i; i++) { + for (size_t j = 0; j < range_j; j++) { + for (size_t k = 0; k < range_k; k++) { + for (size_t l = 0; l < range_l; l++) { + for (size_t m = 0; m < range_m; m += tile_m) { + task(argument, i, j, k, l, m, min(range_m - m, tile_m)); + } + } + } + } + } +} + void pthreadpool_parallelize_5d_tile_2d( pthreadpool_t threadpool, pthreadpool_task_5d_tile_2d_t task, diff --git a/src/threadpool-object.h b/src/threadpool-object.h index 7b643c6..9870e8a 100644 --- a/src/threadpool-object.h +++ b/src/threadpool-object.h @@ -179,6 +179,36 @@ struct pthreadpool_2d_tile_2d_with_uarch_params { struct fxdiv_divisor_size_t tile_range_j; }; +struct pthreadpool_3d_params { + /** + * FXdiv divisor for the range_j argument passed to the pthreadpool_parallelize_3d function. + */ + struct fxdiv_divisor_size_t range_j; + /** + * FXdiv divisor for the range_k argument passed to the pthreadpool_parallelize_3d function. + */ + struct fxdiv_divisor_size_t range_k; +}; + +struct pthreadpool_3d_tile_1d_params { + /** + * Copy of the range_k argument passed to the pthreadpool_parallelize_3d_tile_1d function. + */ + size_t range_k; + /** + * Copy of the tile_k argument passed to the pthreadpool_parallelize_3d_tile_1d function. + */ + size_t tile_k; + /** + * FXdiv divisor for the range_j argument passed to the pthreadpool_parallelize_3d_tile_1d function. + */ + struct fxdiv_divisor_size_t range_j; + /** + * FXdiv divisor for the divide_round_up(range_k, tile_k) value. + */ + struct fxdiv_divisor_size_t tile_range_k; +}; + struct pthreadpool_3d_tile_2d_params { /** * Copy of the range_j argument passed to the pthreadpool_parallelize_3d_tile_2d function. @@ -241,6 +271,52 @@ struct pthreadpool_3d_tile_2d_with_uarch_params { struct fxdiv_divisor_size_t tile_range_k; }; +struct pthreadpool_4d_params { + /** + * Copy of the range_k argument passed to the pthreadpool_parallelize_4d function. + */ + size_t range_k; + /** + * FXdiv divisor for the range_j argument passed to the pthreadpool_parallelize_4d function. + */ + struct fxdiv_divisor_size_t range_j; + /** + * FXdiv divisor for the range_k * range_l value. + */ + struct fxdiv_divisor_size_t range_kl; + /** + * FXdiv divisor for the range_l argument passed to the pthreadpool_parallelize_4d function. + */ + struct fxdiv_divisor_size_t range_l; +}; + +struct pthreadpool_4d_tile_1d_params { + /** + * Copy of the range_k argument passed to the pthreadpool_parallelize_4d_tile_1d function. + */ + size_t range_k; + /** + * Copy of the range_l argument passed to the pthreadpool_parallelize_4d_tile_1d function. + */ + size_t range_l; + /** + * Copy of the tile_l argument passed to the pthreadpool_parallelize_4d_tile_1d function. + */ + size_t tile_l; + /** + * FXdiv divisor for the range_j argument passed to the pthreadpool_parallelize_4d_tile_1d function. + */ + struct fxdiv_divisor_size_t range_j; + /** + * FXdiv divisor for the range_k * divide_round_up(range_l, tile_l) value. + */ + struct fxdiv_divisor_size_t tile_range_kl; + /** + * FXdiv divisor for the divide_round_up(range_l, tile_l) value. + */ + struct fxdiv_divisor_size_t tile_range_l; +}; + struct pthreadpool_4d_tile_2d_params { /** * Copy of the range_k argument passed to the pthreadpool_parallelize_4d_tile_2d function. @@ -311,6 +387,60 @@ struct pthreadpool_4d_tile_2d_with_uarch_params { struct fxdiv_divisor_size_t tile_range_l; }; +struct pthreadpool_5d_params { + /** + * Copy of the range_l argument passed to the pthreadpool_parallelize_5d function. + */ + size_t range_l; + /** + * FXdiv divisor for the range_j argument passed to the pthreadpool_parallelize_5d function. + */ + struct fxdiv_divisor_size_t range_j; + /** + * FXdiv divisor for the range_k argument passed to the pthreadpool_parallelize_5d function. + */ + struct fxdiv_divisor_size_t range_k; + /** + * FXdiv divisor for the range_l * range_m value. + */ + struct fxdiv_divisor_size_t range_lm; + /** + * FXdiv divisor for the range_m argument passed to the pthreadpool_parallelize_5d function. + */ + struct fxdiv_divisor_size_t range_m; +}; + +struct pthreadpool_5d_tile_1d_params { + /** + * Copy of the range_k argument passed to the pthreadpool_parallelize_5d_tile_1d function. + */ + size_t range_k; + /** + * Copy of the range_m argument passed to the pthreadpool_parallelize_5d_tile_1d function. + */ + size_t range_m; + /** + * Copy of the tile_m argument passed to the pthreadpool_parallelize_5d_tile_1d function. + */ + size_t tile_m; + /** + * FXdiv divisor for the range_j argument passed to the pthreadpool_parallelize_5d_tile_1d function. + */ + struct fxdiv_divisor_size_t range_j; + /** + * FXdiv divisor for the range_k * range_l value. + */ + struct fxdiv_divisor_size_t range_kl; + /** + * FXdiv divisor for the range_l argument passed to the pthreadpool_parallelize_5d_tile_1d function. + */ + struct fxdiv_divisor_size_t range_l; + /** + * FXdiv divisor for the divide_round_up(range_m, tile_m) value. + */ + struct fxdiv_divisor_size_t tile_range_m; +}; + struct pthreadpool_5d_tile_2d_params { /** * Copy of the range_l argument passed to the pthreadpool_parallelize_5d_tile_2d function. @@ -434,10 +564,16 @@ struct PTHREADPOOL_CACHELINE_ALIGNED pthreadpool { struct pthreadpool_2d_tile_1d_params parallelize_2d_tile_1d; struct pthreadpool_2d_tile_2d_params parallelize_2d_tile_2d; struct pthreadpool_2d_tile_2d_with_uarch_params parallelize_2d_tile_2d_with_uarch; + struct pthreadpool_3d_params parallelize_3d; + struct pthreadpool_3d_tile_1d_params parallelize_3d_tile_1d; struct pthreadpool_3d_tile_2d_params parallelize_3d_tile_2d; struct pthreadpool_3d_tile_2d_with_uarch_params parallelize_3d_tile_2d_with_uarch; + struct pthreadpool_4d_params parallelize_4d; + struct pthreadpool_4d_tile_1d_params parallelize_4d_tile_1d; struct pthreadpool_4d_tile_2d_params parallelize_4d_tile_2d; struct pthreadpool_4d_tile_2d_with_uarch_params parallelize_4d_tile_2d_with_uarch; + struct pthreadpool_5d_params parallelize_5d; + struct pthreadpool_5d_tile_1d_params parallelize_5d_tile_1d; struct pthreadpool_5d_tile_2d_params parallelize_5d_tile_2d; struct pthreadpool_6d_tile_2d_params parallelize_6d_tile_2d; } params; @@ -555,6 +691,14 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_with_uarch_f struct pthreadpool* threadpool, struct thread_info* thread); +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_1d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_fastpath( struct pthreadpool* threadpool, struct thread_info* thread); @@ -563,6 +707,14 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_with_uarch_f struct pthreadpool* threadpool, struct thread_info* thread); +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_1d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_fastpath( struct pthreadpool* threadpool, struct thread_info* thread); @@ -571,6 +723,14 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_with_uarch_f struct pthreadpool* threadpool, struct thread_info* thread); +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_5d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + +PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_5d_tile_1d_fastpath( + struct pthreadpool* threadpool, + struct thread_info* thread); + PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_5d_tile_2d_fastpath( struct pthreadpool* threadpool, struct thread_info* thread); diff --git a/test/pthreadpool.cc b/test/pthreadpool.cc index b8a6803..f822506 100644 --- a/test/pthreadpool.cc +++ b/test/pthreadpool.cc @@ -23,17 +23,44 @@ const size_t kParallelize2DTile2DRangeI = 53; const size_t kParallelize2DTile2DRangeJ = 59; const size_t kParallelize2DTile2DTileI = 5; const size_t kParallelize2DTile2DTileJ = 7; +const size_t kParallelize3DRangeI = 13; +const size_t kParallelize3DRangeJ = 17; +const size_t kParallelize3DRangeK = 19; +const size_t kParallelize3DTile1DRangeI = 17; +const size_t kParallelize3DTile1DRangeJ = 19; +const size_t kParallelize3DTile1DRangeK = 23; +const size_t kParallelize3DTile1DTileK = 5; const size_t kParallelize3DTile2DRangeI = 19; const size_t kParallelize3DTile2DRangeJ = 23; const size_t kParallelize3DTile2DRangeK = 29; const size_t kParallelize3DTile2DTileJ = 2; const size_t kParallelize3DTile2DTileK = 3; +const size_t kParallelize4DRangeI = 11; +const size_t kParallelize4DRangeJ = 13; +const size_t kParallelize4DRangeK = 17; +const size_t kParallelize4DRangeL = 19; +const size_t kParallelize4DTile1DRangeI = 13; +const size_t kParallelize4DTile1DRangeJ = 17; +const size_t kParallelize4DTile1DRangeK = 19; +const size_t kParallelize4DTile1DRangeL = 23; +const size_t kParallelize4DTile1DTileL = 5; const size_t kParallelize4DTile2DRangeI = 17; const size_t kParallelize4DTile2DRangeJ = 19; const size_t kParallelize4DTile2DRangeK = 23; const size_t kParallelize4DTile2DRangeL = 29; const size_t kParallelize4DTile2DTileK = 2; const size_t kParallelize4DTile2DTileL = 3; +const size_t kParallelize5DRangeI = 7; +const size_t kParallelize5DRangeJ = 11; +const size_t kParallelize5DRangeK = 13; +const size_t kParallelize5DRangeL = 17; +const size_t kParallelize5DRangeM = 19; +const size_t kParallelize5DTile1DRangeI = 11; +const size_t kParallelize5DTile1DRangeJ = 13; +const size_t kParallelize5DTile1DRangeK = 17; +const size_t kParallelize5DTile1DRangeL = 19; +const size_t kParallelize5DTile1DRangeM = 23; +const size_t kParallelize5DTile1DTileM = 5; const size_t kParallelize5DTile2DRangeI = 13; const size_t kParallelize5DTile2DRangeJ = 17; const size_t kParallelize5DTile2DRangeK = 19; @@ -2286,60 +2313,21 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolWorkStealing) { EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ); } -static void ComputeNothing3DTile2D(void*, size_t, size_t, size_t, size_t, size_t) { -} - -TEST(Parallelize3DTile2D, SingleThreadPoolCompletes) { - auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); - ASSERT_TRUE(threadpool.get()); - - pthreadpool_parallelize_3d_tile_2d(threadpool.get(), - ComputeNothing3DTile2D, - nullptr, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, - 0 /* flags */); -} - -TEST(Parallelize3DTile2D, MultiThreadPoolCompletes) { - auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); - ASSERT_TRUE(threadpool.get()); - - if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { - GTEST_SKIP(); - } - - pthreadpool_parallelize_3d_tile_2d( - threadpool.get(), - ComputeNothing3DTile2D, - nullptr, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, - 0 /* flags */); -} - -static void CheckBounds3DTile2D(void*, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { - EXPECT_LT(i, kParallelize3DTile2DRangeI); - EXPECT_LT(start_j, kParallelize3DTile2DRangeJ); - EXPECT_LT(start_k, kParallelize3DTile2DRangeK); - EXPECT_LE(start_j + tile_j, kParallelize3DTile2DRangeJ); - EXPECT_LE(start_k + tile_k, kParallelize3DTile2DRangeK); +static void ComputeNothing3D(void*, size_t, size_t, size_t) { } -TEST(Parallelize3DTile2D, SingleThreadPoolAllItemsInBounds) { +TEST(Parallelize3D, SingleThreadPoolCompletes) { auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_3d_tile_2d( - threadpool.get(), - CheckBounds3DTile2D, + pthreadpool_parallelize_3d(threadpool.get(), + ComputeNothing3D, nullptr, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, 0 /* flags */); } -TEST(Parallelize3DTile2D, MultiThreadPoolAllItemsInBounds) { +TEST(Parallelize3D, MultiThreadPoolCompletes) { auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -2347,41 +2335,33 @@ TEST(Parallelize3DTile2D, MultiThreadPoolAllItemsInBounds) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d( + pthreadpool_parallelize_3d( threadpool.get(), - CheckBounds3DTile2D, + ComputeNothing3D, nullptr, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, 0 /* flags */); } -static void CheckTiling3DTile2D(void*, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { - EXPECT_GT(tile_j, 0); - EXPECT_LE(tile_j, kParallelize3DTile2DTileJ); - EXPECT_EQ(start_j % kParallelize3DTile2DTileJ, 0); - EXPECT_EQ(tile_j, std::min(kParallelize3DTile2DTileJ, kParallelize3DTile2DRangeJ - start_j)); - - EXPECT_GT(tile_k, 0); - EXPECT_LE(tile_k, kParallelize3DTile2DTileK); - EXPECT_EQ(start_k % kParallelize3DTile2DTileK, 0); - EXPECT_EQ(tile_k, std::min(kParallelize3DTile2DTileK, kParallelize3DTile2DRangeK - start_k)); +static void CheckBounds3D(void*, size_t i, size_t j, size_t k) { + EXPECT_LT(i, kParallelize3DRangeI); + EXPECT_LT(j, kParallelize3DRangeJ); + EXPECT_LT(k, kParallelize3DRangeK); } -TEST(Parallelize3DTile2D, SingleThreadPoolUniformTiling) { +TEST(Parallelize3D, SingleThreadPoolAllItemsInBounds) { auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_3d_tile_2d( + pthreadpool_parallelize_3d( threadpool.get(), - CheckTiling3DTile2D, + CheckBounds3D, nullptr, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, 0 /* flags */); } -TEST(Parallelize3DTile2D, MultiThreadPoolUniformTiling) { +TEST(Parallelize3D, MultiThreadPoolAllItemsInBounds) { auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -2389,42 +2369,36 @@ TEST(Parallelize3DTile2D, MultiThreadPoolUniformTiling) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d( + pthreadpool_parallelize_3d( threadpool.get(), - CheckTiling3DTile2D, + CheckBounds3D, nullptr, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, 0 /* flags */); } -static void SetTrue3DTile2D(std::atomic_bool* processed_indicators, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { - for (size_t j = start_j; j < start_j + tile_j; j++) { - for (size_t k = start_k; k < start_k + tile_k; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; - processed_indicators[linear_idx].store(true, std::memory_order_relaxed); - } - } +static void SetTrue3D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t k) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); } -TEST(Parallelize3DTile2D, SingleThreadPoolAllItemsProcessed) { - std::vector indicators(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +TEST(Parallelize3D, SingleThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_3d_tile_2d( + pthreadpool_parallelize_3d( threadpool.get(), - reinterpret_cast(SetTrue3DTile2D), + reinterpret_cast(SetTrue3D), static_cast(indicators.data()), - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, 0 /* flags */); - for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + for (size_t i = 0; i < kParallelize3DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) << "Element (" << i << ", " << j << ", " << k << ") not processed"; } @@ -2432,8 +2406,8 @@ TEST(Parallelize3DTile2D, SingleThreadPoolAllItemsProcessed) { } } -TEST(Parallelize3DTile2D, MultiThreadPoolAllItemsProcessed) { - std::vector indicators(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +TEST(Parallelize3D, MultiThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -2442,18 +2416,17 @@ TEST(Parallelize3DTile2D, MultiThreadPoolAllItemsProcessed) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d( + pthreadpool_parallelize_3d( threadpool.get(), - reinterpret_cast(SetTrue3DTile2D), + reinterpret_cast(SetTrue3D), static_cast(indicators.data()), - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, 0 /* flags */); - for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + for (size_t i = 0; i < kParallelize3DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) << "Element (" << i << ", " << j << ", " << k << ") not processed"; } @@ -2461,33 +2434,28 @@ TEST(Parallelize3DTile2D, MultiThreadPoolAllItemsProcessed) { } } -static void Increment3DTile2D(std::atomic_int* processed_counters, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { - for (size_t j = start_j; j < start_j + tile_j; j++) { - for (size_t k = start_k; k < start_k + tile_k; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; - processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); - } - } +static void Increment3D(std::atomic_int* processed_counters, size_t i, size_t j, size_t k) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); } -TEST(Parallelize3DTile2D, SingleThreadPoolEachItemProcessedOnce) { - std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +TEST(Parallelize3D, SingleThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_3d_tile_2d( + pthreadpool_parallelize_3d( threadpool.get(), - reinterpret_cast(Increment3DTile2D), + reinterpret_cast(Increment3D), static_cast(counters.data()), - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, 0 /* flags */); - for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + for (size_t i = 0; i < kParallelize3DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) << "Element (" << i << ", " << j << ", " << k << ") was processed " << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; @@ -2496,8 +2464,8 @@ TEST(Parallelize3DTile2D, SingleThreadPoolEachItemProcessedOnce) { } } -TEST(Parallelize3DTile2D, MultiThreadPoolEachItemProcessedOnce) { - std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +TEST(Parallelize3D, MultiThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -2506,18 +2474,17 @@ TEST(Parallelize3DTile2D, MultiThreadPoolEachItemProcessedOnce) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d( + pthreadpool_parallelize_3d( threadpool.get(), - reinterpret_cast(Increment3DTile2D), + reinterpret_cast(Increment3D), static_cast(counters.data()), - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, 0 /* flags */); - for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + for (size_t i = 0; i < kParallelize3DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) << "Element (" << i << ", " << j << ", " << k << ") was processed " << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; @@ -2526,26 +2493,25 @@ TEST(Parallelize3DTile2D, MultiThreadPoolEachItemProcessedOnce) { } } -TEST(Parallelize3DTile2D, SingleThreadPoolEachItemProcessedMultipleTimes) { - std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +TEST(Parallelize3D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { - pthreadpool_parallelize_3d_tile_2d( + pthreadpool_parallelize_3d( threadpool.get(), - reinterpret_cast(Increment3DTile2D), + reinterpret_cast(Increment3D), static_cast(counters.data()), - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, - 0 /* flags */); + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); } - for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + for (size_t i = 0; i < kParallelize3DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) << "Element (" << i << ", " << j << ", " << k << ") was processed " << counters[linear_idx].load(std::memory_order_relaxed) << " times " @@ -2555,8 +2521,8 @@ TEST(Parallelize3DTile2D, SingleThreadPoolEachItemProcessedMultipleTimes) { } } -TEST(Parallelize3DTile2D, MultiThreadPoolEachItemProcessedMultipleTimes) { - std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +TEST(Parallelize3D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -2566,19 +2532,18 @@ TEST(Parallelize3DTile2D, MultiThreadPoolEachItemProcessedMultipleTimes) { } for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { - pthreadpool_parallelize_3d_tile_2d( + pthreadpool_parallelize_3d( threadpool.get(), - reinterpret_cast(Increment3DTile2D), + reinterpret_cast(Increment3D), static_cast(counters.data()), - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, - 0 /* flags */); + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); } - for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + for (size_t i = 0; i < kParallelize3DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) << "Element (" << i << ", " << j << ", " << k << ") was processed " << counters[linear_idx].load(std::memory_order_relaxed) << " times " @@ -2588,15 +2553,11 @@ TEST(Parallelize3DTile2D, MultiThreadPoolEachItemProcessedMultipleTimes) { } } -static void IncrementSame3DTile2D(std::atomic_int* num_processed_items, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { - for (size_t j = start_j; j < start_j + tile_j; j++) { - for (size_t k = start_k; k < start_k + tile_k; k++) { - num_processed_items->fetch_add(1, std::memory_order_relaxed); - } - } +static void IncrementSame3D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); } -TEST(Parallelize3DTile2D, MultiThreadPoolHighContention) { +TEST(Parallelize3D, MultiThreadPoolHighContention) { std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); @@ -2606,27 +2567,26 @@ TEST(Parallelize3DTile2D, MultiThreadPoolHighContention) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d( + pthreadpool_parallelize_3d( threadpool.get(), - reinterpret_cast(IncrementSame3DTile2D), + reinterpret_cast(IncrementSame3D), static_cast(&num_processed_items), - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, 0 /* flags */); - EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); } -static void WorkImbalance3DTile2D(std::atomic_int* num_processed_items, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { - num_processed_items->fetch_add(tile_j * tile_k, std::memory_order_relaxed); - if (i == 0 && start_j == 0 && start_k == 0) { +static void WorkImbalance3D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + if (i == 0 && j == 0 && k == 0) { /* Spin-wait until all items are computed */ - while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK) { + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK) { std::atomic_thread_fence(std::memory_order_acquire); } } } -TEST(Parallelize3DTile2D, MultiThreadPoolWorkStealing) { +TEST(Parallelize3D, MultiThreadPoolWorkStealing) { std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); @@ -2636,33 +2596,31 @@ TEST(Parallelize3DTile2D, MultiThreadPoolWorkStealing) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d( + pthreadpool_parallelize_3d( threadpool.get(), - reinterpret_cast(WorkImbalance3DTile2D), + reinterpret_cast(WorkImbalance3D), static_cast(&num_processed_items), - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, 0 /* flags */); - EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); } -static void ComputeNothing3DTile2DWithUArch(void*, uint32_t, size_t, size_t, size_t, size_t, size_t) { +static void ComputeNothing3DTile1D(void*, size_t, size_t, size_t, size_t) { } -TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolCompletes) { +TEST(Parallelize3DTile1D, SingleThreadPoolCompletes) { auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_3d_tile_2d_with_uarch(threadpool.get(), - ComputeNothing3DTile2DWithUArch, + pthreadpool_parallelize_3d_tile_1d(threadpool.get(), + ComputeNothing3DTile1D, nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, 0 /* flags */); } -TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolCompletes) { +TEST(Parallelize3DTile1D, MultiThreadPoolCompletes) { auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -2670,37 +2628,36 @@ TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolCompletes) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_3d_tile_1d( threadpool.get(), - ComputeNothing3DTile2DWithUArch, + ComputeNothing3DTile1D, nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, 0 /* flags */); } -static void CheckUArch3DTile2DWithUArch(void*, uint32_t uarch_index, size_t, size_t, size_t, size_t, size_t) { - if (uarch_index != kDefaultUArchIndex) { - EXPECT_LE(uarch_index, kMaxUArchIndex); - } +static void CheckBounds3DTile1D(void*, size_t i, size_t j, size_t start_k, size_t tile_k) { + EXPECT_LT(i, kParallelize3DTile1DRangeI); + EXPECT_LT(j, kParallelize3DTile1DRangeJ); + EXPECT_LT(start_k, kParallelize3DTile1DRangeK); + EXPECT_LE(start_k + tile_k, kParallelize3DTile1DRangeK); } -TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolUArchInBounds) { +TEST(Parallelize3DTile1D, SingleThreadPoolAllItemsInBounds) { auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_3d_tile_1d( threadpool.get(), - CheckUArch3DTile2DWithUArch, + CheckBounds3DTile1D, nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, 0 /* flags */); } -TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolUArchInBounds) { +TEST(Parallelize3DTile1D, MultiThreadPoolAllItemsInBounds) { auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -2708,39 +2665,36 @@ TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolUArchInBounds) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_3d_tile_1d( threadpool.get(), - CheckUArch3DTile2DWithUArch, + CheckBounds3DTile1D, nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, 0 /* flags */); } -static void CheckBounds3DTile2DWithUArch(void*, uint32_t, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { - EXPECT_LT(i, kParallelize3DTile2DRangeI); - EXPECT_LT(start_j, kParallelize3DTile2DRangeJ); - EXPECT_LT(start_k, kParallelize3DTile2DRangeK); - EXPECT_LE(start_j + tile_j, kParallelize3DTile2DRangeJ); - EXPECT_LE(start_k + tile_k, kParallelize3DTile2DRangeK); +static void CheckTiling3DTile1D(void*, size_t i, size_t j, size_t start_k, size_t tile_k) { + EXPECT_GT(tile_k, 0); + EXPECT_LE(tile_k, kParallelize3DTile1DTileK); + EXPECT_EQ(start_k % kParallelize3DTile1DTileK, 0); + EXPECT_EQ(tile_k, std::min(kParallelize3DTile1DTileK, kParallelize3DTile1DRangeK - start_k)); } -TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolAllItemsInBounds) { +TEST(Parallelize3DTile1D, SingleThreadPoolUniformTiling) { auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_3d_tile_1d( threadpool.get(), - CheckBounds3DTile2DWithUArch, + CheckTiling3DTile1D, nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, 0 /* flags */); } -TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolAllItemsInBounds) { +TEST(Parallelize3DTile1D, MultiThreadPoolUniformTiling) { auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -2748,43 +2702,50 @@ TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolAllItemsInBounds) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_3d_tile_1d( threadpool.get(), - CheckBounds3DTile2DWithUArch, + CheckTiling3DTile1D, nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, 0 /* flags */); } -static void CheckTiling3DTile2DWithUArch(void*, uint32_t, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { - EXPECT_GT(tile_j, 0); - EXPECT_LE(tile_j, kParallelize3DTile2DTileJ); - EXPECT_EQ(start_j % kParallelize3DTile2DTileJ, 0); - EXPECT_EQ(tile_j, std::min(kParallelize3DTile2DTileJ, kParallelize3DTile2DRangeJ - start_j)); - - EXPECT_GT(tile_k, 0); - EXPECT_LE(tile_k, kParallelize3DTile2DTileK); - EXPECT_EQ(start_k % kParallelize3DTile2DTileK, 0); - EXPECT_EQ(tile_k, std::min(kParallelize3DTile2DTileK, kParallelize3DTile2DRangeK - start_k)); +static void SetTrue3DTile1D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t start_k, size_t tile_k) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); + } } -TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolUniformTiling) { +TEST(Parallelize3DTile1D, SingleThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_3d_tile_1d( threadpool.get(), - CheckTiling3DTile2DWithUArch, - nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + reinterpret_cast(SetTrue3DTile1D), + static_cast(indicators.data()), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ") not processed"; + } + } + } } -TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolUniformTiling) { +TEST(Parallelize3DTile1D, MultiThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -2792,17 +2753,319 @@ TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolUniformTiling) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_3d_tile_1d( threadpool.get(), - CheckTiling3DTile2DWithUArch, - nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + reinterpret_cast(SetTrue3DTile1D), + static_cast(indicators.data()), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ") not processed"; + } + } + } } -static void SetTrue3DTile2DWithUArch(std::atomic_bool* processed_indicators, uint32_t, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { +static void Increment3DTile1D(std::atomic_int* processed_counters, size_t i, size_t j, size_t start_k, size_t tile_k) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); + } +} + +TEST(Parallelize3DTile1D, SingleThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast(Increment3DTile1D), + static_cast(counters.data()), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } +} + +TEST(Parallelize3DTile1D, MultiThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast(Increment3DTile1D), + static_cast(counters.data()), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } +} + +TEST(Parallelize3DTile1D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast(Increment3DTile1D), + static_cast(counters.data()), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } +} + +TEST(Parallelize3DTile1D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast(Increment3DTile1D), + static_cast(counters.data()), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } +} + +static void IncrementSame3DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t start_k, size_t tile_k) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + } +} + +TEST(Parallelize3DTile1D, MultiThreadPoolHighContention) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast(IncrementSame3DTile1D), + static_cast(&num_processed_items), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); +} + +static void WorkImbalance3DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t start_k, size_t tile_k) { + num_processed_items->fetch_add(tile_k, std::memory_order_relaxed); + if (i == 0 && j == 0 && start_k == 0) { + /* Spin-wait until all items are computed */ + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK) { + std::atomic_thread_fence(std::memory_order_acquire); + } + } +} + +TEST(Parallelize3DTile1D, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast(WorkImbalance3DTile1D), + static_cast(&num_processed_items), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); +} + +static void ComputeNothing3DTile2D(void*, size_t, size_t, size_t, size_t, size_t) { +} + +TEST(Parallelize3DTile2D, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_2d(threadpool.get(), + ComputeNothing3DTile2D, + nullptr, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +TEST(Parallelize3DTile2D, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d( + threadpool.get(), + ComputeNothing3DTile2D, + nullptr, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +static void CheckBounds3DTile2D(void*, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { + EXPECT_LT(i, kParallelize3DTile2DRangeI); + EXPECT_LT(start_j, kParallelize3DTile2DRangeJ); + EXPECT_LT(start_k, kParallelize3DTile2DRangeK); + EXPECT_LE(start_j + tile_j, kParallelize3DTile2DRangeJ); + EXPECT_LE(start_k + tile_k, kParallelize3DTile2DRangeK); +} + +TEST(Parallelize3DTile2D, SingleThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_2d( + threadpool.get(), + CheckBounds3DTile2D, + nullptr, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +TEST(Parallelize3DTile2D, MultiThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d( + threadpool.get(), + CheckBounds3DTile2D, + nullptr, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +static void CheckTiling3DTile2D(void*, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { + EXPECT_GT(tile_j, 0); + EXPECT_LE(tile_j, kParallelize3DTile2DTileJ); + EXPECT_EQ(start_j % kParallelize3DTile2DTileJ, 0); + EXPECT_EQ(tile_j, std::min(kParallelize3DTile2DTileJ, kParallelize3DTile2DRangeJ - start_j)); + + EXPECT_GT(tile_k, 0); + EXPECT_LE(tile_k, kParallelize3DTile2DTileK); + EXPECT_EQ(start_k % kParallelize3DTile2DTileK, 0); + EXPECT_EQ(tile_k, std::min(kParallelize3DTile2DTileK, kParallelize3DTile2DRangeK - start_k)); +} + +TEST(Parallelize3DTile2D, SingleThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_2d( + threadpool.get(), + CheckTiling3DTile2D, + nullptr, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +TEST(Parallelize3DTile2D, MultiThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d( + threadpool.get(), + CheckTiling3DTile2D, + nullptr, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +static void SetTrue3DTile2D(std::atomic_bool* processed_indicators, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { for (size_t j = start_j; j < start_j + tile_j; j++) { for (size_t k = start_k; k < start_k + tile_k; k++) { const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; @@ -2811,17 +3074,16 @@ static void SetTrue3DTile2DWithUArch(std::atomic_bool* processed_indicators, uin } } -TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolAllItemsProcessed) { +TEST(Parallelize3DTile2D, SingleThreadPoolAllItemsProcessed) { std::vector indicators(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_3d_tile_2d( threadpool.get(), - reinterpret_cast(SetTrue3DTile2DWithUArch), + reinterpret_cast(SetTrue3DTile2D), static_cast(indicators.data()), - kDefaultUArchIndex, kMaxUArchIndex, kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, 0 /* flags */); @@ -2837,7 +3099,7 @@ TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolAllItemsProcessed) { } } -TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolAllItemsProcessed) { +TEST(Parallelize3DTile2D, MultiThreadPoolAllItemsProcessed) { std::vector indicators(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); @@ -2847,11 +3109,10 @@ TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolAllItemsProcessed) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_3d_tile_2d( threadpool.get(), - reinterpret_cast(SetTrue3DTile2DWithUArch), + reinterpret_cast(SetTrue3DTile2D), static_cast(indicators.data()), - kDefaultUArchIndex, kMaxUArchIndex, kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, 0 /* flags */); @@ -2867,44 +3128,1488 @@ TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolAllItemsProcessed) { } } -static void Increment3DTile2DWithUArch(std::atomic_int* processed_counters, uint32_t, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { - for (size_t j = start_j; j < start_j + tile_j; j++) { - for (size_t k = start_k; k < start_k + tile_k; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; +static void Increment3DTile2D(std::atomic_int* processed_counters, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { + for (size_t j = start_j; j < start_j + tile_j; j++) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); + } + } +} + +TEST(Parallelize3DTile2D, SingleThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_2d( + threadpool.get(), + reinterpret_cast(Increment3DTile2D), + static_cast(counters.data()), + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } +} + +TEST(Parallelize3DTile2D, MultiThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d( + threadpool.get(), + reinterpret_cast(Increment3DTile2D), + static_cast(counters.data()), + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } +} + +TEST(Parallelize3DTile2D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_3d_tile_2d( + threadpool.get(), + reinterpret_cast(Increment3DTile2D), + static_cast(counters.data()), + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } +} + +TEST(Parallelize3DTile2D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_3d_tile_2d( + threadpool.get(), + reinterpret_cast(Increment3DTile2D), + static_cast(counters.data()), + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } +} + +static void IncrementSame3DTile2D(std::atomic_int* num_processed_items, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { + for (size_t j = start_j; j < start_j + tile_j; j++) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + } + } +} + +TEST(Parallelize3DTile2D, MultiThreadPoolHighContention) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d( + threadpool.get(), + reinterpret_cast(IncrementSame3DTile2D), + static_cast(&num_processed_items), + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +} + +static void WorkImbalance3DTile2D(std::atomic_int* num_processed_items, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { + num_processed_items->fetch_add(tile_j * tile_k, std::memory_order_relaxed); + if (i == 0 && start_j == 0 && start_k == 0) { + /* Spin-wait until all items are computed */ + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK) { + std::atomic_thread_fence(std::memory_order_acquire); + } + } +} + +TEST(Parallelize3DTile2D, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d( + threadpool.get(), + reinterpret_cast(WorkImbalance3DTile2D), + static_cast(&num_processed_items), + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +} + +static void ComputeNothing3DTile2DWithUArch(void*, uint32_t, size_t, size_t, size_t, size_t, size_t) { +} + +TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_2d_with_uarch(threadpool.get(), + ComputeNothing3DTile2DWithUArch, + nullptr, + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + ComputeNothing3DTile2DWithUArch, + nullptr, + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +static void CheckUArch3DTile2DWithUArch(void*, uint32_t uarch_index, size_t, size_t, size_t, size_t, size_t) { + if (uarch_index != kDefaultUArchIndex) { + EXPECT_LE(uarch_index, kMaxUArchIndex); + } +} + +TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolUArchInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + CheckUArch3DTile2DWithUArch, + nullptr, + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolUArchInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + CheckUArch3DTile2DWithUArch, + nullptr, + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +static void CheckBounds3DTile2DWithUArch(void*, uint32_t, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { + EXPECT_LT(i, kParallelize3DTile2DRangeI); + EXPECT_LT(start_j, kParallelize3DTile2DRangeJ); + EXPECT_LT(start_k, kParallelize3DTile2DRangeK); + EXPECT_LE(start_j + tile_j, kParallelize3DTile2DRangeJ); + EXPECT_LE(start_k + tile_k, kParallelize3DTile2DRangeK); +} + +TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + CheckBounds3DTile2DWithUArch, + nullptr, + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + CheckBounds3DTile2DWithUArch, + nullptr, + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +static void CheckTiling3DTile2DWithUArch(void*, uint32_t, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { + EXPECT_GT(tile_j, 0); + EXPECT_LE(tile_j, kParallelize3DTile2DTileJ); + EXPECT_EQ(start_j % kParallelize3DTile2DTileJ, 0); + EXPECT_EQ(tile_j, std::min(kParallelize3DTile2DTileJ, kParallelize3DTile2DRangeJ - start_j)); + + EXPECT_GT(tile_k, 0); + EXPECT_LE(tile_k, kParallelize3DTile2DTileK); + EXPECT_EQ(start_k % kParallelize3DTile2DTileK, 0); + EXPECT_EQ(tile_k, std::min(kParallelize3DTile2DTileK, kParallelize3DTile2DRangeK - start_k)); +} + +TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + CheckTiling3DTile2DWithUArch, + nullptr, + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + CheckTiling3DTile2DWithUArch, + nullptr, + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); +} + +static void SetTrue3DTile2DWithUArch(std::atomic_bool* processed_indicators, uint32_t, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { + for (size_t j = start_j; j < start_j + tile_j; j++) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); + } + } +} + +TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + reinterpret_cast(SetTrue3DTile2DWithUArch), + static_cast(indicators.data()), + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ") not processed"; + } + } + } +} + +TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + reinterpret_cast(SetTrue3DTile2DWithUArch), + static_cast(indicators.data()), + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ") not processed"; + } + } + } +} + +static void Increment3DTile2DWithUArch(std::atomic_int* processed_counters, uint32_t, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { + for (size_t j = start_j; j < start_j + tile_j; j++) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); + } + } +} + +TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + reinterpret_cast(Increment3DTile2DWithUArch), + static_cast(counters.data()), + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } +} + +TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + reinterpret_cast(Increment3DTile2DWithUArch), + static_cast(counters.data()), + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } +} + +TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + reinterpret_cast(Increment3DTile2DWithUArch), + static_cast(counters.data()), + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } +} + +TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + reinterpret_cast(Increment3DTile2DWithUArch), + static_cast(counters.data()), + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } +} + +static void IncrementSame3DTile2DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { + for (size_t j = start_j; j < start_j + tile_j; j++) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + } + } +} + +TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolHighContention) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + reinterpret_cast(IncrementSame3DTile2DWithUArch), + static_cast(&num_processed_items), + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +} + +static void WorkImbalance3DTile2DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { + num_processed_items->fetch_add(tile_j * tile_k, std::memory_order_relaxed); + if (i == 0 && start_j == 0 && start_k == 0) { + /* Spin-wait until all items are computed */ + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK) { + std::atomic_thread_fence(std::memory_order_acquire); + } + } +} + +TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_2d_with_uarch( + threadpool.get(), + reinterpret_cast(WorkImbalance3DTile2DWithUArch), + static_cast(&num_processed_items), + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, + kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +} + +static void ComputeNothing4D(void*, size_t, size_t, size_t, size_t) { +} + +TEST(Parallelize4D, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d(threadpool.get(), + ComputeNothing4D, + nullptr, + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); +} + +TEST(Parallelize4D, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d( + threadpool.get(), + ComputeNothing4D, + nullptr, + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); +} + +static void CheckBounds4D(void*, size_t i, size_t j, size_t k, size_t l) { + EXPECT_LT(i, kParallelize4DRangeI); + EXPECT_LT(j, kParallelize4DRangeJ); + EXPECT_LT(k, kParallelize4DRangeK); + EXPECT_LT(l, kParallelize4DRangeL); +} + +TEST(Parallelize4D, SingleThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d( + threadpool.get(), + CheckBounds4D, + nullptr, + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); +} + +TEST(Parallelize4D, MultiThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d( + threadpool.get(), + CheckBounds4D, + nullptr, + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); +} + +static void SetTrue4D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t k, size_t l) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); +} + +TEST(Parallelize4D, SingleThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast(SetTrue4D), + static_cast(indicators.data()), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") not processed"; + } + } + } + } +} + +TEST(Parallelize4D, MultiThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast(SetTrue4D), + static_cast(indicators.data()), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") not processed"; + } + } + } + } +} + +static void Increment4D(std::atomic_int* processed_counters, size_t i, size_t j, size_t k, size_t l) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); +} + +TEST(Parallelize4D, SingleThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast(Increment4D), + static_cast(counters.data()), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } +} + +TEST(Parallelize4D, MultiThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast(Increment4D), + static_cast(counters.data()), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } +} + +TEST(Parallelize4D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast(Increment4D), + static_cast(counters.data()), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize4DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } + } +} + +TEST(Parallelize4D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast(Increment4D), + static_cast(counters.data()), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize4DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } + } +} + +static void IncrementSame4D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t l) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); +} + +TEST(Parallelize4D, MultiThreadPoolHighContention) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast(IncrementSame4D), + static_cast(&num_processed_items), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); +} + +static void WorkImbalance4D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t l) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + if (i == 0 && j == 0 && k == 0 && l == 0) { + /* Spin-wait until all items are computed */ + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL) { + std::atomic_thread_fence(std::memory_order_acquire); + } + } +} + +TEST(Parallelize4D, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast(WorkImbalance4D), + static_cast(&num_processed_items), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); +} + +static void ComputeNothing4DTile1D(void*, size_t, size_t, size_t, size_t, size_t) { +} + +TEST(Parallelize4DTile1D, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_1d(threadpool.get(), + ComputeNothing4DTile1D, + nullptr, + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); +} + +TEST(Parallelize4DTile1D, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + ComputeNothing4DTile1D, + nullptr, + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); +} + +static void CheckBounds4DTile1D(void*, size_t i, size_t j, size_t k, size_t start_l, size_t tile_l) { + EXPECT_LT(i, kParallelize4DTile1DRangeI); + EXPECT_LT(j, kParallelize4DTile1DRangeJ); + EXPECT_LT(k, kParallelize4DTile1DRangeK); + EXPECT_LT(start_l, kParallelize4DTile1DRangeL); + EXPECT_LE(start_l + tile_l, kParallelize4DTile1DRangeL); +} + +TEST(Parallelize4DTile1D, SingleThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + CheckBounds4DTile1D, + nullptr, + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); +} + +TEST(Parallelize4DTile1D, MultiThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + CheckBounds4DTile1D, + nullptr, + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); +} + +static void CheckTiling4DTile1D(void*, size_t i, size_t j, size_t k, size_t start_l, size_t tile_l) { + EXPECT_GT(tile_l, 0); + EXPECT_LE(tile_l, kParallelize4DTile1DTileL); + EXPECT_EQ(start_l % kParallelize4DTile1DTileL, 0); + EXPECT_EQ(tile_l, std::min(kParallelize4DTile1DTileL, kParallelize4DTile1DRangeL - start_l)); +} + +TEST(Parallelize4DTile1D, SingleThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + CheckTiling4DTile1D, + nullptr, + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); +} + +TEST(Parallelize4DTile1D, MultiThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + CheckTiling4DTile1D, + nullptr, + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); +} + +static void SetTrue4DTile1D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t k, size_t start_l, size_t tile_l) { + for (size_t l = start_l; l < start_l + tile_l; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); + } +} + +TEST(Parallelize4DTile1D, SingleThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast(SetTrue4DTile1D), + static_cast(indicators.data()), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile1DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") not processed"; + } + } + } + } +} + +TEST(Parallelize4DTile1D, MultiThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast(SetTrue4DTile1D), + static_cast(indicators.data()), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile1DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") not processed"; + } + } + } + } +} + +static void Increment4DTile1D(std::atomic_int* processed_counters, size_t i, size_t j, size_t k, size_t start_l, size_t tile_l) { + for (size_t l = start_l; l < start_l + tile_l; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); + } +} + +TEST(Parallelize4DTile1D, SingleThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast(Increment4DTile1D), + static_cast(counters.data()), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile1DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } +} + +TEST(Parallelize4DTile1D, MultiThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast(Increment4DTile1D), + static_cast(counters.data()), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile1DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } +} + +TEST(Parallelize4DTile1D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast(Increment4DTile1D), + static_cast(counters.data()), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize4DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile1DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } + } +} + +TEST(Parallelize4DTile1D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast(Increment4DTile1D), + static_cast(counters.data()), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize4DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile1DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } + } +} + +static void IncrementSame4DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t start_l, size_t tile_l) { + for (size_t l = start_l; l < start_l + tile_l; l++) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + } +} + +TEST(Parallelize4DTile1D, MultiThreadPoolHighContention) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast(IncrementSame4DTile1D), + static_cast(&num_processed_items), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); +} + +static void WorkImbalance4DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t start_l, size_t tile_l) { + num_processed_items->fetch_add(tile_l, std::memory_order_relaxed); + if (i == 0 && j == 0 && k == 0 && start_l == 0) { + /* Spin-wait until all items are computed */ + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL) { + std::atomic_thread_fence(std::memory_order_acquire); + } + } +} + +TEST(Parallelize4DTile1D, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast(WorkImbalance4DTile1D), + static_cast(&num_processed_items), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); +} + +static void ComputeNothing4DTile2D(void*, size_t, size_t, size_t, size_t, size_t, size_t) { +} + +TEST(Parallelize4DTile2D, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_2d(threadpool.get(), + ComputeNothing4DTile2D, + nullptr, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + 0 /* flags */); +} + +TEST(Parallelize4DTile2D, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_2d( + threadpool.get(), + ComputeNothing4DTile2D, + nullptr, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + 0 /* flags */); +} + +static void CheckBounds4DTile2D(void*, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { + EXPECT_LT(i, kParallelize4DTile2DRangeI); + EXPECT_LT(j, kParallelize4DTile2DRangeJ); + EXPECT_LT(start_k, kParallelize4DTile2DRangeK); + EXPECT_LT(start_l, kParallelize4DTile2DRangeL); + EXPECT_LE(start_k + tile_k, kParallelize4DTile2DRangeK); + EXPECT_LE(start_l + tile_l, kParallelize4DTile2DRangeL); +} + +TEST(Parallelize4DTile2D, SingleThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_2d( + threadpool.get(), + CheckBounds4DTile2D, + nullptr, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + 0 /* flags */); +} + +TEST(Parallelize4DTile2D, MultiThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_2d( + threadpool.get(), + CheckBounds4DTile2D, + nullptr, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + 0 /* flags */); +} + +static void CheckTiling4DTile2D(void*, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { + EXPECT_GT(tile_k, 0); + EXPECT_LE(tile_k, kParallelize4DTile2DTileK); + EXPECT_EQ(start_k % kParallelize4DTile2DTileK, 0); + EXPECT_EQ(tile_k, std::min(kParallelize4DTile2DTileK, kParallelize4DTile2DRangeK - start_k)); + + EXPECT_GT(tile_l, 0); + EXPECT_LE(tile_l, kParallelize4DTile2DTileL); + EXPECT_EQ(start_l % kParallelize4DTile2DTileL, 0); + EXPECT_EQ(tile_l, std::min(kParallelize4DTile2DTileL, kParallelize4DTile2DRangeL - start_l)); +} + +TEST(Parallelize4DTile2D, SingleThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_2d( + threadpool.get(), + CheckTiling4DTile2D, + nullptr, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + 0 /* flags */); +} + +TEST(Parallelize4DTile2D, MultiThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_2d( + threadpool.get(), + CheckTiling4DTile2D, + nullptr, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + 0 /* flags */); +} + +static void SetTrue4DTile2D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + for (size_t l = start_l; l < start_l + tile_l; l++) { + const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); + } + } +} + +TEST(Parallelize4DTile2D, SingleThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_2d( + threadpool.get(), + reinterpret_cast(SetTrue4DTile2D), + static_cast(indicators.data()), + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile2DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile2DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") not processed"; + } + } + } + } +} + +TEST(Parallelize4DTile2D, MultiThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_2d( + threadpool.get(), + reinterpret_cast(SetTrue4DTile2D), + static_cast(indicators.data()), + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile2DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile2DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") not processed"; + } + } + } + } +} + +static void Increment4DTile2D(std::atomic_int* processed_counters, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + for (size_t l = start_l; l < start_l + tile_l; l++) { + const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); } } } -TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolEachItemProcessedOnce) { - std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +TEST(Parallelize4DTile2D, SingleThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_4d_tile_2d( threadpool.get(), - reinterpret_cast(Increment3DTile2DWithUArch), + reinterpret_cast(Increment4DTile2D), static_cast(counters.data()), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); - for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; - EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) - << "Element (" << i << ", " << j << ", " << k << ") was processed " - << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + for (size_t i = 0; i < kParallelize4DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile2DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile2DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } } } } } -TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolEachItemProcessedOnce) { - std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +TEST(Parallelize4DTile2D, MultiThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -2913,59 +4618,61 @@ TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolEachItemProcessedOnce) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_4d_tile_2d( threadpool.get(), - reinterpret_cast(Increment3DTile2DWithUArch), + reinterpret_cast(Increment4DTile2D), static_cast(counters.data()), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); - for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; - EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) - << "Element (" << i << ", " << j << ", " << k << ") was processed " - << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + for (size_t i = 0; i < kParallelize4DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile2DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile2DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } } } } } -TEST(Parallelize3DTile2DWithUArch, SingleThreadPoolEachItemProcessedMultipleTimes) { - std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +TEST(Parallelize4DTile2D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_4d_tile_2d( threadpool.get(), - reinterpret_cast(Increment3DTile2DWithUArch), + reinterpret_cast(Increment4DTile2D), static_cast(counters.data()), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); } - for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; - EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) - << "Element (" << i << ", " << j << ", " << k << ") was processed " - << counters[linear_idx].load(std::memory_order_relaxed) << " times " - << "(expected: " << kIncrementIterations << ")"; + for (size_t i = 0; i < kParallelize4DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile2DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile2DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } } } } } -TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolEachItemProcessedMultipleTimes) { - std::vector counters(kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); +TEST(Parallelize4DTile2D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -2975,38 +4682,39 @@ TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolEachItemProcessedMultipleTimes } for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_4d_tile_2d( threadpool.get(), - reinterpret_cast(Increment3DTile2DWithUArch), + reinterpret_cast(Increment4DTile2D), static_cast(counters.data()), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); } - for (size_t i = 0; i < kParallelize3DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize3DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize3DTile2DRangeK; k++) { - const size_t linear_idx = (i * kParallelize3DTile2DRangeJ + j) * kParallelize3DTile2DRangeK + k; - EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) - << "Element (" << i << ", " << j << ", " << k << ") was processed " - << counters[linear_idx].load(std::memory_order_relaxed) << " times " - << "(expected: " << kIncrementIterations << ")"; + for (size_t i = 0; i < kParallelize4DTile2DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile2DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile2DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile2DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } } } } } -static void IncrementSame3DTile2DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { - for (size_t j = start_j; j < start_j + tile_j; j++) { - for (size_t k = start_k; k < start_k + tile_k; k++) { +static void IncrementSame4DTile2D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + for (size_t l = start_l; l < start_l + tile_l; l++) { num_processed_items->fetch_add(1, std::memory_order_relaxed); } } } -TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolHighContention) { +TEST(Parallelize4DTile2D, MultiThreadPoolHighContention) { std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); @@ -3016,28 +4724,27 @@ TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolHighContention) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_4d_tile_2d( threadpool.get(), - reinterpret_cast(IncrementSame3DTile2DWithUArch), + reinterpret_cast(IncrementSame4DTile2D), static_cast(&num_processed_items), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); - EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); } -static void WorkImbalance3DTile2DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t i, size_t start_j, size_t start_k, size_t tile_j, size_t tile_k) { - num_processed_items->fetch_add(tile_j * tile_k, std::memory_order_relaxed); - if (i == 0 && start_j == 0 && start_k == 0) { +static void WorkImbalance4DTile2D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { + num_processed_items->fetch_add(tile_k * tile_l, std::memory_order_relaxed); + if (i == 0 && j == 0 && start_k == 0 && start_l == 0) { /* Spin-wait until all items are computed */ - while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK) { + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL) { std::atomic_thread_fence(std::memory_order_acquire); } } } -TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolWorkStealing) { +TEST(Parallelize4DTile2D, MultiThreadPoolWorkStealing) { std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); @@ -3047,33 +4754,71 @@ TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolWorkStealing) { GTEST_SKIP(); } - pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_parallelize_4d_tile_2d( threadpool.get(), - reinterpret_cast(WorkImbalance3DTile2DWithUArch), + reinterpret_cast(WorkImbalance4DTile2D), static_cast(&num_processed_items), + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); +} + +static void ComputeNothing4DTile2DWithUArch(void*, uint32_t, size_t, size_t, size_t, size_t, size_t, size_t) { +} + +TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_2d_with_uarch(threadpool.get(), + ComputeNothing4DTile2DWithUArch, + nullptr, kDefaultUArchIndex, kMaxUArchIndex, - kParallelize3DTile2DRangeI, kParallelize3DTile2DRangeJ, kParallelize3DTile2DRangeK, - kParallelize3DTile2DTileJ, kParallelize3DTile2DTileK, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); - EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); } -static void ComputeNothing4DTile2D(void*, size_t, size_t, size_t, size_t, size_t, size_t) { +TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_2d_with_uarch( + threadpool.get(), + ComputeNothing4DTile2DWithUArch, + nullptr, + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + 0 /* flags */); } -TEST(Parallelize4DTile2D, SingleThreadPoolCompletes) { +static void CheckUArch4DTile2DWithUArch(void*, uint32_t uarch_index, size_t, size_t, size_t, size_t, size_t, size_t) { + if (uarch_index != kDefaultUArchIndex) { + EXPECT_LE(uarch_index, kMaxUArchIndex); + } +} + +TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolUArchInBounds) { auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_4d_tile_2d(threadpool.get(), - ComputeNothing4DTile2D, + pthreadpool_parallelize_4d_tile_2d_with_uarch( + threadpool.get(), + CheckUArch4DTile2DWithUArch, nullptr, + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); } -TEST(Parallelize4DTile2D, MultiThreadPoolCompletes) { +TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolUArchInBounds) { auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -3081,16 +4826,17 @@ TEST(Parallelize4DTile2D, MultiThreadPoolCompletes) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_4d_tile_2d_with_uarch( threadpool.get(), - ComputeNothing4DTile2D, + CheckUArch4DTile2DWithUArch, nullptr, + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); } -static void CheckBounds4DTile2D(void*, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { +static void CheckBounds4DTile2DWithUArch(void*, uint32_t, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { EXPECT_LT(i, kParallelize4DTile2DRangeI); EXPECT_LT(j, kParallelize4DTile2DRangeJ); EXPECT_LT(start_k, kParallelize4DTile2DRangeK); @@ -3099,20 +4845,21 @@ static void CheckBounds4DTile2D(void*, size_t i, size_t j, size_t start_k, size_ EXPECT_LE(start_l + tile_l, kParallelize4DTile2DRangeL); } -TEST(Parallelize4DTile2D, SingleThreadPoolAllItemsInBounds) { +TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolAllItemsInBounds) { auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_4d_tile_2d_with_uarch( threadpool.get(), - CheckBounds4DTile2D, + CheckBounds4DTile2DWithUArch, nullptr, + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); } -TEST(Parallelize4DTile2D, MultiThreadPoolAllItemsInBounds) { +TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolAllItemsInBounds) { auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -3120,16 +4867,17 @@ TEST(Parallelize4DTile2D, MultiThreadPoolAllItemsInBounds) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_4d_tile_2d_with_uarch( threadpool.get(), - CheckBounds4DTile2D, + CheckBounds4DTile2DWithUArch, nullptr, + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); } -static void CheckTiling4DTile2D(void*, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { +static void CheckTiling4DTile2DWithUArch(void*, uint32_t, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { EXPECT_GT(tile_k, 0); EXPECT_LE(tile_k, kParallelize4DTile2DTileK); EXPECT_EQ(start_k % kParallelize4DTile2DTileK, 0); @@ -3141,20 +4889,21 @@ static void CheckTiling4DTile2D(void*, size_t i, size_t j, size_t start_k, size_ EXPECT_EQ(tile_l, std::min(kParallelize4DTile2DTileL, kParallelize4DTile2DRangeL - start_l)); } -TEST(Parallelize4DTile2D, SingleThreadPoolUniformTiling) { +TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolUniformTiling) { auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_4d_tile_2d_with_uarch( threadpool.get(), - CheckTiling4DTile2D, + CheckTiling4DTile2DWithUArch, nullptr, + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); } -TEST(Parallelize4DTile2D, MultiThreadPoolUniformTiling) { +TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolUniformTiling) { auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -3162,16 +4911,17 @@ TEST(Parallelize4DTile2D, MultiThreadPoolUniformTiling) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_4d_tile_2d_with_uarch( threadpool.get(), - CheckTiling4DTile2D, + CheckTiling4DTile2DWithUArch, nullptr, + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); } -static void SetTrue4DTile2D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { +static void SetTrue4DTile2DWithUArch(std::atomic_bool* processed_indicators, uint32_t, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { for (size_t k = start_k; k < start_k + tile_k; k++) { for (size_t l = start_l; l < start_l + tile_l; l++) { const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; @@ -3180,16 +4930,17 @@ static void SetTrue4DTile2D(std::atomic_bool* processed_indicators, size_t i, si } } -TEST(Parallelize4DTile2D, SingleThreadPoolAllItemsProcessed) { +TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolAllItemsProcessed) { std::vector indicators(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_4d_tile_2d_with_uarch( threadpool.get(), - reinterpret_cast(SetTrue4DTile2D), + reinterpret_cast(SetTrue4DTile2DWithUArch), static_cast(indicators.data()), + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); @@ -3207,7 +4958,7 @@ TEST(Parallelize4DTile2D, SingleThreadPoolAllItemsProcessed) { } } -TEST(Parallelize4DTile2D, MultiThreadPoolAllItemsProcessed) { +TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolAllItemsProcessed) { std::vector indicators(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); @@ -3217,10 +4968,11 @@ TEST(Parallelize4DTile2D, MultiThreadPoolAllItemsProcessed) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_4d_tile_2d_with_uarch( threadpool.get(), - reinterpret_cast(SetTrue4DTile2D), + reinterpret_cast(SetTrue4DTile2DWithUArch), static_cast(indicators.data()), + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); @@ -3238,7 +4990,7 @@ TEST(Parallelize4DTile2D, MultiThreadPoolAllItemsProcessed) { } } -static void Increment4DTile2D(std::atomic_int* processed_counters, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { +static void Increment4DTile2DWithUArch(std::atomic_int* processed_counters, uint32_t, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { for (size_t k = start_k; k < start_k + tile_k; k++) { for (size_t l = start_l; l < start_l + tile_l; l++) { const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; @@ -3247,16 +4999,17 @@ static void Increment4DTile2D(std::atomic_int* processed_counters, size_t i, siz } } -TEST(Parallelize4DTile2D, SingleThreadPoolEachItemProcessedOnce) { +TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolEachItemProcessedOnce) { std::vector counters(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_4d_tile_2d_with_uarch( threadpool.get(), - reinterpret_cast(Increment4DTile2D), + reinterpret_cast(Increment4DTile2DWithUArch), static_cast(counters.data()), + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); @@ -3275,7 +5028,7 @@ TEST(Parallelize4DTile2D, SingleThreadPoolEachItemProcessedOnce) { } } -TEST(Parallelize4DTile2D, MultiThreadPoolEachItemProcessedOnce) { +TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolEachItemProcessedOnce) { std::vector counters(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); @@ -3285,10 +5038,11 @@ TEST(Parallelize4DTile2D, MultiThreadPoolEachItemProcessedOnce) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_4d_tile_2d_with_uarch( threadpool.get(), - reinterpret_cast(Increment4DTile2D), + reinterpret_cast(Increment4DTile2DWithUArch), static_cast(counters.data()), + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); @@ -3307,17 +5061,18 @@ TEST(Parallelize4DTile2D, MultiThreadPoolEachItemProcessedOnce) { } } -TEST(Parallelize4DTile2D, SingleThreadPoolEachItemProcessedMultipleTimes) { +TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolEachItemProcessedMultipleTimes) { std::vector counters(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_4d_tile_2d_with_uarch( threadpool.get(), - reinterpret_cast(Increment4DTile2D), + reinterpret_cast(Increment4DTile2DWithUArch), static_cast(counters.data()), + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); @@ -3338,7 +5093,7 @@ TEST(Parallelize4DTile2D, SingleThreadPoolEachItemProcessedMultipleTimes) { } } -TEST(Parallelize4DTile2D, MultiThreadPoolEachItemProcessedMultipleTimes) { +TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolEachItemProcessedMultipleTimes) { std::vector counters(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); @@ -3349,10 +5104,11 @@ TEST(Parallelize4DTile2D, MultiThreadPoolEachItemProcessedMultipleTimes) { } for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_4d_tile_2d_with_uarch( threadpool.get(), - reinterpret_cast(Increment4DTile2D), + reinterpret_cast(Increment4DTile2DWithUArch), static_cast(counters.data()), + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); @@ -3373,7 +5129,7 @@ TEST(Parallelize4DTile2D, MultiThreadPoolEachItemProcessedMultipleTimes) { } } -static void IncrementSame4DTile2D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { +static void IncrementSame4DTile2DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { for (size_t k = start_k; k < start_k + tile_k; k++) { for (size_t l = start_l; l < start_l + tile_l; l++) { num_processed_items->fetch_add(1, std::memory_order_relaxed); @@ -3381,7 +5137,7 @@ static void IncrementSame4DTile2D(std::atomic_int* num_processed_items, size_t i } } -TEST(Parallelize4DTile2D, MultiThreadPoolHighContention) { +TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolHighContention) { std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); @@ -3391,17 +5147,18 @@ TEST(Parallelize4DTile2D, MultiThreadPoolHighContention) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_4d_tile_2d_with_uarch( threadpool.get(), - reinterpret_cast(IncrementSame4DTile2D), + reinterpret_cast(IncrementSame4DTile2DWithUArch), static_cast(&num_processed_items), + kDefaultUArchIndex, kMaxUArchIndex, kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, 0 /* flags */); EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); } -static void WorkImbalance4DTile2D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { +static void WorkImbalance4DTile2DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { num_processed_items->fetch_add(tile_k * tile_l, std::memory_order_relaxed); if (i == 0 && j == 0 && start_k == 0 && start_l == 0) { /* Spin-wait until all items are computed */ @@ -3411,7 +5168,298 @@ static void WorkImbalance4DTile2D(std::atomic_int* num_processed_items, size_t i } } -TEST(Parallelize4DTile2D, MultiThreadPoolWorkStealing) { +TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_2d_with_uarch( + threadpool.get(), + reinterpret_cast(WorkImbalance4DTile2DWithUArch), + static_cast(&num_processed_items), + kDefaultUArchIndex, kMaxUArchIndex, + kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, + kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); +} + +static void ComputeNothing5D(void*, size_t, size_t, size_t, size_t, size_t) { +} + +TEST(Parallelize5D, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d(threadpool.get(), + ComputeNothing5D, + nullptr, + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); +} + +TEST(Parallelize5D, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d( + threadpool.get(), + ComputeNothing5D, + nullptr, + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); +} + +static void CheckBounds5D(void*, size_t i, size_t j, size_t k, size_t l, size_t m) { + EXPECT_LT(i, kParallelize5DRangeI); + EXPECT_LT(j, kParallelize5DRangeJ); + EXPECT_LT(k, kParallelize5DRangeK); + EXPECT_LT(l, kParallelize5DRangeL); + EXPECT_LT(m, kParallelize5DRangeM); +} + +TEST(Parallelize5D, SingleThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d( + threadpool.get(), + CheckBounds5D, + nullptr, + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); +} + +TEST(Parallelize5D, MultiThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d( + threadpool.get(), + CheckBounds5D, + nullptr, + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); +} + +static void SetTrue5D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t k, size_t l, size_t m) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); +} + +TEST(Parallelize5D, SingleThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast(SetTrue5D), + static_cast(indicators.data()), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize5DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") not processed"; + } + } + } + } + } +} + +TEST(Parallelize5D, MultiThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast(SetTrue5D), + static_cast(indicators.data()), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize5DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") not processed"; + } + } + } + } + } +} + +static void Increment5D(std::atomic_int* processed_counters, size_t i, size_t j, size_t k, size_t l, size_t m) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); +} + +TEST(Parallelize5D, SingleThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast(Increment5D), + static_cast(counters.data()), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize5DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } + } +} + +TEST(Parallelize5D, MultiThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast(Increment5D), + static_cast(counters.data()), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize5DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } + } +} + +TEST(Parallelize5D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + for (size_t iteration = 0; iteration < kIncrementIterations5D; iteration++) { + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast(Increment5D), + static_cast(counters.data()), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize5DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations5D) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations5D << ")"; + } + } + } + } + } +} + +TEST(Parallelize5D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + for (size_t iteration = 0; iteration < kIncrementIterations5D; iteration++) { + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast(Increment5D), + static_cast(counters.data()), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize5DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations5D) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations5D << ")"; + } + } + } + } + } +} + +static void IncrementSame5D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t l, size_t m) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); +} + +TEST(Parallelize5D, MultiThreadPoolHighContention) { std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); @@ -3421,33 +5469,28 @@ TEST(Parallelize4DTile2D, MultiThreadPoolWorkStealing) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d( + pthreadpool_parallelize_5d( threadpool.get(), - reinterpret_cast(WorkImbalance4DTile2D), + reinterpret_cast(IncrementSame5D), static_cast(&num_processed_items), - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, 0 /* flags */); - EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); } -static void ComputeNothing4DTile2DWithUArch(void*, uint32_t, size_t, size_t, size_t, size_t, size_t, size_t) { +static void WorkImbalance5D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t l, size_t m) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + if (i == 0 && j == 0 && k == 0 && l == 0 && m == 0) { + /* Spin-wait until all items are computed */ + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM) { + std::atomic_thread_fence(std::memory_order_acquire); + } + } } -TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolCompletes) { - auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); - ASSERT_TRUE(threadpool.get()); - - pthreadpool_parallelize_4d_tile_2d_with_uarch(threadpool.get(), - ComputeNothing4DTile2DWithUArch, - nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, - 0 /* flags */); -} +TEST(Parallelize5D, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); -TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolCompletes) { auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -3455,37 +5498,31 @@ TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolCompletes) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_parallelize_5d( threadpool.get(), - ComputeNothing4DTile2DWithUArch, - nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + reinterpret_cast(WorkImbalance5D), + static_cast(&num_processed_items), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); } -static void CheckUArch4DTile2DWithUArch(void*, uint32_t uarch_index, size_t, size_t, size_t, size_t, size_t, size_t) { - if (uarch_index != kDefaultUArchIndex) { - EXPECT_LE(uarch_index, kMaxUArchIndex); - } +static void ComputeNothing5DTile1D(void*, size_t, size_t, size_t, size_t, size_t, size_t) { } -TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolUArchInBounds) { +TEST(Parallelize5DTile1D, SingleThreadPoolCompletes) { auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_4d_tile_2d_with_uarch( - threadpool.get(), - CheckUArch4DTile2DWithUArch, + pthreadpool_parallelize_5d_tile_1d(threadpool.get(), + ComputeNothing5DTile1D, nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); } -TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolUArchInBounds) { +TEST(Parallelize5DTile1D, MultiThreadPoolCompletes) { auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -3493,40 +5530,38 @@ TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolUArchInBounds) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - CheckUArch4DTile2DWithUArch, + ComputeNothing5DTile1D, nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); } -static void CheckBounds4DTile2DWithUArch(void*, uint32_t, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { - EXPECT_LT(i, kParallelize4DTile2DRangeI); - EXPECT_LT(j, kParallelize4DTile2DRangeJ); - EXPECT_LT(start_k, kParallelize4DTile2DRangeK); - EXPECT_LT(start_l, kParallelize4DTile2DRangeL); - EXPECT_LE(start_k + tile_k, kParallelize4DTile2DRangeK); - EXPECT_LE(start_l + tile_l, kParallelize4DTile2DRangeL); +static void CheckBounds5DTile1D(void*, size_t i, size_t j, size_t k, size_t l, size_t start_m, size_t tile_m) { + EXPECT_LT(i, kParallelize5DTile1DRangeI); + EXPECT_LT(j, kParallelize5DTile1DRangeJ); + EXPECT_LT(k, kParallelize5DTile1DRangeK); + EXPECT_LT(l, kParallelize5DTile1DRangeL); + EXPECT_LT(start_m, kParallelize5DTile1DRangeM); + EXPECT_LE(start_m + tile_m, kParallelize5DTile1DRangeM); } -TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolAllItemsInBounds) { +TEST(Parallelize5DTile1D, SingleThreadPoolAllItemsInBounds) { auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - CheckBounds4DTile2DWithUArch, + CheckBounds5DTile1D, nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); } -TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolAllItemsInBounds) { +TEST(Parallelize5DTile1D, MultiThreadPoolAllItemsInBounds) { auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -3534,43 +5569,36 @@ TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolAllItemsInBounds) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - CheckBounds4DTile2DWithUArch, + CheckBounds5DTile1D, nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); } -static void CheckTiling4DTile2DWithUArch(void*, uint32_t, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { - EXPECT_GT(tile_k, 0); - EXPECT_LE(tile_k, kParallelize4DTile2DTileK); - EXPECT_EQ(start_k % kParallelize4DTile2DTileK, 0); - EXPECT_EQ(tile_k, std::min(kParallelize4DTile2DTileK, kParallelize4DTile2DRangeK - start_k)); - - EXPECT_GT(tile_l, 0); - EXPECT_LE(tile_l, kParallelize4DTile2DTileL); - EXPECT_EQ(start_l % kParallelize4DTile2DTileL, 0); - EXPECT_EQ(tile_l, std::min(kParallelize4DTile2DTileL, kParallelize4DTile2DRangeL - start_l)); +static void CheckTiling5DTile1D(void*, size_t i, size_t j, size_t k, size_t l, size_t start_m, size_t tile_m) { + EXPECT_GT(tile_m, 0); + EXPECT_LE(tile_m, kParallelize5DTile1DTileM); + EXPECT_EQ(start_m % kParallelize5DTile1DTileM, 0); + EXPECT_EQ(tile_m, std::min(kParallelize5DTile1DTileM, kParallelize5DTile1DRangeM - start_m)); } -TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolUniformTiling) { +TEST(Parallelize5DTile1D, SingleThreadPoolUniformTiling) { auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - CheckTiling4DTile2DWithUArch, + CheckTiling5DTile1D, nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); } -TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolUniformTiling) { +TEST(Parallelize5DTile1D, MultiThreadPoolUniformTiling) { auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -3578,55 +5606,53 @@ TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolUniformTiling) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - CheckTiling4DTile2DWithUArch, + CheckTiling5DTile1D, nullptr, - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); } -static void SetTrue4DTile2DWithUArch(std::atomic_bool* processed_indicators, uint32_t, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { - for (size_t k = start_k; k < start_k + tile_k; k++) { - for (size_t l = start_l; l < start_l + tile_l; l++) { - const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; - processed_indicators[linear_idx].store(true, std::memory_order_relaxed); - } +static void SetTrue5DTile1D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t k, size_t l, size_t start_m, size_t tile_m) { + for (size_t m = start_m; m < start_m + tile_m; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); } } -TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolAllItemsProcessed) { - std::vector indicators(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); +TEST(Parallelize5DTile1D, SingleThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - reinterpret_cast(SetTrue4DTile2DWithUArch), + reinterpret_cast(SetTrue5DTile1D), static_cast(indicators.data()), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); - for (size_t i = 0; i < kParallelize4DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize4DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize4DTile2DRangeK; k++) { - for (size_t l = 0; l < kParallelize4DTile2DRangeL; l++) { - const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; - EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) - << "Element (" << i << ", " << j << ", " << k << ", " << l << ") not processed"; + for (size_t i = 0; i < kParallelize5DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DTile1DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DTile1DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") not processed"; + } } } } } } -TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolAllItemsProcessed) { - std::vector indicators(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); +TEST(Parallelize5DTile1D, MultiThreadPoolAllItemsProcessed) { + std::vector indicators(kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -3635,68 +5661,68 @@ TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolAllItemsProcessed) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - reinterpret_cast(SetTrue4DTile2DWithUArch), + reinterpret_cast(SetTrue5DTile1D), static_cast(indicators.data()), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); - for (size_t i = 0; i < kParallelize4DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize4DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize4DTile2DRangeK; k++) { - for (size_t l = 0; l < kParallelize4DTile2DRangeL; l++) { - const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; - EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) - << "Element (" << i << ", " << j << ", " << k << ", " << l << ") not processed"; + for (size_t i = 0; i < kParallelize5DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DTile1DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DTile1DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") not processed"; + } } } } } } -static void Increment4DTile2DWithUArch(std::atomic_int* processed_counters, uint32_t, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { - for (size_t k = start_k; k < start_k + tile_k; k++) { - for (size_t l = start_l; l < start_l + tile_l; l++) { - const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; - processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); - } +static void Increment5DTile1D(std::atomic_int* processed_counters, size_t i, size_t j, size_t k, size_t l, size_t start_m, size_t tile_m) { + for (size_t m = start_m; m < start_m + tile_m; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); } } -TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolEachItemProcessedOnce) { - std::vector counters(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); +TEST(Parallelize5DTile1D, SingleThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - reinterpret_cast(Increment4DTile2DWithUArch), + reinterpret_cast(Increment5DTile1D), static_cast(counters.data()), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); - for (size_t i = 0; i < kParallelize4DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize4DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize4DTile2DRangeK; k++) { - for (size_t l = 0; l < kParallelize4DTile2DRangeL; l++) { - const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; - EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) - << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " - << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + for (size_t i = 0; i < kParallelize5DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DTile1DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DTile1DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } } } } } } -TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolEachItemProcessedOnce) { - std::vector counters(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); +TEST(Parallelize5DTile1D, MultiThreadPoolEachItemProcessedOnce) { + std::vector counters(kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -3705,63 +5731,65 @@ TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolEachItemProcessedOnce) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - reinterpret_cast(Increment4DTile2DWithUArch), + reinterpret_cast(Increment5DTile1D), static_cast(counters.data()), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); - for (size_t i = 0; i < kParallelize4DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize4DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize4DTile2DRangeK; k++) { - for (size_t l = 0; l < kParallelize4DTile2DRangeL; l++) { - const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; - EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) - << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " - << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + for (size_t i = 0; i < kParallelize5DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DTile1DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DTile1DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } } } } } } -TEST(Parallelize4DTile2DWithUArch, SingleThreadPoolEachItemProcessedMultipleTimes) { - std::vector counters(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); +TEST(Parallelize5DTile1D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); - for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { - pthreadpool_parallelize_4d_tile_2d_with_uarch( + for (size_t iteration = 0; iteration < kIncrementIterations5D; iteration++) { + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - reinterpret_cast(Increment4DTile2DWithUArch), + reinterpret_cast(Increment5DTile1D), static_cast(counters.data()), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); } - for (size_t i = 0; i < kParallelize4DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize4DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize4DTile2DRangeK; k++) { - for (size_t l = 0; l < kParallelize4DTile2DRangeL; l++) { - const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; - EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) - << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " - << counters[linear_idx].load(std::memory_order_relaxed) << " times " - << "(expected: " << kIncrementIterations << ")"; + for (size_t i = 0; i < kParallelize5DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DTile1DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DTile1DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations5D) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations5D << ")"; + } } } } } } -TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolEachItemProcessedMultipleTimes) { - std::vector counters(kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); +TEST(Parallelize5DTile1D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector counters(kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); ASSERT_TRUE(threadpool.get()); @@ -3770,41 +5798,40 @@ TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolEachItemProcessedMultipleTimes GTEST_SKIP(); } - for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { - pthreadpool_parallelize_4d_tile_2d_with_uarch( + for (size_t iteration = 0; iteration < kIncrementIterations5D; iteration++) { + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - reinterpret_cast(Increment4DTile2DWithUArch), + reinterpret_cast(Increment5DTile1D), static_cast(counters.data()), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); } - for (size_t i = 0; i < kParallelize4DTile2DRangeI; i++) { - for (size_t j = 0; j < kParallelize4DTile2DRangeJ; j++) { - for (size_t k = 0; k < kParallelize4DTile2DRangeK; k++) { - for (size_t l = 0; l < kParallelize4DTile2DRangeL; l++) { - const size_t linear_idx = ((i * kParallelize4DTile2DRangeJ + j) * kParallelize4DTile2DRangeK + k) * kParallelize4DTile2DRangeL + l; - EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) - << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " - << counters[linear_idx].load(std::memory_order_relaxed) << " times " - << "(expected: " << kIncrementIterations << ")"; + for (size_t i = 0; i < kParallelize5DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DTile1DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DTile1DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations5D) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations5D << ")"; + } } } } } } -static void IncrementSame4DTile2DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { - for (size_t k = start_k; k < start_k + tile_k; k++) { - for (size_t l = start_l; l < start_l + tile_l; l++) { - num_processed_items->fetch_add(1, std::memory_order_relaxed); - } +static void IncrementSame5DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t l, size_t start_m, size_t tile_m) { + for (size_t m = start_m; m < start_m + tile_m; m++) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); } } -TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolHighContention) { +TEST(Parallelize5DTile1D, MultiThreadPoolHighContention) { std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); @@ -3814,28 +5841,27 @@ TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolHighContention) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - reinterpret_cast(IncrementSame4DTile2DWithUArch), + reinterpret_cast(IncrementSame5DTile1D), static_cast(&num_processed_items), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); - EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); } -static void WorkImbalance4DTile2DWithUArch(std::atomic_int* num_processed_items, uint32_t, size_t i, size_t j, size_t start_k, size_t start_l, size_t tile_k, size_t tile_l) { - num_processed_items->fetch_add(tile_k * tile_l, std::memory_order_relaxed); - if (i == 0 && j == 0 && start_k == 0 && start_l == 0) { +static void WorkImbalance5DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t l, size_t start_m, size_t tile_m) { + num_processed_items->fetch_add(tile_m, std::memory_order_relaxed); + if (i == 0 && j == 0 && k == 0 && l == 0 && start_m == 0) { /* Spin-wait until all items are computed */ - while (num_processed_items->load(std::memory_order_relaxed) != kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL) { + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM) { std::atomic_thread_fence(std::memory_order_acquire); } } } -TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolWorkStealing) { +TEST(Parallelize5DTile1D, MultiThreadPoolWorkStealing) { std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); @@ -3845,15 +5871,14 @@ TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolWorkStealing) { GTEST_SKIP(); } - pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_parallelize_5d_tile_1d( threadpool.get(), - reinterpret_cast(WorkImbalance4DTile2DWithUArch), + reinterpret_cast(WorkImbalance5DTile1D), static_cast(&num_processed_items), - kDefaultUArchIndex, kMaxUArchIndex, - kParallelize4DTile2DRangeI, kParallelize4DTile2DRangeJ, kParallelize4DTile2DRangeK, kParallelize4DTile2DRangeL, - kParallelize4DTile2DTileK, kParallelize4DTile2DTileL, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, 0 /* flags */); - EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); } static void ComputeNothing5DTile2D(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t) { -- cgit v1.2.3 From 18a7156cb9be8e534acefade42e46d4209600c35 Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Tue, 26 May 2020 10:03:31 -0700 Subject: Use cpuinfo_get_current_uarch_index_with_default for parallelization with uarch index --- cmake/DownloadCpuinfo.cmake | 4 ++-- src/fastpath.c | 8 ++++---- src/portable-api.c | 16 ++++++++-------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cmake/DownloadCpuinfo.cmake b/cmake/DownloadCpuinfo.cmake index 25213a0..e6f2893 100644 --- a/cmake/DownloadCpuinfo.cmake +++ b/cmake/DownloadCpuinfo.cmake @@ -4,8 +4,8 @@ PROJECT(cpuinfo-download NONE) INCLUDE(ExternalProject) ExternalProject_Add(cpuinfo - URL https://github.com/pytorch/cpuinfo/archive/0cc563acb9baac39f2c1349bc42098c4a1da59e3.tar.gz - URL_HASH SHA256=80625d0b69a3d69b70c2236f30db2c542d0922ccf9bb51a61bc39c49fac91a35 + URL https://github.com/pytorch/cpuinfo/archive/19b9316c71e4e45b170a664bf62ddefd7ac9feb5.zip + URL_HASH SHA256=e0a485c072de957668eb324c49d726dc0fd736cfb9436b334325f20d93085003 SOURCE_DIR "${CMAKE_BINARY_DIR}/cpuinfo-source" BINARY_DIR "${CMAKE_BINARY_DIR}/cpuinfo" CONFIGURE_COMMAND "" diff --git a/src/fastpath.c b/src/fastpath.c index b4e40c5..6abbebe 100644 --- a/src/fastpath.c +++ b/src/fastpath.c @@ -71,7 +71,7 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_with_uarch_fastpath( const uint32_t default_uarch_index = threadpool->params.parallelize_1d_with_uarch.default_uarch_index; uint32_t uarch_index = default_uarch_index; #if PTHREADPOOL_USE_CPUINFO - uarch_index = cpuinfo_get_current_uarch_index(); + uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index); if (uarch_index > threadpool->params.parallelize_1d_with_uarch.max_uarch_index) { uarch_index = default_uarch_index; } @@ -307,7 +307,7 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_with_uarch_f const uint32_t default_uarch_index = threadpool->params.parallelize_2d_tile_2d_with_uarch.default_uarch_index; uint32_t uarch_index = default_uarch_index; #if PTHREADPOOL_USE_CPUINFO - uarch_index = cpuinfo_get_current_uarch_index(); + uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index); if (uarch_index > threadpool->params.parallelize_2d_tile_2d_with_uarch.max_uarch_index) { uarch_index = default_uarch_index; } @@ -540,7 +540,7 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_with_uarch_f const uint32_t default_uarch_index = threadpool->params.parallelize_3d_tile_2d_with_uarch.default_uarch_index; uint32_t uarch_index = default_uarch_index; #if PTHREADPOOL_USE_CPUINFO - uarch_index = cpuinfo_get_current_uarch_index(); + uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index); if (uarch_index > threadpool->params.parallelize_3d_tile_2d_with_uarch.max_uarch_index) { uarch_index = default_uarch_index; } @@ -804,7 +804,7 @@ PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_with_uarch_f const uint32_t default_uarch_index = threadpool->params.parallelize_4d_tile_2d_with_uarch.default_uarch_index; uint32_t uarch_index = default_uarch_index; #if PTHREADPOOL_USE_CPUINFO - uarch_index = cpuinfo_get_current_uarch_index(); + uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index); if (uarch_index > threadpool->params.parallelize_4d_tile_2d_with_uarch.max_uarch_index) { uarch_index = default_uarch_index; } diff --git a/src/portable-api.c b/src/portable-api.c index 6a8ccf2..ef36578 100644 --- a/src/portable-api.c +++ b/src/portable-api.c @@ -70,7 +70,7 @@ static void thread_parallelize_1d_with_uarch(struct pthreadpool* threadpool, str const uint32_t default_uarch_index = threadpool->params.parallelize_1d_with_uarch.default_uarch_index; uint32_t uarch_index = default_uarch_index; #if PTHREADPOOL_USE_CPUINFO - uarch_index = cpuinfo_get_current_uarch_index(); + uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index); if (uarch_index > threadpool->params.parallelize_1d_with_uarch.max_uarch_index) { uarch_index = default_uarch_index; } @@ -281,7 +281,7 @@ static void thread_parallelize_2d_tile_2d_with_uarch(struct pthreadpool* threadp const uint32_t default_uarch_index = threadpool->params.parallelize_2d_tile_2d_with_uarch.default_uarch_index; uint32_t uarch_index = default_uarch_index; #if PTHREADPOOL_USE_CPUINFO - uarch_index = cpuinfo_get_current_uarch_index(); + uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index); if (uarch_index > threadpool->params.parallelize_2d_tile_2d_with_uarch.max_uarch_index) { uarch_index = default_uarch_index; } @@ -494,7 +494,7 @@ static void thread_parallelize_3d_tile_2d_with_uarch(struct pthreadpool* threadp const uint32_t default_uarch_index = threadpool->params.parallelize_3d_tile_2d_with_uarch.default_uarch_index; uint32_t uarch_index = default_uarch_index; #if PTHREADPOOL_USE_CPUINFO - uarch_index = cpuinfo_get_current_uarch_index(); + uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index); if (uarch_index > threadpool->params.parallelize_3d_tile_2d_with_uarch.max_uarch_index) { uarch_index = default_uarch_index; } @@ -738,7 +738,7 @@ static void thread_parallelize_4d_tile_2d_with_uarch(struct pthreadpool* threadp const uint32_t default_uarch_index = threadpool->params.parallelize_4d_tile_2d_with_uarch.default_uarch_index; uint32_t uarch_index = default_uarch_index; #if PTHREADPOOL_USE_CPUINFO - uarch_index = cpuinfo_get_current_uarch_index(); + uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index); if (uarch_index > threadpool->params.parallelize_4d_tile_2d_with_uarch.max_uarch_index) { uarch_index = default_uarch_index; } @@ -1131,7 +1131,7 @@ void pthreadpool_parallelize_1d_with_uarch( uint32_t uarch_index = default_uarch_index; #if PTHREADPOOL_USE_CPUINFO - uarch_index = cpuinfo_get_current_uarch_index(); + uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index); if (uarch_index > max_uarch_index) { uarch_index = default_uarch_index; } @@ -1363,7 +1363,7 @@ void pthreadpool_parallelize_2d_tile_2d_with_uarch( uint32_t uarch_index = default_uarch_index; #if PTHREADPOOL_USE_CPUINFO - uarch_index = cpuinfo_get_current_uarch_index(); + uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index); if (uarch_index > max_uarch_index) { uarch_index = default_uarch_index; } @@ -1577,7 +1577,7 @@ void pthreadpool_parallelize_3d_tile_2d_with_uarch( uint32_t uarch_index = default_uarch_index; #if PTHREADPOOL_USE_CPUINFO - uarch_index = cpuinfo_get_current_uarch_index(); + uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index); if (uarch_index > max_uarch_index) { uarch_index = default_uarch_index; } @@ -1812,7 +1812,7 @@ void pthreadpool_parallelize_4d_tile_2d_with_uarch( uint32_t uarch_index = default_uarch_index; #if PTHREADPOOL_USE_CPUINFO - uarch_index = cpuinfo_get_current_uarch_index(); + uarch_index = cpuinfo_get_current_uarch_index_with_default(default_uarch_index); if (uarch_index > max_uarch_index) { uarch_index = default_uarch_index; } -- cgit v1.2.3 From e1642461b3b0217d23d6664d839a060f54e4e652 Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Wed, 10 Jun 2020 12:54:14 -0700 Subject: Recognize "armv7a" cpu as Linux/ARM in Bazel build --- BUILD.bazel | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/BUILD.bazel b/BUILD.bazel index 0bad9a7..c9b986f 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -66,6 +66,7 @@ cc_library( }) + select({ ":linux_arm": ["-DPTHREADPOOL_USE_CPUINFO=1"], ":linux_armhf": ["-DPTHREADPOOL_USE_CPUINFO=1"], + ":linux_armv7a": ["-DPTHREADPOOL_USE_CPUINFO=1"], ":linux_aarch64": ["-DPTHREADPOOL_USE_CPUINFO=1"], ":android_armv7": ["-DPTHREADPOOL_USE_CPUINFO=1"], ":android_arm64": ["-DPTHREADPOOL_USE_CPUINFO=1"], @@ -132,6 +133,7 @@ cc_library( ] + select({ ":linux_arm": ["@cpuinfo"], ":linux_armhf": ["@cpuinfo"], + ":linux_armv7a": ["@cpuinfo"], ":linux_aarch64": ["@cpuinfo"], ":android_armv7": ["@cpuinfo"], ":android_arm64": ["@cpuinfo"], @@ -247,6 +249,11 @@ config_setting( values = {"cpu": "armhf"}, ) +config_setting( + name = "linux_armv7a", + values = {"cpu": "armv7a"}, +) + config_setting( name = "linux_aarch64", values = {"cpu": "aarch64"}, -- cgit v1.2.3 From 029c88620802e1361ccf41d1970bd5b07fd6b7bb Mon Sep 17 00:00:00 2001 From: Marat Dukhan Date: Mon, 15 Jun 2020 15:41:33 -0700 Subject: Recognize "armeabi" cpu as Linux/ARM in Bazel build --- BUILD.bazel | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/BUILD.bazel b/BUILD.bazel index c9b986f..0b832cf 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -65,6 +65,7 @@ cc_library( "//conditions:default": [], }) + select({ ":linux_arm": ["-DPTHREADPOOL_USE_CPUINFO=1"], + ":linux_armeabi": ["-DPTHREADPOOL_USE_CPUINFO=1"], ":linux_armhf": ["-DPTHREADPOOL_USE_CPUINFO=1"], ":linux_armv7a": ["-DPTHREADPOOL_USE_CPUINFO=1"], ":linux_aarch64": ["-DPTHREADPOOL_USE_CPUINFO=1"], @@ -132,6 +133,7 @@ cc_library( "@FXdiv", ] + select({ ":linux_arm": ["@cpuinfo"], + ":linux_armeabi": ["@cpuinfo"], ":linux_armhf": ["@cpuinfo"], ":linux_armv7a": ["@cpuinfo"], ":linux_aarch64": ["@cpuinfo"], @@ -244,6 +246,11 @@ config_setting( values = {"cpu": "arm"}, ) +config_setting( + name = "linux_armeabi", + values = {"cpu": "armeabi"}, +) + config_setting( name = "linux_armhf", values = {"cpu": "armhf"}, -- cgit v1.2.3