Fast path using atomic decrement instead of atomic compare-and-swap

50% higher throughput on x86 (disabled on other platforms)

5 files changed, 1117 insertions, 42 deletions

diff --git a/BUILD.bazel b/BUILD.bazel
index af1401b..fa00807 100644
--- a/BUILD.bazel
+++ b/BUILD.bazel
@@ -34,6 +34,10 @@ PORTABLE_SRCS = [
     "src/portable-api.c",
 ]
 
+ARCH_SPECIFIC_SRCS = [
+    "src/fastpath.c",
+]
+
 PTHREADS_IMPL_SRCS = PORTABLE_SRCS + ["src/pthreads.c"]
 
 GCD_IMPL_SRCS = PORTABLE_SRCS + ["src/gcd.c"]
@@ -54,9 +58,23 @@ cc_library(
         ":macos_x86": INTERNAL_HDRS + GCD_IMPL_SRCS,
         ":macos_x86_64": INTERNAL_HDRS + GCD_IMPL_SRCS,
         ":ios": INTERNAL_HDRS + GCD_IMPL_SRCS,
+        ":watchos": INTERNAL_HDRS + GCD_IMPL_SRCS,
+        ":tvos": INTERNAL_HDRS + GCD_IMPL_SRCS,
         ":windows_x86_64": INTERNAL_HDRS + WINDOWS_IMPL_SRCS,
-        ":windows_x86_64_msvc": INTERNAL_HDRS + WINDOWS_IMPL_SRCS,
         "//conditions:default": INTERNAL_HDRS + PTHREADS_IMPL_SRCS,
+    }) + select({
+        ":linux_x86_64": ARCH_SPECIFIC_SRCS,
+        ":android_x86": ARCH_SPECIFIC_SRCS,
+        ":android_x86_64": ARCH_SPECIFIC_SRCS,
+        ":windows_x86_64": ARCH_SPECIFIC_SRCS,
+        ":macos_x86": ARCH_SPECIFIC_SRCS,
+        ":macos_x86_64": ARCH_SPECIFIC_SRCS,
+        ":ios_x86": ARCH_SPECIFIC_SRCS,
+        ":ios_x86_64": ARCH_SPECIFIC_SRCS,
+        ":watchos_x86": ARCH_SPECIFIC_SRCS,
+        ":watchos_x86_64": ARCH_SPECIFIC_SRCS,
+        ":tvos_x86_64": ARCH_SPECIFIC_SRCS,
+        "//conditions:default": [],
     }),
     copts = [
         "-std=gnu11",
@@ -96,6 +114,19 @@ cc_library(
             "-DPTHREADPOOL_USE_EVENT=1",
         ],
         "//conditions:default": [],
+    }) + select({
+        ":linux_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"],
+        ":android_x86": ["-DPTHREADPOOL_USE_FASTPATH=1"],
+        ":android_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"],
+        ":windows_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"],
+        ":macos_x86": ["-DPTHREADPOOL_USE_FASTPATH=1"],
+        ":macos_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"],
+        ":ios_x86": ["-DPTHREADPOOL_USE_FASTPATH=1"],
+        ":ios_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"],
+        ":watchos_x86": ["-DPTHREADPOOL_USE_FASTPATH=1"],
+        ":watchos_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"],
+        ":tvos_x86_64": ["-DPTHREADPOOL_USE_FASTPATH=1"],
+        "//conditions:default": ["-DPTHREADPOOL_USE_FASTPATH=0"],
     }),
     hdrs = [
         "include/pthreadpool.h",
@@ -220,6 +251,11 @@ config_setting(
 )
 
 config_setting(
+    name = "linux_x86_64",
+    values = {"cpu": "k8"},
+)
+
+config_setting(
     name = "linux_arm",
     values = {"cpu": "arm"},
 )
@@ -235,6 +271,22 @@ config_setting(
 )
 
 config_setting(
+    name = "android_x86",
+    values = {
+        "crosstool_top": "//external:android/crosstool",
+        "cpu": "x86",
+    },
+)
+
+config_setting(
+    name = "android_x86_64",
+    values = {
+        "crosstool_top": "//external:android/crosstool",
+        "cpu": "x86_64",
+    },
+)
+
+config_setting(
     name = "android_armv7",
     values = {
         "crosstool_top": "//external:android/crosstool",
@@ -278,16 +330,65 @@ config_setting(
 )
 
 config_setting(
-    name = "windows_x86_64",
+    name = "ios_x86",
     values = {
-        "cpu": "x64_windows",
+        "apple_platform_type": "ios",
+        "cpu": "ios_i386",
+    },
+)
+
+config_setting(
+    name = "ios_x86_64",
+    values = {
+        "apple_platform_type": "ios",
+        "cpu": "ios_x86_64",
     },
 )
 
 config_setting(
-    name = "windows_x86_64_msvc",
+    name = "watchos",
     values = {
-        "cpu": "x64_windows_msvc",
+        "crosstool_top": "@bazel_tools//tools/cpp:toolchain",
+        "apple_platform_type": "watchos",
+    },
+)
+
+config_setting(
+    name = "watchos_x86",
+    values = {
+        "apple_platform_type": "watchos",
+        "cpu": "watchos_i386",
+    },
+)
+
+config_setting(
+    name = "watchos_x86_64",
+    values = {
+        "apple_platform_type": "watchos",
+        "cpu": "watchos_x86_64",
+    },
+)
+
+config_setting(
+    name = "tvos",
+    values = {
+        "crosstool_top": "@bazel_tools//tools/cpp:toolchain",
+        "apple_platform_type": "tvos",
+    },
+)
+
+config_setting(
+    name = "tvos_x86_64",
+    values = {
+        "apple_platform_type": "tvos",
+        "cpu": "tvos_x86_64",
+    },
+)
+
+config_setting(
+    name = "windows_x86_64",
+    values = {
+        "cpu": "x64_windows",
     },
 )
 
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 51b0105..0db3264 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -9,6 +9,11 @@ SET_PROPERTY(CACHE PTHREADPOOL_LIBRARY_TYPE PROPERTY STRINGS default static shar
 OPTION(PTHREADPOOL_ALLOW_DEPRECATED_API "Enable deprecated API functions" ON)
 SET(PTHREADPOOL_SYNC_PRIMITIVE "default" CACHE STRING "Synchronization primitive (condvar, futex, gcd, event, or default) for worker threads")
 SET_PROPERTY(CACHE PTHREADPOOL_SYNC_PRIMITIVE PROPERTY STRINGS default condvar futex gcd event)
+IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^(i[3-6]86|AMD64|x86(_64)?)$")
+  OPTION(PTHREADPOOL_ENABLE_FASTPATH "Enable fast path using atomic decrement instead of atomic compare-and-swap" ON)
+ELSE()
+  OPTION(PTHREADPOOL_ENABLE_FASTPATH "Enable fast path using atomic decrement instead of atomic compare-and-swap" OFF)
+ENDIF()
 IF("${CMAKE_SOURCE_DIR}" STREQUAL "${PROJECT_SOURCE_DIR}")
   OPTION(PTHREADPOOL_BUILD_TESTS "Build pthreadpool unit tests" ON)
   OPTION(PTHREADPOOL_BUILD_BENCHMARKS "Build pthreadpool micro-benchmarks" ON)
@@ -76,6 +81,9 @@ ELSE()
   ELSE()
     LIST(APPEND PTHREADPOOL_SRCS src/pthreads.c)
   ENDIF()
+  IF(PTHREADPOOL_ENABLE_FASTPATH)
+    LIST(APPEND PTHREADPOOL_SRCS src/fastpath.c)
+  ENDIF()
 ENDIF()
 
 ADD_LIBRARY(pthreadpool_interface INTERFACE)
@@ -114,6 +122,11 @@ ELSEIF(PTHREADPOOL_SYNC_PRIMITIVE STREQUAL "event")
 ELSEIF(NOT PTHREADPOOL_SYNC_PRIMITIVE STREQUAL "default")
   MESSAGE(FATAL_ERROR "Unsupported synchronization primitive ${PTHREADPOOL_SYNC_PRIMITIVE}")
 ENDIF()
+IF(PTHREADPOOL_ENABLE_FASTPATH)
+  TARGET_COMPILE_DEFINITIONS(pthreadpool PRIVATE PTHREADPOOL_ENABLE_FASTPATH=1)
+ELSE()
+  TARGET_COMPILE_DEFINITIONS(pthreadpool PRIVATE PTHREADPOOL_ENABLE_FASTPATH=0)
+ENDIF()
 
 SET_TARGET_PROPERTIES(pthreadpool PROPERTIES
   C_STANDARD 11
diff --git a/src/fastpath.c b/src/fastpath.c
new file mode 100644
index 0000000..1a5066a
--- /dev/null
+++ b/src/fastpath.c
@@ -0,0 +1,793 @@
+/* Standard C headers */
+#include <assert.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if PTHREADPOOL_USE_CPUINFO
+	#include <cpuinfo.h>
+#endif
+
+/* Dependencies */
+#include <fxdiv.h>
+
+/* Public library header */
+#include <pthreadpool.h>
+
+/* Internal library headers */
+#include "threadpool-atomics.h"
+#include "threadpool-common.h"
+#include "threadpool-object.h"
+#include "threadpool-utils.h"
+
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_1d_t task = (pthreadpool_task_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, range_start++);
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			task(argument, index);
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_with_uarch_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_1d_with_id_t task = (pthreadpool_task_1d_with_id_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const uint32_t default_uarch_index = threadpool->params.parallelize_1d_with_uarch.default_uarch_index;
+	uint32_t uarch_index = default_uarch_index;
+	#if PTHREADPOOL_USE_CPUINFO
+		uarch_index = cpuinfo_get_current_uarch_index();
+		if (uarch_index > threadpool->params.parallelize_1d_with_uarch.max_uarch_index) {
+			uarch_index = default_uarch_index;
+		}
+	#endif
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, uarch_index, range_start++);
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			task(argument, uarch_index, index);
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_tile_1d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_1d_tile_1d_t task = (pthreadpool_task_1d_tile_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	const size_t tile = threadpool->params.parallelize_1d_tile_1d.tile;
+	size_t tile_start = range_start * tile;
+
+	const size_t range = threadpool->params.parallelize_1d_tile_1d.range;
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, tile_start, min(range - tile_start, tile));
+		tile_start += tile;
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t tile_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			const size_t tile_start = tile_index * tile;
+			task(argument, tile_start, min(range - tile_start, tile));
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_2d_t task = (pthreadpool_task_2d_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_2d.range_j;
+	const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(range_start, range_j);
+	size_t i = index_i_j.quotient;
+	size_t j = index_i_j.remainder;
+
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, i, j);
+		if (++j == range_j.value) {
+			j = 0;
+			i += 1;
+		}
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(linear_index, range_j);
+			task(argument, index_i_j.quotient, index_i_j.remainder);
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_1d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_2d_tile_1d_t task = (pthreadpool_task_2d_tile_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_2d_tile_1d.tile_range_j;
+	const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(range_start, tile_range_j);
+	const size_t tile_j = threadpool->params.parallelize_2d_tile_1d.tile_j;
+	size_t i = tile_index_i_j.quotient;
+	size_t start_j = tile_index_i_j.remainder * tile_j;
+
+	const size_t range_j = threadpool->params.parallelize_2d_tile_1d.range_j;
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, i, start_j, min(range_j - start_j, tile_j));
+		start_j += tile_j;
+		if (start_j >= range_j) {
+			start_j = 0;
+			i += 1;
+		}
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(linear_index, tile_range_j);
+			const size_t start_j = tile_index_i_j.remainder * tile_j;
+			task(argument, tile_index_i_j.quotient, start_j, min(range_j - start_j, tile_j));
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_2d_tile_2d_t task = (pthreadpool_task_2d_tile_2d_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_2d_tile_2d.tile_range_j;
+	const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(range_start, tile_range_j);
+	const size_t tile_i = threadpool->params.parallelize_2d_tile_2d.tile_i;
+	const size_t tile_j = threadpool->params.parallelize_2d_tile_2d.tile_j;
+	size_t start_i = tile_index_i_j.quotient * tile_i;
+	size_t start_j = tile_index_i_j.remainder * tile_j;
+
+	const size_t range_i = threadpool->params.parallelize_2d_tile_2d.range_i;
+	const size_t range_j = threadpool->params.parallelize_2d_tile_2d.range_j;
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, start_i, start_j, min(range_i - start_i, tile_i), min(range_j - start_j, tile_j));
+		start_j += tile_j;
+		if (start_j >= range_j) {
+			start_j = 0;
+			start_i += tile_i;
+		}
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(linear_index, tile_range_j);
+			const size_t start_i = tile_index_i_j.quotient * tile_i;
+			const size_t start_j = tile_index_i_j.remainder * tile_j;
+			task(argument, start_i, start_j, min(range_i - start_i, tile_i), min(range_j - start_j, tile_j));
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_with_uarch_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_2d_tile_2d_with_id_t task = (pthreadpool_task_2d_tile_2d_with_id_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const uint32_t default_uarch_index = threadpool->params.parallelize_2d_tile_2d_with_uarch.default_uarch_index;
+	uint32_t uarch_index = default_uarch_index;
+	#if PTHREADPOOL_USE_CPUINFO
+		uarch_index = cpuinfo_get_current_uarch_index();
+		if (uarch_index > threadpool->params.parallelize_2d_tile_2d_with_uarch.max_uarch_index) {
+			uarch_index = default_uarch_index;
+		}
+	#endif
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_2d_tile_2d_with_uarch.tile_range_j;
+	const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	const struct fxdiv_result_size_t index = fxdiv_divide_size_t(range_start, tile_range_j);
+	const size_t range_i = threadpool->params.parallelize_2d_tile_2d_with_uarch.range_i;
+	const size_t tile_i = threadpool->params.parallelize_2d_tile_2d_with_uarch.tile_i;
+	const size_t range_j = threadpool->params.parallelize_2d_tile_2d_with_uarch.range_j;
+	const size_t tile_j = threadpool->params.parallelize_2d_tile_2d_with_uarch.tile_j;
+	size_t start_i = index.quotient * tile_i;
+	size_t start_j = index.remainder * tile_j;
+
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, uarch_index, start_i, start_j, min(range_i - start_i, tile_i), min(range_j - start_j, tile_j));
+		start_j += tile_j;
+		if (start_j >= range_j) {
+			start_j = 0;
+			start_i += tile_i;
+		}
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(linear_index, tile_range_j);
+			const size_t start_i = tile_index_i_j.quotient * tile_i;
+			const size_t start_j = tile_index_i_j.remainder * tile_j;
+			task(argument, uarch_index, start_i, start_j, min(range_i - start_i, tile_i), min(range_j - start_j, tile_j));
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_3d_tile_2d_t task = (pthreadpool_task_3d_tile_2d_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	const struct fxdiv_divisor_size_t tile_range_k = threadpool->params.parallelize_3d_tile_2d.tile_range_k;
+	const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(range_start, tile_range_k);
+	const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_3d_tile_2d.tile_range_j;
+	const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, tile_range_j);
+	const size_t tile_j = threadpool->params.parallelize_3d_tile_2d.tile_j;
+	const size_t tile_k = threadpool->params.parallelize_3d_tile_2d.tile_k;
+	size_t i = tile_index_i_j.quotient;
+	size_t start_j = tile_index_i_j.remainder * tile_j;
+	size_t start_k = tile_index_ij_k.remainder * tile_k;
+
+	const size_t range_k = threadpool->params.parallelize_3d_tile_2d.range_k;
+	const size_t range_j = threadpool->params.parallelize_3d_tile_2d.range_j;
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, i, start_j, start_k, min(range_j - start_j, tile_j), min(range_k - start_k, tile_k));
+		start_k += tile_k;
+		if (start_k >= range_k) {
+			start_k = 0;
+			start_j += tile_j;
+			if (start_j >= range_j) {
+				start_j = 0;
+				i += 1;
+			}
+		}
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(linear_index, tile_range_k);
+			const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, tile_range_j);
+			const size_t start_j = tile_index_i_j.remainder * tile_j;
+			const size_t start_k = tile_index_ij_k.remainder * tile_k;
+			task(argument, tile_index_i_j.quotient, start_j, start_k, min(range_j - start_j, tile_j), min(range_k - start_k, tile_k));
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_with_uarch_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_3d_tile_2d_with_id_t task = (pthreadpool_task_3d_tile_2d_with_id_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const uint32_t default_uarch_index = threadpool->params.parallelize_3d_tile_2d_with_uarch.default_uarch_index;
+	uint32_t uarch_index = default_uarch_index;
+	#if PTHREADPOOL_USE_CPUINFO
+		uarch_index = cpuinfo_get_current_uarch_index();
+		if (uarch_index > threadpool->params.parallelize_3d_tile_2d_with_uarch.max_uarch_index) {
+			uarch_index = default_uarch_index;
+		}
+	#endif
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	const struct fxdiv_divisor_size_t tile_range_k = threadpool->params.parallelize_3d_tile_2d_with_uarch.tile_range_k;
+	const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(range_start, tile_range_k);
+	const struct fxdiv_divisor_size_t tile_range_j = threadpool->params.parallelize_3d_tile_2d_with_uarch.tile_range_j;
+	const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, tile_range_j);
+	const size_t tile_j = threadpool->params.parallelize_3d_tile_2d_with_uarch.tile_j;
+	const size_t tile_k = threadpool->params.parallelize_3d_tile_2d_with_uarch.tile_k;
+	size_t i = tile_index_i_j.quotient;
+	size_t start_j = tile_index_i_j.remainder * tile_j;
+	size_t start_k = tile_index_ij_k.remainder * tile_k;
+
+	const size_t range_k = threadpool->params.parallelize_3d_tile_2d_with_uarch.range_k;
+	const size_t range_j = threadpool->params.parallelize_3d_tile_2d_with_uarch.range_j;
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, uarch_index, i, start_j, start_k, min(range_j - start_j, tile_j), min(range_k - start_k, tile_k));
+		start_k += tile_k;
+		if (start_k >= range_k) {
+			start_k = 0;
+			start_j += tile_j;
+			if (start_j >= range_j) {
+				start_j = 0;
+				i += 1;
+			}
+		}
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			const struct fxdiv_result_size_t tile_index_ij_k = fxdiv_divide_size_t(linear_index, tile_range_k);
+			const struct fxdiv_result_size_t tile_index_i_j = fxdiv_divide_size_t(tile_index_ij_k.quotient, tile_range_j);
+			const size_t start_j = tile_index_i_j.remainder * tile_j;
+			const size_t start_k = tile_index_ij_k.remainder * tile_k;
+			task(argument, uarch_index, tile_index_i_j.quotient, start_j, start_k, min(range_j - start_j, tile_j), min(range_k - start_k, tile_k));
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_4d_tile_2d_t task = (pthreadpool_task_4d_tile_2d_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	const struct fxdiv_divisor_size_t tile_range_kl = threadpool->params.parallelize_4d_tile_2d.tile_range_kl;
+	const struct fxdiv_result_size_t tile_index_ij_kl = fxdiv_divide_size_t(range_start, tile_range_kl);
+	const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_4d_tile_2d.range_j;
+	const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_kl.quotient, range_j);
+	const struct fxdiv_divisor_size_t tile_range_l = threadpool->params.parallelize_4d_tile_2d.tile_range_l;
+	const struct fxdiv_result_size_t tile_index_k_l = fxdiv_divide_size_t(tile_index_ij_kl.remainder, tile_range_l);
+	const size_t tile_k = threadpool->params.parallelize_4d_tile_2d.tile_k;
+	const size_t tile_l = threadpool->params.parallelize_4d_tile_2d.tile_l;
+	size_t i = index_i_j.quotient;
+	size_t j = index_i_j.remainder;
+	size_t start_k = tile_index_k_l.quotient * tile_k;
+	size_t start_l = tile_index_k_l.remainder * tile_l;
+
+	const size_t range_l = threadpool->params.parallelize_4d_tile_2d.range_l;
+	const size_t range_k = threadpool->params.parallelize_4d_tile_2d.range_k;
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, i, j, start_k, start_l, min(range_k - start_k, tile_k), min(range_l - start_l, tile_l));
+		start_l += tile_l;
+		if (start_l >= range_l) {
+			start_l = 0;
+			start_k += tile_k;
+			if (start_k >= range_k) {
+				start_k = 0;
+				if (++j == range_j.value) {
+					j = 0;
+					i += 1;
+				}
+			}
+		}
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			const struct fxdiv_result_size_t tile_index_ij_kl = fxdiv_divide_size_t(linear_index, tile_range_kl);
+			const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_kl.quotient, range_j);
+			const struct fxdiv_result_size_t tile_index_k_l = fxdiv_divide_size_t(tile_index_ij_kl.remainder, tile_range_l);
+			const size_t start_k = tile_index_k_l.quotient * tile_k;
+			const size_t start_l = tile_index_k_l.remainder * tile_l;
+			task(argument, index_i_j.quotient, index_i_j.remainder, start_k, start_l, min(range_k - start_k, tile_k), min(range_l - start_l, tile_l));
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_with_uarch_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_4d_tile_2d_with_id_t task = (pthreadpool_task_4d_tile_2d_with_id_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const uint32_t default_uarch_index = threadpool->params.parallelize_4d_tile_2d_with_uarch.default_uarch_index;
+	uint32_t uarch_index = default_uarch_index;
+	#if PTHREADPOOL_USE_CPUINFO
+		uarch_index = cpuinfo_get_current_uarch_index();
+		if (uarch_index > threadpool->params.parallelize_4d_tile_2d_with_uarch.max_uarch_index) {
+			uarch_index = default_uarch_index;
+		}
+	#endif
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	const struct fxdiv_divisor_size_t tile_range_kl = threadpool->params.parallelize_4d_tile_2d_with_uarch.tile_range_kl;
+	const struct fxdiv_result_size_t tile_index_ij_kl = fxdiv_divide_size_t(range_start, tile_range_kl);
+	const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_4d_tile_2d_with_uarch.range_j;
+	const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_kl.quotient, range_j);
+	const struct fxdiv_divisor_size_t tile_range_l = threadpool->params.parallelize_4d_tile_2d_with_uarch.tile_range_l;
+	const struct fxdiv_result_size_t tile_index_k_l = fxdiv_divide_size_t(tile_index_ij_kl.remainder, tile_range_l);
+	const size_t tile_k = threadpool->params.parallelize_4d_tile_2d_with_uarch.tile_k;
+	const size_t tile_l = threadpool->params.parallelize_4d_tile_2d_with_uarch.tile_l;
+	size_t i = index_i_j.quotient;
+	size_t j = index_i_j.remainder;
+	size_t start_k = tile_index_k_l.quotient * tile_k;
+	size_t start_l = tile_index_k_l.remainder * tile_l;
+
+	const size_t range_l = threadpool->params.parallelize_4d_tile_2d_with_uarch.range_l;
+	const size_t range_k = threadpool->params.parallelize_4d_tile_2d_with_uarch.range_k;
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, uarch_index, i, j, start_k, start_l, min(range_k - start_k, tile_k), min(range_l - start_l, tile_l));
+		start_l += tile_l;
+		if (start_l >= range_l) {
+			start_l = 0;
+			start_k += tile_k;
+			if (start_k >= range_k) {
+				start_k = 0;
+				if (++j == range_j.value) {
+					j = 0;
+					i += 1;
+				}
+			}
+		}
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			const struct fxdiv_result_size_t tile_index_ij_kl = fxdiv_divide_size_t(linear_index, tile_range_kl);
+			const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(tile_index_ij_kl.quotient, range_j);
+			const struct fxdiv_result_size_t tile_index_k_l = fxdiv_divide_size_t(tile_index_ij_kl.remainder, tile_range_l);
+			const size_t start_k = tile_index_k_l.quotient * tile_k;
+			const size_t start_l = tile_index_k_l.remainder * tile_l;
+			task(argument, uarch_index, index_i_j.quotient, index_i_j.remainder, start_k, start_l, min(range_k - start_k, tile_k), min(range_l - start_l, tile_l));
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_5d_tile_2d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_5d_tile_2d_t task = (pthreadpool_task_5d_tile_2d_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	const struct fxdiv_divisor_size_t tile_range_lm = threadpool->params.parallelize_5d_tile_2d.tile_range_lm;
+	const struct fxdiv_result_size_t tile_index_ijk_lm = fxdiv_divide_size_t(range_start, tile_range_lm);
+	const struct fxdiv_divisor_size_t range_k = threadpool->params.parallelize_5d_tile_2d.range_k;
+	const struct fxdiv_result_size_t index_ij_k = fxdiv_divide_size_t(tile_index_ijk_lm.quotient, range_k);
+	const struct fxdiv_divisor_size_t tile_range_m = threadpool->params.parallelize_5d_tile_2d.tile_range_m;
+	const struct fxdiv_result_size_t tile_index_l_m = fxdiv_divide_size_t(tile_index_ijk_lm.remainder, tile_range_m);
+	const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_5d_tile_2d.range_j;
+	const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_k.quotient, range_j);
+	const size_t tile_l = threadpool->params.parallelize_5d_tile_2d.tile_l;
+	const size_t tile_m = threadpool->params.parallelize_5d_tile_2d.tile_m;
+	size_t i = index_i_j.quotient;
+	size_t j = index_i_j.remainder;
+	size_t k = index_ij_k.remainder;
+	size_t start_l = tile_index_l_m.quotient * tile_l;
+	size_t start_m = tile_index_l_m.remainder * tile_m;
+
+	const size_t range_m = threadpool->params.parallelize_5d_tile_2d.range_m;
+	const size_t range_l = threadpool->params.parallelize_5d_tile_2d.range_l;
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, i, j, k, start_l, start_m, min(range_l - start_l, tile_l), min(range_m - start_m, tile_m));
+		start_m += tile_m;
+		if (start_m >= range_m) {
+			start_m = 0;
+			start_l += tile_l;
+			if (start_l >= range_l) {
+				start_l = 0;
+				if (++k == range_k.value) {
+					k = 0;
+					if (++j == range_j.value) {
+						j = 0;
+						i += 1;
+					}
+				}
+			}
+		}
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			const struct fxdiv_result_size_t tile_index_ijk_lm = fxdiv_divide_size_t(linear_index, tile_range_lm);
+			const struct fxdiv_result_size_t index_ij_k = fxdiv_divide_size_t(tile_index_ijk_lm.quotient, range_k);
+			const struct fxdiv_result_size_t tile_index_l_m = fxdiv_divide_size_t(tile_index_ijk_lm.remainder, tile_range_m);
+			const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_k.quotient, range_j);
+			const size_t start_l = tile_index_l_m.quotient * tile_l;
+			const size_t start_m = tile_index_l_m.remainder * tile_m;
+			task(argument, index_i_j.quotient, index_i_j.remainder, index_ij_k.remainder,
+				start_l, start_m, min(range_l - start_l, tile_l), min(range_m - start_m, tile_m));
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_6d_tile_2d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread)
+{
+	assert(threadpool != NULL);
+	assert(thread != NULL);
+
+	const pthreadpool_task_6d_tile_2d_t task = (pthreadpool_task_6d_tile_2d_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
+	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
+	const size_t threads_count = threadpool->threads_count.value;
+	const size_t range_threshold = -threads_count;
+
+	/* Process thread's own range of items */
+	const size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
+	const struct fxdiv_divisor_size_t tile_range_mn = threadpool->params.parallelize_6d_tile_2d.tile_range_mn;
+	const struct fxdiv_result_size_t tile_index_ijkl_mn = fxdiv_divide_size_t(range_start, tile_range_mn);
+	const struct fxdiv_divisor_size_t range_kl = threadpool->params.parallelize_6d_tile_2d.range_kl;
+	const struct fxdiv_result_size_t index_ij_kl = fxdiv_divide_size_t(tile_index_ijkl_mn.quotient, range_kl);
+	const struct fxdiv_divisor_size_t tile_range_n = threadpool->params.parallelize_6d_tile_2d.tile_range_n;
+	const struct fxdiv_result_size_t tile_index_m_n = fxdiv_divide_size_t(tile_index_ijkl_mn.remainder, tile_range_n);
+	const struct fxdiv_divisor_size_t range_j = threadpool->params.parallelize_6d_tile_2d.range_j;
+	const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_kl.quotient, range_j);
+	const struct fxdiv_divisor_size_t range_l = threadpool->params.parallelize_6d_tile_2d.range_l;
+	const struct fxdiv_result_size_t index_k_l = fxdiv_divide_size_t(index_ij_kl.remainder, range_l);
+	const size_t tile_m = threadpool->params.parallelize_6d_tile_2d.tile_m;
+	const size_t tile_n = threadpool->params.parallelize_6d_tile_2d.tile_n;
+	size_t i = index_i_j.quotient;
+	size_t j = index_i_j.remainder;
+	size_t k = index_k_l.quotient;
+	size_t l = index_k_l.remainder;
+	size_t start_m = tile_index_m_n.quotient * tile_m;
+	size_t start_n = tile_index_m_n.remainder * tile_n;
+
+	const size_t range_n = threadpool->params.parallelize_6d_tile_2d.range_n;
+	const size_t range_m = threadpool->params.parallelize_6d_tile_2d.range_m;
+	const size_t range_k = threadpool->params.parallelize_6d_tile_2d.range_k;
+	while (pthreadpool_decrement_fetch_relaxed_size_t(&thread->range_length) < range_threshold) {
+		task(argument, i, j, k, l, start_m, start_n, min(range_m - start_m, tile_m), min(range_n - start_n, tile_n));
+		start_n += tile_n;
+		if (start_n >= range_n) {
+			start_n = 0;
+			start_m += tile_m;
+			if (start_m >= range_m) {
+				start_m = 0;
+				if (++l == range_l.value) {
+					l = 0;
+					if (++k == range_k) {
+						k = 0;
+						if (++j == range_j.value) {
+							j = 0;
+							i += 1;
+						}
+					}
+				}
+			}
+		}
+	}
+
+	/* There still may be other threads with work */
+	const size_t thread_number = thread->thread_number;
+	for (size_t tid = modulo_decrement(thread_number, threads_count);
+		tid != thread_number;
+		tid = modulo_decrement(tid, threads_count))
+	{
+		struct thread_info* other_thread = &threadpool->threads[tid];
+		while (pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_length) < range_threshold) {
+			const size_t linear_index = pthreadpool_decrement_fetch_relaxed_size_t(&other_thread->range_end);
+			const struct fxdiv_result_size_t tile_index_ijkl_mn = fxdiv_divide_size_t(linear_index, tile_range_mn);
+			const struct fxdiv_result_size_t index_ij_kl = fxdiv_divide_size_t(tile_index_ijkl_mn.quotient, range_kl);
+			const struct fxdiv_result_size_t tile_index_m_n = fxdiv_divide_size_t(tile_index_ijkl_mn.remainder, tile_range_n);
+			const struct fxdiv_result_size_t index_i_j = fxdiv_divide_size_t(index_ij_kl.quotient, range_j);
+			const struct fxdiv_result_size_t index_k_l = fxdiv_divide_size_t(index_ij_kl.remainder, range_l);
+			const size_t start_m = tile_index_m_n.quotient * tile_m;
+			const size_t start_n = tile_index_m_n.remainder * tile_n;
+			task(argument, index_i_j.quotient, index_i_j.remainder, index_k_l.quotient, index_k_l.remainder,
+				start_m, start_n, min(range_m - start_m, tile_m), min(range_n - start_n, tile_n));
+		}
+	}
+
+	/* Make changes by this thread visible to other threads */
+	pthreadpool_fence_release();
+}
diff --git a/src/portable-api.c b/src/portable-api.c
index 84d6eda..6b16674 100644
--- a/src/portable-api.c
+++ b/src/portable-api.c
@@ -35,6 +35,7 @@ static void thread_parallelize_1d(struct pthreadpool* threadpool, struct thread_
 
 	const pthreadpool_task_1d_t task = (pthreadpool_task_1d_t) pthreadpool_load_relaxed_void_p(&threadpool->task);
 	void *const argument = pthreadpool_load_relaxed_void_p(&threadpool->argument);
+
 	/* Process thread's own range of items */
 	size_t range_start = pthreadpool_load_relaxed_size_t(&thread->range_start);
 	while (pthreadpool_try_decrement_relaxed_size_t(&thread->range_length)) {
@@ -740,7 +741,8 @@ void pthreadpool_parallelize_1d(
 	size_t range,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || range <= 1) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || range <= 1) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 		struct fpu_state saved_fpu_state = { 0 };
 		if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
@@ -754,8 +756,15 @@ void pthreadpool_parallelize_1d(
 			set_fpu_state(saved_fpu_state);
 		}
 	} else {
+		thread_function_t parallelize_1d = &thread_parallelize_1d;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (range < range_threshold) {
+				parallelize_1d = &pthreadpool_thread_parallelize_1d_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_1d, NULL, 0,
+			threadpool, parallelize_1d, NULL, 0,
 			(void*) task, argument, range, flags);
 	}
 }
@@ -769,7 +778,8 @@ void pthreadpool_parallelize_1d_with_uarch(
 	size_t range,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || range <= 1) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || range <= 1) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 
 		uint32_t uarch_index = default_uarch_index;
@@ -796,8 +806,15 @@ void pthreadpool_parallelize_1d_with_uarch(
 			.default_uarch_index = default_uarch_index,
 			.max_uarch_index = max_uarch_index,
 		};
+		thread_function_t parallelize_1d_with_uarch = &thread_parallelize_1d_with_uarch;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (range < range_threshold) {
+				parallelize_1d_with_uarch = &pthreadpool_thread_parallelize_1d_with_uarch_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_1d_with_uarch, &params, sizeof(params),
+			threadpool, parallelize_1d_with_uarch, &params, sizeof(params),
 			task, argument, range, flags);
 	}
 }
@@ -810,7 +827,8 @@ void pthreadpool_parallelize_1d_tile_1d(
 	size_t tile,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || range <= tile) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || range <= tile) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 		struct fpu_state saved_fpu_state = { 0 };
 		if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
@@ -824,13 +842,21 @@ void pthreadpool_parallelize_1d_tile_1d(
 			set_fpu_state(saved_fpu_state);
 		}
 	} else {
+		const size_t tile_range = divide_round_up(range, tile);
 		const struct pthreadpool_1d_tile_1d_params params = {
 			.range = range,
 			.tile = tile,
 		};
+		thread_function_t parallelize_1d_tile_1d = &thread_parallelize_1d_tile_1d;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (range < range_threshold) {
+				parallelize_1d_tile_1d = &pthreadpool_thread_parallelize_1d_tile_1d_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_1d_tile_1d, &params, sizeof(params),
-			task, argument, divide_round_up(range, tile), flags);
+			threadpool, parallelize_1d_tile_1d, &params, sizeof(params),
+			task, argument, tile_range, flags);
 	}
 }
 
@@ -842,7 +868,8 @@ void pthreadpool_parallelize_2d(
 	size_t range_j,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || (range_i | range_j) <= 1) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i | range_j) <= 1) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 		struct fpu_state saved_fpu_state = { 0 };
 		if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
@@ -858,12 +885,20 @@ void pthreadpool_parallelize_2d(
 			set_fpu_state(saved_fpu_state);
 		}
 	} else {
+		const size_t range = range_i * range_j;
 		const struct pthreadpool_2d_params params = {
 			.range_j = fxdiv_init_size_t(range_j),
 		};
+		thread_function_t parallelize_2d = &thread_parallelize_2d;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (range < range_threshold) {
+				parallelize_2d = &pthreadpool_thread_parallelize_2d_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_2d, &params, sizeof(params),
-			task, argument, range_i * range_j, flags);
+			threadpool, parallelize_2d, &params, sizeof(params),
+			task, argument, range, flags);
 	}
 }
 
@@ -876,7 +911,8 @@ void pthreadpool_parallelize_2d_tile_1d(
 	size_t tile_j,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || (range_i <= 1 && range_j <= tile_j)) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i <= 1 && range_j <= tile_j)) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 		struct fpu_state saved_fpu_state = { 0 };
 		if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
@@ -893,14 +929,22 @@ void pthreadpool_parallelize_2d_tile_1d(
 		}
 	} else {
 		const size_t tile_range_j = divide_round_up(range_j, tile_j);
+		const size_t tile_range = range_i * tile_range_j;
 		const struct pthreadpool_2d_tile_1d_params params = {
 			.range_j = range_j,
 			.tile_j = tile_j,
 			.tile_range_j = fxdiv_init_size_t(tile_range_j),
 		};
+		thread_function_t parallelize_2d_tile_1d = &thread_parallelize_2d_tile_1d;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (tile_range < range_threshold) {
+				parallelize_2d_tile_1d = &pthreadpool_thread_parallelize_2d_tile_1d_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_2d_tile_1d, &params, sizeof(params),
-			task, argument, range_i * tile_range_j, flags);
+			threadpool, parallelize_2d_tile_1d, &params, sizeof(params),
+			task, argument, tile_range, flags);
 	}
 }
 
@@ -914,7 +958,8 @@ void pthreadpool_parallelize_2d_tile_2d(
 	size_t tile_j,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || (range_i <= tile_i && range_j <= tile_j)) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i <= tile_i && range_j <= tile_j)) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 		struct fpu_state saved_fpu_state = { 0 };
 		if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
@@ -932,6 +977,7 @@ void pthreadpool_parallelize_2d_tile_2d(
 	} else {
 		const size_t tile_range_i = divide_round_up(range_i, tile_i);
 		const size_t tile_range_j = divide_round_up(range_j, tile_j);
+		const size_t tile_range = tile_range_i * tile_range_j;
 		const struct pthreadpool_2d_tile_2d_params params = {
 			.range_i = range_i,
 			.tile_i = tile_i,
@@ -939,9 +985,16 @@ void pthreadpool_parallelize_2d_tile_2d(
 			.tile_j = tile_j,
 			.tile_range_j = fxdiv_init_size_t(tile_range_j),
 		};
+		thread_function_t parallelize_2d_tile_2d = &thread_parallelize_2d_tile_2d;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (tile_range < range_threshold) {
+				parallelize_2d_tile_2d = &pthreadpool_thread_parallelize_2d_tile_2d_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_2d_tile_2d, &params, sizeof(params),
-			task, argument, tile_range_i * tile_range_j, flags);
+			threadpool, parallelize_2d_tile_2d, &params, sizeof(params),
+			task, argument, tile_range, flags);
 	}
 }
 
@@ -957,7 +1010,8 @@ void pthreadpool_parallelize_2d_tile_2d_with_uarch(
 	size_t tile_j,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || (range_i <= tile_i && range_j <= tile_j)) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i <= tile_i && range_j <= tile_j)) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 
 		uint32_t uarch_index = default_uarch_index;
@@ -984,6 +1038,7 @@ void pthreadpool_parallelize_2d_tile_2d_with_uarch(
 	} else {
 		const size_t tile_range_i = divide_round_up(range_i, tile_i);
 		const size_t tile_range_j = divide_round_up(range_j, tile_j);
+		const size_t tile_range = tile_range_i * tile_range_j;
 		const struct pthreadpool_2d_tile_2d_with_uarch_params params = {
 			.default_uarch_index = default_uarch_index,
 			.max_uarch_index = max_uarch_index,
@@ -993,9 +1048,16 @@ void pthreadpool_parallelize_2d_tile_2d_with_uarch(
 			.tile_j = tile_j,
 			.tile_range_j = fxdiv_init_size_t(tile_range_j),
 		};
+		thread_function_t parallelize_2d_tile_2d_with_uarch = &thread_parallelize_2d_tile_2d_with_uarch;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (tile_range < range_threshold) {
+				parallelize_2d_tile_2d_with_uarch = &pthreadpool_thread_parallelize_2d_tile_2d_with_uarch_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_2d_tile_2d_with_uarch, &params, sizeof(params),
-			task, argument, tile_range_i * tile_range_j, flags);
+			threadpool, parallelize_2d_tile_2d_with_uarch, &params, sizeof(params),
+			task, argument, tile_range, flags);
 	}
 }
 
@@ -1010,7 +1072,8 @@ void pthreadpool_parallelize_3d_tile_2d(
 	size_t tile_k,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || (range_i <= 1 && range_j <= tile_j && range_k <= tile_k)) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i <= 1 && range_j <= tile_j && range_k <= tile_k)) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 		struct fpu_state saved_fpu_state = { 0 };
 		if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
@@ -1030,6 +1093,7 @@ void pthreadpool_parallelize_3d_tile_2d(
 	} else {
 		const size_t tile_range_j = divide_round_up(range_j, tile_j);
 		const size_t tile_range_k = divide_round_up(range_k, tile_k);
+		const size_t tile_range = range_i * tile_range_j * tile_range_k;
 		const struct pthreadpool_3d_tile_2d_params params = {
 			.range_j = range_j,
 			.tile_j = tile_j,
@@ -1038,9 +1102,16 @@ void pthreadpool_parallelize_3d_tile_2d(
 			.tile_range_j = fxdiv_init_size_t(tile_range_j),
 			.tile_range_k = fxdiv_init_size_t(tile_range_k),
 		};
+		thread_function_t parallelize_3d_tile_2d = &thread_parallelize_3d_tile_2d;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (tile_range < range_threshold) {
+				parallelize_3d_tile_2d = &pthreadpool_thread_parallelize_3d_tile_2d_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_3d_tile_2d, &params, sizeof(params),
-			task, argument, range_i * tile_range_j * tile_range_k, flags);
+			threadpool, parallelize_3d_tile_2d, &params, sizeof(params),
+			task, argument, tile_range, flags);
 	}
 }
 
@@ -1057,7 +1128,8 @@ void pthreadpool_parallelize_3d_tile_2d_with_uarch(
 	size_t tile_k,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || (range_i <= 1 && range_j <= tile_j && range_k <= tile_k)) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || (range_i <= 1 && range_j <= tile_j && range_k <= tile_k)) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 
 		uint32_t uarch_index = default_uarch_index;
@@ -1086,6 +1158,7 @@ void pthreadpool_parallelize_3d_tile_2d_with_uarch(
 	} else {
 		const size_t tile_range_j = divide_round_up(range_j, tile_j);
 		const size_t tile_range_k = divide_round_up(range_k, tile_k);
+		const size_t tile_range = range_i * tile_range_j * tile_range_k;
 		const struct pthreadpool_3d_tile_2d_with_uarch_params params = {
 			.default_uarch_index = default_uarch_index,
 			.max_uarch_index = max_uarch_index,
@@ -1096,9 +1169,16 @@ void pthreadpool_parallelize_3d_tile_2d_with_uarch(
 			.tile_range_j = fxdiv_init_size_t(tile_range_j),
 			.tile_range_k = fxdiv_init_size_t(tile_range_k),
 		};
+		thread_function_t parallelize_3d_tile_2d_with_uarch = &thread_parallelize_3d_tile_2d_with_uarch;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (tile_range < range_threshold) {
+				parallelize_3d_tile_2d_with_uarch = &pthreadpool_thread_parallelize_3d_tile_2d_with_uarch_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_3d_tile_2d_with_uarch, &params, sizeof(params),
-			task, argument, range_i * tile_range_j * tile_range_k, flags);
+			threadpool, parallelize_3d_tile_2d_with_uarch, &params, sizeof(params),
+			task, argument, tile_range, flags);
 	}
 }
 
@@ -1114,7 +1194,8 @@ void pthreadpool_parallelize_4d_tile_2d(
 	size_t tile_l,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || ((range_i | range_j) <= 1 && range_k <= tile_k && range_l <= tile_l)) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j) <= 1 && range_k <= tile_k && range_l <= tile_l)) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 		struct fpu_state saved_fpu_state = { 0 };
 		if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
@@ -1137,6 +1218,7 @@ void pthreadpool_parallelize_4d_tile_2d(
 	} else {
 		const size_t tile_range_l = divide_round_up(range_l, tile_l);
 		const size_t tile_range_kl = divide_round_up(range_k, tile_k) * tile_range_l;
+		const size_t tile_range = range_i * range_j * tile_range_kl;
 		const struct pthreadpool_4d_tile_2d_params params = {
 			.range_k = range_k,
 			.tile_k = tile_k,
@@ -1146,9 +1228,16 @@ void pthreadpool_parallelize_4d_tile_2d(
 			.tile_range_kl = fxdiv_init_size_t(tile_range_kl),
 			.tile_range_l = fxdiv_init_size_t(tile_range_l),
 		};
+		thread_function_t parallelize_4d_tile_2d = &thread_parallelize_4d_tile_2d;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (tile_range < range_threshold) {
+				parallelize_4d_tile_2d = &pthreadpool_thread_parallelize_4d_tile_2d_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_4d_tile_2d, &params, sizeof(params),
-			task, argument, range_i * range_j * tile_range_kl, flags);
+			threadpool, parallelize_4d_tile_2d, &params, sizeof(params),
+			task, argument, tile_range, flags);
 	}
 }
 
@@ -1166,7 +1255,8 @@ void pthreadpool_parallelize_4d_tile_2d_with_uarch(
 	size_t tile_l,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || ((range_i | range_j) <= 1 && range_k <= tile_k && range_l <= tile_l)) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j) <= 1 && range_k <= tile_k && range_l <= tile_l)) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 
 		uint32_t uarch_index = default_uarch_index;
@@ -1198,6 +1288,7 @@ void pthreadpool_parallelize_4d_tile_2d_with_uarch(
 	} else {
 		const size_t tile_range_l = divide_round_up(range_l, tile_l);
 		const size_t tile_range_kl = divide_round_up(range_k, tile_k) * tile_range_l;
+		const size_t tile_range = range_i * range_j * tile_range_kl;
 		const struct pthreadpool_4d_tile_2d_with_uarch_params params = {
 			.default_uarch_index = default_uarch_index,
 			.max_uarch_index = max_uarch_index,
@@ -1209,9 +1300,16 @@ void pthreadpool_parallelize_4d_tile_2d_with_uarch(
 			.tile_range_kl = fxdiv_init_size_t(tile_range_kl),
 			.tile_range_l = fxdiv_init_size_t(tile_range_l),
 		};
+		thread_function_t parallelize_4d_tile_2d_with_uarch = &thread_parallelize_4d_tile_2d_with_uarch;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (tile_range < range_threshold) {
+				parallelize_4d_tile_2d_with_uarch = &pthreadpool_thread_parallelize_4d_tile_2d_with_uarch_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_4d_tile_2d_with_uarch, &params, sizeof(params),
-			task, argument, range_i * range_j * tile_range_kl, flags);
+			threadpool, parallelize_4d_tile_2d_with_uarch, &params, sizeof(params),
+			task, argument, tile_range, flags);
 	}
 }
 
@@ -1228,7 +1326,8 @@ void pthreadpool_parallelize_5d_tile_2d(
 	size_t tile_m,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || ((range_i | range_j | range_k) <= 1 && range_l <= tile_l && range_m <= tile_m)) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j | range_k) <= 1 && range_l <= tile_l && range_m <= tile_m)) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 		struct fpu_state saved_fpu_state = { 0 };
 		if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
@@ -1253,6 +1352,7 @@ void pthreadpool_parallelize_5d_tile_2d(
 	} else {
 		const size_t tile_range_m = divide_round_up(range_m, tile_m);
 		const size_t tile_range_lm = divide_round_up(range_l, tile_l) * tile_range_m;
+		const size_t tile_range = range_i * range_j * range_k * tile_range_lm;
 		const struct pthreadpool_5d_tile_2d_params params = {
 			.range_l = range_l,
 			.tile_l = tile_l,
@@ -1263,9 +1363,16 @@ void pthreadpool_parallelize_5d_tile_2d(
 			.tile_range_lm = fxdiv_init_size_t(tile_range_lm),
 			.tile_range_m = fxdiv_init_size_t(tile_range_m),
 		};
+		thread_function_t parallelize_5d_tile_2d = &thread_parallelize_5d_tile_2d;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (tile_range < range_threshold) {
+				parallelize_5d_tile_2d = &pthreadpool_thread_parallelize_5d_tile_2d_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_5d_tile_2d, &params, sizeof(params),
-			task, argument, range_i * range_j * range_k * tile_range_lm, flags);
+			threadpool, parallelize_5d_tile_2d, &params, sizeof(params),
+			task, argument, tile_range, flags);
 	}
 }
 
@@ -1283,7 +1390,8 @@ void pthreadpool_parallelize_6d_tile_2d(
 	size_t tile_n,
 	uint32_t flags)
 {
-	if (threadpool == NULL || threadpool->threads_count.value <= 1 || ((range_i | range_j | range_k | range_l) <= 1 && range_m <= tile_m && range_n <= tile_n)) {
+	size_t threads_count;
+	if (threadpool == NULL || (threads_count = threadpool->threads_count.value) <= 1 || ((range_i | range_j | range_k | range_l) <= 1 && range_m <= tile_m && range_n <= tile_n)) {
 		/* No thread pool used: execute task sequentially on the calling thread */
 		struct fpu_state saved_fpu_state = { 0 };
 		if (flags & PTHREADPOOL_FLAG_DISABLE_DENORMALS) {
@@ -1311,6 +1419,7 @@ void pthreadpool_parallelize_6d_tile_2d(
 		const size_t range_kl = range_k * range_l;
 		const size_t tile_range_n = divide_round_up(range_n, tile_n);
 		const size_t tile_range_mn = divide_round_up(range_m, tile_m) * tile_range_n;
+		const size_t tile_range = range_i * range_j * range_kl * tile_range_mn;
 		const struct pthreadpool_6d_tile_2d_params params = {
 			.range_k = range_k,
 			.range_m = range_m,
@@ -1323,8 +1432,15 @@ void pthreadpool_parallelize_6d_tile_2d(
 			.tile_range_mn = fxdiv_init_size_t(tile_range_mn),
 			.tile_range_n = fxdiv_init_size_t(tile_range_n),
 		};
+		thread_function_t parallelize_6d_tile_2d = &thread_parallelize_6d_tile_2d;
+		#if PTHREADPOOL_USE_FASTPATH
+			const size_t range_threshold = -threads_count;
+			if (tile_range < range_threshold) {
+				parallelize_6d_tile_2d = &pthreadpool_thread_parallelize_6d_tile_2d_fastpath;
+			}
+		#endif
 		pthreadpool_parallelize(
-			threadpool, &thread_parallelize_6d_tile_2d, &params, sizeof(params),
-			task, argument, range_i * range_j * range_kl * tile_range_mn, flags);
+			threadpool, parallelize_6d_tile_2d, &params, sizeof(params),
+			task, argument, tile_range, flags);
 	}
 }
diff --git a/src/threadpool-object.h b/src/threadpool-object.h
index 239d116..7b643c6 100644
--- a/src/threadpool-object.h
+++ b/src/threadpool-object.h
@@ -526,3 +526,55 @@ PTHREADPOOL_INTERNAL void pthreadpool_parallelize(
 	void* context,
 	size_t linear_range,
 	uint32_t flags);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_with_uarch_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_1d_tile_1d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_1d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_2d_tile_2d_with_uarch_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_3d_tile_2d_with_uarch_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_4d_tile_2d_with_uarch_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_5d_tile_2d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);
+
+PTHREADPOOL_INTERNAL void pthreadpool_thread_parallelize_6d_tile_2d_fastpath(
+	struct pthreadpool* threadpool,
+	struct thread_info* thread);