diff options
Diffstat (limited to 'test/pthreadpool.cc')
| -rw-r--r-- | test/pthreadpool.cc | 2025 |
1 files changed, 2025 insertions, 0 deletions
diff --git a/test/pthreadpool.cc b/test/pthreadpool.cc index b8a6803..f822506 100644 --- a/test/pthreadpool.cc +++ b/test/pthreadpool.cc @@ -23,17 +23,44 @@ const size_t kParallelize2DTile2DRangeI = 53; const size_t kParallelize2DTile2DRangeJ = 59; const size_t kParallelize2DTile2DTileI = 5; const size_t kParallelize2DTile2DTileJ = 7; +const size_t kParallelize3DRangeI = 13; +const size_t kParallelize3DRangeJ = 17; +const size_t kParallelize3DRangeK = 19; +const size_t kParallelize3DTile1DRangeI = 17; +const size_t kParallelize3DTile1DRangeJ = 19; +const size_t kParallelize3DTile1DRangeK = 23; +const size_t kParallelize3DTile1DTileK = 5; const size_t kParallelize3DTile2DRangeI = 19; const size_t kParallelize3DTile2DRangeJ = 23; const size_t kParallelize3DTile2DRangeK = 29; const size_t kParallelize3DTile2DTileJ = 2; const size_t kParallelize3DTile2DTileK = 3; +const size_t kParallelize4DRangeI = 11; +const size_t kParallelize4DRangeJ = 13; +const size_t kParallelize4DRangeK = 17; +const size_t kParallelize4DRangeL = 19; +const size_t kParallelize4DTile1DRangeI = 13; +const size_t kParallelize4DTile1DRangeJ = 17; +const size_t kParallelize4DTile1DRangeK = 19; +const size_t kParallelize4DTile1DRangeL = 23; +const size_t kParallelize4DTile1DTileL = 5; const size_t kParallelize4DTile2DRangeI = 17; const size_t kParallelize4DTile2DRangeJ = 19; const size_t kParallelize4DTile2DRangeK = 23; const size_t kParallelize4DTile2DRangeL = 29; const size_t kParallelize4DTile2DTileK = 2; const size_t kParallelize4DTile2DTileL = 3; +const size_t kParallelize5DRangeI = 7; +const size_t kParallelize5DRangeJ = 11; +const size_t kParallelize5DRangeK = 13; +const size_t kParallelize5DRangeL = 17; +const size_t kParallelize5DRangeM = 19; +const size_t kParallelize5DTile1DRangeI = 11; +const size_t kParallelize5DTile1DRangeJ = 13; +const size_t kParallelize5DTile1DRangeK = 17; +const size_t kParallelize5DTile1DRangeL = 19; +const size_t kParallelize5DTile1DRangeM = 23; +const size_t kParallelize5DTile1DTileM = 5; const size_t kParallelize5DTile2DRangeI = 13; const size_t kParallelize5DTile2DRangeJ = 17; const size_t kParallelize5DTile2DRangeK = 19; @@ -2286,6 +2313,646 @@ TEST(Parallelize2DTile2DWithUArch, MultiThreadPoolWorkStealing) { EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize2DTile2DRangeI * kParallelize2DTile2DRangeJ); } +static void ComputeNothing3D(void*, size_t, size_t, size_t) { +} + +TEST(Parallelize3D, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d(threadpool.get(), + ComputeNothing3D, + nullptr, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); +} + +TEST(Parallelize3D, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d( + threadpool.get(), + ComputeNothing3D, + nullptr, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); +} + +static void CheckBounds3D(void*, size_t i, size_t j, size_t k) { + EXPECT_LT(i, kParallelize3DRangeI); + EXPECT_LT(j, kParallelize3DRangeJ); + EXPECT_LT(k, kParallelize3DRangeK); +} + +TEST(Parallelize3D, SingleThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d( + threadpool.get(), + CheckBounds3D, + nullptr, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); +} + +TEST(Parallelize3D, MultiThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d( + threadpool.get(), + CheckBounds3D, + nullptr, + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); +} + +static void SetTrue3D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t k) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); +} + +TEST(Parallelize3D, SingleThreadPoolAllItemsProcessed) { + std::vector<std::atomic_bool> indicators(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_t>(SetTrue3D), + static_cast<void*>(indicators.data()), + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ") not processed"; + } + } + } +} + +TEST(Parallelize3D, MultiThreadPoolAllItemsProcessed) { + std::vector<std::atomic_bool> indicators(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_t>(SetTrue3D), + static_cast<void*>(indicators.data()), + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ") not processed"; + } + } + } +} + +static void Increment3D(std::atomic_int* processed_counters, size_t i, size_t j, size_t k) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); +} + +TEST(Parallelize3D, SingleThreadPoolEachItemProcessedOnce) { + std::vector<std::atomic_int> counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_t>(Increment3D), + static_cast<void*>(counters.data()), + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } +} + +TEST(Parallelize3D, MultiThreadPoolEachItemProcessedOnce) { + std::vector<std::atomic_int> counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_t>(Increment3D), + static_cast<void*>(counters.data()), + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } +} + +TEST(Parallelize3D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector<std::atomic_int> counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_3d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_t>(Increment3D), + static_cast<void*>(counters.data()), + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize3DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } +} + +TEST(Parallelize3D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector<std::atomic_int> counters(kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_3d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_t>(Increment3D), + static_cast<void*>(counters.data()), + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize3DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DRangeJ + j) * kParallelize3DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } +} + +static void IncrementSame3D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); +} + +TEST(Parallelize3D, MultiThreadPoolHighContention) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_t>(IncrementSame3D), + static_cast<void*>(&num_processed_items), + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); +} + +static void WorkImbalance3D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + if (i == 0 && j == 0 && k == 0) { + /* Spin-wait until all items are computed */ + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK) { + std::atomic_thread_fence(std::memory_order_acquire); + } + } +} + +TEST(Parallelize3D, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_t>(WorkImbalance3D), + static_cast<void*>(&num_processed_items), + kParallelize3DRangeI, kParallelize3DRangeJ, kParallelize3DRangeK, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DRangeI * kParallelize3DRangeJ * kParallelize3DRangeK); +} + +static void ComputeNothing3DTile1D(void*, size_t, size_t, size_t, size_t) { +} + +TEST(Parallelize3DTile1D, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_1d(threadpool.get(), + ComputeNothing3DTile1D, + nullptr, + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); +} + +TEST(Parallelize3DTile1D, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + ComputeNothing3DTile1D, + nullptr, + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); +} + +static void CheckBounds3DTile1D(void*, size_t i, size_t j, size_t start_k, size_t tile_k) { + EXPECT_LT(i, kParallelize3DTile1DRangeI); + EXPECT_LT(j, kParallelize3DTile1DRangeJ); + EXPECT_LT(start_k, kParallelize3DTile1DRangeK); + EXPECT_LE(start_k + tile_k, kParallelize3DTile1DRangeK); +} + +TEST(Parallelize3DTile1D, SingleThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + CheckBounds3DTile1D, + nullptr, + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); +} + +TEST(Parallelize3DTile1D, MultiThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + CheckBounds3DTile1D, + nullptr, + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); +} + +static void CheckTiling3DTile1D(void*, size_t i, size_t j, size_t start_k, size_t tile_k) { + EXPECT_GT(tile_k, 0); + EXPECT_LE(tile_k, kParallelize3DTile1DTileK); + EXPECT_EQ(start_k % kParallelize3DTile1DTileK, 0); + EXPECT_EQ(tile_k, std::min<size_t>(kParallelize3DTile1DTileK, kParallelize3DTile1DRangeK - start_k)); +} + +TEST(Parallelize3DTile1D, SingleThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + CheckTiling3DTile1D, + nullptr, + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); +} + +TEST(Parallelize3DTile1D, MultiThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + CheckTiling3DTile1D, + nullptr, + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); +} + +static void SetTrue3DTile1D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t start_k, size_t tile_k) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); + } +} + +TEST(Parallelize3DTile1D, SingleThreadPoolAllItemsProcessed) { + std::vector<std::atomic_bool> indicators(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(SetTrue3DTile1D), + static_cast<void*>(indicators.data()), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ") not processed"; + } + } + } +} + +TEST(Parallelize3DTile1D, MultiThreadPoolAllItemsProcessed) { + std::vector<std::atomic_bool> indicators(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(SetTrue3DTile1D), + static_cast<void*>(indicators.data()), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ") not processed"; + } + } + } +} + +static void Increment3DTile1D(std::atomic_int* processed_counters, size_t i, size_t j, size_t start_k, size_t tile_k) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); + } +} + +TEST(Parallelize3DTile1D, SingleThreadPoolEachItemProcessedOnce) { + std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(Increment3DTile1D), + static_cast<void*>(counters.data()), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } +} + +TEST(Parallelize3DTile1D, MultiThreadPoolEachItemProcessedOnce) { + std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(Increment3DTile1D), + static_cast<void*>(counters.data()), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } +} + +TEST(Parallelize3DTile1D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(Increment3DTile1D), + static_cast<void*>(counters.data()), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } +} + +TEST(Parallelize3DTile1D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector<std::atomic_int> counters(kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(Increment3DTile1D), + static_cast<void*>(counters.data()), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize3DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize3DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize3DTile1DRangeK; k++) { + const size_t linear_idx = (i * kParallelize3DTile1DRangeJ + j) * kParallelize3DTile1DRangeK + k; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } +} + +static void IncrementSame3DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t start_k, size_t tile_k) { + for (size_t k = start_k; k < start_k + tile_k; k++) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + } +} + +TEST(Parallelize3DTile1D, MultiThreadPoolHighContention) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(IncrementSame3DTile1D), + static_cast<void*>(&num_processed_items), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); +} + +static void WorkImbalance3DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t start_k, size_t tile_k) { + num_processed_items->fetch_add(tile_k, std::memory_order_relaxed); + if (i == 0 && j == 0 && start_k == 0) { + /* Spin-wait until all items are computed */ + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK) { + std::atomic_thread_fence(std::memory_order_acquire); + } + } +} + +TEST(Parallelize3DTile1D, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_3d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_3d_tile_1d_t>(WorkImbalance3DTile1D), + static_cast<void*>(&num_processed_items), + kParallelize3DTile1DRangeI, kParallelize3DTile1DRangeJ, kParallelize3DTile1DRangeK, + kParallelize3DTile1DTileK, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile1DRangeI * kParallelize3DTile1DRangeJ * kParallelize3DTile1DRangeK); +} + static void ComputeNothing3DTile2D(void*, size_t, size_t, size_t, size_t, size_t) { } @@ -3058,6 +3725,672 @@ TEST(Parallelize3DTile2DWithUArch, MultiThreadPoolWorkStealing) { EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize3DTile2DRangeI * kParallelize3DTile2DRangeJ * kParallelize3DTile2DRangeK); } +static void ComputeNothing4D(void*, size_t, size_t, size_t, size_t) { +} + +TEST(Parallelize4D, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d(threadpool.get(), + ComputeNothing4D, + nullptr, + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); +} + +TEST(Parallelize4D, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d( + threadpool.get(), + ComputeNothing4D, + nullptr, + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); +} + +static void CheckBounds4D(void*, size_t i, size_t j, size_t k, size_t l) { + EXPECT_LT(i, kParallelize4DRangeI); + EXPECT_LT(j, kParallelize4DRangeJ); + EXPECT_LT(k, kParallelize4DRangeK); + EXPECT_LT(l, kParallelize4DRangeL); +} + +TEST(Parallelize4D, SingleThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d( + threadpool.get(), + CheckBounds4D, + nullptr, + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); +} + +TEST(Parallelize4D, MultiThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d( + threadpool.get(), + CheckBounds4D, + nullptr, + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); +} + +static void SetTrue4D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t k, size_t l) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); +} + +TEST(Parallelize4D, SingleThreadPoolAllItemsProcessed) { + std::vector<std::atomic_bool> indicators(kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_t>(SetTrue4D), + static_cast<void*>(indicators.data()), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") not processed"; + } + } + } + } +} + +TEST(Parallelize4D, MultiThreadPoolAllItemsProcessed) { + std::vector<std::atomic_bool> indicators(kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_t>(SetTrue4D), + static_cast<void*>(indicators.data()), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") not processed"; + } + } + } + } +} + +static void Increment4D(std::atomic_int* processed_counters, size_t i, size_t j, size_t k, size_t l) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); +} + +TEST(Parallelize4D, SingleThreadPoolEachItemProcessedOnce) { + std::vector<std::atomic_int> counters(kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_t>(Increment4D), + static_cast<void*>(counters.data()), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } +} + +TEST(Parallelize4D, MultiThreadPoolEachItemProcessedOnce) { + std::vector<std::atomic_int> counters(kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_t>(Increment4D), + static_cast<void*>(counters.data()), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } +} + +TEST(Parallelize4D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector<std::atomic_int> counters(kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_t>(Increment4D), + static_cast<void*>(counters.data()), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize4DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } + } +} + +TEST(Parallelize4D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector<std::atomic_int> counters(kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_t>(Increment4D), + static_cast<void*>(counters.data()), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize4DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DRangeJ + j) * kParallelize4DRangeK + k) * kParallelize4DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } + } +} + +static void IncrementSame4D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t l) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); +} + +TEST(Parallelize4D, MultiThreadPoolHighContention) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_t>(IncrementSame4D), + static_cast<void*>(&num_processed_items), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); +} + +static void WorkImbalance4D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t l) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + if (i == 0 && j == 0 && k == 0 && l == 0) { + /* Spin-wait until all items are computed */ + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL) { + std::atomic_thread_fence(std::memory_order_acquire); + } + } +} + +TEST(Parallelize4D, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_t>(WorkImbalance4D), + static_cast<void*>(&num_processed_items), + kParallelize4DRangeI, kParallelize4DRangeJ, kParallelize4DRangeK, kParallelize4DRangeL, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DRangeI * kParallelize4DRangeJ * kParallelize4DRangeK * kParallelize4DRangeL); +} + +static void ComputeNothing4DTile1D(void*, size_t, size_t, size_t, size_t, size_t) { +} + +TEST(Parallelize4DTile1D, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_1d(threadpool.get(), + ComputeNothing4DTile1D, + nullptr, + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); +} + +TEST(Parallelize4DTile1D, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + ComputeNothing4DTile1D, + nullptr, + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); +} + +static void CheckBounds4DTile1D(void*, size_t i, size_t j, size_t k, size_t start_l, size_t tile_l) { + EXPECT_LT(i, kParallelize4DTile1DRangeI); + EXPECT_LT(j, kParallelize4DTile1DRangeJ); + EXPECT_LT(k, kParallelize4DTile1DRangeK); + EXPECT_LT(start_l, kParallelize4DTile1DRangeL); + EXPECT_LE(start_l + tile_l, kParallelize4DTile1DRangeL); +} + +TEST(Parallelize4DTile1D, SingleThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + CheckBounds4DTile1D, + nullptr, + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); +} + +TEST(Parallelize4DTile1D, MultiThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + CheckBounds4DTile1D, + nullptr, + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); +} + +static void CheckTiling4DTile1D(void*, size_t i, size_t j, size_t k, size_t start_l, size_t tile_l) { + EXPECT_GT(tile_l, 0); + EXPECT_LE(tile_l, kParallelize4DTile1DTileL); + EXPECT_EQ(start_l % kParallelize4DTile1DTileL, 0); + EXPECT_EQ(tile_l, std::min<size_t>(kParallelize4DTile1DTileL, kParallelize4DTile1DRangeL - start_l)); +} + +TEST(Parallelize4DTile1D, SingleThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + CheckTiling4DTile1D, + nullptr, + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); +} + +TEST(Parallelize4DTile1D, MultiThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + CheckTiling4DTile1D, + nullptr, + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); +} + +static void SetTrue4DTile1D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t k, size_t start_l, size_t tile_l) { + for (size_t l = start_l; l < start_l + tile_l; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); + } +} + +TEST(Parallelize4DTile1D, SingleThreadPoolAllItemsProcessed) { + std::vector<std::atomic_bool> indicators(kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_tile_1d_t>(SetTrue4DTile1D), + static_cast<void*>(indicators.data()), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile1DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") not processed"; + } + } + } + } +} + +TEST(Parallelize4DTile1D, MultiThreadPoolAllItemsProcessed) { + std::vector<std::atomic_bool> indicators(kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_tile_1d_t>(SetTrue4DTile1D), + static_cast<void*>(indicators.data()), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile1DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") not processed"; + } + } + } + } +} + +static void Increment4DTile1D(std::atomic_int* processed_counters, size_t i, size_t j, size_t k, size_t start_l, size_t tile_l) { + for (size_t l = start_l; l < start_l + tile_l; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); + } +} + +TEST(Parallelize4DTile1D, SingleThreadPoolEachItemProcessedOnce) { + std::vector<std::atomic_int> counters(kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_tile_1d_t>(Increment4DTile1D), + static_cast<void*>(counters.data()), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile1DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } +} + +TEST(Parallelize4DTile1D, MultiThreadPoolEachItemProcessedOnce) { + std::vector<std::atomic_int> counters(kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_tile_1d_t>(Increment4DTile1D), + static_cast<void*>(counters.data()), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize4DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile1DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } +} + +TEST(Parallelize4DTile1D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector<std::atomic_int> counters(kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_tile_1d_t>(Increment4DTile1D), + static_cast<void*>(counters.data()), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize4DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile1DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } + } +} + +TEST(Parallelize4DTile1D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector<std::atomic_int> counters(kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + for (size_t iteration = 0; iteration < kIncrementIterations; iteration++) { + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_tile_1d_t>(Increment4DTile1D), + static_cast<void*>(counters.data()), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize4DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize4DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize4DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize4DTile1DRangeL; l++) { + const size_t linear_idx = ((i * kParallelize4DTile1DRangeJ + j) * kParallelize4DTile1DRangeK + k) * kParallelize4DTile1DRangeL + l; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations << ")"; + } + } + } + } +} + +static void IncrementSame4DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t start_l, size_t tile_l) { + for (size_t l = start_l; l < start_l + tile_l; l++) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + } +} + +TEST(Parallelize4DTile1D, MultiThreadPoolHighContention) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_tile_1d_t>(IncrementSame4DTile1D), + static_cast<void*>(&num_processed_items), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); +} + +static void WorkImbalance4DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t start_l, size_t tile_l) { + num_processed_items->fetch_add(tile_l, std::memory_order_relaxed); + if (i == 0 && j == 0 && k == 0 && start_l == 0) { + /* Spin-wait until all items are computed */ + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL) { + std::atomic_thread_fence(std::memory_order_acquire); + } + } +} + +TEST(Parallelize4DTile1D, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_4d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_4d_tile_1d_t>(WorkImbalance4DTile1D), + static_cast<void*>(&num_processed_items), + kParallelize4DTile1DRangeI, kParallelize4DTile1DRangeJ, kParallelize4DTile1DRangeK, kParallelize4DTile1DRangeL, + kParallelize4DTile1DTileL, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DTile1DRangeI * kParallelize4DTile1DRangeJ * kParallelize4DTile1DRangeK * kParallelize4DTile1DRangeL); +} + static void ComputeNothing4DTile2D(void*, size_t, size_t, size_t, size_t, size_t, size_t) { } @@ -3856,6 +5189,698 @@ TEST(Parallelize4DTile2DWithUArch, MultiThreadPoolWorkStealing) { EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize4DTile2DRangeI * kParallelize4DTile2DRangeJ * kParallelize4DTile2DRangeK * kParallelize4DTile2DRangeL); } +static void ComputeNothing5D(void*, size_t, size_t, size_t, size_t, size_t) { +} + +TEST(Parallelize5D, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d(threadpool.get(), + ComputeNothing5D, + nullptr, + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); +} + +TEST(Parallelize5D, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d( + threadpool.get(), + ComputeNothing5D, + nullptr, + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); +} + +static void CheckBounds5D(void*, size_t i, size_t j, size_t k, size_t l, size_t m) { + EXPECT_LT(i, kParallelize5DRangeI); + EXPECT_LT(j, kParallelize5DRangeJ); + EXPECT_LT(k, kParallelize5DRangeK); + EXPECT_LT(l, kParallelize5DRangeL); + EXPECT_LT(m, kParallelize5DRangeM); +} + +TEST(Parallelize5D, SingleThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d( + threadpool.get(), + CheckBounds5D, + nullptr, + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); +} + +TEST(Parallelize5D, MultiThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d( + threadpool.get(), + CheckBounds5D, + nullptr, + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); +} + +static void SetTrue5D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t k, size_t l, size_t m) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); +} + +TEST(Parallelize5D, SingleThreadPoolAllItemsProcessed) { + std::vector<std::atomic_bool> indicators(kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_t>(SetTrue5D), + static_cast<void*>(indicators.data()), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize5DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") not processed"; + } + } + } + } + } +} + +TEST(Parallelize5D, MultiThreadPoolAllItemsProcessed) { + std::vector<std::atomic_bool> indicators(kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_t>(SetTrue5D), + static_cast<void*>(indicators.data()), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize5DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") not processed"; + } + } + } + } + } +} + +static void Increment5D(std::atomic_int* processed_counters, size_t i, size_t j, size_t k, size_t l, size_t m) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); +} + +TEST(Parallelize5D, SingleThreadPoolEachItemProcessedOnce) { + std::vector<std::atomic_int> counters(kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_t>(Increment5D), + static_cast<void*>(counters.data()), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize5DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } + } +} + +TEST(Parallelize5D, MultiThreadPoolEachItemProcessedOnce) { + std::vector<std::atomic_int> counters(kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_t>(Increment5D), + static_cast<void*>(counters.data()), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize5DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } + } +} + +TEST(Parallelize5D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector<std::atomic_int> counters(kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + for (size_t iteration = 0; iteration < kIncrementIterations5D; iteration++) { + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_t>(Increment5D), + static_cast<void*>(counters.data()), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize5DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations5D) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations5D << ")"; + } + } + } + } + } +} + +TEST(Parallelize5D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector<std::atomic_int> counters(kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + for (size_t iteration = 0; iteration < kIncrementIterations5D; iteration++) { + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_t>(Increment5D), + static_cast<void*>(counters.data()), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize5DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DRangeJ + j) * kParallelize5DRangeK + k) * kParallelize5DRangeL + l) * kParallelize5DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations5D) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations5D << ")"; + } + } + } + } + } +} + +static void IncrementSame5D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t l, size_t m) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); +} + +TEST(Parallelize5D, MultiThreadPoolHighContention) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_t>(IncrementSame5D), + static_cast<void*>(&num_processed_items), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); +} + +static void WorkImbalance5D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t l, size_t m) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + if (i == 0 && j == 0 && k == 0 && l == 0 && m == 0) { + /* Spin-wait until all items are computed */ + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM) { + std::atomic_thread_fence(std::memory_order_acquire); + } + } +} + +TEST(Parallelize5D, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_t>(WorkImbalance5D), + static_cast<void*>(&num_processed_items), + kParallelize5DRangeI, kParallelize5DRangeJ, kParallelize5DRangeK, kParallelize5DRangeL, kParallelize5DRangeM, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize5DRangeI * kParallelize5DRangeJ * kParallelize5DRangeK * kParallelize5DRangeL * kParallelize5DRangeM); +} + +static void ComputeNothing5DTile1D(void*, size_t, size_t, size_t, size_t, size_t, size_t) { +} + +TEST(Parallelize5DTile1D, SingleThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d_tile_1d(threadpool.get(), + ComputeNothing5DTile1D, + nullptr, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); +} + +TEST(Parallelize5DTile1D, MultiThreadPoolCompletes) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + ComputeNothing5DTile1D, + nullptr, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); +} + +static void CheckBounds5DTile1D(void*, size_t i, size_t j, size_t k, size_t l, size_t start_m, size_t tile_m) { + EXPECT_LT(i, kParallelize5DTile1DRangeI); + EXPECT_LT(j, kParallelize5DTile1DRangeJ); + EXPECT_LT(k, kParallelize5DTile1DRangeK); + EXPECT_LT(l, kParallelize5DTile1DRangeL); + EXPECT_LT(start_m, kParallelize5DTile1DRangeM); + EXPECT_LE(start_m + tile_m, kParallelize5DTile1DRangeM); +} + +TEST(Parallelize5DTile1D, SingleThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + CheckBounds5DTile1D, + nullptr, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); +} + +TEST(Parallelize5DTile1D, MultiThreadPoolAllItemsInBounds) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + CheckBounds5DTile1D, + nullptr, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); +} + +static void CheckTiling5DTile1D(void*, size_t i, size_t j, size_t k, size_t l, size_t start_m, size_t tile_m) { + EXPECT_GT(tile_m, 0); + EXPECT_LE(tile_m, kParallelize5DTile1DTileM); + EXPECT_EQ(start_m % kParallelize5DTile1DTileM, 0); + EXPECT_EQ(tile_m, std::min<size_t>(kParallelize5DTile1DTileM, kParallelize5DTile1DRangeM - start_m)); +} + +TEST(Parallelize5DTile1D, SingleThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + CheckTiling5DTile1D, + nullptr, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); +} + +TEST(Parallelize5DTile1D, MultiThreadPoolUniformTiling) { + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + CheckTiling5DTile1D, + nullptr, + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); +} + +static void SetTrue5DTile1D(std::atomic_bool* processed_indicators, size_t i, size_t j, size_t k, size_t l, size_t start_m, size_t tile_m) { + for (size_t m = start_m; m < start_m + tile_m; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + processed_indicators[linear_idx].store(true, std::memory_order_relaxed); + } +} + +TEST(Parallelize5DTile1D, SingleThreadPoolAllItemsProcessed) { + std::vector<std::atomic_bool> indicators(kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_tile_1d_t>(SetTrue5DTile1D), + static_cast<void*>(indicators.data()), + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize5DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DTile1DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DTile1DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") not processed"; + } + } + } + } + } +} + +TEST(Parallelize5DTile1D, MultiThreadPoolAllItemsProcessed) { + std::vector<std::atomic_bool> indicators(kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_tile_1d_t>(SetTrue5DTile1D), + static_cast<void*>(indicators.data()), + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize5DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DTile1DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DTile1DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + EXPECT_TRUE(indicators[linear_idx].load(std::memory_order_relaxed)) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") not processed"; + } + } + } + } + } +} + +static void Increment5DTile1D(std::atomic_int* processed_counters, size_t i, size_t j, size_t k, size_t l, size_t start_m, size_t tile_m) { + for (size_t m = start_m; m < start_m + tile_m; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + processed_counters[linear_idx].fetch_add(1, std::memory_order_relaxed); + } +} + +TEST(Parallelize5DTile1D, SingleThreadPoolEachItemProcessedOnce) { + std::vector<std::atomic_int> counters(kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_tile_1d_t>(Increment5DTile1D), + static_cast<void*>(counters.data()), + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize5DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DTile1DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DTile1DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } + } +} + +TEST(Parallelize5DTile1D, MultiThreadPoolEachItemProcessedOnce) { + std::vector<std::atomic_int> counters(kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_tile_1d_t>(Increment5DTile1D), + static_cast<void*>(counters.data()), + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); + + for (size_t i = 0; i < kParallelize5DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DTile1DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DTile1DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), 1) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times (expected: 1)"; + } + } + } + } + } +} + +TEST(Parallelize5DTile1D, SingleThreadPoolEachItemProcessedMultipleTimes) { + std::vector<std::atomic_int> counters(kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(1), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + for (size_t iteration = 0; iteration < kIncrementIterations5D; iteration++) { + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_tile_1d_t>(Increment5DTile1D), + static_cast<void*>(counters.data()), + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize5DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DTile1DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DTile1DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations5D) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations5D << ")"; + } + } + } + } + } +} + +TEST(Parallelize5DTile1D, MultiThreadPoolEachItemProcessedMultipleTimes) { + std::vector<std::atomic_int> counters(kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + for (size_t iteration = 0; iteration < kIncrementIterations5D; iteration++) { + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_tile_1d_t>(Increment5DTile1D), + static_cast<void*>(counters.data()), + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); + } + + for (size_t i = 0; i < kParallelize5DTile1DRangeI; i++) { + for (size_t j = 0; j < kParallelize5DTile1DRangeJ; j++) { + for (size_t k = 0; k < kParallelize5DTile1DRangeK; k++) { + for (size_t l = 0; l < kParallelize5DTile1DRangeL; l++) { + for (size_t m = 0; m < kParallelize5DTile1DRangeM; m++) { + const size_t linear_idx = (((i * kParallelize5DTile1DRangeJ + j) * kParallelize5DTile1DRangeK + k) * kParallelize5DTile1DRangeL + l) * kParallelize5DTile1DRangeM + m; + EXPECT_EQ(counters[linear_idx].load(std::memory_order_relaxed), kIncrementIterations5D) + << "Element (" << i << ", " << j << ", " << k << ", " << l << ", " << m << ") was processed " + << counters[linear_idx].load(std::memory_order_relaxed) << " times " + << "(expected: " << kIncrementIterations5D << ")"; + } + } + } + } + } +} + +static void IncrementSame5DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t l, size_t start_m, size_t tile_m) { + for (size_t m = start_m; m < start_m + tile_m; m++) { + num_processed_items->fetch_add(1, std::memory_order_relaxed); + } +} + +TEST(Parallelize5DTile1D, MultiThreadPoolHighContention) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_tile_1d_t>(IncrementSame5DTile1D), + static_cast<void*>(&num_processed_items), + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); +} + +static void WorkImbalance5DTile1D(std::atomic_int* num_processed_items, size_t i, size_t j, size_t k, size_t l, size_t start_m, size_t tile_m) { + num_processed_items->fetch_add(tile_m, std::memory_order_relaxed); + if (i == 0 && j == 0 && k == 0 && l == 0 && start_m == 0) { + /* Spin-wait until all items are computed */ + while (num_processed_items->load(std::memory_order_relaxed) != kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM) { + std::atomic_thread_fence(std::memory_order_acquire); + } + } +} + +TEST(Parallelize5DTile1D, MultiThreadPoolWorkStealing) { + std::atomic_int num_processed_items = ATOMIC_VAR_INIT(0); + + auto_pthreadpool_t threadpool(pthreadpool_create(0), pthreadpool_destroy); + ASSERT_TRUE(threadpool.get()); + + if (pthreadpool_get_threads_count(threadpool.get()) <= 1) { + GTEST_SKIP(); + } + + pthreadpool_parallelize_5d_tile_1d( + threadpool.get(), + reinterpret_cast<pthreadpool_task_5d_tile_1d_t>(WorkImbalance5DTile1D), + static_cast<void*>(&num_processed_items), + kParallelize5DTile1DRangeI, kParallelize5DTile1DRangeJ, kParallelize5DTile1DRangeK, kParallelize5DTile1DRangeL, kParallelize5DTile1DRangeM, + kParallelize5DTile1DTileM, + 0 /* flags */); + EXPECT_EQ(num_processed_items.load(std::memory_order_relaxed), kParallelize5DTile1DRangeI * kParallelize5DTile1DRangeJ * kParallelize5DTile1DRangeK * kParallelize5DTile1DRangeL * kParallelize5DTile1DRangeM); +} + static void ComputeNothing5DTile2D(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t) { } |