summaryrefslogtreecommitdiff
path: root/mali_kbase/mmu/mali_kbase_mmu.c
diff options
context:
space:
mode:
Diffstat (limited to 'mali_kbase/mmu/mali_kbase_mmu.c')
-rw-r--r--mali_kbase/mmu/mali_kbase_mmu.c118
1 files changed, 74 insertions, 44 deletions
diff --git a/mali_kbase/mmu/mali_kbase_mmu.c b/mali_kbase/mmu/mali_kbase_mmu.c
index cc4de07..2783e04 100644
--- a/mali_kbase/mmu/mali_kbase_mmu.c
+++ b/mali_kbase/mmu/mali_kbase_mmu.c
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
*
- * (C) COPYRIGHT 2010-2023 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2024 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@@ -776,7 +776,7 @@ static bool page_fault_try_alloc(struct kbase_context *kctx, struct kbase_va_reg
return false;
}
- if (kctx->kbdev->pagesize_2mb && new_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE) {
+ if (kbase_is_large_pages_enabled() && new_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE) {
root_pool = &kctx->mem_pools.large[region->gpu_alloc->group_id];
*grow_2mb_pool = true;
} else {
@@ -923,7 +923,7 @@ void kbase_mmu_page_fault_worker(struct work_struct *data)
int err;
bool grown = false;
size_t pages_to_grow;
- bool grow_2mb_pool;
+ bool grow_2mb_pool = false;
struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL };
int i;
size_t current_backed_size;
@@ -1093,7 +1093,7 @@ void kbase_mmu_page_fault_worker(struct work_struct *data)
}
page_fault_retry:
- if (kbdev->pagesize_2mb) {
+ if (kbase_is_large_pages_enabled()) {
/* Preallocate (or re-allocate) memory for the sub-allocation structs if necessary */
for (i = 0; i != ARRAY_SIZE(prealloc_sas); ++i) {
if (!prealloc_sas[i]) {
@@ -1180,10 +1180,14 @@ page_fault_retry:
*/
op_param.mmu_sync_info = mmu_sync_info;
op_param.kctx_id = kctx->id;
- /* Can safely skip the invalidate for all levels in case
- * of duplicate page faults.
+ /* Usually it is safe to skip the MMU cache invalidate for all levels
+ * in case of duplicate page faults. But for the pathological scenario
+ * where the faulty VA gets mapped by the time page fault worker runs it
+ * becomes imperative to invalidate MMU cache for all levels, otherwise
+ * there is a possibility of repeated page faults on GPUs which supports
+ * fine grained MMU cache invalidation.
*/
- op_param.flush_skip_levels = 0xF;
+ op_param.flush_skip_levels = 0x0;
op_param.vpfn = fault_pfn;
op_param.nr = 1;
spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_flags);
@@ -1217,10 +1221,14 @@ page_fault_retry:
/* See comment [1] about UNLOCK usage */
op_param.mmu_sync_info = mmu_sync_info;
op_param.kctx_id = kctx->id;
- /* Can safely skip the invalidate for all levels in case
- * of duplicate page faults.
+ /* Usually it is safe to skip the MMU cache invalidate for all levels
+ * in case of duplicate page faults. But for the pathological scenario
+ * where the faulty VA gets mapped by the time page fault worker runs it
+ * becomes imperative to invalidate MMU cache for all levels, otherwise
+ * there is a possibility of repeated page faults on GPUs which supports
+ * fine grained MMU cache invalidation.
*/
- op_param.flush_skip_levels = 0xF;
+ op_param.flush_skip_levels = 0x0;
op_param.vpfn = fault_pfn;
op_param.nr = 1;
spin_lock_irqsave(&kbdev->hwaccess_lock, hwaccess_flags);
@@ -1382,7 +1390,7 @@ page_fault_retry:
* Otherwise fail the allocation.
*/
if (pages_to_grow > 0) {
- if (kbdev->pagesize_2mb && grow_2mb_pool) {
+ if (kbase_is_large_pages_enabled() && grow_2mb_pool) {
/* Round page requirement up to nearest 2 MB */
struct kbase_mem_pool *const lp_mem_pool =
&kctx->mem_pools.large[group_id];
@@ -1595,6 +1603,7 @@ static int mmu_get_lowest_valid_pgd(struct kbase_device *kbdev, struct kbase_mmu
return err;
}
+KBASE_ALLOW_ERROR_INJECTION_TEST_API(mmu_get_lowest_valid_pgd, ERRNO);
/*
* On success, sets out_pgd to the PGD for the specified level of translation
@@ -1700,8 +1709,16 @@ static void mmu_insert_pages_failure_recovery(struct kbase_device *kbdev,
mmu_mode->entries_invalidate(&page[idx], pcount);
if (!num_of_valid_entries) {
+ mmu_mode->set_num_valid_entries(page, 0);
+
kbase_kunmap(p, page);
+ /* No CPU and GPU cache maintenance is done here as caller would do the
+ * complete flush of GPU cache and invalidation of TLB before the PGD
+ * page is freed. CPU cache flush would be done when the PGD page is
+ * returned to the memory pool.
+ */
+
kbase_mmu_add_to_free_pgds_list(mmut, p);
kbase_mmu_update_and_free_parent_pgds(kbdev, mmut, pgds, vpfn, level,
@@ -1728,7 +1745,8 @@ next:
* going to happen to these pages at this stage. They might return
* movable once they are returned to a memory pool.
*/
- if (kbase_is_page_migration_enabled() && !ignore_page_migration && phys) {
+ if (kbase_is_page_migration_enabled() && !ignore_page_migration && phys &&
+ !is_huge(*phys) && !is_partial(*phys)) {
const u64 num_pages = (to_vpfn - from_vpfn) / GPU_PAGES_PER_CPU_PAGE;
u64 i;
@@ -2525,6 +2543,7 @@ int kbase_mmu_insert_pages(struct kbase_device *kbdev, struct kbase_mmu_table *m
}
KBASE_EXPORT_TEST_API(kbase_mmu_insert_pages);
+KBASE_ALLOW_ERROR_INJECTION_TEST_API(kbase_mmu_insert_pages, ERRNO);
int kbase_mmu_insert_pages_skip_status_update(struct kbase_device *kbdev,
struct kbase_mmu_table *mmut, u64 vpfn,
@@ -2582,6 +2601,7 @@ int kbase_mmu_insert_aliased_pages(struct kbase_device *kbdev, struct kbase_mmu_
return 0;
}
+KBASE_ALLOW_ERROR_INJECTION_TEST_API(kbase_mmu_insert_aliased_pages, ERRNO);
void kbase_mmu_update(struct kbase_device *kbdev, struct kbase_mmu_table *mmut, int as_nr)
{
@@ -2720,15 +2740,25 @@ static void kbase_mmu_update_and_free_parent_pgds(struct kbase_device *kbdev,
kbdev->mmu_mode->entries_invalidate(&current_page[index], 1);
if (current_valid_entries == 1 && current_level != MIDGARD_MMU_LEVEL(0)) {
+ kbdev->mmu_mode->set_num_valid_entries(current_page, 0);
+
kbase_kunmap(p, current_page);
- /* Ensure the cacheline containing the last valid entry
- * of PGD is invalidated from the GPU cache, before the
- * PGD page is freed.
- */
- kbase_mmu_sync_pgd_gpu(kbdev, mmut->kctx,
- current_pgd + (index * sizeof(u64)), sizeof(u64),
- flush_op);
+ /* Check if fine grained GPU cache maintenance is being used */
+ if (flush_op == KBASE_MMU_OP_FLUSH_PT) {
+ /* Ensure the invalidated PTE is visible in memory right away */
+ kbase_mmu_sync_pgd_cpu(kbdev,
+ kbase_dma_addr(p) + (index * sizeof(u64)),
+ sizeof(u64));
+ /* Invalidate the GPU cache for the whole PGD page and not just for
+ * the cacheline containing the invalidated PTE, as the PGD page is
+ * going to be freed. There is an extremely remote possibility that
+ * other cachelines (containing all invalid PTEs) of PGD page are
+ * also present in the GPU cache.
+ */
+ kbase_mmu_sync_pgd_gpu(kbdev, mmut->kctx, current_pgd,
+ 512 * sizeof(u64), KBASE_MMU_OP_FLUSH_PT);
+ }
kbase_mmu_add_to_free_pgds_list(mmut, p);
} else {
@@ -2832,8 +2862,7 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase
phys_addr_t pgd = mmut->pgd;
struct page *p = phys_to_page(pgd);
- if (count > nr)
- count = nr;
+ count = MIN(nr, count);
/* need to check if this is a 2MB page or a small page */
for (level = MIDGARD_MMU_TOPLEVEL; level <= MIDGARD_MMU_BOTTOMLEVEL; level++) {
@@ -2844,23 +2873,12 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase
if (mmu_mode->ate_is_valid(page[index], level))
break; /* keep the mapping */
else if (!mmu_mode->pte_is_valid(page[index], level)) {
- /* nothing here, advance */
- switch (level) {
- case MIDGARD_MMU_LEVEL(0):
- count = 134217728;
- break;
- case MIDGARD_MMU_LEVEL(1):
- count = 262144;
- break;
- case MIDGARD_MMU_LEVEL(2):
- count = 512;
- break;
- case MIDGARD_MMU_LEVEL(3):
- count = 1;
- break;
- }
- if (count > nr)
- count = nr;
+ dev_warn(kbdev->dev, "Invalid PTE found @ level %d for VA %llx",
+ level, vpfn << PAGE_SHIFT);
+ /* nothing here, advance to the next PTE of the current level */
+ count = (1 << ((3 - level) * 9));
+ count -= (vpfn & (count - 1));
+ count = MIN(nr, count);
goto next;
}
next_pgd = mmu_mode->pte_to_phy_addr(
@@ -2915,14 +2933,25 @@ static int kbase_mmu_teardown_pgd_pages(struct kbase_device *kbdev, struct kbase
mmu_mode->entries_invalidate(&page[index], pcount);
if (!num_of_valid_entries) {
+ mmu_mode->set_num_valid_entries(page, 0);
+
kbase_kunmap(p, page);
- /* Ensure the cacheline(s) containing the last valid entries
- * of PGD is invalidated from the GPU cache, before the
- * PGD page is freed.
- */
- kbase_mmu_sync_pgd_gpu(kbdev, mmut->kctx, pgd + (index * sizeof(u64)),
- pcount * sizeof(u64), flush_op);
+ /* Check if fine grained GPU cache maintenance is being used */
+ if (flush_op == KBASE_MMU_OP_FLUSH_PT) {
+ /* Ensure the invalidated ATEs are visible in memory right away */
+ kbase_mmu_sync_pgd_cpu(kbdev,
+ kbase_dma_addr(p) + (index * sizeof(u64)),
+ pcount * sizeof(u64));
+ /* Invalidate the GPU cache for the whole PGD page and not just for
+ * the cachelines containing the invalidated ATEs, as the PGD page
+ * is going to be freed. There is an extremely remote possibility
+ * that other cachelines (containing all invalid ATEs) of PGD page
+ * are also present in the GPU cache.
+ */
+ kbase_mmu_sync_pgd_gpu(kbdev, mmut->kctx, pgd, 512 * sizeof(u64),
+ KBASE_MMU_OP_FLUSH_PT);
+ }
kbase_mmu_add_to_free_pgds_list(mmut, p);
@@ -3069,6 +3098,7 @@ int kbase_mmu_teardown_pages(struct kbase_device *kbdev, struct kbase_mmu_table
return mmu_teardown_pages(kbdev, mmut, vpfn, phys, nr_phys_pages, nr_virt_pages, as_nr,
false);
}
+KBASE_EXPORT_TEST_API(kbase_mmu_teardown_pages);
int kbase_mmu_teardown_imported_pages(struct kbase_device *kbdev, struct kbase_mmu_table *mmut,
u64 vpfn, struct tagged_addr *phys, size_t nr_phys_pages,
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-06 06:21:33 +0000