diff options
author | Sidath Senanayake <sidaths@google.com> | 2021-04-14 19:14:30 +0100 |
---|---|---|
committer | Sidath Senanayake <sidaths@google.com> | 2021-04-14 19:53:59 +0100 |
commit | 8037b534570814775d79aeddd06b76e5ee941f59 (patch) | |
tree | bd03fb25466475adb31333ff487350e1bd352fa1 /mali_kbase/mali_kbase_mem.c | |
parent | 5ddb2927918bdf131080bc488eb1fb455d1cdfb7 (diff) | |
download | gpu-8037b534570814775d79aeddd06b76e5ee941f59.tar.gz |
Mali Valhall DDK r30p0 KMD
Provenance
4a09e9e58 (collaborate/EAC/v_r30p0)
VX504X08X-BU-00000-r30p0-01eac0 - Valhall Android DDK
VX504X08X-BU-60000-r30p0-01eac0 - Valhall Android Document Bundle
VX504X08X-DC-11001-r30p0-01eac0 - Valhall Android DDK Software Errata
VX504X08X-SW-99006-r30p0-01eac0 - Valhall Android Renderscript AOSP parts
Signed-off-by: Sidath Senanayake <sidaths@google.com>
Change-Id: Ie30d6ece7ebe221c8e4c736af05ad05dcfc15127
Diffstat (limited to 'mali_kbase/mali_kbase_mem.c')
-rw-r--r-- | mali_kbase/mali_kbase_mem.c | 327 |
1 files changed, 250 insertions, 77 deletions
diff --git a/mali_kbase/mali_kbase_mem.c b/mali_kbase/mali_kbase_mem.c index c054205..fd992e2 100644 --- a/mali_kbase/mali_kbase_mem.c +++ b/mali_kbase/mali_kbase_mem.c @@ -1,12 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 /* * - * (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved. + * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved. * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. + * of such GNU license. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of @@ -17,8 +17,6 @@ * along with this program; if not, you can access it online at * http://www.gnu.org/licenses/gpl-2.0.html. * - * SPDX-License-Identifier: GPL-2.0 - * */ /** @@ -106,22 +104,28 @@ static struct rb_root *kbase_gpu_va_to_rbtree(struct kbase_context *kctx, u64 gpu_pfn) { struct rb_root *rbtree = NULL; + struct kbase_reg_zone *exec_va_zone = + kbase_ctx_reg_zone_get(kctx, KBASE_REG_ZONE_EXEC_VA); /* The gpu_pfn can only be greater than the starting pfn of the EXEC_VA * zone if this has been initialized. */ - if (gpu_pfn >= kctx->exec_va_start) + if (gpu_pfn >= exec_va_zone->base_pfn) rbtree = &kctx->reg_rbtree_exec; else { u64 same_va_end; #ifdef CONFIG_64BIT - if (kbase_ctx_flag(kctx, KCTX_COMPAT)) + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { #endif /* CONFIG_64BIT */ same_va_end = KBASE_REG_ZONE_CUSTOM_VA_BASE; #ifdef CONFIG_64BIT - else - same_va_end = kctx->same_va_end; + } else { + struct kbase_reg_zone *same_va_zone = + kbase_ctx_reg_zone_get(kctx, + KBASE_REG_ZONE_SAME_VA); + same_va_end = kbase_reg_zone_end_pfn(same_va_zone); + } #endif /* CONFIG_64BIT */ if (gpu_pfn >= same_va_end) @@ -346,7 +350,8 @@ static struct kbase_va_region *kbase_region_tracker_find_region_meeting_reqs( } /** - * @brief Remove a region object from the global list. + * Remove a region object from the global list. + * @reg: Region object to remove * * The region reg is removed, possibly by merging with other free and * compatible adjacent regions. It must be called with the context @@ -651,7 +656,7 @@ exit: return err; } -/** +/* * @brief Initialize the internal region tracker data structure. */ static void kbase_region_tracker_ds_init(struct kbase_context *kctx, @@ -731,21 +736,24 @@ int kbase_region_tracker_init(struct kbase_context *kctx) u64 custom_va_size = KBASE_REG_ZONE_CUSTOM_VA_SIZE; u64 gpu_va_limit = (1ULL << kctx->kbdev->gpu_props.mmu.va_bits) >> PAGE_SHIFT; u64 same_va_pages; + u64 same_va_base = 1u; int err; /* Take the lock as kbase_free_alloced_region requires it */ kbase_gpu_vm_lock(kctx); - same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - 1; + same_va_pages = (1ULL << (same_va_bits - PAGE_SHIFT)) - same_va_base; /* all have SAME_VA */ - same_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_same, 1, - same_va_pages, - KBASE_REG_ZONE_SAME_VA); + same_va_reg = + kbase_alloc_free_region(&kctx->reg_rbtree_same, same_va_base, + same_va_pages, KBASE_REG_ZONE_SAME_VA); if (!same_va_reg) { err = -ENOMEM; goto fail_unlock; } + kbase_ctx_reg_zone_init(kctx, KBASE_REG_ZONE_SAME_VA, same_va_base, + same_va_pages); #ifdef CONFIG_64BIT /* 32-bit clients have custom VA zones */ @@ -771,17 +779,23 @@ int kbase_region_tracker_init(struct kbase_context *kctx) err = -ENOMEM; goto fail_free_same_va; } + kbase_ctx_reg_zone_init(kctx, KBASE_REG_ZONE_CUSTOM_VA, + KBASE_REG_ZONE_CUSTOM_VA_BASE, + custom_va_size); #ifdef CONFIG_64BIT } else { custom_va_size = 0; } #endif + /* EXEC_VA zone's codepaths are slightly easier when its base_pfn is + * initially U64_MAX + */ + kbase_ctx_reg_zone_init(kctx, KBASE_REG_ZONE_EXEC_VA, U64_MAX, 0u); + /* Other zones are 0: kbase_create_context() uses vzalloc */ kbase_region_tracker_ds_init(kctx, same_va_reg, custom_va_reg); - kctx->same_va_end = same_va_pages + 1; - kctx->gpu_va_end = kctx->same_va_end + custom_va_size; - kctx->exec_va_start = U64_MAX; + kctx->gpu_va_end = same_va_base + same_va_pages + custom_va_size; kctx->jit_va = false; #if MALI_USE_CSF @@ -798,44 +812,147 @@ fail_unlock: return err; } +static bool kbase_has_exec_va_zone_locked(struct kbase_context *kctx) +{ + struct kbase_reg_zone *exec_va_zone; + + lockdep_assert_held(&kctx->reg_lock); + exec_va_zone = kbase_ctx_reg_zone_get(kctx, KBASE_REG_ZONE_EXEC_VA); + + return (exec_va_zone->base_pfn != U64_MAX); +} + +bool kbase_has_exec_va_zone(struct kbase_context *kctx) +{ + bool has_exec_va_zone; + + kbase_gpu_vm_lock(kctx); + has_exec_va_zone = kbase_has_exec_va_zone_locked(kctx); + kbase_gpu_vm_unlock(kctx); + + return has_exec_va_zone; +} + +/** + * Determine if any allocations have been made on a context's region tracker + * @kctx: KBase context + * + * Check the context to determine if any allocations have been made yet from + * any of its zones. This check should be done before resizing a zone, e.g. to + * make space to add a second zone. + * + * Whilst a zone without allocations can be resized whilst other zones have + * allocations, we still check all of @kctx 's zones anyway: this is a stronger + * guarantee and should be adhered to when creating new zones anyway. + * + * Allocations from kbdev zones are not counted. + * + * Return: true if any allocs exist on any zone, false otherwise + */ +bool kbase_region_tracker_has_allocs(struct kbase_context *kctx) +{ + unsigned int zone_idx; + + lockdep_assert_held(&kctx->reg_lock); + + for (zone_idx = 0; zone_idx < KBASE_REG_ZONE_MAX; ++zone_idx) { + struct kbase_reg_zone *zone; + struct kbase_va_region *reg; + u64 zone_base_addr; + unsigned long zone_bits = KBASE_REG_ZONE(zone_idx); + unsigned long reg_zone; + + zone = kbase_ctx_reg_zone_get(kctx, zone_bits); + zone_base_addr = zone->base_pfn << PAGE_SHIFT; + + reg = kbase_region_tracker_find_region_base_address( + kctx, zone_base_addr); + + if (!zone->va_size_pages) { + WARN(reg, + "Should not have found a region that starts at 0x%.16llx for zone 0x%lx", + (unsigned long long)zone_base_addr, zone_bits); + continue; + } + + if (WARN(!reg, + "There should always be a region that starts at 0x%.16llx for zone 0x%lx, couldn't find it", + (unsigned long long)zone_base_addr, zone_bits)) + return true; /* Safest return value */ + + reg_zone = reg->flags & KBASE_REG_ZONE_MASK; + if (WARN(reg_zone != zone_bits, + "The region that starts at 0x%.16llx should be in zone 0x%lx but was found in the wrong zone 0x%lx", + (unsigned long long)zone_base_addr, zone_bits, + reg_zone)) + return true; /* Safest return value */ + + /* Unless the region is completely free, of the same size as + * the original zone, then it has allocs + */ + if ((!(reg->flags & KBASE_REG_FREE)) || + (reg->nr_pages != zone->va_size_pages)) + return true; + } + + /* All zones are the same size as originally made, so there are no + * allocs + */ + return false; +} + #ifdef CONFIG_64BIT static int kbase_region_tracker_init_jit_64(struct kbase_context *kctx, u64 jit_va_pages) { - struct kbase_va_region *same_va; + struct kbase_va_region *same_va_reg; + struct kbase_reg_zone *same_va_zone; + u64 same_va_zone_base_addr; + const unsigned long same_va_zone_bits = KBASE_REG_ZONE_SAME_VA; struct kbase_va_region *custom_va_reg; + u64 jit_va_start; lockdep_assert_held(&kctx->reg_lock); - /* First verify that a JIT_VA zone has not been created already. */ - if (kctx->jit_va) - return -EINVAL; - /* - * Modify the same VA free region after creation. Be careful to ensure - * that allocations haven't been made as they could cause an overlap - * to happen with existing same VA allocations and the custom VA zone. + * Modify the same VA free region after creation. The caller has + * ensured that allocations haven't been made, as any allocations could + * cause an overlap to happen with existing same VA allocations and the + * custom VA zone. */ - same_va = kbase_region_tracker_find_region_base_address(kctx, - PAGE_SIZE); - if (!same_va) + same_va_zone = kbase_ctx_reg_zone_get(kctx, same_va_zone_bits); + same_va_zone_base_addr = same_va_zone->base_pfn << PAGE_SHIFT; + + same_va_reg = kbase_region_tracker_find_region_base_address( + kctx, same_va_zone_base_addr); + if (WARN(!same_va_reg, + "Already found a free region at the start of every zone, but now cannot find any region for zone base 0x%.16llx zone 0x%lx", + (unsigned long long)same_va_zone_base_addr, same_va_zone_bits)) return -ENOMEM; - if (same_va->nr_pages < jit_va_pages || kctx->same_va_end < jit_va_pages) + /* kbase_region_tracker_has_allocs() in the caller has already ensured + * that all of the zones have no allocs, so no need to check that again + * on same_va_reg + */ + WARN_ON((!(same_va_reg->flags & KBASE_REG_FREE)) || + same_va_reg->nr_pages != same_va_zone->va_size_pages); + + if (same_va_reg->nr_pages < jit_va_pages || + same_va_zone->va_size_pages < jit_va_pages) return -ENOMEM; /* It's safe to adjust the same VA zone now */ - same_va->nr_pages -= jit_va_pages; - kctx->same_va_end -= jit_va_pages; + same_va_reg->nr_pages -= jit_va_pages; + same_va_zone->va_size_pages -= jit_va_pages; + jit_va_start = kbase_reg_zone_end_pfn(same_va_zone); /* * Create a custom VA zone at the end of the VA for allocations which * JIT can use so it doesn't have to allocate VA from the kernel. */ - custom_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_custom, - kctx->same_va_end, - jit_va_pages, - KBASE_REG_ZONE_CUSTOM_VA); + custom_va_reg = + kbase_alloc_free_region(&kctx->reg_rbtree_custom, jit_va_start, + jit_va_pages, KBASE_REG_ZONE_CUSTOM_VA); /* * The context will be destroyed if we fail here so no point @@ -843,6 +960,11 @@ static int kbase_region_tracker_init_jit_64(struct kbase_context *kctx, */ if (!custom_va_reg) return -ENOMEM; + /* Since this is 64-bit, the custom zone will not have been + * initialized, so initialize it now + */ + kbase_ctx_reg_zone_init(kctx, KBASE_REG_ZONE_CUSTOM_VA, jit_va_start, + jit_va_pages); kbase_region_tracker_insert(custom_va_reg); return 0; @@ -871,6 +993,23 @@ int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages, kbase_gpu_vm_lock(kctx); + /* Verify that a JIT_VA zone has not been created already. */ + if (kctx->jit_va) { + err = -EINVAL; + goto exit_unlock; + } + + /* If in 64-bit, we always lookup the SAME_VA zone. To ensure it has no + * allocs, we can ensure there are no allocs anywhere. + * + * This check is also useful in 32-bit, just to make sure init of the + * zone is always done before any allocs. + */ + if (kbase_region_tracker_has_allocs(kctx)) { + err = -ENOMEM; + goto exit_unlock; + } + #ifdef CONFIG_64BIT if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) err = kbase_region_tracker_init_jit_64(kctx, jit_va_pages); @@ -892,6 +1031,7 @@ int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages, #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ } +exit_unlock: kbase_gpu_vm_unlock(kctx); return err; @@ -899,24 +1039,33 @@ int kbase_region_tracker_init_jit(struct kbase_context *kctx, u64 jit_va_pages, int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages) { - struct kbase_va_region *shrinking_va_reg; struct kbase_va_region *exec_va_reg; - u64 exec_va_start, exec_va_base_addr; + struct kbase_reg_zone *exec_va_zone; + struct kbase_reg_zone *target_zone; + struct kbase_va_region *target_reg; + u64 target_zone_base_addr; + unsigned long target_zone_bits; + u64 exec_va_start; int err; - /* The EXEC_VA zone shall be created by making space at the end of the - * address space. Firstly, verify that the number of EXEC_VA pages - * requested by the client is reasonable and then make sure that it is - * not greater than the address space itself before calculating the base - * address of the new zone. + /* The EXEC_VA zone shall be created by making space either: + * - for 64-bit clients, at the end of the process's address space + * - for 32-bit clients, in the CUSTOM zone + * + * Firstly, verify that the number of EXEC_VA pages requested by the + * client is reasonable and then make sure that it is not greater than + * the address space itself before calculating the base address of the + * new zone. */ if (exec_va_pages == 0 || exec_va_pages > KBASE_REG_ZONE_EXEC_VA_MAX_PAGES) return -EINVAL; kbase_gpu_vm_lock(kctx); - /* First verify that a JIT_VA zone has not been created already. */ - if (kctx->jit_va) { + /* Verify that we've not already created a EXEC_VA zone, and that the + * EXEC_VA zone must come before JIT's CUSTOM_VA. + */ + if (kbase_has_exec_va_zone_locked(kctx) || kctx->jit_va) { err = -EPERM; goto exit_unlock; } @@ -926,28 +1075,50 @@ int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages goto exit_unlock; } - exec_va_start = kctx->gpu_va_end - exec_va_pages; - exec_va_base_addr = exec_va_start << PAGE_SHIFT; - - shrinking_va_reg = kbase_region_tracker_find_region_enclosing_address(kctx, - exec_va_base_addr); - if (!shrinking_va_reg) { + /* Verify no allocations have already been made */ + if (kbase_region_tracker_has_allocs(kctx)) { err = -ENOMEM; goto exit_unlock; } - /* Make sure that the EXEC_VA region is still uninitialized */ - if ((shrinking_va_reg->flags & KBASE_REG_ZONE_MASK) == - KBASE_REG_ZONE_EXEC_VA) { - err = -EPERM; +#ifdef CONFIG_64BIT + if (kbase_ctx_flag(kctx, KCTX_COMPAT)) { +#endif + /* 32-bit client: take from CUSTOM_VA zone */ + target_zone_bits = KBASE_REG_ZONE_CUSTOM_VA; +#ifdef CONFIG_64BIT + } else { + /* 64-bit client: take from SAME_VA zone */ + target_zone_bits = KBASE_REG_ZONE_SAME_VA; + } +#endif + target_zone = kbase_ctx_reg_zone_get(kctx, target_zone_bits); + target_zone_base_addr = target_zone->base_pfn << PAGE_SHIFT; + + target_reg = kbase_region_tracker_find_region_base_address( + kctx, target_zone_base_addr); + if (WARN(!target_reg, + "Already found a free region at the start of every zone, but now cannot find any region for zone base 0x%.16llx zone 0x%lx", + (unsigned long long)target_zone_base_addr, target_zone_bits)) { + err = -ENOMEM; goto exit_unlock; } + /* kbase_region_tracker_has_allocs() above has already ensured that all + * of the zones have no allocs, so no need to check that again on + * target_reg + */ + WARN_ON((!(target_reg->flags & KBASE_REG_FREE)) || + target_reg->nr_pages != target_zone->va_size_pages); - if (shrinking_va_reg->nr_pages <= exec_va_pages) { + if (target_reg->nr_pages <= exec_va_pages || + target_zone->va_size_pages <= exec_va_pages) { err = -ENOMEM; goto exit_unlock; } + /* Taken from the end of the target zone */ + exec_va_start = kbase_reg_zone_end_pfn(target_zone) - exec_va_pages; + exec_va_reg = kbase_alloc_free_region(&kctx->reg_rbtree_exec, exec_va_start, exec_va_pages, @@ -956,13 +1127,17 @@ int kbase_region_tracker_init_exec(struct kbase_context *kctx, u64 exec_va_pages err = -ENOMEM; goto exit_unlock; } + /* Update EXEC_VA zone + * + * not using kbase_ctx_reg_zone_init() - it was already initialized + */ + exec_va_zone = kbase_ctx_reg_zone_get(kctx, KBASE_REG_ZONE_EXEC_VA); + exec_va_zone->base_pfn = exec_va_start; + exec_va_zone->va_size_pages = exec_va_pages; - shrinking_va_reg->nr_pages -= exec_va_pages; -#ifdef CONFIG_64BIT - if (!kbase_ctx_flag(kctx, KCTX_COMPAT)) - kctx->same_va_end -= exec_va_pages; -#endif - kctx->exec_va_start = exec_va_start; + /* Update target zone and corresponding region */ + target_reg->nr_pages -= exec_va_pages; + target_zone->va_size_pages -= exec_va_pages; kbase_region_tracker_insert(exec_va_reg); err = 0; @@ -1113,7 +1288,11 @@ void kbase_mem_term(struct kbase_device *kbdev) KBASE_EXPORT_TEST_API(kbase_mem_term); /** - * @brief Allocate a free region object. + * Allocate a free region object. + * @rbtree: Backlink to the red-black tree of memory regions. + * @start_pfn: The Page Frame Number in GPU virtual address space. + * @nr_pages: The size of the region in pages. + * @zone: KBASE_REG_ZONE_CUSTOM_VA or KBASE_REG_ZONE_SAME_VA * * The allocated object is not part of any list yet, and is flagged as * KBASE_REG_FREE. No mapping is allocated yet. @@ -1186,7 +1365,8 @@ static struct kbase_context *kbase_reg_flags_to_kctx( } /** - * @brief Free a region object. + * Free a region object. + * @reg: Region * * The described region must be freed of any mapping. * @@ -1782,7 +1962,9 @@ int kbase_mem_free_region(struct kbase_context *kctx, struct kbase_va_region *re KBASE_EXPORT_TEST_API(kbase_mem_free_region); /** - * @brief Free the region from the GPU and unregister it. + * Free the region from the GPU and unregister it. + * @kctx: KBase context + * @gpu_addr: GPU address to free * * This function implements the free operation on a memory segment. * It will loudly fail if called with outstanding mappings. @@ -2957,7 +3139,8 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags, } /** - * @brief Acquire the per-context region list lock + * Acquire the per-context region list lock + * @kctx: KBase context */ void kbase_gpu_vm_lock(struct kbase_context *kctx) { @@ -2968,7 +3151,8 @@ void kbase_gpu_vm_lock(struct kbase_context *kctx) KBASE_EXPORT_TEST_API(kbase_gpu_vm_lock); /** - * @brief Release the per-context region list lock + * Release the per-context region list lock + * @kctx: KBase context */ void kbase_gpu_vm_unlock(struct kbase_context *kctx) { @@ -4266,17 +4450,6 @@ void kbase_jit_report_update_pressure(struct kbase_context *kctx, } #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */ -bool kbase_has_exec_va_zone(struct kbase_context *kctx) -{ - bool has_exec_va_zone; - - kbase_gpu_vm_lock(kctx); - has_exec_va_zone = (kctx->exec_va_start != U64_MAX); - kbase_gpu_vm_unlock(kctx); - - return has_exec_va_zone; -} - #if MALI_USE_CSF static void kbase_jd_user_buf_unpin_pages(struct kbase_mem_phy_alloc *alloc) { |