diff options
author | Jörg Wagner <jorwag@google.com> | 2023-12-14 09:44:26 +0000 |
---|---|---|
committer | Jörg Wagner <jorwag@google.com> | 2023-12-14 09:44:26 +0000 |
commit | 049a542207ed694271316782397b78b2e202086a (patch) | |
tree | 105e9378d4d5062dc72109fdd4a77c915bd9425d /mali_kbase/mali_kbase_mem_linux.c | |
parent | e61eb93296e9f940b32d4ad4b0c3a5557cbeaf17 (diff) | |
download | gpu-049a542207ed694271316782397b78b2e202086a.tar.gz |
Update KMD to r47p0
Provenance: ipdelivery@ad01e50d640910a99224382bb227e6d4de627657
Change-Id: I19ac9bce34a5c5a319c1b4a388e8b037b3dfe6e7
Diffstat (limited to 'mali_kbase/mali_kbase_mem_linux.c')
-rw-r--r-- | mali_kbase/mali_kbase_mem_linux.c | 1066 |
1 files changed, 472 insertions, 594 deletions
diff --git a/mali_kbase/mali_kbase_mem_linux.c b/mali_kbase/mali_kbase_mem_linux.c index 232f777..6838fba 100644 --- a/mali_kbase/mali_kbase_mem_linux.c +++ b/mali_kbase/mali_kbase_mem_linux.c @@ -48,7 +48,7 @@ #include <mali_kbase_reset_gpu.h> #if ((KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE) || \ - (KERNEL_VERSION(5, 0, 0) > LINUX_VERSION_CODE)) + (KERNEL_VERSION(5, 0, 0) > LINUX_VERSION_CODE)) /* Enable workaround for ion for kernels prior to v5.0.0 and from v5.3.0 * onwards. * @@ -84,6 +84,18 @@ #define IR_THRESHOLD_STEPS (256u) +/* + * fully_backed_gpf_memory - enable full physical backing of all grow-on-GPU-page-fault + * allocations in the kernel. + */ +static bool fully_backed_gpf_memory; +module_param(fully_backed_gpf_memory, bool, 0444); +MODULE_PARM_DESC( + fully_backed_gpf_memory, + "Enable the full physical backing of all grow-on-GPU-page-fault allocations in the kernel." + "Note that this should only be enabled for testing worst case memory usage " + "and should not be enabled in production"); + #if MALI_USE_CSF static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, struct vm_area_struct *vma); static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, struct vm_area_struct *vma); @@ -92,8 +104,7 @@ static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, struct v static int kbase_vmap_phy_pages(struct kbase_context *kctx, struct kbase_va_region *reg, u64 offset_bytes, size_t size, struct kbase_vmap_struct *map, kbase_vmap_flag vmap_flags); -static void kbase_vunmap_phy_pages(struct kbase_context *kctx, - struct kbase_vmap_struct *map); +static void kbase_vunmap_phy_pages(struct kbase_context *kctx, struct kbase_vmap_struct *map); static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_struct *vma); @@ -118,8 +129,7 @@ static bool is_process_exiting(struct vm_area_struct *vma) * one of the event memory pages. The enclosing region, if found, shouldn't * have been marked as free. */ -static struct kbase_va_region *kbase_find_event_mem_region( - struct kbase_context *kctx, u64 gpu_addr) +static struct kbase_va_region *kbase_find_event_mem_region(struct kbase_context *kctx, u64 gpu_addr) { #if MALI_USE_CSF u64 gpu_pfn = gpu_addr >> PAGE_SHIFT; @@ -128,8 +138,7 @@ static struct kbase_va_region *kbase_find_event_mem_region( lockdep_assert_held(&kctx->reg_lock); list_for_each_entry(reg, &kctx->csf.event_pages_head, link) { - if ((reg->start_pfn <= gpu_pfn) && - (gpu_pfn < (reg->start_pfn + reg->nr_pages))) { + if ((reg->start_pfn <= gpu_pfn) && (gpu_pfn < (reg->start_pfn + reg->nr_pages))) { if (WARN_ON(reg->flags & KBASE_REG_FREE)) return NULL; @@ -139,6 +148,9 @@ static struct kbase_va_region *kbase_find_event_mem_region( return reg; } } +#else + CSTD_UNUSED(kctx); + CSTD_UNUSED(gpu_addr); #endif return NULL; @@ -179,8 +191,8 @@ static struct kbase_va_region *kbase_find_event_mem_region( * On success, the region will also be forced into a certain kind: * - It will no longer be growable */ -static int kbase_phy_alloc_mapping_init(struct kbase_context *kctx, - struct kbase_va_region *reg, size_t vsize, size_t size) +static int kbase_phy_alloc_mapping_init(struct kbase_context *kctx, struct kbase_va_region *reg, + size_t vsize, size_t size) { size_t size_bytes = (size << PAGE_SHIFT); struct kbase_vmap_struct *kern_mapping; @@ -191,7 +203,7 @@ static int kbase_phy_alloc_mapping_init(struct kbase_context *kctx, * Only support KBASE_MEM_TYPE_NATIVE allocations */ if (vsize != size || reg->cpu_alloc->permanent_map != NULL || - reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) + reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) return -EINVAL; kern_mapping = kzalloc(sizeof(*kern_mapping), GFP_KERNEL); @@ -214,8 +226,7 @@ vmap_fail: return err; } -void kbase_phy_alloc_mapping_term(struct kbase_context *kctx, - struct kbase_mem_phy_alloc *alloc) +void kbase_phy_alloc_mapping_term(struct kbase_context *kctx, struct kbase_mem_phy_alloc *alloc) { WARN_ON(!alloc->permanent_map); kbase_vunmap_phy_pages(kctx, alloc->permanent_map); @@ -224,9 +235,8 @@ void kbase_phy_alloc_mapping_term(struct kbase_context *kctx, alloc->permanent_map = NULL; } -void *kbase_phy_alloc_mapping_get(struct kbase_context *kctx, - u64 gpu_addr, - struct kbase_vmap_struct **out_kern_mapping) +void *kbase_phy_alloc_mapping_get(struct kbase_context *kctx, u64 gpu_addr, + struct kbase_vmap_struct **out_kern_mapping) { struct kbase_va_region *reg; void *kern_mem_ptr = NULL; @@ -242,8 +252,7 @@ void *kbase_phy_alloc_mapping_get(struct kbase_context *kctx, reg = kbase_find_event_mem_region(kctx, gpu_addr); if (!reg) { - reg = kbase_region_tracker_find_region_enclosing_address( - kctx, gpu_addr); + reg = kbase_region_tracker_find_region_enclosing_address(kctx, gpu_addr); } if (kbase_is_region_invalid_or_free(reg)) @@ -268,8 +277,7 @@ out_unlock: return kern_mem_ptr; } -void kbase_phy_alloc_mapping_put(struct kbase_context *kctx, - struct kbase_vmap_struct *kern_mapping) +void kbase_phy_alloc_mapping_put(struct kbase_context *kctx, struct kbase_vmap_struct *kern_mapping) { WARN_ON(!kctx); WARN_ON(!kern_mapping); @@ -298,8 +306,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages KBASE_DEBUG_ASSERT(gpu_va); dev = kctx->kbdev->dev; - dev_dbg(dev, - "Allocating %lld va_pages, %lld commit_pages, %lld extension, 0x%llX flags\n", + dev_dbg(dev, "Allocating %lld va_pages, %lld commit_pages, %lld extension, 0x%llX flags\n", va_pages, commit_pages, extension, *flags); #if MALI_USE_CSF @@ -310,15 +317,11 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages *gpu_va = 0; /* return 0 on failure */ #endif else - dev_dbg(dev, - "Keeping requested GPU VA of 0x%llx\n", - (unsigned long long)*gpu_va); + dev_dbg(dev, "Keeping requested GPU VA of 0x%llx\n", (unsigned long long)*gpu_va); if (!kbase_check_alloc_flags(*flags)) { - dev_warn(dev, - "%s called with bad flags (%llx)", - __func__, - (unsigned long long)*flags); + dev_warn(dev, "%s called with bad flags (%llx)", __func__, + (unsigned long long)*flags); goto bad_flags; } @@ -327,37 +330,36 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages /* Mask coherency flags if infinite cache is enabled to prevent * the skipping of syncs from BASE side. */ - *flags &= ~(BASE_MEM_COHERENT_SYSTEM_REQUIRED | - BASE_MEM_COHERENT_SYSTEM); + *flags &= ~(BASE_MEM_COHERENT_SYSTEM_REQUIRED | BASE_MEM_COHERENT_SYSTEM); } #endif if ((*flags & BASE_MEM_UNCACHED_GPU) != 0 && - (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0) { + (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0) { /* Remove COHERENT_SYSTEM_REQUIRED flag if uncached GPU mapping is requested */ *flags &= ~BASE_MEM_COHERENT_SYSTEM_REQUIRED; } if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && - !kbase_device_is_cpu_coherent(kctx->kbdev)) { - dev_warn(dev, "%s call required coherent mem when unavailable", - __func__); + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + dev_warn(dev, "%s call required coherent mem when unavailable", __func__); goto bad_flags; } if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && - !kbase_device_is_cpu_coherent(kctx->kbdev)) { + !kbase_device_is_cpu_coherent(kctx->kbdev)) { /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ *flags &= ~BASE_MEM_COHERENT_SYSTEM; } - if (kbase_check_alloc_sizes(kctx, *flags, va_pages, commit_pages, - extension)) + if (kbase_check_alloc_sizes(kctx, *flags, va_pages, commit_pages, extension)) goto bad_sizes; -#ifdef CONFIG_MALI_MEMORY_FULLY_BACKED - /* Ensure that memory is fully physically-backed. */ - if (*flags & BASE_MEM_GROW_ON_GPF) - commit_pages = va_pages; -#endif + /* Ensure that memory is fully physically-backed if full physical backing + * has been requested for GPF allocations. + */ + if (unlikely(fully_backed_gpf_memory)) { + if (*flags & BASE_MEM_GROW_ON_GPF) + commit_pages = va_pages; + } /* find out which VA zone to use */ if (*flags & BASE_MEM_SAME_VA) @@ -388,8 +390,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages if (kbase_update_region_flags(kctx, reg, *flags) != 0) goto invalid_flags; - if (kbase_reg_prepare_native(reg, kctx, - kbase_mem_group_id_get(*flags)) != 0) { + if (kbase_reg_prepare_native(reg, kctx, kbase_mem_group_id_get(*flags)) != 0) { dev_err(dev, "Failed to prepare region"); goto prepare_failed; } @@ -404,11 +405,11 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages *flags &= ~BASE_MEM_CACHED_CPU; if (*flags & BASE_MEM_GROW_ON_GPF) { - unsigned int const ir_threshold = atomic_read( - &kctx->kbdev->memdev.ir_threshold); + unsigned int const ir_threshold = + (unsigned int)atomic_read(&kctx->kbdev->memdev.ir_threshold); - reg->threshold_pages = ((va_pages * ir_threshold) + - (IR_THRESHOLD_STEPS / 2)) / IR_THRESHOLD_STEPS; + reg->threshold_pages = + ((va_pages * ir_threshold) + (IR_THRESHOLD_STEPS / 2)) / IR_THRESHOLD_STEPS; } else reg->threshold_pages = 0; @@ -427,8 +428,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages if (kbase_alloc_phy_pages(reg, va_pages, commit_pages) != 0) { dev_warn(dev, "Failed to allocate %lld pages (va_pages=%lld)", - (unsigned long long)commit_pages, - (unsigned long long)va_pages); + (unsigned long long)commit_pages, (unsigned long long)va_pages); goto no_mem; } reg->initial_commit = commit_pages; @@ -441,8 +441,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages * kbase_alloc_phy_pages(). If we move that to setup pages * earlier, also move this call too */ - int err = kbase_phy_alloc_mapping_init(kctx, reg, va_pages, - commit_pages); + int err = kbase_phy_alloc_mapping_init(kctx, reg, va_pages, commit_pages); if (err < 0) { kbase_gpu_vm_unlock(kctx); goto no_kern_mapping; @@ -469,7 +468,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages cookie = cookie_nr + PFN_DOWN(BASE_MEM_COOKIE_BASE); cookie <<= PAGE_SHIFT; - *gpu_va = (u64) cookie; + *gpu_va = (u64)cookie; } else /* we control the VA */ { size_t align = 1; @@ -479,8 +478,8 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages * the SAME_VA zone inside kbase_context_get_unmapped_area(). */ if (kctx->kbdev->gpu_props.mmu.va_bits > 33) { - if (va_pages >= (SZ_2M / SZ_4K)) - align = (SZ_2M / SZ_4K); + if (va_pages >= NUM_PAGES_IN_2MB_LARGE_PAGE) + align = NUM_PAGES_IN_2MB_LARGE_PAGE; } if (*gpu_va) align = 1; @@ -489,8 +488,7 @@ struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages align = 1; #endif /* !MALI_USE_CSF */ } - if (kbase_gpu_mmap(kctx, reg, *gpu_va, va_pages, align, - mmu_sync_info) != 0) { + if (kbase_gpu_mmap(kctx, reg, *gpu_va, va_pages, align, mmu_sync_info) != 0) { dev_warn(dev, "Failed to map memory on GPU"); kbase_gpu_vm_unlock(kctx); goto no_mmap; @@ -544,8 +542,7 @@ bad_flags: } KBASE_EXPORT_TEST_API(kbase_mem_alloc); -int kbase_mem_query(struct kbase_context *kctx, - u64 gpu_addr, u64 query, u64 * const out) +int kbase_mem_query(struct kbase_context *kctx, u64 gpu_addr, u64 query, u64 *const out) { struct kbase_va_region *reg; int ret = -EINVAL; @@ -581,8 +578,7 @@ int kbase_mem_query(struct kbase_context *kctx, case KBASE_MEM_QUERY_VA_SIZE: *out = reg->nr_pages; break; - case KBASE_MEM_QUERY_FLAGS: - { + case KBASE_MEM_QUERY_FLAGS: { *out = 0; if (KBASE_REG_CPU_WR & reg->flags) *out |= BASE_MEM_PROT_CPU_WR; @@ -600,6 +596,8 @@ int kbase_mem_query(struct kbase_context *kctx, *out |= BASE_MEM_COHERENT_SYSTEM; if (KBASE_REG_SHARE_IN & reg->flags) *out |= BASE_MEM_COHERENT_LOCAL; + if (KBASE_REG_DONT_NEED & reg->flags) + *out |= BASE_MEM_DONT_NEED; if (mali_kbase_supports_mem_grow_on_gpf(kctx->api_version)) { /* Prior to this version, this was known about by * user-side but we did not return them. Returning @@ -642,8 +640,7 @@ int kbase_mem_query(struct kbase_context *kctx, *out |= kbase_mem_group_id_set(reg->cpu_alloc->group_id); - WARN(*out & ~BASE_MEM_FLAGS_QUERYABLE, - "BASE_MEM_FLAGS_QUERYABLE needs updating\n"); + WARN(*out & ~BASE_MEM_FLAGS_QUERYABLE, "BASE_MEM_FLAGS_QUERYABLE needs updating\n"); *out &= BASE_MEM_FLAGS_QUERYABLE; break; } @@ -667,9 +664,8 @@ out_unlock: * * Return: Number of pages which can be freed or SHRINK_EMPTY if no page remains. */ -static -unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, - struct shrink_control *sc) +static unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, + struct shrink_control *sc) { struct kbase_context *kctx = container_of(s, struct kbase_context, reclaim); int evict_nents = atomic_read(&kctx->evict_nents); @@ -683,7 +679,7 @@ unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, dev_err(kctx->kbdev->dev, "invalid evict_nents(%d)", evict_nents); nr_freeable_items = 0; } else { - nr_freeable_items = evict_nents; + nr_freeable_items = (unsigned long)evict_nents; } #if KERNEL_VERSION(4, 19, 0) <= LINUX_VERSION_CODE @@ -713,9 +709,8 @@ unsigned long kbase_mem_evictable_reclaim_count_objects(struct shrinker *s, * The eviction list itself is guarded by the eviction lock and the MMU updates * are protected by their own lock. */ -static -unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s, - struct shrink_control *sc) +static unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s, + struct shrink_control *sc) { struct kbase_context *kctx; struct kbase_mem_phy_alloc *alloc; @@ -732,8 +727,7 @@ unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s, if (!alloc->reg) continue; - err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg, - 0, alloc->nents); + err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg, 0, alloc->nents); /* Failed to remove GPU mapping, proceed to next one. */ if (err != 0) @@ -772,15 +766,12 @@ int kbase_mem_evictable_init(struct kbase_context *kctx) INIT_LIST_HEAD(&kctx->evict_list); mutex_init(&kctx->jit_evict_lock); - atomic_set(&kctx->evict_nents, 0); - kctx->reclaim.count_objects = kbase_mem_evictable_reclaim_count_objects; kctx->reclaim.scan_objects = kbase_mem_evictable_reclaim_scan_objects; kctx->reclaim.seeks = DEFAULT_SEEKS; /* Kernel versions prior to 3.1 : * struct shrinker does not define batch */ - kctx->reclaim.batch = 0; #if KERNEL_VERSION(6, 0, 0) > LINUX_VERSION_CODE register_shrinker(&kctx->reclaim); #else @@ -805,14 +796,10 @@ void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc) int __maybe_unused new_page_count; kbase_process_page_usage_dec(kctx, alloc->nents); - new_page_count = atomic_sub_return(alloc->nents, - &kctx->used_pages); + new_page_count = atomic_sub_return(alloc->nents, &kctx->used_pages); atomic_sub(alloc->nents, &kctx->kbdev->memdev.used_pages); - KBASE_TLSTREAM_AUX_PAGESALLOC( - kbdev, - kctx->id, - (u64)new_page_count); + KBASE_TLSTREAM_AUX_PAGESALLOC(kbdev, kctx->id, (u64)new_page_count); kbase_trace_gpu_mem_usage_dec(kbdev, kctx, alloc->nents); } @@ -820,15 +807,13 @@ void kbase_mem_evictable_mark_reclaim(struct kbase_mem_phy_alloc *alloc) * kbase_mem_evictable_unmark_reclaim - Mark the pages as no longer reclaimable. * @alloc: The physical allocation */ -static -void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc) +static void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc) { struct kbase_context *kctx = alloc->imported.native.kctx; struct kbase_device *kbdev = kctx->kbdev; int __maybe_unused new_page_count; - new_page_count = atomic_add_return(alloc->nents, - &kctx->used_pages); + new_page_count = atomic_add_return(alloc->nents, &kctx->used_pages); atomic_add(alloc->nents, &kctx->kbdev->memdev.used_pages); /* Increase mm counters so that the allocation is accounted for @@ -836,14 +821,11 @@ void kbase_mem_evictable_unmark_reclaim(struct kbase_mem_phy_alloc *alloc) */ kbase_process_page_usage_inc(kctx, alloc->nents); - KBASE_TLSTREAM_AUX_PAGESALLOC( - kbdev, - kctx->id, - (u64)new_page_count); + KBASE_TLSTREAM_AUX_PAGESALLOC(kbdev, kctx->id, (u64)new_page_count); kbase_trace_gpu_mem_usage_inc(kbdev, kctx, alloc->nents); } -int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) +void kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) { struct kbase_context *kctx = gpu_alloc->imported.native.kctx; @@ -852,8 +834,7 @@ int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) /* Memory is in the process of transitioning to the shrinker, and * should ignore migration attempts */ - kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg, - 0, gpu_alloc->nents); + kbase_mem_shrink_cpu_mapping(kctx, gpu_alloc->reg, 0, gpu_alloc->nents); mutex_lock(&kctx->jit_evict_lock); /* This allocation can't already be on a list. */ @@ -874,7 +855,6 @@ int kbase_mem_evictable_make(struct kbase_mem_phy_alloc *gpu_alloc) kbase_mem_evictable_mark_reclaim(gpu_alloc); gpu_alloc->reg->flags |= KBASE_REG_DONT_NEED; - return 0; } bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) @@ -908,8 +888,7 @@ bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) /* If the region is still alive ... */ if (gpu_alloc->reg) { /* ... allocate replacement backing ... */ - err = kbase_alloc_phy_pages_helper(gpu_alloc, - gpu_alloc->evicted); + err = kbase_alloc_phy_pages_helper(gpu_alloc, gpu_alloc->evicted); /* * ... and grow the mapping back to its @@ -917,8 +896,7 @@ bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) */ if (!err) err = kbase_mem_grow_gpu_mapping( - kctx, gpu_alloc->reg, - gpu_alloc->evicted, 0, mmu_sync_info); + kctx, gpu_alloc->reg, gpu_alloc->evicted, 0, mmu_sync_info); gpu_alloc->evicted = 0; @@ -940,32 +918,35 @@ bool kbase_mem_evictable_unmake(struct kbase_mem_phy_alloc *gpu_alloc) return (err == 0); } -int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, unsigned int mask) +/** kbase_mem_flags_change_imported_umm - Change memory flags for imported UMM region + * @kctx: Pointer to kbase context. + * @flags: Base memory flags to modify. + * @reg: Pointer to region whose flags shall be modified. + * + * Return: 0 on success, error code otherwise. + */ +static int kbase_mem_flags_change_imported_umm(struct kbase_context *kctx, unsigned int flags, + struct kbase_va_region *reg) { - struct kbase_va_region *reg; - int ret = -EINVAL; unsigned int real_flags = 0; unsigned int new_flags = 0; - bool prev_needed, new_needed; - - KBASE_DEBUG_ASSERT(kctx); - - if (!gpu_addr) - return -EINVAL; + int ret = -EINVAL; - if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) - return -EINVAL; + if (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM) + return ret; - /* nuke other bits */ - flags &= mask; + if (flags & ~(BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL)) + return ret; - /* check for only supported flags */ - if (flags & ~(BASE_MEM_FLAGS_MODIFIABLE)) - goto out; + /* Don't change outer sharability if platform is not coherent */ + if ((BASE_MEM_COHERENT_SYSTEM & flags) && !kbase_device_is_cpu_coherent(kctx->kbdev)) + return ret; - /* mask covers bits we don't support? */ - if (mask & ~(BASE_MEM_FLAGS_MODIFIABLE)) - goto out; + /* shareability flags are ignored for GPU uncached memory + * instead of causing an error. + */ + if (!(reg->flags & KBASE_REG_GPU_CACHED)) + return 0; /* convert flags */ if (BASE_MEM_COHERENT_SYSTEM & flags) @@ -973,79 +954,13 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in else if (BASE_MEM_COHERENT_LOCAL & flags) real_flags |= KBASE_REG_SHARE_IN; - /* now we can lock down the context, and find the region */ - down_write(kbase_mem_get_process_mmap_lock()); - kbase_gpu_vm_lock(kctx); - - /* Validate the region */ - reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); - if (kbase_is_region_invalid_or_free(reg)) - goto out_unlock; - - /* There is no use case to support MEM_FLAGS_CHANGE ioctl for allocations - * that have NO_USER_FREE flag set, to mark them as evictable/reclaimable. - * This would usually include JIT allocations, Tiler heap related allocations - * & GPU queue ringbuffer and none of them needs to be explicitly marked - * as evictable by Userspace. - */ - if (kbase_va_region_is_no_user_free(reg)) - goto out_unlock; - - /* Is the region being transitioning between not needed and needed? */ - prev_needed = (KBASE_REG_DONT_NEED & reg->flags) == KBASE_REG_DONT_NEED; - new_needed = (BASE_MEM_DONT_NEED & flags) == BASE_MEM_DONT_NEED; - if (prev_needed != new_needed) { - /* Aliased allocations can't be shrunk as the code doesn't - * support looking up: - * - all physical pages assigned to different GPU VAs - * - CPU mappings for the physical pages at different vm_pgoff - * (==GPU VA) locations. - */ - if (atomic_read(®->cpu_alloc->gpu_mappings) > 1) - goto out_unlock; - - if (atomic_read(®->cpu_alloc->kernel_mappings) > 0) - goto out_unlock; - - if (new_needed) { - /* Only native allocations can be marked not needed */ - if (reg->cpu_alloc->type != KBASE_MEM_TYPE_NATIVE) { - ret = -EINVAL; - goto out_unlock; - } - ret = kbase_mem_evictable_make(reg->gpu_alloc); - if (ret) - goto out_unlock; - } else { - kbase_mem_evictable_unmake(reg->gpu_alloc); - } - } - - /* limit to imported memory */ - if (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM) - goto out_unlock; - - /* shareability flags are ignored for GPU uncached memory */ - if (!(reg->flags & KBASE_REG_GPU_CACHED)) { - ret = 0; - goto out_unlock; - } - /* no change? */ - if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) { - ret = 0; - goto out_unlock; - } + if (real_flags == (reg->flags & (KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH))) + return 0; new_flags = reg->flags & ~(KBASE_REG_SHARE_IN | KBASE_REG_SHARE_BOTH); new_flags |= real_flags; - /* Currently supporting only imported memory */ - if (reg->gpu_alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM) { - ret = -EINVAL; - goto out_unlock; - } - if (IS_ENABLED(CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND)) { /* Future use will use the new flags, existing mapping * will NOT be updated as memory should not be in use @@ -1069,17 +984,13 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in * addresses from buffer with different shareability * properties. */ - dev_dbg(kctx->kbdev->dev, - "Updating page tables on mem flag change\n"); - ret = kbase_mmu_update_pages(kctx, reg->start_pfn, - kbase_get_gpu_phy_pages(reg), - kbase_reg_current_backed_size(reg), - new_flags, - reg->gpu_alloc->group_id); + dev_dbg(kctx->kbdev->dev, "Updating page tables on mem flag change\n"); + ret = kbase_mmu_update_pages(kctx, reg->start_pfn, kbase_get_gpu_phy_pages(reg), + kbase_reg_current_backed_size(reg), new_flags, + reg->gpu_alloc->group_id); if (ret) dev_warn(kctx->kbdev->dev, - "Failed to update GPU page tables on flag change: %d\n", - ret); + "Failed to update GPU page tables on flag change: %d\n", ret); } else WARN_ON(!reg->gpu_alloc->imported.umm.current_mapping_usage_count); @@ -1087,17 +998,128 @@ int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned in if (!ret) reg->flags = new_flags; + return ret; +} + +/** kbase_mem_flags_change_native - Change memory flags for native region + * @kctx: Pointer to kbase context. + * @flags: Base memory flags to modify. + * @reg: Pointer to region whose flags shall be modified. + * + * Return: 0 on success, error code otherwise. + */ +static int kbase_mem_flags_change_native(struct kbase_context *kctx, unsigned int flags, + struct kbase_va_region *reg) +{ + bool kbase_reg_dont_need_flag = (KBASE_REG_DONT_NEED & reg->flags); + bool requested_dont_need_flag = (BASE_MEM_DONT_NEED & flags); + int ret = 0; + + if (reg->gpu_alloc->type != KBASE_MEM_TYPE_NATIVE) + return -EINVAL; + + /* Nothing to do if flag doesn't change. */ + if (kbase_reg_dont_need_flag ^ requested_dont_need_flag) { + /* Changes of the "evictable" property cannot be applied to regions + * which are members of memory aliases, as the code doesn't support + * looking up all physical pages assigned to different GPU VAs + * and all CPU mappings associated with those physical pages. + */ + if (atomic_read(®->gpu_alloc->gpu_mappings) > 1) + return -EINVAL; + + if (atomic_read(®->cpu_alloc->kernel_mappings) > 0) + return -EINVAL; + + if (requested_dont_need_flag) + kbase_mem_evictable_make(reg->gpu_alloc); + else + ret = kbase_mem_evictable_unmake(reg->gpu_alloc) ? 0 : -ENOMEM; + } + + return ret; +} + +int kbase_mem_flags_change(struct kbase_context *kctx, u64 gpu_addr, unsigned int flags, + unsigned int mask) +{ + struct kbase_va_region *reg; + int ret = -EINVAL; + + KBASE_DEBUG_ASSERT(kctx); + + if (!gpu_addr) + return ret; + + if ((gpu_addr & ~PAGE_MASK) && (gpu_addr >= PAGE_SIZE)) + return ret; + + flags &= mask; + + /* Lock down the context, and find the region */ + down_write(kbase_mem_get_process_mmap_lock()); + kbase_gpu_vm_lock(kctx); + + /* Validate the region */ + reg = kbase_region_tracker_find_region_base_address(kctx, gpu_addr); + if (kbase_is_region_invalid_or_free(reg)) + goto out_unlock; + + /* There is no use case to support MEM_FLAGS_CHANGE ioctl for allocations + * that have NO_USER_FREE flag set, to mark them as evictable/reclaimable. + * This would usually include JIT allocations, Tiler heap related allocations + * & GPU queue ringbuffer and none of them needs to be explicitly marked + * as evictable by Userspace. + */ + if (kbase_va_region_is_no_user_free(reg)) + goto out_unlock; + + /* Different memory flags are allowed for different memory types, hence + * this step of the validation process must be done at this point and + * cannot be done earlier. + * + * Once the region has been found and validated, the actions to take depend + * on the memory type of the region. + */ + switch (reg->gpu_alloc->type) { + case KBASE_MEM_TYPE_NATIVE: { + if ((flags & ~(BASE_MEM_FLAGS_MODIFIABLE_NATIVE)) || + (mask & ~(BASE_MEM_FLAGS_MODIFIABLE_NATIVE))) { + ret = -EINVAL; + break; + } + + ret = kbase_mem_flags_change_native(kctx, flags, reg); + break; + } + case KBASE_MEM_TYPE_IMPORTED_UMM: { + if ((flags & ~(BASE_MEM_FLAGS_MODIFIABLE_IMPORTED_UMM)) || + (mask & ~(BASE_MEM_FLAGS_MODIFIABLE_IMPORTED_UMM))) { + ret = -EINVAL; + break; + } + + ret = kbase_mem_flags_change_imported_umm(kctx, flags, reg); + break; + } + default: { + /* Other memory types are not supported. Return error. */ + ret = -EINVAL; + break; + } + } + out_unlock: kbase_gpu_vm_unlock(kctx); up_write(kbase_mem_get_process_mmap_lock()); -out: + return ret; } #define KBASE_MEM_IMPORT_HAVE_PAGES (1UL << BASE_MEM_FLAGS_NR_BITS) -int kbase_mem_do_sync_imported(struct kbase_context *kctx, - struct kbase_va_region *reg, enum kbase_sync_type sync_fn) +int kbase_mem_do_sync_imported(struct kbase_context *kctx, struct kbase_va_region *reg, + enum kbase_sync_type sync_fn) { int ret = -EINVAL; struct dma_buf __maybe_unused *dma_buf; @@ -1131,16 +1153,15 @@ int kbase_mem_do_sync_imported(struct kbase_context *kctx, switch (sync_fn) { case KBASE_SYNC_TO_DEVICE: - dev_dbg(kctx->kbdev->dev, - "Syncing imported buffer at GPU VA %llx to GPU\n", + dev_dbg(kctx->kbdev->dev, "Syncing imported buffer at GPU VA %llx to GPU\n", reg->start_pfn); #ifdef KBASE_MEM_ION_SYNC_WORKAROUND if (!WARN_ON(!reg->gpu_alloc->imported.umm.dma_attachment)) { - struct dma_buf_attachment *attachment = reg->gpu_alloc->imported.umm.dma_attachment; + struct dma_buf_attachment *attachment = + reg->gpu_alloc->imported.umm.dma_attachment; struct sg_table *sgt = reg->gpu_alloc->imported.umm.sgt; - dma_sync_sg_for_device(attachment->dev, sgt->sgl, - sgt->nents, dir); + dma_sync_sg_for_device(attachment->dev, sgt->sgl, (int)sgt->nents, dir); ret = 0; } #else @@ -1148,16 +1169,15 @@ int kbase_mem_do_sync_imported(struct kbase_context *kctx, #endif /* KBASE_MEM_ION_SYNC_WORKAROUND */ break; case KBASE_SYNC_TO_CPU: - dev_dbg(kctx->kbdev->dev, - "Syncing imported buffer at GPU VA %llx to CPU\n", + dev_dbg(kctx->kbdev->dev, "Syncing imported buffer at GPU VA %llx to CPU\n", reg->start_pfn); #ifdef KBASE_MEM_ION_SYNC_WORKAROUND if (!WARN_ON(!reg->gpu_alloc->imported.umm.dma_attachment)) { - struct dma_buf_attachment *attachment = reg->gpu_alloc->imported.umm.dma_attachment; + struct dma_buf_attachment *attachment = + reg->gpu_alloc->imported.umm.dma_attachment; struct sg_table *sgt = reg->gpu_alloc->imported.umm.sgt; - dma_sync_sg_for_cpu(attachment->dev, sgt->sgl, - sgt->nents, dir); + dma_sync_sg_for_cpu(attachment->dev, sgt->sgl, (int)sgt->nents, dir); ret = 0; } #else @@ -1167,8 +1187,7 @@ int kbase_mem_do_sync_imported(struct kbase_context *kctx, } if (unlikely(ret)) - dev_warn(kctx->kbdev->dev, - "Failed to sync mem region %pK at GPU VA %llx: %d\n", + dev_warn(kctx->kbdev->dev, "Failed to sync mem region %pK at GPU VA %llx: %d\n", reg, reg->start_pfn, ret); return ret; @@ -1187,8 +1206,13 @@ static void kbase_mem_umm_unmap_attachment(struct kbase_context *kctx, { struct tagged_addr *pa = alloc->pages; - dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, - alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); +#if (KERNEL_VERSION(6, 1, 55) <= LINUX_VERSION_CODE) + dma_buf_unmap_attachment_unlocked(alloc->imported.umm.dma_attachment, + alloc->imported.umm.sgt, DMA_BIDIRECTIONAL); +#else + dma_buf_unmap_attachment(alloc->imported.umm.dma_attachment, alloc->imported.umm.sgt, + DMA_BIDIRECTIONAL); +#endif alloc->imported.umm.sgt = NULL; kbase_remove_dma_buf_usage(kctx, alloc); @@ -1207,8 +1231,7 @@ static void kbase_mem_umm_unmap_attachment(struct kbase_context *kctx, * * Return: 0 on success, or negative error code */ -static int kbase_mem_umm_map_attachment(struct kbase_context *kctx, - struct kbase_va_region *reg) +static int kbase_mem_umm_map_attachment(struct kbase_context *kctx, struct kbase_va_region *reg) { struct sg_table *sgt; struct scatterlist *s; @@ -1221,8 +1244,12 @@ static int kbase_mem_umm_map_attachment(struct kbase_context *kctx, WARN_ON_ONCE(alloc->type != KBASE_MEM_TYPE_IMPORTED_UMM); WARN_ON_ONCE(alloc->imported.umm.sgt); - sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment, - DMA_BIDIRECTIONAL); +#if (KERNEL_VERSION(6, 1, 55) <= LINUX_VERSION_CODE) + sgt = dma_buf_map_attachment_unlocked(alloc->imported.umm.dma_attachment, + DMA_BIDIRECTIONAL); +#else + sgt = dma_buf_map_attachment(alloc->imported.umm.dma_attachment, DMA_BIDIRECTIONAL); +#endif if (IS_ERR_OR_NULL(sgt)) return -EINVAL; @@ -1234,26 +1261,23 @@ static int kbase_mem_umm_map_attachment(struct kbase_context *kctx, for_each_sg(sgt->sgl, s, sgt->nents, i) { size_t j, pages = PFN_UP(sg_dma_len(s)); - WARN_ONCE(sg_dma_len(s) & (PAGE_SIZE-1), - "sg_dma_len(s)=%u is not a multiple of PAGE_SIZE\n", - sg_dma_len(s)); + WARN_ONCE(sg_dma_len(s) & (PAGE_SIZE - 1), + "sg_dma_len(s)=%u is not a multiple of PAGE_SIZE\n", sg_dma_len(s)); - WARN_ONCE(sg_dma_address(s) & (PAGE_SIZE-1), - "sg_dma_address(s)=%llx is not aligned to PAGE_SIZE\n", - (unsigned long long) sg_dma_address(s)); + WARN_ONCE(sg_dma_address(s) & (PAGE_SIZE - 1), + "sg_dma_address(s)=%llx is not aligned to PAGE_SIZE\n", + (unsigned long long)sg_dma_address(s)); for (j = 0; (j < pages) && (count < reg->nr_pages); j++, count++) - *pa++ = as_tagged(sg_dma_address(s) + - (j << PAGE_SHIFT)); - WARN_ONCE(j < pages, - "sg list from dma_buf_map_attachment > dma_buf->size=%zu\n", - alloc->imported.umm.dma_buf->size); + *pa++ = as_tagged(sg_dma_address(s) + (j << PAGE_SHIFT)); + WARN_ONCE(j < pages, "sg list from dma_buf_map_attachment > dma_buf->size=%zu\n", + alloc->imported.umm.dma_buf->size); } if (!(reg->flags & KBASE_REG_IMPORT_PAD) && - WARN_ONCE(count < reg->nr_pages, - "sg list from dma_buf_map_attachment < dma_buf->size=%zu\n", - alloc->imported.umm.dma_buf->size)) { + WARN_ONCE(count < reg->nr_pages, + "sg list from dma_buf_map_attachment < dma_buf->size=%zu\n", + alloc->imported.umm.dma_buf->size)) { err = -EINVAL; goto err_unmap_attachment; } @@ -1270,12 +1294,11 @@ err_unmap_attachment: return err; } -int kbase_mem_umm_map(struct kbase_context *kctx, - struct kbase_va_region *reg) +int kbase_mem_umm_map(struct kbase_context *kctx, struct kbase_va_region *reg) { int err; struct kbase_mem_phy_alloc *alloc; - unsigned long gwt_mask = ~0; + unsigned long gwt_mask = ~0UL; /* Calls to this function are inherently asynchronous, with respect to * MMU operations. @@ -1289,10 +1312,9 @@ int kbase_mem_umm_map(struct kbase_context *kctx, alloc->imported.umm.current_mapping_usage_count++; if (alloc->imported.umm.current_mapping_usage_count != 1) { if (IS_ENABLED(CONFIG_MALI_DMA_BUF_LEGACY_COMPAT) || - alloc->imported.umm.need_sync) { + alloc->imported.umm.need_sync) { if (!kbase_is_region_invalid_or_free(reg)) { - err = kbase_mem_do_sync_imported(kctx, reg, - KBASE_SYNC_TO_DEVICE); + err = kbase_mem_do_sync_imported(kctx, reg, KBASE_SYNC_TO_DEVICE); WARN_ON_ONCE(err); } } @@ -1316,8 +1338,7 @@ int kbase_mem_umm_map(struct kbase_context *kctx, if (err) goto bad_insert; - if (reg->flags & KBASE_REG_IMPORT_PAD && - !WARN_ON(reg->nr_pages < alloc->nents)) { + if (reg->flags & KBASE_REG_IMPORT_PAD && !WARN_ON(reg->nr_pages < alloc->nents)) { /* For padded imported dma-buf memory, map the dummy aliasing * page from the end of the dma-buf pages, to the end of the * region using a read only mapping. @@ -1347,16 +1368,15 @@ bad_map_attachment: return err; } -void kbase_mem_umm_unmap(struct kbase_context *kctx, - struct kbase_va_region *reg, struct kbase_mem_phy_alloc *alloc) +void kbase_mem_umm_unmap(struct kbase_context *kctx, struct kbase_va_region *reg, + struct kbase_mem_phy_alloc *alloc) { alloc->imported.umm.current_mapping_usage_count--; if (alloc->imported.umm.current_mapping_usage_count) { if (IS_ENABLED(CONFIG_MALI_DMA_BUF_LEGACY_COMPAT) || - alloc->imported.umm.need_sync) { + alloc->imported.umm.need_sync) { if (!kbase_is_region_invalid_or_free(reg)) { - int err = kbase_mem_do_sync_imported(kctx, reg, - KBASE_SYNC_TO_CPU); + int err = kbase_mem_do_sync_imported(kctx, reg, KBASE_SYNC_TO_CPU); WARN_ON_ONCE(err); } } @@ -1375,20 +1395,18 @@ void kbase_mem_umm_unmap(struct kbase_context *kctx, kbase_mem_umm_unmap_attachment(kctx, alloc); } -static int get_umm_memory_group_id(struct kbase_context *kctx, - struct dma_buf *dma_buf) +static int get_umm_memory_group_id(struct kbase_context *kctx, struct dma_buf *dma_buf) { int group_id = BASE_MEM_GROUP_DEFAULT; if (kctx->kbdev->mgm_dev->ops.mgm_get_import_memory_id) { struct memory_group_manager_import_data mgm_import_data; - mgm_import_data.type = - MEMORY_GROUP_MANAGER_IMPORT_TYPE_DMA_BUF; + mgm_import_data.type = MEMORY_GROUP_MANAGER_IMPORT_TYPE_DMA_BUF; mgm_import_data.u.dma_buf = dma_buf; - group_id = kctx->kbdev->mgm_dev->ops.mgm_get_import_memory_id( - kctx->kbdev->mgm_dev, &mgm_import_data); + group_id = kctx->kbdev->mgm_dev->ops.mgm_get_import_memory_id(kctx->kbdev->mgm_dev, + &mgm_import_data); } return group_id; @@ -1409,8 +1427,8 @@ static int get_umm_memory_group_id(struct kbase_context *kctx, * This function imports a dma-buf into kctx, and created a kbase_va_region * object that wraps the dma-buf. */ -static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, - int fd, u64 *va_pages, u64 *flags, u32 padding) +static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, int fd, u64 *va_pages, + u64 *flags, u32 padding) { struct kbase_va_region *reg; struct dma_buf *dma_buf; @@ -1488,8 +1506,7 @@ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, group_id = get_umm_memory_group_id(kctx, dma_buf); - reg->gpu_alloc = kbase_alloc_create(kctx, *va_pages, - KBASE_MEM_TYPE_IMPORTED_UMM, group_id); + reg->gpu_alloc = kbase_alloc_create(kctx, *va_pages, KBASE_MEM_TYPE_IMPORTED_UMM, group_id); if (IS_ERR_OR_NULL(reg->gpu_alloc)) goto no_alloc; @@ -1502,8 +1519,8 @@ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, reg->gpu_alloc->nents = 0; reg->flags &= ~KBASE_REG_FREE; - reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */ - reg->flags &= ~KBASE_REG_GROWABLE; /* UMM cannot be grown */ + reg->flags |= KBASE_REG_GPU_NX; /* UMM is always No eXecute */ + reg->flags &= ~KBASE_REG_GROWABLE; /* UMM cannot be grown */ if (*flags & BASE_MEM_PROTECTED) reg->flags |= KBASE_REG_PROTECTED; @@ -1527,8 +1544,7 @@ static struct kbase_va_region *kbase_mem_from_umm(struct kbase_context *kctx, err = kbase_mem_umm_map_attachment(kctx, reg); if (err) { - dev_warn(kctx->kbdev->dev, - "Failed to map dma-buf %pK on GPU: %d\n", + dev_warn(kctx->kbdev->dev, "Failed to map dma-buf %pK on GPU: %d\n", dma_buf, err); goto error_out; } @@ -1549,60 +1565,53 @@ no_alloc: u32 kbase_get_cache_line_alignment(struct kbase_device *kbdev) { - u32 cpu_cache_line_size = cache_line_size(); - u32 gpu_cache_line_size = - (1UL << kbdev->gpu_props.props.l2_props.log2_line_size); + u32 cpu_cache_line_size = (u32)cache_line_size(); + u32 gpu_cache_line_size = (1UL << kbdev->gpu_props.log2_line_size); - return ((cpu_cache_line_size > gpu_cache_line_size) ? - cpu_cache_line_size : - gpu_cache_line_size); + return ((cpu_cache_line_size > gpu_cache_line_size) ? cpu_cache_line_size : + gpu_cache_line_size); } -static struct kbase_va_region *kbase_mem_from_user_buffer( - struct kbase_context *kctx, unsigned long address, - unsigned long size, u64 *va_pages, u64 *flags) +static struct kbase_va_region *kbase_mem_from_user_buffer(struct kbase_context *kctx, + unsigned long address, unsigned long size, + u64 *va_pages, u64 *flags) { - long i, dma_mapped_pages; struct kbase_va_region *reg; - long faulted_pages; enum kbase_memory_zone zone = CUSTOM_VA_ZONE; bool shared_zone = false; u32 cache_line_alignment = kbase_get_cache_line_alignment(kctx->kbdev); struct kbase_alloc_import_user_buf *user_buf; - struct page **pages = NULL; - struct tagged_addr *pa; - struct device *dev; int write; - enum dma_data_direction dma_dir; + long faulted_pages; /* Flag supported only for dma-buf imported memory */ if (*flags & BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP) return NULL; if ((address & (cache_line_alignment - 1)) != 0 || - (size & (cache_line_alignment - 1)) != 0) { + (size & (cache_line_alignment - 1)) != 0) { if (*flags & BASE_MEM_UNCACHED_GPU) { - dev_warn(kctx->kbdev->dev, - "User buffer is not cache line aligned and marked as GPU uncached\n"); + dev_warn( + kctx->kbdev->dev, + "User buffer is not cache line aligned and marked as GPU uncached\n"); goto bad_size; } /* Coherency must be enabled to handle partial cache lines */ - if (*flags & (BASE_MEM_COHERENT_SYSTEM | - BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { + if (*flags & (BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_SYSTEM_REQUIRED)) { /* Force coherent system required flag, import will * then fail if coherency isn't available */ *flags |= BASE_MEM_COHERENT_SYSTEM_REQUIRED; } else { - dev_warn(kctx->kbdev->dev, - "User buffer is not cache line aligned and no coherency enabled\n"); + dev_warn( + kctx->kbdev->dev, + "User buffer is not cache line aligned and no coherency enabled\n"); goto bad_size; } } - *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) - - PFN_DOWN(address); + *va_pages = (PAGE_ALIGN(address + size) >> PAGE_SHIFT) - PFN_DOWN(address); if (!*va_pages) goto bad_size; @@ -1636,9 +1645,8 @@ static struct kbase_va_region *kbase_mem_from_user_buffer( if (!reg) goto no_region; - reg->gpu_alloc = kbase_alloc_create( - kctx, *va_pages, KBASE_MEM_TYPE_IMPORTED_USER_BUF, - BASE_MEM_GROUP_DEFAULT); + reg->gpu_alloc = kbase_alloc_create(kctx, *va_pages, KBASE_MEM_TYPE_IMPORTED_USER_BUF, + BASE_MEM_GROUP_DEFAULT); if (IS_ERR_OR_NULL(reg->gpu_alloc)) goto no_alloc_obj; @@ -1657,130 +1665,36 @@ static struct kbase_va_region *kbase_mem_from_user_buffer( user_buf->address = address; user_buf->nr_pages = *va_pages; user_buf->mm = current->mm; + user_buf->current_mapping_usage_count = 0; + kbase_user_buf_empty_init(reg); kbase_mem_mmgrab(); if (reg->gpu_alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE) user_buf->pages = vmalloc(*va_pages * sizeof(struct page *)); else - user_buf->pages = kmalloc_array(*va_pages, - sizeof(struct page *), GFP_KERNEL); + user_buf->pages = kmalloc_array(*va_pages, sizeof(struct page *), GFP_KERNEL); if (!user_buf->pages) goto no_page_array; - /* If the region is coherent with the CPU then the memory is imported - * and mapped onto the GPU immediately. - * Otherwise get_user_pages is called as a sanity check, but with - * NULL as the pages argument which will fault the pages, but not - * pin them. The memory will then be pinned only around the jobs that - * specify the region as an external resource. - */ - if (reg->flags & KBASE_REG_SHARE_BOTH) { - pages = user_buf->pages; - *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; - } - - down_read(kbase_mem_get_process_mmap_lock()); + reg->gpu_alloc->nents = 0; + reg->extension = 0; write = reg->flags & (KBASE_REG_CPU_WR | KBASE_REG_GPU_WR); - dma_dir = write ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; - -#if KERNEL_VERSION(5, 9, 0) > LINUX_VERSION_CODE - faulted_pages = get_user_pages(address, *va_pages, - write ? FOLL_WRITE : 0, pages, NULL); -#else - /* pin_user_pages function cannot be called with pages param NULL. - * get_user_pages function will be used instead because it is safe to be - * used with NULL pages param as long as it doesn't have FOLL_GET flag. - */ - if (pages != NULL) { - faulted_pages = - pin_user_pages(address, *va_pages, write ? FOLL_WRITE : 0, pages, NULL); - } else { - faulted_pages = - get_user_pages(address, *va_pages, write ? FOLL_WRITE : 0, pages, NULL); - } -#endif + down_read(kbase_mem_get_process_mmap_lock()); + faulted_pages = + kbase_get_user_pages(address, *va_pages, write ? FOLL_WRITE : 0, NULL, NULL); up_read(kbase_mem_get_process_mmap_lock()); if (faulted_pages != *va_pages) goto fault_mismatch; - reg->gpu_alloc->nents = 0; - reg->extension = 0; - - pa = kbase_get_gpu_phy_pages(reg); - dev = kctx->kbdev->dev; - - if (pages) { - /* Top bit signifies that this was pinned on import */ - user_buf->current_mapping_usage_count |= PINNED_ON_IMPORT; - - /* Manual CPU cache synchronization. - * - * The driver disables automatic CPU cache synchronization because the - * memory pages that enclose the imported region may also contain - * sub-regions which are not imported and that are allocated and used - * by the user process. This may be the case of memory at the beginning - * of the first page and at the end of the last page. Automatic CPU cache - * synchronization would force some operations on those memory allocations, - * unbeknown to the user process: in particular, a CPU cache invalidate - * upon unmapping would destroy the content of dirty CPU caches and cause - * the user process to lose CPU writes to the non-imported sub-regions. - * - * When the GPU claims ownership of the imported memory buffer, it shall - * commit CPU writes for the whole of all pages that enclose the imported - * region, otherwise the initial content of memory would be wrong. - */ - for (i = 0; i < faulted_pages; i++) { - dma_addr_t dma_addr; -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - dma_addr = dma_map_page(dev, pages[i], 0, PAGE_SIZE, dma_dir); -#else - dma_addr = dma_map_page_attrs(dev, pages[i], 0, PAGE_SIZE, dma_dir, - DMA_ATTR_SKIP_CPU_SYNC); -#endif - if (dma_mapping_error(dev, dma_addr)) - goto unwind_dma_map; - - user_buf->dma_addrs[i] = dma_addr; - pa[i] = as_tagged(page_to_phys(pages[i])); - - dma_sync_single_for_device(dev, dma_addr, PAGE_SIZE, dma_dir); - } - - reg->gpu_alloc->nents = faulted_pages; - } + if (reg->flags & KBASE_REG_SHARE_BOTH) + *flags |= KBASE_MEM_IMPORT_HAVE_PAGES; return reg; -unwind_dma_map: - dma_mapped_pages = i; - /* Run the unmap loop in the same order as map loop, and perform again - * CPU cache synchronization to re-write the content of dirty CPU caches - * to memory. This precautionary measure is kept here to keep this code - * aligned with kbase_jd_user_buf_map() to allow for a potential refactor - * in the future. - */ - for (i = 0; i < dma_mapped_pages; i++) { - dma_addr_t dma_addr = user_buf->dma_addrs[i]; - - dma_sync_single_for_device(dev, dma_addr, PAGE_SIZE, dma_dir); -#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE) - dma_unmap_page(dev, dma_addr, PAGE_SIZE, dma_dir); -#else - dma_unmap_page_attrs(dev, dma_addr, PAGE_SIZE, dma_dir, DMA_ATTR_SKIP_CPU_SYNC); -#endif - } fault_mismatch: - if (pages) { - /* In this case, the region was not yet in the region tracker, - * and so there are no CPU mappings to remove before we unpin - * the page - */ - for (i = 0; i < faulted_pages; i++) - kbase_unpin_user_buf_page(pages[i]); - } no_page_array: invalid_flags: kbase_mem_phy_alloc_put(reg->cpu_alloc); @@ -1792,10 +1706,8 @@ bad_size: return NULL; } - -u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, - u64 nents, struct base_mem_aliasing_info *ai, - u64 *num_pages) +u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, u64 nents, + struct base_mem_aliasing_info *ai, u64 *num_pages) { struct kbase_va_region *reg; u64 gpu_va; @@ -1815,19 +1727,17 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, KBASE_DEBUG_ASSERT(num_pages); /* mask to only allowed flags */ - *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | - BASE_MEM_COHERENT_SYSTEM | BASE_MEM_COHERENT_LOCAL | - BASE_MEM_PROT_CPU_RD | BASE_MEM_COHERENT_SYSTEM_REQUIRED); + *flags &= (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR | BASE_MEM_COHERENT_SYSTEM | + BASE_MEM_COHERENT_LOCAL | BASE_MEM_PROT_CPU_RD | + BASE_MEM_COHERENT_SYSTEM_REQUIRED); if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) { - dev_warn(kctx->kbdev->dev, - "%s called with bad flags (%llx)", - __func__, - (unsigned long long)*flags); + dev_warn(kctx->kbdev->dev, "%s called with bad flags (%llx)", __func__, + (unsigned long long)*flags); goto bad_flags; } coherent = (*flags & BASE_MEM_COHERENT_SYSTEM) != 0 || - (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0; + (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0; if (!stride) goto bad_stride; @@ -1865,8 +1775,7 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, goto no_reg; /* zero-sized page array, as we don't need one/can support one */ - reg->gpu_alloc = kbase_alloc_create(kctx, 0, KBASE_MEM_TYPE_ALIAS, - BASE_MEM_GROUP_DEFAULT); + reg->gpu_alloc = kbase_alloc_create(kctx, 0, KBASE_MEM_TYPE_ALIAS, BASE_MEM_GROUP_DEFAULT); if (IS_ERR_OR_NULL(reg->gpu_alloc)) goto no_alloc_obj; @@ -1877,7 +1786,8 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, reg->gpu_alloc->imported.alias.nents = nents; reg->gpu_alloc->imported.alias.stride = stride; - reg->gpu_alloc->imported.alias.aliased = vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents); + reg->gpu_alloc->imported.alias.aliased = + vzalloc(sizeof(*reg->gpu_alloc->imported.alias.aliased) * nents); if (!reg->gpu_alloc->imported.alias.aliased) goto no_aliased_array; @@ -1886,8 +1796,7 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, /* validate and add src handles */ for (i = 0; i < nents; i++) { if (ai[i].handle.basep.handle < BASE_MEM_FIRST_FREE_ADDRESS) { - if (ai[i].handle.basep.handle != - BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) + if (ai[i].handle.basep.handle != BASEP_MEM_WRITE_ALLOC_PAGES_HANDLE) goto bad_handle; /* unsupported magic handle */ if (!ai[i].length) goto bad_handle; /* must be > 0 */ @@ -1901,8 +1810,7 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, struct kbase_mem_phy_alloc *alloc; aliasing_reg = kbase_region_tracker_find_region_base_address( - kctx, - (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT); + kctx, (ai[i].handle.basep.handle >> PAGE_SHIFT) << PAGE_SHIFT); /* validate found region */ if (kbase_is_region_invalid_or_free(aliasing_reg)) @@ -1946,7 +1854,8 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, if (ai[i].offset + ai[i].length > alloc->nents) goto bad_handle; /* beyond end */ - reg->gpu_alloc->imported.alias.aliased[i].alloc = kbase_mem_phy_alloc_get(alloc); + reg->gpu_alloc->imported.alias.aliased[i].alloc = + kbase_mem_phy_alloc_get(alloc); reg->gpu_alloc->imported.alias.aliased[i].length = ai[i].length; reg->gpu_alloc->imported.alias.aliased[i].offset = ai[i].offset; @@ -1981,8 +1890,7 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride, gpu_va <<= PAGE_SHIFT; } else { /* we control the VA */ - if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1, - mmu_sync_info) != 0) { + if (kbase_gpu_mmap(kctx, reg, 0, *num_pages, 1, mmu_sync_info) != 0) { dev_warn(kctx->kbdev->dev, "Failed to map memory on GPU"); goto no_mmap; } @@ -2020,8 +1928,7 @@ bad_flags: } int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, - void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, - u64 *flags) + void __user *phandle, u32 padding, u64 *gpu_va, u64 *va_pages, u64 *flags) { struct kbase_va_region *reg; @@ -2035,32 +1942,28 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, KBASE_DEBUG_ASSERT(va_pages); KBASE_DEBUG_ASSERT(flags); - if ((!kbase_ctx_flag(kctx, KCTX_COMPAT)) && - kbase_ctx_flag(kctx, KCTX_FORCE_SAME_VA)) + if ((!kbase_ctx_flag(kctx, KCTX_COMPAT)) && kbase_ctx_flag(kctx, KCTX_FORCE_SAME_VA)) *flags |= BASE_MEM_SAME_VA; if (!kbase_check_import_flags(*flags)) { - dev_warn(kctx->kbdev->dev, - "%s called with bad flags (%llx)", - __func__, - (unsigned long long)*flags); + dev_warn(kctx->kbdev->dev, "%s called with bad flags (%llx)", __func__, + (unsigned long long)*flags); goto bad_flags; } if ((*flags & BASE_MEM_UNCACHED_GPU) != 0 && - (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0) { + (*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0) { /* Remove COHERENT_SYSTEM_REQUIRED flag if uncached GPU mapping is requested */ *flags &= ~BASE_MEM_COHERENT_SYSTEM_REQUIRED; } if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 && - !kbase_device_is_cpu_coherent(kctx->kbdev)) { - dev_warn(kctx->kbdev->dev, - "%s call required coherent mem when unavailable", - __func__); + !kbase_device_is_cpu_coherent(kctx->kbdev)) { + dev_warn(kctx->kbdev->dev, "%s call required coherent mem when unavailable", + __func__); goto bad_flags; } if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 && - !kbase_device_is_cpu_coherent(kctx->kbdev)) { + !kbase_device_is_cpu_coherent(kctx->kbdev)) { /* Remove COHERENT_SYSTEM flag if coherent mem is unavailable */ *flags &= ~BASE_MEM_COHERENT_SYSTEM; } @@ -2069,8 +1972,7 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, goto bad_flags; } if ((padding != 0) && (type != BASE_MEM_IMPORT_TYPE_UMM)) { - dev_warn(kctx->kbdev->dev, - "padding is only supported for UMM"); + dev_warn(kctx->kbdev->dev, "padding is only supported for UMM"); goto bad_flags; } @@ -2081,16 +1983,13 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, if (get_user(fd, (int __user *)phandle)) reg = NULL; else - reg = kbase_mem_from_umm(kctx, fd, va_pages, flags, - padding); - } - break; + reg = kbase_mem_from_umm(kctx, fd, va_pages, flags, padding); + } break; case BASE_MEM_IMPORT_TYPE_USER_BUFFER: { struct base_mem_import_user_buffer user_buffer; void __user *uptr; - if (copy_from_user(&user_buffer, phandle, - sizeof(user_buffer))) { + if (copy_from_user(&user_buffer, phandle, sizeof(user_buffer))) { reg = NULL; } else { #if IS_ENABLED(CONFIG_COMPAT) @@ -2100,9 +1999,8 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, #endif uptr = u64_to_user_ptr(user_buffer.ptr); - reg = kbase_mem_from_user_buffer(kctx, - (unsigned long)uptr, user_buffer.length, - va_pages, flags); + reg = kbase_mem_from_user_buffer(kctx, (unsigned long)uptr, + user_buffer.length, va_pages, flags); } break; } @@ -2132,10 +2030,9 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type, *gpu_va += PFN_DOWN(BASE_MEM_COOKIE_BASE); *gpu_va <<= PAGE_SHIFT; - } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES) { + } else if (*flags & KBASE_MEM_IMPORT_HAVE_PAGES) { /* we control the VA, mmap now to the GPU */ - if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1, mmu_sync_info) != - 0) + if (kbase_gpu_mmap(kctx, reg, 0, *va_pages, 1, mmu_sync_info) != 0) goto no_gpu_va; /* return real GPU VA */ *gpu_va = reg->start_pfn << PAGE_SHIFT; @@ -2168,9 +2065,8 @@ bad_flags: return -ENOMEM; } -int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, - struct kbase_va_region *reg, u64 new_pages, - u64 old_pages, +int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, struct kbase_va_region *reg, + u64 new_pages, u64 old_pages, enum kbase_caller_mmu_sync_info mmu_sync_info) { struct tagged_addr *phy_pages; @@ -2188,9 +2084,8 @@ int kbase_mem_grow_gpu_mapping(struct kbase_context *kctx, return ret; } -void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, - struct kbase_va_region *reg, - u64 new_pages, u64 old_pages) +void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, struct kbase_va_region *reg, + u64 new_pages, u64 old_pages) { u64 gpu_va_start = reg->start_pfn; @@ -2199,8 +2094,8 @@ void kbase_mem_shrink_cpu_mapping(struct kbase_context *kctx, return; unmap_mapping_range(kctx->kfile->filp->f_inode->i_mapping, - (gpu_va_start + new_pages)<<PAGE_SHIFT, - (old_pages - new_pages)<<PAGE_SHIFT, 1); + (loff_t)(gpu_va_start + new_pages) << PAGE_SHIFT, + (loff_t)(old_pages - new_pages) << PAGE_SHIFT, 1); } int kbase_mem_shrink_gpu_mapping(struct kbase_context *const kctx, @@ -2231,10 +2126,15 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) const enum kbase_caller_mmu_sync_info mmu_sync_info = CALLER_MMU_ASYNC; KBASE_DEBUG_ASSERT(kctx); - KBASE_DEBUG_ASSERT(gpu_addr != 0); + + if (unlikely(gpu_addr == 0)) { + dev_warn(kctx->kbdev->dev, "kbase:mem_commit: gpu_addr is 0"); + return -EINVAL; + } if (gpu_addr & ~PAGE_MASK) { - dev_warn(kctx->kbdev->dev, "kbase:mem_commit: gpu_addr: passed parameter is invalid"); + dev_warn(kctx->kbdev->dev, + "kbase:mem_commit: gpu_addr: passed parameter is invalid"); return -EINVAL; } @@ -2283,14 +2183,22 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) if (kbase_va_region_is_no_user_free(reg)) goto out_unlock; -#ifdef CONFIG_MALI_MEMORY_FULLY_BACKED - /* Reject resizing commit size */ - if (reg->flags & KBASE_REG_PF_GROW) - new_pages = reg->nr_pages; -#endif + if (unlikely(fully_backed_gpf_memory)) { + if ((reg->flags & KBASE_REG_PF_GROW) && (reg->nr_pages != reg->initial_commit)) { + dev_err(kctx->kbdev->dev, + "GPF region has different physical and VA page size!"); + res = -EINVAL; + } else { + /* The condition should already be satisfied, so no need to change + * the commit size. + */ + res = 0; + } + goto out_unlock; + } + /* no change */ if (new_pages == reg->gpu_alloc->nents) { - /* no change */ res = 0; goto out_unlock; } @@ -2312,11 +2220,9 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) goto out_unlock; } if (reg->cpu_alloc != reg->gpu_alloc) { - if (kbase_alloc_phy_pages_helper( - reg->gpu_alloc, delta) != 0) { + if (kbase_alloc_phy_pages_helper(reg->gpu_alloc, delta) != 0) { res = -ENOMEM; - kbase_free_phy_pages_helper(reg->cpu_alloc, - delta); + kbase_free_phy_pages_helper(reg->cpu_alloc, delta); goto out_unlock; } } @@ -2324,15 +2230,13 @@ int kbase_mem_commit(struct kbase_context *kctx, u64 gpu_addr, u64 new_pages) /* No update required for CPU mappings, that's done on fault. */ /* Update GPU mapping. */ - res = kbase_mem_grow_gpu_mapping(kctx, reg, new_pages, - old_pages, mmu_sync_info); + res = kbase_mem_grow_gpu_mapping(kctx, reg, new_pages, old_pages, mmu_sync_info); /* On error free the new pages */ if (res) { kbase_free_phy_pages_helper(reg->cpu_alloc, delta); if (reg->cpu_alloc != reg->gpu_alloc) - kbase_free_phy_pages_helper(reg->gpu_alloc, - delta); + kbase_free_phy_pages_helper(reg->gpu_alloc, delta); res = -ENOMEM; goto out_unlock; } @@ -2352,8 +2256,8 @@ out_unlock: return res; } -int kbase_mem_shrink(struct kbase_context *const kctx, - struct kbase_va_region *const reg, u64 new_pages) +int kbase_mem_shrink(struct kbase_context *const kctx, struct kbase_va_region *const reg, + u64 new_pages) { u64 delta, old_pages; int err; @@ -2374,7 +2278,7 @@ int kbase_mem_shrink(struct kbase_context *const kctx, if (kctx->kbdev->pagesize_2mb) { struct tagged_addr *start_free = reg->gpu_alloc->pages + new_pages; - /* Move the end of new committed range to a valid location. + /* Move the end of new commited range to a valid location. * This mirrors the adjustment done inside kbase_free_phy_pages_helper(). */ while (delta && is_huge(*start_free) && !is_huge_head(*start_free)) { @@ -2388,12 +2292,10 @@ int kbase_mem_shrink(struct kbase_context *const kctx, } /* Update the GPU mapping */ - err = kbase_mem_shrink_gpu_mapping(kctx, reg, - new_pages, old_pages); + err = kbase_mem_shrink_gpu_mapping(kctx, reg, new_pages, old_pages); if (err >= 0) { /* Update all CPU mapping(s) */ - kbase_mem_shrink_cpu_mapping(kctx, reg, - new_pages, old_pages); + kbase_mem_shrink_cpu_mapping(kctx, reg, new_pages, old_pages); kbase_free_phy_pages_helper(reg->cpu_alloc, delta); if (reg->cpu_alloc != reg->gpu_alloc) @@ -2403,7 +2305,6 @@ int kbase_mem_shrink(struct kbase_context *const kctx, return err; } - static void kbase_cpu_vm_open(struct vm_area_struct *vma) { struct kbase_cpu_mapping *map = vma->vm_private_data; @@ -2450,17 +2351,15 @@ static void kbase_cpu_vm_close(struct vm_area_struct *vma) } static struct kbase_aliased *get_aliased_alloc(struct vm_area_struct *vma, - struct kbase_va_region *reg, - pgoff_t *start_off, - size_t nr_pages) + struct kbase_va_region *reg, pgoff_t *start_off, + size_t nr_pages) { - struct kbase_aliased *aliased = - reg->cpu_alloc->imported.alias.aliased; + struct kbase_aliased *aliased = reg->cpu_alloc->imported.alias.aliased; + + CSTD_UNUSED(vma); - if (!reg->cpu_alloc->imported.alias.stride || - reg->nr_pages < (*start_off + nr_pages)) { + if (!reg->cpu_alloc->imported.alias.stride || reg->nr_pages < (*start_off + nr_pages)) return NULL; - } while (*start_off >= reg->cpu_alloc->imported.alias.stride) { aliased++; @@ -2481,8 +2380,7 @@ static struct kbase_aliased *get_aliased_alloc(struct vm_area_struct *vma, } #if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) -static vm_fault_t kbase_cpu_vm_fault(struct vm_area_struct *vma, - struct vm_fault *vmf) +static vm_fault_t kbase_cpu_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { #else static vm_fault_t kbase_cpu_vm_fault(struct vm_fault *vmf) @@ -2517,14 +2415,14 @@ static vm_fault_t kbase_cpu_vm_fault(struct vm_fault *vmf) if (unlikely(map->region->cpu_alloc->type == KBASE_MEM_TYPE_ALIAS)) { struct kbase_aliased *aliased = - get_aliased_alloc(vma, map->region, &map_start_pgoff, 1); + get_aliased_alloc(vma, map->region, &map_start_pgoff, 1); if (!aliased) goto exit; nents = aliased->length; pages = aliased->alloc->pages + aliased->offset; - } else { + } else { nents = map->alloc->nents; pages = map->alloc->pages; } @@ -2546,15 +2444,16 @@ static vm_fault_t kbase_cpu_vm_fault(struct vm_fault *vmf) addr = (pgoff_t)(vma->vm_start >> PAGE_SHIFT); mgm_dev = map->kctx->kbdev->mgm_dev; while (i < nents && (addr < vma->vm_end >> PAGE_SHIFT)) { - - ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, - map->alloc->group_id, vma, addr << PAGE_SHIFT, - PFN_DOWN(as_phys_addr_t(pages[i])), vma->vm_page_prot); + ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, map->alloc->group_id, vma, + addr << PAGE_SHIFT, + PFN_DOWN(as_phys_addr_t(pages[i])), + vma->vm_page_prot); if (ret != VM_FAULT_NOPAGE) goto exit; - i++; addr++; + i++; + addr++; } exit: @@ -2562,19 +2461,13 @@ exit: return ret; } -const struct vm_operations_struct kbase_vm_ops = { - .open = kbase_cpu_vm_open, - .close = kbase_cpu_vm_close, - .fault = kbase_cpu_vm_fault -}; +const struct vm_operations_struct kbase_vm_ops = { .open = kbase_cpu_vm_open, + .close = kbase_cpu_vm_close, + .fault = kbase_cpu_vm_fault }; -static int kbase_cpu_mmap(struct kbase_context *kctx, - struct kbase_va_region *reg, - struct vm_area_struct *vma, - void *kaddr, - size_t nr_pages, - unsigned long aligned_offset, - int free_on_close) +static int kbase_cpu_mmap(struct kbase_context *kctx, struct kbase_va_region *reg, + struct vm_area_struct *vma, void *kaddr, size_t nr_pages, + unsigned long aligned_offset, int free_on_close) { struct kbase_cpu_mapping *map; int err = 0; @@ -2606,10 +2499,8 @@ static int kbase_cpu_mmap(struct kbase_context *kctx, vma->vm_private_data = map; if (reg->cpu_alloc->type == KBASE_MEM_TYPE_ALIAS && nr_pages) { - pgoff_t rel_pgoff = vma->vm_pgoff - reg->start_pfn + - (aligned_offset >> PAGE_SHIFT); - struct kbase_aliased *aliased = - get_aliased_alloc(vma, reg, &rel_pgoff, nr_pages); + pgoff_t rel_pgoff = vma->vm_pgoff - reg->start_pfn + (aligned_offset >> PAGE_SHIFT); + struct kbase_aliased *aliased = get_aliased_alloc(vma, reg, &rel_pgoff, nr_pages); if (!aliased) { err = -EINVAL; @@ -2619,7 +2510,7 @@ static int kbase_cpu_mmap(struct kbase_context *kctx, } if (!(reg->flags & KBASE_REG_CPU_CACHED) && - (reg->flags & (KBASE_REG_CPU_WR|KBASE_REG_CPU_RD))) { + (reg->flags & (KBASE_REG_CPU_WR | KBASE_REG_CPU_RD))) { /* We can't map vmalloc'd memory uncached. * Other memory will have been returned from * kbase_mem_pool which would be @@ -2657,7 +2548,7 @@ static int kbase_cpu_mmap(struct kbase_context *kctx, list_add(&map->mappings_list, &map->alloc->mappings); kbase_file_inc_cpu_mapping_count(kctx->kfile); - out: +out: return err; } @@ -2678,14 +2569,12 @@ static void kbase_free_unused_jit_allocations(struct kbase_context *kctx) ; } -static int kbase_mmu_dump_mmap(struct kbase_context *kctx, - struct vm_area_struct *vma, - struct kbase_va_region **const reg, - void **const kmap_addr) +static int kbase_mmu_dump_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, + struct kbase_va_region **const reg, void **const kmap_addr) { struct kbase_va_region *new_reg; void *kaddr; - u32 nr_pages; + size_t nr_pages; size_t size; int err = 0; @@ -2711,8 +2600,8 @@ static int kbase_mmu_dump_mmap(struct kbase_context *kctx, goto out; } - new_reg->cpu_alloc = kbase_alloc_create(kctx, 0, KBASE_MEM_TYPE_RAW, - BASE_MEM_GROUP_DEFAULT); + new_reg->cpu_alloc = + kbase_alloc_create(kctx, 0, KBASE_MEM_TYPE_RAW, BASE_MEM_GROUP_DEFAULT); if (IS_ERR_OR_NULL(new_reg->cpu_alloc)) { err = -ENOMEM; new_reg->cpu_alloc = NULL; @@ -2756,10 +2645,8 @@ void kbase_os_mem_map_unlock(struct kbase_context *kctx) up_read(kbase_mem_get_process_mmap_lock()); } -static int kbasep_reg_mmap(struct kbase_context *kctx, - struct vm_area_struct *vma, - struct kbase_va_region **regm, - size_t *nr_pages, size_t *aligned_offset) +static int kbasep_reg_mmap(struct kbase_context *kctx, struct vm_area_struct *vma, + struct kbase_va_region **regm, size_t *nr_pages, size_t *aligned_offset) { unsigned int cookie = vma->vm_pgoff - PFN_DOWN(BASE_MEM_COOKIE_BASE); @@ -2796,18 +2683,16 @@ static int kbasep_reg_mmap(struct kbase_context *kctx, (vma->vm_flags & VM_WRITE && !(reg->flags & KBASE_REG_CPU_WR))) { /* VM flags inconsistent with region flags */ err = -EPERM; - dev_err(kctx->kbdev->dev, "%s:%d inconsistent VM flags\n", - __FILE__, __LINE__); + dev_err(kctx->kbdev->dev, "%s:%d inconsistent VM flags\n", __FILE__, __LINE__); goto out; } /* adjust down nr_pages to what we have physically */ *nr_pages = kbase_reg_current_backed_size(reg); - if (kbase_gpu_mmap(kctx, reg, vma->vm_start + *aligned_offset, - reg->nr_pages, 1, mmu_sync_info) != 0) { - dev_err(kctx->kbdev->dev, "%s:%d\n", __FILE__, __LINE__); + if (kbase_gpu_mmap(kctx, reg, vma->vm_start + *aligned_offset, reg->nr_pages, 1, + mmu_sync_info) != 0) { /* Unable to map in GPU space. */ - WARN_ON(1); + dev_err(kctx->kbdev->dev, "%s:%d\n", __FILE__, __LINE__); err = -ENOMEM; goto out; } @@ -2826,7 +2711,7 @@ static int kbasep_reg_mmap(struct kbase_context *kctx, * offset so that when user space trims the mapping the beginning of * the trimmed VMA has the correct vm_pgoff; */ - vma->vm_pgoff = reg->start_pfn - ((*aligned_offset)>>PAGE_SHIFT); + vma->vm_pgoff = reg->start_pfn - ((*aligned_offset) >> PAGE_SHIFT); out: *regm = reg; dev_dbg(kctx->kbdev->dev, "%s done\n", __func__); @@ -2834,8 +2719,7 @@ out: return err; } -int kbase_context_mmap(struct kbase_context *const kctx, - struct vm_area_struct *const vma) +int kbase_context_mmap(struct kbase_context *const kctx, struct vm_area_struct *const vma) { struct kbase_va_region *reg = NULL; void *kaddr = NULL; @@ -2900,8 +2784,7 @@ int kbase_context_mmap(struct kbase_context *const kctx, kbase_gpu_vm_unlock(kctx); err = kbase_csf_cpu_mmap_user_reg_page(kctx, vma); goto out; - case PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE) ... - PFN_DOWN(BASE_MEM_COOKIE_BASE) - 1: { + case PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE)... PFN_DOWN(BASE_MEM_COOKIE_BASE) - 1: { kbase_gpu_vm_unlock(kctx); mutex_lock(&kctx->csf.lock); err = kbase_csf_cpu_mmap_user_io_pages(kctx, vma); @@ -2909,10 +2792,8 @@ int kbase_context_mmap(struct kbase_context *const kctx, goto out; } #endif - case PFN_DOWN(BASE_MEM_COOKIE_BASE) ... - PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { - err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, - &aligned_offset); + case PFN_DOWN(BASE_MEM_COOKIE_BASE)... PFN_DOWN(BASE_MEM_FIRST_FREE_ADDRESS) - 1: { + err = kbasep_reg_mmap(kctx, vma, ®, &nr_pages, &aligned_offset); if (err != 0) goto out_unlock; /* free the region on munmap */ @@ -2920,58 +2801,54 @@ int kbase_context_mmap(struct kbase_context *const kctx, break; } default: { - reg = kbase_region_tracker_find_region_enclosing_address(kctx, - (u64)vma->vm_pgoff << PAGE_SHIFT); + reg = kbase_region_tracker_find_region_enclosing_address( + kctx, (u64)vma->vm_pgoff << PAGE_SHIFT); if (!kbase_is_region_invalid_or_free(reg)) { /* will this mapping overflow the size of the region? */ - if (nr_pages > (reg->nr_pages - - (vma->vm_pgoff - reg->start_pfn))) { + if (nr_pages > (reg->nr_pages - (vma->vm_pgoff - reg->start_pfn))) { err = -ENOMEM; goto out_unlock; } - if ((vma->vm_flags & VM_READ && - !(reg->flags & KBASE_REG_CPU_RD)) || - (vma->vm_flags & VM_WRITE && - !(reg->flags & KBASE_REG_CPU_WR))) { + if ((vma->vm_flags & VM_READ && !(reg->flags & KBASE_REG_CPU_RD)) || + (vma->vm_flags & VM_WRITE && !(reg->flags & KBASE_REG_CPU_WR))) { /* VM flags inconsistent with region flags */ err = -EPERM; - dev_err(dev, "%s:%d inconsistent VM flags\n", - __FILE__, __LINE__); + dev_err(dev, "%s:%d inconsistent VM flags\n", __FILE__, __LINE__); goto out_unlock; } - if (KBASE_MEM_TYPE_IMPORTED_UMM == - reg->cpu_alloc->type) { + if (reg->cpu_alloc->type == KBASE_MEM_TYPE_IMPORTED_UMM) { if (0 != (vma->vm_pgoff - reg->start_pfn)) { err = -EINVAL; - dev_warn(dev, "%s:%d attempt to do a partial map in a dma_buf: non-zero offset to dma_buf mapping!\n", + dev_warn( + dev, + "%s:%d attempt to do a partial map in a dma_buf: non-zero offset to dma_buf mapping!\n", __FILE__, __LINE__); goto out_unlock; } - err = dma_buf_mmap( - reg->cpu_alloc->imported.umm.dma_buf, - vma, vma->vm_pgoff - reg->start_pfn); + err = dma_buf_mmap(reg->cpu_alloc->imported.umm.dma_buf, vma, + vma->vm_pgoff - reg->start_pfn); goto out_unlock; } if (reg->cpu_alloc->type == KBASE_MEM_TYPE_ALIAS) { /* initial params check for aliased dumping map */ if (nr_pages > reg->gpu_alloc->imported.alias.stride || - !reg->gpu_alloc->imported.alias.stride || - !nr_pages) { + !reg->gpu_alloc->imported.alias.stride || !nr_pages) { err = -EINVAL; dev_warn(dev, "mmap aliased: invalid params!\n"); goto out_unlock; } } else if (reg->cpu_alloc->nents < - (vma->vm_pgoff - reg->start_pfn + nr_pages)) { + (vma->vm_pgoff - reg->start_pfn + nr_pages)) { /* limit what we map to the amount currently backed */ if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents) nr_pages = 0; else - nr_pages = reg->cpu_alloc->nents - (vma->vm_pgoff - reg->start_pfn); + nr_pages = reg->cpu_alloc->nents - + (vma->vm_pgoff - reg->start_pfn); } } else { err = -ENOMEM; @@ -2980,8 +2857,7 @@ int kbase_context_mmap(struct kbase_context *const kctx, } /* default */ } /* switch */ - err = kbase_cpu_mmap(kctx, reg, vma, kaddr, nr_pages, aligned_offset, - free_on_close); + err = kbase_cpu_mmap(kctx, reg, vma, kaddr, nr_pages, aligned_offset, free_on_close); #if defined(CONFIG_MALI_VECTOR_DUMP) if (vma->vm_pgoff == PFN_DOWN(BASE_MEM_MMU_DUMP_HANDLE)) { /* MMU dump - userspace should now have a reference on @@ -3013,15 +2889,15 @@ out: KBASE_EXPORT_TEST_API(kbase_context_mmap); -void kbase_sync_mem_regions(struct kbase_context *kctx, - struct kbase_vmap_struct *map, enum kbase_sync_type dest) +void kbase_sync_mem_regions(struct kbase_context *kctx, struct kbase_vmap_struct *map, + enum kbase_sync_type dest) { size_t i; off_t const offset = map->offset_in_page; - size_t const page_count = PFN_UP(offset + map->size); + size_t const page_count = PFN_UP((size_t)offset + map->size); /* Sync first page */ - size_t sz = MIN(((size_t) PAGE_SIZE - offset), map->size); + size_t sz = MIN((PAGE_SIZE - (size_t)offset), map->size); struct tagged_addr cpu_pa = map->cpu_pages[0]; struct tagged_addr gpu_pa = map->gpu_pages[0]; @@ -3038,7 +2914,7 @@ void kbase_sync_mem_regions(struct kbase_context *kctx, if (page_count > 1) { cpu_pa = map->cpu_pages[page_count - 1]; gpu_pa = map->gpu_pages[page_count - 1]; - sz = ((offset + map->size - 1) & ~PAGE_MASK) + 1; + sz = (((size_t)offset + map->size - 1) & ~PAGE_MASK) + 1; kbase_sync_single(kctx, cpu_pa, gpu_pa, 0, sz, dest); } } @@ -3070,7 +2946,7 @@ static void kbase_vmap_phy_pages_migrate_count_increment(struct tagged_addr *pag struct page *p = as_page(pages[i]); struct kbase_page_metadata *page_md = kbase_page_private(p); - /* Skip the 4KB page that is part of a large page, as the large page is + /* Skip the small page that is part of a large page, as the large page is * excluded from the migration process. */ if (is_huge(pages[i]) || is_partial(pages[i])) @@ -3122,7 +2998,7 @@ static void kbase_vunmap_phy_pages_migrate_count_decrement(struct tagged_addr *p struct page *p = as_page(pages[i]); struct kbase_page_metadata *page_md = kbase_page_private(p); - /* Skip the 4KB page that is part of a large page, as the large page is + /* Skip the small page that is part of a large page, as the large page is * excluded from the migration process. */ if (is_huge(pages[i]) || is_partial(pages[i])) @@ -3179,7 +3055,7 @@ static int kbase_vmap_phy_pages(struct kbase_context *kctx, struct kbase_va_regi if ((vmap_flags & KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING) && (page_count > (KBASE_PERMANENTLY_MAPPED_MEM_LIMIT_PAGES - - atomic_read(&kctx->permanent_mapped_pages)))) { + (size_t)atomic_read(&kctx->permanent_mapped_pages)))) { dev_warn( kctx->kbdev->dev, "Request for %llu more pages mem needing a permanent mapping would breach limit %lu, currently at %d pages", @@ -3307,7 +3183,7 @@ out_unlock: } void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, - struct kbase_vmap_struct *map) + struct kbase_vmap_struct *map) { /* 0 is specified for prot_request to indicate no access checks should * be made. @@ -3319,8 +3195,7 @@ void *kbase_vmap(struct kbase_context *kctx, u64 gpu_addr, size_t size, } KBASE_EXPORT_TEST_API(kbase_vmap); -static void kbase_vunmap_phy_pages(struct kbase_context *kctx, - struct kbase_vmap_struct *map) +static void kbase_vunmap_phy_pages(struct kbase_context *kctx, struct kbase_vmap_struct *map) { void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK); @@ -3331,7 +3206,7 @@ static void kbase_vunmap_phy_pages(struct kbase_context *kctx, * haven't been released yet. */ if (kbase_is_page_migration_enabled() && !kbase_mem_is_imported(map->gpu_alloc->type)) { - const size_t page_count = PFN_UP(map->offset_in_page + map->size); + const size_t page_count = PFN_UP((size_t)map->offset_in_page + map->size); struct tagged_addr *pages_array = map->cpu_pages; kbase_vunmap_phy_pages_migrate_count_decrement(pages_array, page_count); @@ -3340,9 +3215,9 @@ static void kbase_vunmap_phy_pages(struct kbase_context *kctx, if (map->flags & KBASE_VMAP_FLAG_SYNC_NEEDED) kbase_sync_mem_regions(kctx, map, KBASE_SYNC_TO_DEVICE); if (map->flags & KBASE_VMAP_FLAG_PERMANENT_MAP_ACCOUNTING) { - size_t page_count = PFN_UP(map->offset_in_page + map->size); + size_t page_count = PFN_UP((size_t)map->offset_in_page + map->size); - WARN_ON(page_count > atomic_read(&kctx->permanent_mapped_pages)); + WARN_ON(page_count > (size_t)atomic_read(&kctx->permanent_mapped_pages)); atomic_sub(page_count, &kctx->permanent_mapped_pages); } @@ -3433,8 +3308,7 @@ static int kbase_tracking_page_setup(struct kbase_context *kctx, struct vm_area_ } #if MALI_USE_CSF -static unsigned long get_queue_doorbell_pfn(struct kbase_device *kbdev, - struct kbase_queue *queue) +static unsigned long get_queue_doorbell_pfn(struct kbase_device *kbdev, struct kbase_queue *queue) { lockdep_assert_held(&kbdev->csf.reg_lock); @@ -3454,18 +3328,25 @@ static unsigned long get_queue_doorbell_pfn(struct kbase_device *kbdev, static int #if (KERNEL_VERSION(5, 13, 0) <= LINUX_VERSION_CODE || \ - KERNEL_VERSION(5, 11, 0) > LINUX_VERSION_CODE) + KERNEL_VERSION(5, 11, 0) > LINUX_VERSION_CODE) kbase_csf_user_io_pages_vm_mremap(struct vm_area_struct *vma) +{ #else kbase_csf_user_io_pages_vm_mremap(struct vm_area_struct *vma, unsigned long flags) -#endif { + CSTD_UNUSED(flags); +#endif + CSTD_UNUSED(vma); + pr_debug("Unexpected call to mremap method for User IO pages mapping vma\n"); return -EINVAL; } static int kbase_csf_user_io_pages_vm_split(struct vm_area_struct *vma, unsigned long addr) { + CSTD_UNUSED(vma); + CSTD_UNUSED(addr); + pr_debug("Unexpected call to split method for User IO pages mapping vma\n"); return -EINVAL; } @@ -3516,8 +3397,7 @@ static void kbase_csf_user_io_pages_vm_close(struct vm_area_struct *vma) } #if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) -static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_area_struct *vma, - struct vm_fault *vmf) +static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { #else static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_fault *vmf) @@ -3565,25 +3445,25 @@ static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_fault *vmf) if (vmf->address == doorbell_cpu_addr) { #endif doorbell_page_pfn = get_queue_doorbell_pfn(kbdev, queue); - ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, - KBASE_MEM_GROUP_CSF_IO, vma, doorbell_cpu_addr, - doorbell_page_pfn, doorbell_pgprot); + ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, KBASE_MEM_GROUP_CSF_IO, vma, + doorbell_cpu_addr, doorbell_page_pfn, + doorbell_pgprot); } else { /* Map the Input page */ input_cpu_addr = doorbell_cpu_addr + PAGE_SIZE; input_page_pfn = PFN_DOWN(as_phys_addr_t(queue->phys[0])); - ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, - KBASE_MEM_GROUP_CSF_IO, vma, input_cpu_addr, - input_page_pfn, input_page_pgprot); + ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, KBASE_MEM_GROUP_CSF_IO, vma, + input_cpu_addr, input_page_pfn, + input_page_pgprot); if (ret != VM_FAULT_NOPAGE) goto exit; /* Map the Output page */ output_cpu_addr = input_cpu_addr + PAGE_SIZE; output_page_pfn = PFN_DOWN(as_phys_addr_t(queue->phys[1])); - ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, - KBASE_MEM_GROUP_CSF_IO, vma, output_cpu_addr, - output_page_pfn, output_page_pgprot); + ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, KBASE_MEM_GROUP_CSF_IO, vma, + output_cpu_addr, output_page_pfn, + output_page_pgprot); } exit: @@ -3607,11 +3487,9 @@ static const struct vm_operations_struct kbase_csf_user_io_pages_vm_ops = { * input/output pages & Hw doorbell page. The caller should have validated that * vma->vm_pgoff maps to the range of csf cookies. */ -static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, - struct vm_area_struct *vma) +static int kbase_csf_cpu_mmap_user_io_pages(struct kbase_context *kctx, struct vm_area_struct *vma) { - unsigned long cookie = - vma->vm_pgoff - PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE); + unsigned long cookie = vma->vm_pgoff - PFN_DOWN(BASEP_MEM_CSF_USER_IO_PAGES_HANDLE); size_t nr_pages = vma_pages(vma); struct kbase_queue *queue; int err = 0; @@ -3744,19 +3622,22 @@ static void kbase_csf_user_reg_vm_close(struct vm_area_struct *vma) */ static int #if ((KERNEL_VERSION(5, 13, 0) <= LINUX_VERSION_CODE) || \ - (KERNEL_VERSION(5, 11, 0) > LINUX_VERSION_CODE)) + (KERNEL_VERSION(5, 11, 0) > LINUX_VERSION_CODE)) kbase_csf_user_reg_vm_mremap(struct vm_area_struct *vma) +{ #else kbase_csf_user_reg_vm_mremap(struct vm_area_struct *vma, unsigned long flags) -#endif { + CSTD_UNUSED(flags); +#endif + CSTD_UNUSED(vma); + pr_debug("Unexpected call to mremap method for USER page mapping vma\n"); return -EINVAL; } #if (KERNEL_VERSION(4, 11, 0) > LINUX_VERSION_CODE) -static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_area_struct *vma, - struct vm_fault *vmf) +static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { #else static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_fault *vmf) @@ -3796,10 +3677,8 @@ static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_fault *vmf) spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags); list_move_tail(&kctx->csf.user_reg.link, &kbdev->csf.user_reg.list); - ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, - KBASE_MEM_GROUP_CSF_FW, vma, - vma->vm_start, pfn, - vma->vm_page_prot); + ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev, KBASE_MEM_GROUP_CSF_FW, vma, + vma->vm_start, pfn, vma->vm_page_prot); mutex_unlock(&kbdev->csf.reg_lock); @@ -3813,8 +3692,7 @@ static const struct vm_operations_struct kbase_csf_user_reg_vm_ops = { .fault = kbase_csf_user_reg_vm_fault }; -static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, - struct vm_area_struct *vma) +static int kbase_csf_cpu_mmap_user_reg_page(struct kbase_context *kctx, struct vm_area_struct *vma) { size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start); struct kbase_device *kbdev = kctx->kbdev; |