Merge r36p0 from upstream into android-gs-pixel-5.10-gs101

Bug: 220942030
Test: boot to Home, Camera, Video, Chrome
Signed-off-by: Siddharth Kapoor <ksiddharth@google.com>
Change-Id: I8cf3a6243b092a05299acf956f5293facd1ed3cc

1 files changed, 86 insertions, 33 deletions

diff --git a/mali_kbase/mali_kbase_mem_linux.c b/mali_kbase/mali_kbase_mem_linux.c
index 762b5a6..57d61f7 100644
--- a/mali_kbase/mali_kbase_mem_linux.c
+++ b/mali_kbase/mali_kbase_mem_linux.c
@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
 /*
  *
- * (C) COPYRIGHT 2010-2021 ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2010-2022 ARM Limited. All rights reserved.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -289,10 +289,9 @@ void kbase_phy_alloc_mapping_put(struct kbase_context *kctx,
 	 */
 }
 
-struct kbase_va_region *
-kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages,
-		u64 extension, u64 *flags, u64 *gpu_va,
-		enum kbase_caller_mmu_sync_info mmu_sync_info)
+struct kbase_va_region *kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages,
+					u64 extension, u64 *flags, u64 *gpu_va,
+					enum kbase_caller_mmu_sync_info mmu_sync_info)
 {
 	int zone;
 	struct kbase_va_region *reg;
@@ -309,19 +308,21 @@ kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages,
 		va_pages, commit_pages, extension, *flags);
 
 #if MALI_USE_CSF
-	*gpu_va = 0; /* return 0 on failure */
+	if (!(*flags & BASE_MEM_FIXED))
+		*gpu_va = 0; /* return 0 on failure */
 #else
 	if (!(*flags & BASE_MEM_FLAG_MAP_FIXED))
 		*gpu_va = 0; /* return 0 on failure */
+#endif
 	else
-		dev_err(dev,
+		dev_dbg(dev,
 			"Keeping requested GPU VA of 0x%llx\n",
 			(unsigned long long)*gpu_va);
-#endif
 
 	if (!kbase_check_alloc_flags(*flags)) {
 		dev_warn(dev,
-				"kbase_mem_alloc called with bad flags (%llx)",
+				"%s called with bad flags (%llx)",
+				__func__,
 				(unsigned long long)*flags);
 		goto bad_flags;
 	}
@@ -343,7 +344,8 @@ kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages,
 	}
 	if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 &&
 			!kbase_device_is_cpu_coherent(kctx->kbdev)) {
-		dev_warn(dev, "kbase_mem_alloc call required coherent mem when unavailable");
+		dev_warn(dev, "%s call required coherent mem when unavailable",
+			__func__);
 		goto bad_flags;
 	}
 	if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 &&
@@ -366,7 +368,20 @@ kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages,
 	if (*flags & BASE_MEM_SAME_VA) {
 		rbtree = &kctx->reg_rbtree_same;
 		zone = KBASE_REG_ZONE_SAME_VA;
-	} else if ((*flags & BASE_MEM_PROT_GPU_EX) && kbase_has_exec_va_zone(kctx)) {
+	}
+#if MALI_USE_CSF
+	/* fixed va_zone always exists */
+	else if (*flags & (BASE_MEM_FIXED | BASE_MEM_FIXABLE)) {
+		if (*flags & BASE_MEM_PROT_GPU_EX) {
+			rbtree = &kctx->reg_rbtree_exec_fixed;
+			zone = KBASE_REG_ZONE_EXEC_FIXED_VA;
+		} else {
+			rbtree = &kctx->reg_rbtree_fixed;
+			zone = KBASE_REG_ZONE_FIXED_VA;
+		}
+	}
+#endif
+	else if ((*flags & BASE_MEM_PROT_GPU_EX) && kbase_has_exec_va_zone(kctx)) {
 		rbtree = &kctx->reg_rbtree_exec;
 		zone = KBASE_REG_ZONE_EXEC_VA;
 	} else {
@@ -490,6 +505,14 @@ kbase_mem_alloc(struct kbase_context *kctx, u64 va_pages, u64 commit_pages,
 #endif /* MALI_JIT_PRESSURE_LIMIT_BASE */
 
 	kbase_gpu_vm_unlock(kctx);
+
+#if MALI_USE_CSF
+	if (*flags & BASE_MEM_FIXABLE)
+		atomic64_inc(&kctx->num_fixable_allocs);
+	else if (*flags & BASE_MEM_FIXED)
+		atomic64_inc(&kctx->num_fixed_allocs);
+#endif
+
 	return reg;
 
 no_mmap:
@@ -600,6 +623,13 @@ int kbase_mem_query(struct kbase_context *kctx,
 #if MALI_USE_CSF
 		if (KBASE_REG_CSF_EVENT & reg->flags)
 			*out |= BASE_MEM_CSF_EVENT;
+		if (((KBASE_REG_ZONE_MASK & reg->flags) == KBASE_REG_ZONE_FIXED_VA) ||
+		    ((KBASE_REG_ZONE_MASK & reg->flags) == KBASE_REG_ZONE_EXEC_FIXED_VA)) {
+			if (KBASE_REG_FIXED_ADDRESS & reg->flags)
+				*out |= BASE_MEM_FIXED;
+			else
+				*out |= BASE_MEM_FIXABLE;
+		}
 #endif
 		if (KBASE_REG_GPU_VA_SAME_4GB_PAGE & reg->flags)
 			*out |= BASE_MEM_GPU_VA_SAME_4GB_PAGE;
@@ -1649,15 +1679,23 @@ KERNEL_VERSION(4, 5, 0) > LINUX_VERSION_CODE
 	faulted_pages = get_user_pages(address, *va_pages,
 			write ? FOLL_WRITE : 0, pages, NULL);
 #else
+
 	/*
 	 * User buffers should be pinned with FOLL_LONGTERM flag as their usage
 	 * cannot be time bounded. This will make sure that we do not pin pages
 	 * in the CMA region.
+	 * pin_user_pages function cannot be called with pages param NULL.
+	 * get_user_pages function will be used instead because it is safe to be
+	 * used with NULL pages param as long as it doesn't have FOLL_GET flag.
 	 */
-	pages = user_buf->pages;
-	faulted_pages = pin_user_pages(address, *va_pages,
-				       write ? FOLL_WRITE | FOLL_LONGTERM : FOLL_LONGTERM,
-				       pages, NULL);
+	if (pages != NULL) {
+		faulted_pages =
+			pin_user_pages(address, *va_pages,
+				write ? FOLL_WRITE | FOLL_LONGTERM : FOLL_LONGTERM, pages, NULL);
+	} else {
+		faulted_pages =
+			get_user_pages(address, *va_pages, write ? FOLL_WRITE : 0, pages, NULL);
+	}
 #endif
 
 	up_read(kbase_mem_get_process_mmap_lock());
@@ -1750,7 +1788,8 @@ u64 kbase_mem_alias(struct kbase_context *kctx, u64 *flags, u64 stride,
 
 	if (!(*flags & (BASE_MEM_PROT_GPU_RD | BASE_MEM_PROT_GPU_WR))) {
 		dev_warn(kctx->kbdev->dev,
-				"kbase_mem_alias called with bad flags (%llx)",
+				"%s called with bad flags (%llx)",
+				__func__,
 				(unsigned long long)*flags);
 		goto bad_flags;
 	}
@@ -1973,7 +2012,8 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type,
 
 	if (!kbase_check_import_flags(*flags)) {
 		dev_warn(kctx->kbdev->dev,
-				"kbase_mem_import called with bad flags (%llx)",
+				"%s called with bad flags (%llx)",
+				__func__,
 				(unsigned long long)*flags);
 		goto bad_flags;
 	}
@@ -1986,7 +2026,8 @@ int kbase_mem_import(struct kbase_context *kctx, enum base_mem_import_type type,
 	if ((*flags & BASE_MEM_COHERENT_SYSTEM_REQUIRED) != 0 &&
 			!kbase_device_is_cpu_coherent(kctx->kbdev)) {
 		dev_warn(kctx->kbdev->dev,
-				"kbase_mem_import call required coherent mem when unavailable");
+				"%s call required coherent mem when unavailable",
+				__func__);
 		goto bad_flags;
 	}
 	if ((*flags & BASE_MEM_COHERENT_SYSTEM) != 0 &&
@@ -2291,7 +2332,7 @@ out_unlock:
 }
 
 int kbase_mem_shrink(struct kbase_context *const kctx,
-		struct kbase_va_region *const reg, u64 const new_pages)
+		struct kbase_va_region *const reg, u64 new_pages)
 {
 	u64 delta, old_pages;
 	int err;
@@ -2321,6 +2362,18 @@ int kbase_mem_shrink(struct kbase_context *const kctx,
 		kbase_free_phy_pages_helper(reg->cpu_alloc, delta);
 		if (reg->cpu_alloc != reg->gpu_alloc)
 			kbase_free_phy_pages_helper(reg->gpu_alloc, delta);
+#ifdef CONFIG_MALI_2MB_ALLOC
+		if (kbase_reg_current_backed_size(reg) > new_pages) {
+			old_pages = new_pages;
+			new_pages = kbase_reg_current_backed_size(reg);
+
+			/* Update GPU mapping. */
+			err = kbase_mem_grow_gpu_mapping(kctx, reg,
+					new_pages, old_pages, CALLER_MMU_ASYNC);
+		}
+#else
+		WARN_ON(kbase_reg_current_backed_size(reg) != new_pages);
+#endif
 	}
 
 	return err;
@@ -2633,7 +2686,7 @@ static int kbase_mmu_dump_mmap(struct kbase_context *kctx,
 	size_t size;
 	int err = 0;
 
-	dev_dbg(kctx->kbdev->dev, "in kbase_mmu_dump_mmap\n");
+	dev_dbg(kctx->kbdev->dev, "%s\n", __func__);
 	size = (vma->vm_end - vma->vm_start);
 	nr_pages = size >> PAGE_SHIFT;
 
@@ -2678,7 +2731,7 @@ static int kbase_mmu_dump_mmap(struct kbase_context *kctx,
 	*kmap_addr = kaddr;
 	*reg = new_reg;
 
-	dev_dbg(kctx->kbdev->dev, "kbase_mmu_dump_mmap done\n");
+	dev_dbg(kctx->kbdev->dev, "%s done\n", __func__);
 	return 0;
 
 out_no_alloc:
@@ -2718,7 +2771,7 @@ static int kbasep_reg_mmap(struct kbase_context *kctx,
 
 	*aligned_offset = 0;
 
-	dev_dbg(kctx->kbdev->dev, "in kbasep_reg_mmap\n");
+	dev_dbg(kctx->kbdev->dev, "%s\n", __func__);
 
 	/* SAME_VA stuff, fetch the right region */
 	reg = kctx->pending_regions[cookie];
@@ -2775,7 +2828,7 @@ static int kbasep_reg_mmap(struct kbase_context *kctx,
 	vma->vm_pgoff = reg->start_pfn - ((*aligned_offset)>>PAGE_SHIFT);
 out:
 	*regm = reg;
-	dev_dbg(kctx->kbdev->dev, "kbasep_reg_mmap done\n");
+	dev_dbg(kctx->kbdev->dev, "%s done\n", __func__);
 
 	return err;
 }
@@ -2905,8 +2958,7 @@ int kbase_context_mmap(struct kbase_context *const kctx,
 					dev_warn(dev, "mmap aliased: invalid params!\n");
 					goto out_unlock;
 				}
-			}
-			else if (reg->cpu_alloc->nents <
+			} else if (reg->cpu_alloc->nents <
 					(vma->vm_pgoff - reg->start_pfn + nr_pages)) {
 				/* limit what we map to the amount currently backed */
 				if ((vma->vm_pgoff - reg->start_pfn) >= reg->cpu_alloc->nents)
@@ -3123,6 +3175,7 @@ static void kbase_vunmap_phy_pages(struct kbase_context *kctx,
 		struct kbase_vmap_struct *map)
 {
 	void *addr = (void *)((uintptr_t)map->addr & PAGE_MASK);
+
 	vunmap(addr);
 
 	if (map->sync_needed)
@@ -3321,10 +3374,11 @@ static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_fault *vmf)
 	    (vma->vm_pgoff != queue->db_file_offset))
 		return VM_FAULT_SIGBUS;
 
-	mutex_lock(&queue->kctx->csf.lock);
 	kbdev = queue->kctx->kbdev;
 	mgm_dev = kbdev->mgm_dev;
 
+	mutex_lock(&kbdev->csf.reg_lock);
+
 	/* Always map the doorbell page as uncached */
 	doorbell_pgprot = pgprot_device(vma->vm_page_prot);
 
@@ -3351,12 +3405,10 @@ static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_fault *vmf)
 #else
 	if (vmf->address == doorbell_cpu_addr) {
 #endif
-		mutex_lock(&kbdev->csf.reg_lock);
 		doorbell_page_pfn = get_queue_doorbell_pfn(kbdev, queue);
 		ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev,
 			KBASE_MEM_GROUP_CSF_IO, vma, doorbell_cpu_addr,
 			doorbell_page_pfn, doorbell_pgprot);
-		mutex_unlock(&kbdev->csf.reg_lock);
 	} else {
 		/* Map the Input page */
 		input_cpu_addr = doorbell_cpu_addr + PAGE_SIZE;
@@ -3376,7 +3428,7 @@ static vm_fault_t kbase_csf_user_io_pages_vm_fault(struct vm_fault *vmf)
 	}
 
 exit:
-	mutex_unlock(&queue->kctx->csf.lock);
+	mutex_unlock(&kbdev->csf.reg_lock);
 	return ret;
 }
 
@@ -3486,6 +3538,7 @@ static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_fault *vmf)
 	unsigned long pfn = PFN_DOWN(kbdev->reg_start + USER_BASE);
 	size_t nr_pages = PFN_DOWN(vma->vm_end - vma->vm_start);
 	vm_fault_t ret = VM_FAULT_SIGBUS;
+	unsigned long flags;
 
 	/* Few sanity checks up front */
 	if (WARN_ON(nr_pages != 1) ||
@@ -3494,8 +3547,8 @@ static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_fault *vmf)
 			PFN_DOWN(BASEP_MEM_CSF_USER_REG_PAGE_HANDLE)))
 		return VM_FAULT_SIGBUS;
 
-	mutex_lock(&kbdev->pm.lock);
-
+	mutex_lock(&kbdev->csf.reg_lock);
+	spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 	/* Don't map in the actual register page if GPU is powered down.
 	 * Always map in the dummy page in no mali builds.
 	 */
@@ -3505,13 +3558,13 @@ static vm_fault_t kbase_csf_user_reg_vm_fault(struct vm_fault *vmf)
 	if (!kbdev->pm.backend.gpu_powered)
 		pfn = PFN_DOWN(as_phys_addr_t(kbdev->csf.dummy_user_reg_page));
 #endif
+	spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 
 	ret = mgm_dev->ops.mgm_vmf_insert_pfn_prot(mgm_dev,
 						   KBASE_MEM_GROUP_CSF_FW, vma,
 						   vma->vm_start, pfn,
 						   vma->vm_page_prot);
-
-	mutex_unlock(&kbdev->pm.lock);
+	mutex_unlock(&kbdev->csf.reg_lock);
 
 	return ret;
 }