Mali Valhall DDK r27p0 KMD

Provenance:
 7e6f74ec1 (collaborate/EAC/v_r27p0)
 VX504X08X-BU-00000-r27p0-01eac0 - Android DDK
 VX504X08X-SW-99006-r27p0-01eac0 - Android Renderscript AOSP parts
 VX504X08X-BU-60000-r27p0-01eac0 - Android Document Bundle
 VX504X08X-DC-11001-r27p0-01eac0 - Valhall Android DDK Software Errata

Signed-off-by: Sidath Senanayake <sidaths@google.com>
Change-Id: Ib59de731b034cc7e2631e35f1b0063b8f6894ecc

1 files changed, 147 insertions, 1 deletions

diff --git a/mali_kbase/mali_kbase_mem.c b/mali_kbase/mali_kbase_mem.c
index 8cf7e5d..ce52f6a 100644
--- a/mali_kbase/mali_kbase_mem.c
+++ b/mali_kbase/mali_kbase_mem.c
@@ -615,6 +615,7 @@ int kbase_add_va_region_rbtree(struct kbase_device *kbdev,
 		size_t align_offset = align;
 		size_t align_mask = align - 1;
 
+#if !MALI_USE_CSF
 		if ((reg->flags & KBASE_REG_TILER_ALIGN_TOP)) {
 			WARN(align > 1, "%s with align %lx might not be honored for KBASE_REG_TILER_ALIGN_TOP memory",
 					__func__,
@@ -622,6 +623,7 @@ int kbase_add_va_region_rbtree(struct kbase_device *kbdev,
 			align_mask  = reg->extent - 1;
 			align_offset = reg->extent - reg->initial_commit;
 		}
+#endif /* !MALI_USE_CSF */
 
 		tmp = kbase_region_tracker_find_region_meeting_reqs(reg,
 				nr_pages, align_offset, align_mask,
@@ -699,6 +701,9 @@ void kbase_region_tracker_term(struct kbase_context *kctx)
 	kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_same);
 	kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_custom);
 	kbase_region_tracker_erase_rbtree(&kctx->reg_rbtree_exec);
+#if MALI_USE_CSF
+	WARN_ON(!list_empty(&kctx->csf.event_pages_head));
+#endif
 	kbase_gpu_vm_unlock(kctx);
 }
 
@@ -774,6 +779,9 @@ int kbase_region_tracker_init(struct kbase_context *kctx)
 	kctx->exec_va_start = U64_MAX;
 	kctx->jit_va = false;
 
+#if MALI_USE_CSF
+	INIT_LIST_HEAD(&kctx->csf.event_pages_head);
+#endif
 
 	kbase_gpu_vm_unlock(kctx);
 	return 0;
@@ -959,6 +967,34 @@ exit_unlock:
 	return err;
 }
 
+#if MALI_USE_CSF
+void kbase_mcu_shared_interface_region_tracker_term(struct kbase_device *kbdev)
+{
+	kbase_region_tracker_term_rbtree(&kbdev->csf.shared_reg_rbtree);
+}
+
+int kbase_mcu_shared_interface_region_tracker_init(struct kbase_device *kbdev)
+{
+	struct kbase_va_region *shared_reg;
+	u64 shared_reg_start_pfn;
+	u64 shared_reg_size;
+
+	shared_reg_start_pfn = KBASE_REG_ZONE_MCU_SHARED_BASE;
+	shared_reg_size = KBASE_REG_ZONE_MCU_SHARED_SIZE;
+
+	kbdev->csf.shared_reg_rbtree = RB_ROOT;
+
+	shared_reg = kbase_alloc_free_region(&kbdev->csf.shared_reg_rbtree,
+					shared_reg_start_pfn,
+					shared_reg_size,
+					KBASE_REG_ZONE_MCU_SHARED);
+	if (!shared_reg)
+		return -ENOMEM;
+
+	kbase_region_tracker_insert(shared_reg);
+	return 0;
+}
+#endif
 
 int kbase_mem_init(struct kbase_device *kbdev)
 {
@@ -1156,6 +1192,13 @@ static struct kbase_context *kbase_reg_flags_to_kctx(
  */
 void kbase_free_alloced_region(struct kbase_va_region *reg)
 {
+#if MALI_USE_CSF
+	if ((reg->flags & KBASE_REG_ZONE_MASK) ==
+			KBASE_REG_ZONE_MCU_SHARED) {
+		kfree(reg);
+		return;
+	}
+#endif
 	if (!(reg->flags & KBASE_REG_FREE)) {
 		struct kbase_context *kctx = kbase_reg_flags_to_kctx(reg);
 
@@ -1167,6 +1210,10 @@ void kbase_free_alloced_region(struct kbase_va_region *reg)
 
 		dev_dbg(kctx->kbdev->dev, "Freeing memory region %p\n",
 			(void *)reg);
+#if MALI_USE_CSF
+		if (reg->flags & KBASE_REG_CSF_EVENT)
+			kbase_unlink_event_mem_page(kctx, reg);
+#endif
 
 		mutex_lock(&kctx->jit_evict_lock);
 
@@ -1417,7 +1464,7 @@ static struct kbase_cpu_mapping *kbasep_find_enclosing_cpu_mapping(
 	unsigned long map_start;
 	size_t map_size;
 
-	lockdep_assert_held(&current->mm->mmap_sem);
+	lockdep_assert_held(kbase_mem_get_process_mmap_lock());
 
 	if ((uintptr_t) uaddr + size < (uintptr_t) uaddr) /* overflow check */
 		return NULL;
@@ -1848,9 +1895,25 @@ int kbase_update_region_flags(struct kbase_context *kctx,
 		reg->flags |= KBASE_REG_SHARE_IN;
 	}
 
+#if !MALI_USE_CSF
 	if (flags & BASE_MEM_TILER_ALIGN_TOP)
 		reg->flags |= KBASE_REG_TILER_ALIGN_TOP;
+#endif /* !MALI_USE_CSF */
 
+#if MALI_USE_CSF
+	if (flags & BASE_MEM_CSF_EVENT) {
+		reg->flags |= KBASE_REG_CSF_EVENT;
+		reg->flags |= KBASE_REG_PERMANENT_KERNEL_MAPPING;
+
+		if (!(reg->flags & KBASE_REG_SHARE_BOTH)) {
+			/* On non coherent platforms need to map as uncached on
+			 * both sides.
+			 */
+			reg->flags &= ~KBASE_REG_CPU_CACHED;
+			reg->flags &= ~KBASE_REG_GPU_CACHED;
+		}
+	}
+#endif
 
 	/* Set up default MEMATTR usage */
 	if (!(reg->flags & KBASE_REG_GPU_CACHED)) {
@@ -1864,6 +1927,13 @@ int kbase_update_region_flags(struct kbase_context *kctx,
 				"Can't allocate GPU uncached memory due to MMU in Legacy Mode\n");
 			return -EINVAL;
 		}
+#if MALI_USE_CSF
+	} else if (reg->flags & KBASE_REG_CSF_EVENT) {
+		WARN_ON(!(reg->flags & KBASE_REG_SHARE_BOTH));
+
+		reg->flags |=
+			KBASE_REG_MEMATTR_INDEX(AS_MEMATTR_INDEX_SHARED);
+#endif
 	} else if (kctx->kbdev->system_coherency == COHERENCY_ACE &&
 		(reg->flags & KBASE_REG_SHARE_BOTH)) {
 		reg->flags |=
@@ -2522,6 +2592,13 @@ void kbase_free_phy_pages_helper_locked(struct kbase_mem_phy_alloc *alloc,
 	}
 }
 
+#if MALI_USE_CSF
+/**
+ * kbase_jd_user_buf_unpin_pages - Release the pinned pages of a user buffer.
+ * @alloc: The allocation for the imported user buffer.
+ */
+static void kbase_jd_user_buf_unpin_pages(struct kbase_mem_phy_alloc *alloc);
+#endif
 
 void kbase_mem_kref_free(struct kref *kref)
 {
@@ -2589,6 +2666,9 @@ void kbase_mem_kref_free(struct kref *kref)
 		dma_buf_put(alloc->imported.umm.dma_buf);
 		break;
 	case KBASE_MEM_TYPE_IMPORTED_USER_BUF:
+#if MALI_USE_CSF
+		kbase_jd_user_buf_unpin_pages(alloc);
+#endif
 		if (alloc->imported.user_buf.mm)
 			mmdrop(alloc->imported.user_buf.mm);
 		if (alloc->properties & KBASE_MEM_PHY_ALLOC_LARGE)
@@ -2673,12 +2753,14 @@ bool kbase_check_alloc_flags(unsigned long flags)
 			(BASE_MEM_PROT_GPU_WR | BASE_MEM_GROW_ON_GPF)))
 		return false;
 
+#if !MALI_USE_CSF
 	/* GPU executable memory also cannot have the top of its initial
 	 * commit aligned to 'extent'
 	 */
 	if ((flags & BASE_MEM_PROT_GPU_EX) && (flags &
 			BASE_MEM_TILER_ALIGN_TOP))
 		return false;
+#endif /* !MALI_USE_CSF */
 
 	/* To have an allocation lie within a 4GB chunk is required only for
 	 * TLS memory, which will never be used to contain executable code.
@@ -2687,10 +2769,12 @@ bool kbase_check_alloc_flags(unsigned long flags)
 			BASE_MEM_PROT_GPU_EX))
 		return false;
 
+#if !MALI_USE_CSF
 	/* TLS memory should also not be used for tiler heap */
 	if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) && (flags &
 			BASE_MEM_TILER_ALIGN_TOP))
 		return false;
+#endif /* !MALI_USE_CSF */
 
 	/* GPU should have at least read or write access otherwise there is no
 	   reason for allocating. */
@@ -2734,9 +2818,11 @@ bool kbase_check_import_flags(unsigned long flags)
 	if (flags & BASE_MEM_GROW_ON_GPF)
 		return false;
 
+#if !MALI_USE_CSF
 	/* Imported memory cannot be aligned to the end of its initial commit */
 	if (flags & BASE_MEM_TILER_ALIGN_TOP)
 		return false;
+#endif /* !MALI_USE_CSF */
 
 	/* GPU should have at least read or write access otherwise there is no
 	   reason for importing. */
@@ -2791,6 +2877,7 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags,
 		return -EINVAL;
 	}
 
+#if !MALI_USE_CSF
 	if ((flags & BASE_MEM_TILER_ALIGN_TOP) && (test_reg.extent == 0)) {
 		dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_TILER_ALIGN_TOP but extent == 0\n");
 		return -EINVAL;
@@ -2801,7 +2888,14 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags,
 		dev_warn(dev, KBASE_MSG_PRE "neither BASE_MEM_GROW_ON_GPF nor BASE_MEM_TILER_ALIGN_TOP set but extent != 0\n");
 		return -EINVAL;
 	}
+#else
+	if (!(flags & BASE_MEM_GROW_ON_GPF) && test_reg.extent != 0) {
+		dev_warn(dev, KBASE_MSG_PRE "BASE_MEM_GROW_ON_GPF not set but extent != 0\n");
+		return -EINVAL;
+	}
+#endif /* !MALI_USE_CSF */
 
+#if !MALI_USE_CSF
 	/* BASE_MEM_TILER_ALIGN_TOP memory has a number of restrictions */
 	if (flags & BASE_MEM_TILER_ALIGN_TOP) {
 #define KBASE_MSG_PRE_FLAG KBASE_MSG_PRE "BASE_MEM_TILER_ALIGN_TOP and "
@@ -2831,6 +2925,7 @@ int kbase_check_alloc_sizes(struct kbase_context *kctx, unsigned long flags,
 		}
 #undef KBASE_MSG_PRE_FLAG
 	}
+#endif /* !MALI_USE_CSF */
 
 	if ((flags & BASE_MEM_GPU_VA_SAME_4GB_PAGE) &&
 	    (va_pages > (BASE_MEM_PFN_MASK_4GB + 1))) {
@@ -3028,13 +3123,17 @@ static int kbase_jit_debugfs_used_get(struct kbase_jit_debugfs_data *data)
 	struct kbase_context *kctx = data->kctx;
 	struct kbase_va_region *reg;
 
+#if !MALI_USE_CSF
 	mutex_lock(&kctx->jctx.lock);
+#endif /* !MALI_USE_CSF */
 	mutex_lock(&kctx->jit_evict_lock);
 	list_for_each_entry(reg, &kctx->jit_active_head, jit_node) {
 		data->active_value += reg->used_pages;
 	}
 	mutex_unlock(&kctx->jit_evict_lock);
+#if !MALI_USE_CSF
 	mutex_unlock(&kctx->jctx.lock);
+#endif /* !MALI_USE_CSF */
 
 	return 0;
 }
@@ -3051,7 +3150,9 @@ static int kbase_jit_debugfs_trim_get(struct kbase_jit_debugfs_data *data)
 	struct kbase_context *kctx = data->kctx;
 	struct kbase_va_region *reg;
 
+#if !MALI_USE_CSF
 	mutex_lock(&kctx->jctx.lock);
+#endif /* !MALI_USE_CSF */
 	kbase_gpu_vm_lock(kctx);
 	mutex_lock(&kctx->jit_evict_lock);
 	list_for_each_entry(reg, &kctx->jit_active_head, jit_node) {
@@ -3070,7 +3171,9 @@ static int kbase_jit_debugfs_trim_get(struct kbase_jit_debugfs_data *data)
 	}
 	mutex_unlock(&kctx->jit_evict_lock);
 	kbase_gpu_vm_unlock(kctx);
+#if !MALI_USE_CSF
 	mutex_unlock(&kctx->jctx.lock);
+#endif /* !MALI_USE_CSF */
 
 	return 0;
 }
@@ -3177,8 +3280,13 @@ int kbase_jit_init(struct kbase_context *kctx)
 	INIT_LIST_HEAD(&kctx->jit_destroy_head);
 	INIT_WORK(&kctx->jit_work, kbase_jit_destroy_worker);
 
+#if MALI_USE_CSF
+	INIT_LIST_HEAD(&kctx->csf.kcpu_queues.jit_cmds_head);
+	INIT_LIST_HEAD(&kctx->csf.kcpu_queues.jit_blocked_queues);
+#else /* !MALI_USE_CSF */
 	INIT_LIST_HEAD(&kctx->jctx.jit_atoms_head);
 	INIT_LIST_HEAD(&kctx->jctx.jit_pending_alloc);
+#endif /* MALI_USE_CSF */
 	mutex_unlock(&kctx->jit_evict_lock);
 
 	kctx->jit_max_allocations = 0;
@@ -3201,6 +3309,7 @@ static bool meet_size_and_tiler_align_top_requirements(
 	if (walker->nr_pages != info->va_pages)
 		meet_reqs = false;
 
+#if !MALI_USE_CSF
 	if (meet_reqs && (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP)) {
 		size_t align = info->extent;
 		size_t align_mask = align - 1;
@@ -3208,6 +3317,7 @@ static bool meet_size_and_tiler_align_top_requirements(
 		if ((walker->start_pfn + info->commit_pages) & align_mask)
 			meet_reqs = false;
 	}
+#endif /* !MALI_USE_CSF */
 
 	return meet_reqs;
 }
@@ -3226,7 +3336,9 @@ static int kbase_mem_jit_trim_pages_from_region(struct kbase_context *kctx,
 	size_t to_free = 0u;
 	size_t max_allowed_pages = old_pages;
 
+#if !MALI_USE_CSF
 	lockdep_assert_held(&kctx->jctx.lock);
+#endif /* !MALI_USE_CSF */
 	lockdep_assert_held(&kctx->reg_lock);
 
 	/* Is this a JIT allocation that has been reported on? */
@@ -3349,7 +3461,9 @@ static size_t kbase_mem_jit_trim_pages(struct kbase_context *kctx,
 	struct kbase_va_region *reg, *tmp;
 	size_t total_freed = 0;
 
+#if !MALI_USE_CSF
 	lockdep_assert_held(&kctx->jctx.lock);
+#endif /* !MALI_USE_CSF */
 	lockdep_assert_held(&kctx->reg_lock);
 	lockdep_assert_held(&kctx->jit_evict_lock);
 
@@ -3552,7 +3666,9 @@ void kbase_jit_trim_necessary_pages(struct kbase_context *kctx,
 	size_t jit_backing = 0;
 	size_t pages_to_trim = 0;
 
+#if !MALI_USE_CSF
 	lockdep_assert_held(&kctx->jctx.lock);
+#endif /* !MALI_USE_CSF */
 	lockdep_assert_held(&kctx->reg_lock);
 	lockdep_assert_held(&kctx->jit_evict_lock);
 
@@ -3601,7 +3717,11 @@ static bool jit_allow_allocate(struct kbase_context *kctx,
 		const struct base_jit_alloc_info *info,
 		bool ignore_pressure_limit)
 {
+#if MALI_USE_CSF
+	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
+#else
 	lockdep_assert_held(&kctx->jctx.lock);
+#endif
 
 #if MALI_JIT_PRESSURE_LIMIT_BASE
 	if (!ignore_pressure_limit &&
@@ -3687,7 +3807,11 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx,
 	struct kbase_sub_alloc *prealloc_sas[2] = { NULL, NULL };
 	int i;
 
+#if MALI_USE_CSF
+	lockdep_assert_held(&kctx->csf.kcpu_queues.lock);
+#else
 	lockdep_assert_held(&kctx->jctx.lock);
+#endif
 
 	if (!jit_allow_allocate(kctx, info, ignore_pressure_limit))
 		return NULL;
@@ -3818,8 +3942,10 @@ struct kbase_va_region *kbase_jit_allocate(struct kbase_context *kctx,
 				BASEP_MEM_NO_USER_FREE;
 		u64 gpu_addr;
 
+#if !MALI_USE_CSF
 		if (info->flags & BASE_JIT_ALLOC_MEM_TILER_ALIGN_TOP)
 			flags |= BASE_MEM_TILER_ALIGN_TOP;
+#endif /* !MALI_USE_CSF */
 
 		flags |= base_mem_group_id_set(kctx->jit_group_id);
 #if MALI_JIT_PRESSURE_LIMIT_BASE
@@ -4089,7 +4215,9 @@ void kbase_jit_report_update_pressure(struct kbase_context *kctx,
 {
 	u64 diff;
 
+#if !MALI_USE_CSF
 	lockdep_assert_held(&kctx->jctx.lock);
+#endif /* !MALI_USE_CSF */
 
 	trace_mali_jit_report_pressure(reg, new_used_pages,
 		kctx->jit_current_phys_pressure + new_used_pages -
@@ -4131,6 +4259,20 @@ bool kbase_has_exec_va_zone(struct kbase_context *kctx)
 	return has_exec_va_zone;
 }
 
+#if MALI_USE_CSF
+static void kbase_jd_user_buf_unpin_pages(struct kbase_mem_phy_alloc *alloc)
+{
+	if (alloc->nents) {
+		struct page **pages = alloc->imported.user_buf.pages;
+		long i;
+
+		WARN_ON(alloc->nents != alloc->imported.user_buf.nr_pages);
+
+		for (i = 0; i < alloc->nents; i++)
+			put_page(pages[i]);
+	}
+}
+#endif
 
 int kbase_jd_user_buf_pin_pages(struct kbase_context *kctx,
 		struct kbase_va_region *reg)
@@ -4297,12 +4439,16 @@ static void kbase_jd_user_buf_unmap(struct kbase_context *kctx,
 				DMA_BIDIRECTIONAL);
 		if (writeable)
 			set_page_dirty_lock(pages[i]);
+#if !MALI_USE_CSF
 		put_page(pages[i]);
 		pages[i] = NULL;
+#endif
 
 		size -= local_size;
 	}
+#if !MALI_USE_CSF
 	alloc->nents = 0;
+#endif
 }
 
 int kbase_mem_copy_to_pinned_user_pages(struct page **dest_pages,