mali_pixel: Implement SLC partition ref counting

Bug: 313458962
Test: gfxbench
Change-Id: Id2ab03db4b8b3122bc1f87ecb97715fa3a5d80e4
Signed-off-by: Jack Diver <diverj@google.com>

6 files changed, 214 insertions, 5 deletions

diff --git a/common/include/uapi/gpu/arm/midgard/platform/pixel/pixel_memory_group_manager.h b/common/include/uapi/gpu/arm/midgard/platform/pixel/pixel_memory_group_manager.h
index 893cdca..d410f7b 100644
--- a/common/include/uapi/gpu/arm/midgard/platform/pixel/pixel_memory_group_manager.h
+++ b/common/include/uapi/gpu/arm/midgard/platform/pixel/pixel_memory_group_manager.h
@@ -7,4 +7,8 @@
 #ifndef _UAPI_PIXEL_MEMORY_GROUP_MANAGER_H_
 #define _UAPI_PIXEL_MEMORY_GROUP_MANAGER_H_
 
+void pixel_mgm_slc_inc_refcount(struct memory_group_manager_device* mgm_dev);
+
+void pixel_mgm_slc_dec_refcount(struct memory_group_manager_device* mgm_dev);
+
 #endif /* _UAPI_PIXEL_MEMORY_GROUP_MANAGER_H_ */
diff --git a/mali_kbase/platform/pixel/mali_kbase_config_platform.h b/mali_kbase/platform/pixel/mali_kbase_config_platform.h
index 6dabc15..499f6f8 100644
--- a/mali_kbase/platform/pixel/mali_kbase_config_platform.h
+++ b/mali_kbase/platform/pixel/mali_kbase_config_platform.h
@@ -471,12 +471,14 @@ struct pixel_context {
 /**
  * struct pixel_platform_data - Per kbase_context Pixel specific platform data
  *
- * @kctx:  Handle to the parent kctx
- * @stats: Tracks the dvfs metrics for the UID associated with this context
+ * @kctx:     Handle to the parent kctx
+ * @stats:    Tracks the dvfs metrics for the UID associated with this context
+ * @slc_vote: Tracks whether this context is voting for slc
  */
 struct pixel_platform_data {
 	struct kbase_context *kctx;
 	struct gpu_dvfs_metrics_uid_stats* stats;
+	int slc_vote;
 };
 
 #endif /* _KBASE_CONFIG_PLATFORM_H_ */
diff --git a/mali_kbase/platform/pixel/pixel_gpu_slc.c b/mali_kbase/platform/pixel/pixel_gpu_slc.c
index 8e46be1..e8aae75 100644
--- a/mali_kbase/platform/pixel/pixel_gpu_slc.c
+++ b/mali_kbase/platform/pixel/pixel_gpu_slc.c
@@ -17,6 +17,37 @@
 #include "mali_kbase_config_platform.h"
 #include "pixel_gpu_slc.h"
 
+#include <uapi/gpu/arm/midgard/platform/pixel/pixel_memory_group_manager.h>
+
+/**
+ * enum slc_vote_state - Whether a context is voting for SLC
+ */
+enum slc_vote_state {
+	/** @IDLE: Idle, not voting for SLC */
+	IDLE   = 0,
+	/** @VOTING: Active, voting for SLC */
+	VOTING = 1,
+};
+
+/**
+ * transition() - Try to transition from one value to another
+ *
+ * @v:   Value to transition
+ * @old: Starting state to transition from
+ * @new: Destination state to transition to
+ *
+ * Return: Whether the transition was successful
+ */
+static bool transition(int *v, int old, int new)
+{
+	bool const cond = *v == old;
+
+	if (cond)
+		*v = new;
+
+	return cond;
+}
+
 /**
  * gpu_pixel_handle_buffer_liveness_update_ioctl() - See gpu_slc_liveness_update
  *
@@ -57,7 +88,11 @@ int gpu_slc_kctx_init(struct kbase_context *kctx)
  */
 void gpu_slc_kctx_term(struct kbase_context *kctx)
 {
-	(void)kctx;
+	struct pixel_platform_data *pd = kctx->platform_data;
+
+	/* Contexts can be terminated without being idled first */
+	if (transition(&pd->slc_vote, VOTING, IDLE))
+		pixel_mgm_slc_dec_refcount(kctx->kbdev->mgm_dev);
 }
 
 /**
@@ -67,7 +102,12 @@ void gpu_slc_kctx_term(struct kbase_context *kctx)
  */
 void gpu_slc_kctx_active(struct kbase_context *kctx)
 {
-	(void)kctx;
+	struct pixel_platform_data *pd = kctx->platform_data;
+
+	lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
+
+	if (transition(&pd->slc_vote, IDLE, VOTING))
+		pixel_mgm_slc_inc_refcount(kctx->kbdev->mgm_dev);
 }
 
 /**
@@ -77,7 +117,12 @@ void gpu_slc_kctx_active(struct kbase_context *kctx)
  */
 void gpu_slc_kctx_idle(struct kbase_context *kctx)
 {
-	(void)kctx;
+	struct pixel_platform_data *pd = kctx->platform_data;
+
+	lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
+
+	if (transition(&pd->slc_vote, VOTING, IDLE))
+		pixel_mgm_slc_dec_refcount(kctx->kbdev->mgm_dev);
 }
 
 /**
diff --git a/mali_pixel/memory_group_manager.c b/mali_pixel/memory_group_manager.c
index e46856b..bc596bb 100644
--- a/mali_pixel/memory_group_manager.c
+++ b/mali_pixel/memory_group_manager.c
@@ -533,6 +533,22 @@ static vm_fault_t mgm_vmf_insert_pfn_prot(
 	return fault;
 }
 
+void pixel_mgm_slc_inc_refcount(struct memory_group_manager_device* mgm_dev)
+{
+	struct mgm_groups *const data = mgm_dev->data;
+
+	slc_inc_refcount(&data->slc_data);
+}
+EXPORT_SYMBOL_GPL(pixel_mgm_slc_inc_refcount);
+
+void pixel_mgm_slc_dec_refcount(struct memory_group_manager_device* mgm_dev)
+{
+	struct mgm_groups *const data = mgm_dev->data;
+
+	slc_dec_refcount(&data->slc_data);
+}
+EXPORT_SYMBOL_GPL(pixel_mgm_slc_dec_refcount);
+
 static int mgm_initialize_data(struct mgm_groups *mgm_data)
 {
 	int i, ret;
diff --git a/mali_pixel/pixel_slc.c b/mali_pixel/pixel_slc.c
index 62c6908..78f1b74 100644
--- a/mali_pixel/pixel_slc.c
+++ b/mali_pixel/pixel_slc.c
@@ -37,6 +37,50 @@
 #endif
 #define PBHA_BIT_MASK (0xf)
 
+#define PARTITION_DISABLE_HYSTERESIS (msecs_to_jiffies(100))
+
+
+/**
+ * partition_required() - Determine whether we require a partition to be enabled
+ *
+ * @pt: The partition to check.
+ *
+ * Check whether a partition meets the requirements for being enabled.
+ *
+ * Return: True, if the partition is required to be enabled, otherwise false.
+ */
+static bool partition_required(struct slc_partition *pt)
+{
+	lockdep_assert_held(&pt->lock);
+
+	return atomic_read(&pt->refcount);
+}
+
+/**
+ * pixel_atomic_dec_and_lock_irqsave - lock on reaching reference count zero
+ *
+ * @val:   The atomic counter
+ * @lock:  The spinlock in question
+ * @flags: Storage for the current interrupt enable state
+ *
+ * Decrements @val by 1, if the result is 0, locks @lock.
+ *
+ * Return: True if the lock was taken, false for all other cases.
+ */
+static int pixel_atomic_dec_and_lock_irqsave(atomic_t* val, spinlock_t* lock, unsigned long* flags)
+{
+	/* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */
+	if (atomic_add_unless(val, -1, 1))
+		return 0;
+
+	/* Otherwise do it the slow way */
+	spin_lock_irqsave(lock, *flags);
+	if (atomic_dec_and_test(val))
+		return 1;
+	spin_unlock_irqrestore(lock, *flags);
+
+	return 0;
+}
 
 /**
  * slc_wipe_pbha - Clear any set PBHA bits from the pte.
@@ -104,6 +148,84 @@ static void disable_partition(struct slc_data *data, struct slc_partition *pt)
 }
 
 /**
+ * queue_disable_worker - Queue a delayed partition disable op
+ *
+ * @data: The &struct slc_data tracking partition information.
+ */
+static void queue_disable_worker(struct slc_data *data)
+{
+	queue_delayed_work(system_highpri_wq, &data->disable_work, PARTITION_DISABLE_HYSTERESIS);
+}
+
+/**
+ * partition_disable_worker - Callback to lazily disable a partition
+ *
+ * @work: The &struct work_struct dequeued
+ */
+static void partition_disable_worker(struct work_struct *work)
+{
+	struct slc_data* data = container_of(work, struct slc_data, disable_work.work);
+	struct slc_partition *pt = &data->partition;
+	unsigned long flags;
+
+	/* Complete any pending disable ops */
+	spin_lock_irqsave(&pt->lock, flags);
+
+	if (!partition_required(pt))
+		disable_partition(data, pt);
+
+	spin_unlock_irqrestore(&pt->lock, flags);
+}
+
+/**
+ * slc_inc_refcount - Increase the partition reference count.
+ *
+ * @data: The &struct slc_data tracking partition information.
+ *
+ * If this is the first reference being taken, the partition will be enabled.
+ */
+void slc_inc_refcount(struct slc_data *data)
+{
+	struct slc_partition *pt = &data->partition;
+
+	/* Try to re-enable the partition if this is the first reference */
+	if (atomic_inc_return(&pt->refcount) == 1) {
+		unsigned long flags;
+
+		spin_lock_irqsave(&pt->lock, flags);
+
+		/* Enable the partition immediately if it's required */
+		if (partition_required(pt))
+			enable_partition(data, pt);
+
+		spin_unlock_irqrestore(&pt->lock, flags);
+	}
+}
+
+/**
+ * slc_dec_refcount - Decrease the partition reference count.
+ *
+ * @data: The &struct slc_data tracking partition information.
+ *
+ * If this is the last reference being released, the partition will be disabled.
+ */
+void slc_dec_refcount(struct slc_data *data)
+{
+	struct slc_partition *pt = &data->partition;
+	unsigned long flags;
+
+	/* Disable the partition if this was the last reference */
+	if (pixel_atomic_dec_and_lock_irqsave(&pt->refcount, &pt->lock, &flags)) {
+
+		/* Lazily disable the partition if it's no longer required */
+		if (!partition_required(pt))
+			queue_disable_worker(data);
+
+		spin_unlock_irqrestore(&pt->lock, flags);
+	}
+}
+
+/**
  * init_partition - Register and initialize a partition with the SLC driver.
  *
  * @data:  The &struct slc_data tracking partition information.
@@ -141,7 +263,9 @@ static int init_partition(struct slc_data *data, struct slc_partition *pt, u32 i
 		.ptid = ptid,
 		.pbha = pbha,
 		.enabled = false,
+		.refcount = ATOMIC_INIT(0),
 	};
+	spin_lock_init(&pt->lock);
 
 err_exit:
 	return err;
@@ -180,6 +304,8 @@ int slc_init_data(struct slc_data *data, struct device* dev)
 	/* Inherit the platform device */
 	data->dev = dev;
 
+	INIT_DELAYED_WORK(&data->disable_work, partition_disable_worker);
+
 	/* Register our node with the SLC driver.
 	 * This detects our partitions defined within the DT.
 	 */
@@ -209,6 +335,9 @@ err_exit:
  */
 void slc_term_data(struct slc_data *data)
 {
+	/* Ensure all pending disable ops are complete */
+	cancel_delayed_work_sync(&data->disable_work);
+
 	term_partition(data, &data->partition);
 
 	pt_client_unregister(data->pt_handle);
diff --git a/mali_pixel/pixel_slc.h b/mali_pixel/pixel_slc.h
index 40b5ad7..bcaf1ff 100644
--- a/mali_pixel/pixel_slc.h
+++ b/mali_pixel/pixel_slc.h
@@ -45,6 +45,12 @@ struct slc_partition {
 
 	/** @enabled: Is the partition currently enabled */
 	bool enabled;
+
+	/** @refcount: Reference count for this partition */
+	atomic_t refcount;
+
+	/** @lock: Lock protecting enable/disable ops on this partition */
+	spinlock_t lock;
 };
 
 /**
@@ -59,6 +65,9 @@ struct slc_data {
 
 	/** @dev: Inherited pointer to device attached */
 	struct device *dev;
+
+	/** @disable_work: Work item used to queue lazy SLC partition disable ops. */
+	struct delayed_work disable_work;
 };
 
 int slc_init_data(struct slc_data *data, struct device* dev);
@@ -69,4 +78,8 @@ u64 slc_set_pbha(struct slc_data const *data, u64 pte);
 
 u64 slc_wipe_pbha(u64 pte);
 
+void slc_inc_refcount(struct slc_data *data);
+
+void slc_dec_refcount(struct slc_data *data);
+
 #endif /* _PIXEL_SLC_H_ */