Mali Bifrost DDK r3p0 KMD

Provenance:
 7ee75b28d (collaborate/EAC/b_r3p0)
 BX304L01B-BU-00000-r3p0-00rel0
 BX304L06A-BU-00000-r3p0-00rel0
 BX304X07X-BU-00000-r3p0-00rel0

Signed-off-by: Sidath Senanayake <sidaths@google.com>
Change-Id: I7a45a78c9d84463008986343e4cf3d11e467dce8

1 files changed, 95 insertions, 29 deletions

diff --git a/mali_kbase/backend/gpu/mali_kbase_jm_rb.c b/mali_kbase/backend/gpu/mali_kbase_jm_rb.c
index d7b4d3f..caf0e2f 100644
--- a/mali_kbase/backend/gpu/mali_kbase_jm_rb.c
+++ b/mali_kbase/backend/gpu/mali_kbase_jm_rb.c
@@ -719,6 +719,30 @@ static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js)
 	return true;
 }
 
+/**
+ * other_slots_busy - Determine if any job slots other than @js are currently
+ *                    running atoms
+ * @kbdev: Device pointer
+ * @js:    Job slot
+ *
+ * Return: true if any slots other than @js are busy, false otherwise
+ */
+static inline bool other_slots_busy(struct kbase_device *kbdev, int js)
+{
+	int slot;
+
+	for (slot = 0; slot < kbdev->gpu_props.num_job_slots; slot++) {
+		if (slot == js)
+			continue;
+
+		if (kbase_gpu_nr_atoms_on_slot_min(kbdev, slot,
+				KBASE_ATOM_GPU_RB_SUBMITTED))
+			return true;
+	}
+
+	return false;
+}
+
 static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev)
 {
 	return kbdev->protected_mode;
@@ -1010,6 +1034,12 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
 						katom[idx])))
 					break;
 
+				if ((idx == 1) && (kbase_jd_katom_is_protected(
+								katom[0]) !=
+						kbase_jd_katom_is_protected(
+								katom[1])))
+					break;
+
 				if (kbdev->protected_mode_transition)
 					break;
 
@@ -1104,13 +1134,33 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
 
 			case KBASE_ATOM_GPU_RB_READY:
 
-				/* Only submit if head atom or previous atom
-				 * already submitted */
-				if (idx == 1 &&
-					(katom[0]->gpu_rb_state !=
+				if (idx == 1) {
+					/* Only submit if head atom or previous
+					 * atom already submitted */
+					if ((katom[0]->gpu_rb_state !=
 						KBASE_ATOM_GPU_RB_SUBMITTED &&
-					katom[0]->gpu_rb_state !=
+						katom[0]->gpu_rb_state !=
 					KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
+						break;
+
+					/* If intra-slot serialization in use
+					 * then don't submit atom to NEXT slot
+					 */
+					if (kbdev->serialize_jobs &
+						KBASE_SERIALIZE_INTRA_SLOT)
+						break;
+				}
+
+				/* If inter-slot serialization in use then don't
+				 * submit atom if any other slots are in use */
+				if ((kbdev->serialize_jobs &
+						KBASE_SERIALIZE_INTER_SLOT) &&
+						other_slots_busy(kbdev, js))
+					break;
+
+				if ((kbdev->serialize_jobs &
+						KBASE_SERIALIZE_RESET) &&
+						kbase_reset_gpu_active(kbdev))
 					break;
 
 				/* Check if this job needs the cycle counter
@@ -1166,6 +1216,9 @@ void kbase_backend_run_atom(struct kbase_device *kbdev,
 	kbase_backend_slot_update(kbdev);
 }
 
+#define HAS_DEP(katom) (katom->pre_dep || katom->atom_flags & \
+	(KBASE_KATOM_FLAG_X_DEP_BLOCKED | KBASE_KATOM_FLAG_FAIL_BLOCKER))
+
 bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js)
 {
 	struct kbase_jd_atom *katom;
@@ -1178,6 +1231,7 @@ bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js)
 
 	if (next_katom && katom->kctx == next_katom->kctx &&
 		next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED &&
+		HAS_DEP(next_katom) &&
 		(kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL)
 									!= 0 ||
 		kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL)
@@ -1224,12 +1278,12 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
 
 	katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
 	kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0);
-	kbase_tlstream_tl_nret_atom_lpu(
+	KBASE_TLSTREAM_TL_NRET_ATOM_LPU(
 			katom,
 			&kbdev->gpu_props.props.raw_props.js_features[
 				katom->slot_nr]);
-	kbase_tlstream_tl_nret_atom_as(katom, &kbdev->as[kctx->as_nr]);
-	kbase_tlstream_tl_nret_ctx_lpu(
+	KBASE_TLSTREAM_TL_NRET_ATOM_AS(katom, &kbdev->as[kctx->as_nr]);
+	KBASE_TLSTREAM_TL_NRET_CTX_LPU(
 			kctx,
 			&kbdev->gpu_props.props.raw_props.js_features[
 				katom->slot_nr]);
@@ -1244,7 +1298,9 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
 		 * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that
 		 * the atoms on this slot are returned in the correct order.
 		 */
-		if (next_katom && katom->kctx == next_katom->kctx) {
+		if (next_katom && katom->kctx == next_katom->kctx &&
+				next_katom->sched_priority ==
+				katom->sched_priority) {
 			kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
 			kbase_jm_return_atom_to_js(kbdev, next_katom);
 		}
@@ -1270,14 +1326,16 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
 						kbase_gpu_inspect(kbdev, i, 1);
 
 			if (katom_idx0 && katom_idx0->kctx == katom->kctx &&
-				katom_idx0->gpu_rb_state !=
-				KBASE_ATOM_GPU_RB_SUBMITTED) {
+					HAS_DEP(katom_idx0) &&
+					katom_idx0->gpu_rb_state !=
+					KBASE_ATOM_GPU_RB_SUBMITTED) {
 				/* Dequeue katom_idx0 from ringbuffer */
 				kbase_gpu_dequeue_atom(kbdev, i, end_timestamp);
 
 				if (katom_idx1 &&
-					katom_idx1->kctx == katom->kctx &&
-					katom_idx0->gpu_rb_state !=
+						katom_idx1->kctx == katom->kctx
+						&& HAS_DEP(katom_idx1) &&
+						katom_idx0->gpu_rb_state !=
 						KBASE_ATOM_GPU_RB_SUBMITTED) {
 					/* Dequeue katom_idx1 from ringbuffer */
 					kbase_gpu_dequeue_atom(kbdev, i,
@@ -1293,8 +1351,9 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
 
 			} else if (katom_idx1 &&
 					katom_idx1->kctx == katom->kctx &&
+					HAS_DEP(katom_idx1) &&
 					katom_idx1->gpu_rb_state !=
-						KBASE_ATOM_GPU_RB_SUBMITTED) {
+					KBASE_ATOM_GPU_RB_SUBMITTED) {
 				/* Can not dequeue this atom yet - will be
 				 * dequeued when atom at idx0 completes */
 				katom_idx1->event_code = BASE_JD_EVENT_STOPPED;
@@ -1345,7 +1404,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
 			char js_string[16];
 
 			trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
-								js_string),
+							js_string,
+							sizeof(js_string)),
 						ktime_to_ns(*end_timestamp),
 						(u32)next_katom->kctx->id, 0,
 						next_katom->work_id);
@@ -1355,7 +1415,8 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
 			char js_string[16];
 
 			trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
-								js_string),
+							js_string,
+							sizeof(js_string)),
 						ktime_to_ns(ktime_get()), 0, 0,
 						0);
 			kbdev->hwaccess.backend.slot_rb[js].last_context = 0;
@@ -1363,6 +1424,9 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
 	}
 #endif
 
+	if (kbdev->serialize_jobs & KBASE_SERIALIZE_RESET)
+		kbase_reset_gpu_silent(kbdev);
+
 	if (completion_code == BASE_JD_EVENT_STOPPED)
 		katom = kbase_jm_return_atom_to_js(kbdev, katom);
 	else
@@ -1412,6 +1476,7 @@ void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp)
 			 * it will be processed again from the starting state.
 			 */
 			if (keep_in_jm_rb) {
+				kbasep_js_job_check_deref_cores(kbdev, katom);
 				katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
 				katom->affinity = 0;
 				katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK;
@@ -1441,13 +1506,12 @@ static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev,
 					struct kbase_jd_atom *katom,
 					u32 action)
 {
-	struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 	u32 hw_action = action & JS_COMMAND_MASK;
 
 	kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom);
 	kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action,
 							katom->core_req, katom);
-	kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
+	katom->kctx->blocked_js[js][katom->sched_priority] = true;
 }
 
 static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev,
@@ -1455,11 +1519,9 @@ static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev,
 						u32 action,
 						bool disjoint)
 {
-	struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
-
 	katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
 	kbase_gpu_mark_atom_for_return(kbdev, katom);
-	kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
+	katom->kctx->blocked_js[katom->slot_nr][katom->sched_priority] = true;
 
 	if (disjoint)
 		kbase_job_check_enter_disjoint(kbdev, action, katom->core_req,
@@ -1492,8 +1554,6 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
 					struct kbase_jd_atom *katom,
 					u32 action)
 {
-	struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
-
 	struct kbase_jd_atom *katom_idx0;
 	struct kbase_jd_atom *katom_idx1;
 
@@ -1502,12 +1562,18 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
 	bool ret = false;
 
 	int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1;
+	int prio_idx0 = 0, prio_idx1 = 0;
 
 	lockdep_assert_held(&kbdev->hwaccess_lock);
 
 	katom_idx0 = kbase_gpu_inspect(kbdev, js, 0);
 	katom_idx1 = kbase_gpu_inspect(kbdev, js, 1);
 
+	if (katom_idx0)
+		prio_idx0 = katom_idx0->sched_priority;
+	if (katom_idx1)
+		prio_idx1 = katom_idx1->sched_priority;
+
 	if (katom) {
 		katom_idx0_valid = (katom_idx0 == katom);
 		/* If idx0 is to be removed and idx1 is on the same context,
@@ -1522,9 +1588,10 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
 			katom_idx1_valid = false;
 	} else {
 		katom_idx0_valid = (katom_idx0 &&
-					(!kctx || katom_idx0->kctx == kctx));
+				(!kctx || katom_idx0->kctx == kctx));
 		katom_idx1_valid = (katom_idx1 &&
-					(!kctx || katom_idx1->kctx == kctx));
+				(!kctx || katom_idx1->kctx == kctx) &&
+				prio_idx0 == prio_idx1);
 	}
 
 	if (katom_idx0_valid)
@@ -1541,15 +1608,14 @@ bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
 				katom_idx1->event_code =
 						BASE_JD_EVENT_REMOVED_FROM_NEXT;
 				kbase_jm_return_atom_to_js(kbdev, katom_idx1);
-				kbasep_js_clear_submit_allowed(js_devdata,
-							katom_idx1->kctx);
+				katom_idx1->kctx->blocked_js[js][prio_idx1] =
+						true;
 			}
 
 			katom_idx0->event_code =
 						BASE_JD_EVENT_REMOVED_FROM_NEXT;
 			kbase_jm_return_atom_to_js(kbdev, katom_idx0);
-			kbasep_js_clear_submit_allowed(js_devdata,
-							katom_idx0->kctx);
+			katom_idx0->kctx->blocked_js[js][prio_idx0] = true;
 		} else {
 			/* katom_idx0 is on GPU */
 			if (katom_idx1 && katom_idx1->gpu_rb_state ==