ANDROID: add hooks into blk-mq.c for customized I/O scheduler.

Our scheduler uses the ANDROID_OEM_DATA field in the request_queue
structure to customize a set of ops operations for the request queue
applied by the ufs/emmc driver. Adopt fine-grained IO scheduling
strategies based on specific request types and user scenarios

android_rvh_internal_blk_mq_alloc_request: Obtain a new tag.
This process may be scheduled because the tag cannot be allocated
quickly, so a restricted hook is required.

android_vh_internal_blk_mq_free_request: Release individually allocated
tags

android_vh_blk_mq_complete_request: Record the time when the request was
completed

android_vh_blk_mq_add_to_requeue_list: Add requests to a separate
distribution queue

android_rvh_blk_mq_delay_run_hw_queue: The process of calling queue_rq
to handle the request may be scheduled, so restricted hooks need to be
used

android_vh_blk_mq_run_hw_queue: Set need_run to true

android_vh_blk_mq_insert_request: Insert the request into a customized
queue

android_rvh_blk_mq_alloc_rq_map: The process of assigning customized
tags may be scheduled, so restricted hooks need to be used

android_rvh_blk_mq_init_allocated_queue: The customized scheduler is
initialized and needs to allocate some resources. This process may be
scheduled, so restricted hooks need to be used

android_vh_blk_mq_exit_queue: Release resources allocated by the
customized scheduler

android_vh_blk_mq_alloc_tag_set: Get tagset information

Bug: 319582497

Change-Id: I2b16d69a1e7085a4f5f82660b75188c517b01894
Signed-off-by: hao lv <hao.lv5@transsion.com>

3 files changed, 114 insertions, 5 deletions

diff --git a/block/blk-mq.c b/block/blk-mq.c
index 21544b119a7e..f5dc90f25d31 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -353,6 +353,7 @@ static struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data)
 	struct elevator_queue *e = q->elevator;
 	u64 alloc_time_ns = 0;
 	unsigned int tag;
+	bool skip = false;
 
 	/* alloc_time includes depth and tag waits */
 	if (blk_queue_rq_alloc_time(q))
@@ -384,7 +385,9 @@ retry:
 	 * case just retry the hctx assignment and tag allocation as CPU hotplug
 	 * should have migrated us to an online CPU by now.
 	 */
-	tag = blk_mq_get_tag(data);
+	trace_android_rvh_internal_blk_mq_alloc_request(&skip, &tag, data);
+	if (!skip)
+		tag = blk_mq_get_tag(data);
 	if (tag == BLK_MQ_NO_TAG) {
 		if (data->flags & BLK_MQ_REQ_NOWAIT)
 			return NULL;
@@ -496,12 +499,17 @@ static void __blk_mq_free_request(struct request *rq)
 	struct blk_mq_ctx *ctx = rq->mq_ctx;
 	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
 	const int sched_tag = rq->internal_tag;
+	bool skip = false;
 
 	blk_crypto_free_request(rq);
 	blk_pm_mark_last_busy(rq);
 	rq->mq_hctx = NULL;
-	if (rq->tag != BLK_MQ_NO_TAG)
-		blk_mq_put_tag(hctx->tags, ctx, rq->tag);
+
+	trace_android_vh_internal_blk_mq_free_request(&skip, rq, hctx);
+	if (!skip) {
+		if (rq->tag != BLK_MQ_NO_TAG)
+			blk_mq_put_tag(hctx->tags, ctx, rq->tag);
+	}
 	if (sched_tag != BLK_MQ_NO_TAG)
 		blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
 	blk_mq_sched_restart(hctx);
@@ -701,6 +709,11 @@ EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);
  **/
 void blk_mq_complete_request(struct request *rq)
 {
+	bool skip = false;
+
+	trace_android_vh_blk_mq_complete_request(&skip, rq);
+	if (skip)
+		return;
 	if (!blk_mq_complete_request_remote(rq))
 		rq->q->mq_ops->complete(rq);
 }
@@ -827,7 +840,12 @@ void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
 {
 	struct request_queue *q = rq->q;
 	unsigned long flags;
+	bool skip = false;
 
+	trace_android_vh_blk_mq_add_to_requeue_list(&skip, rq,
+						     kick_requeue_list);
+	if (skip)
+		return;
 	/*
 	 * We abuse this flag that is otherwise used by the I/O scheduler to
 	 * request head insertion from the workqueue.
@@ -1593,9 +1611,15 @@ select_cpu:
 static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async,
 					unsigned long msecs)
 {
+	bool skip = false;
+
 	if (unlikely(blk_mq_hctx_stopped(hctx)))
 		return;
 
+	trace_android_rvh_blk_mq_delay_run_hw_queue(&skip, hctx, async);
+	if (skip)
+		return;
+
 	if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
 		int cpu = get_cpu();
 		if (cpumask_test_cpu(cpu, hctx->cpumask)) {
@@ -1651,6 +1675,7 @@ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
 		blk_mq_hctx_has_pending(hctx);
 	hctx_unlock(hctx, srcu_idx);
 
+	trace_android_vh_blk_mq_run_hw_queue(&need_run, hctx);
 	if (need_run)
 		__blk_mq_delay_run_hw_queue(hctx, async, 0);
 }
@@ -1877,9 +1902,14 @@ void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
 			     bool at_head)
 {
 	struct blk_mq_ctx *ctx = rq->mq_ctx;
+	bool skip = false;
 
 	lockdep_assert_held(&ctx->lock);
 
+	trace_android_vh_blk_mq_insert_request(&skip, hctx, rq);
+	if (skip)
+		return;
+
 	__blk_mq_insert_req_list(hctx, rq, at_head);
 	blk_mq_hctx_mark_pending(hctx, ctx);
 }
@@ -2419,12 +2449,15 @@ struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
 {
 	struct blk_mq_tags *tags;
 	int node;
+	bool skip = false;
 
 	node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx);
 	if (node == NUMA_NO_NODE)
 		node = set->numa_node;
 
-	tags = blk_mq_init_tags(nr_tags, reserved_tags, node, flags);
+	trace_android_rvh_blk_mq_alloc_rq_map(&skip, &tags, set, node, flags);
+	if (!skip)
+		tags = blk_mq_init_tags(nr_tags, reserved_tags, node, flags);
 	if (!tags)
 		return NULL;
 
@@ -3362,6 +3395,8 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
 	blk_mq_add_queue_tag_set(set, q);
 	blk_mq_map_swqueue(q);
 
+	trace_android_rvh_blk_mq_init_allocated_queue(q);
+
 	if (elevator_init)
 		elevator_init_mq(q);
 
@@ -3385,6 +3420,7 @@ void blk_mq_exit_queue(struct request_queue *q)
 {
 	struct blk_mq_tag_set *set = q->tag_set;
 
+	trace_android_vh_blk_mq_exit_queue(q);
 	/* Checks hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED. */
 	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
 	/* May clear BLK_MQ_F_TAG_QUEUE_SHARED in hctx->flags. */
@@ -3575,6 +3611,8 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
 	if (ret)
 		goto out_free_mq_map;
 
+	trace_android_vh_blk_mq_alloc_tag_set(set);
+
 	ret = blk_mq_alloc_map_and_requests(set);
 	if (ret)
 		goto out_free_mq_map;
diff --git a/drivers/android/vendor_hooks.c b/drivers/android/vendor_hooks.c
index b5e1c85f1472..88c5f093c6b5 100644
--- a/drivers/android/vendor_hooks.c
+++ b/drivers/android/vendor_hooks.c
@@ -518,6 +518,17 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_vmscan_kswapd_done);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_swap_page_spf);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_exit_check);
 EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_bio_free);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_internal_blk_mq_alloc_request);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_internal_blk_mq_free_request);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_complete_request);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_add_to_requeue_list);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_blk_mq_delay_run_hw_queue);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_run_hw_queue);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_insert_request);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_blk_mq_alloc_rq_map);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_blk_mq_init_allocated_queue);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_exit_queue);
+EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_blk_mq_alloc_tag_set);
 /*
  * For type visibility
  */
diff --git a/include/trace/hooks/block.h b/include/trace/hooks/block.h
index a748dae4671e..39cba4bd0756 100644
--- a/include/trace/hooks/block.h
+++ b/include/trace/hooks/block.h
@@ -14,15 +14,18 @@
 struct blk_mq_tags;
 struct blk_mq_alloc_data;
 struct blk_mq_tag_set;
+struct blk_mq_hw_ctx;
 #else
 /* struct blk_mq_tags */
 #include <../block/blk-mq-tag.h>
 /* struct blk_mq_alloc_data */
 #include <../block/blk-mq.h>
-/* struct blk_mq_tag_set */
+/* struct blk_mq_tag_set struct blk_mq_hw_ctx*/
 #include <linux/blk-mq.h>
 #endif /* __GENKSYMS__ */
 struct bio;
+struct request_queue;
+struct request;
 
 DECLARE_HOOK(android_vh_blk_alloc_rqs,
 	TP_PROTO(size_t *rq_size, struct blk_mq_tag_set *set,
@@ -38,6 +41,63 @@ DECLARE_HOOK(android_vh_bio_free,
 	TP_PROTO(struct bio *bio),
 	TP_ARGS(bio));
 
+DECLARE_RESTRICTED_HOOK(android_rvh_internal_blk_mq_alloc_request,
+	TP_PROTO(bool *skip, int *tag, struct blk_mq_alloc_data *data),
+	TP_ARGS(skip, tag, data), 1);
+
+DECLARE_HOOK(android_vh_internal_blk_mq_free_request,
+	TP_PROTO(bool *skip, struct request *rq, struct blk_mq_hw_ctx *hctx),
+	TP_ARGS(skip, rq, hctx));
+
+DECLARE_HOOK(android_vh_blk_mq_complete_request,
+	TP_PROTO(bool *skip, struct request *rq),
+	TP_ARGS(skip, rq));
+
+DECLARE_HOOK(android_vh_blk_mq_add_to_requeue_list,
+	TP_PROTO(bool *skip, struct request *rq, bool kick_requeue_list),
+	TP_ARGS(skip, rq, kick_requeue_list));
+
+DECLARE_HOOK(android_vh_blk_mq_get_driver_tag,
+	TP_PROTO(struct request *rq),
+	TP_ARGS(rq));
+
+DECLARE_RESTRICTED_HOOK(android_rvh_blk_mq_delay_run_hw_queue,
+	TP_PROTO(bool *skip, struct blk_mq_hw_ctx *hctx, bool async),
+	TP_ARGS(skip, hctx, async), 1);
+
+DECLARE_HOOK(android_vh_blk_mq_run_hw_queue,
+	TP_PROTO(bool *need_run, struct blk_mq_hw_ctx *hctx),
+	TP_ARGS(need_run, hctx));
+
+DECLARE_HOOK(android_vh_blk_mq_insert_request,
+	TP_PROTO(bool *skip, struct blk_mq_hw_ctx *hctx, struct request *rq),
+	TP_ARGS(skip, hctx, rq));
+
+DECLARE_RESTRICTED_HOOK(android_rvh_blk_mq_alloc_rq_map,
+	TP_PROTO(bool *skip, struct blk_mq_tags **tags,
+		struct blk_mq_tag_set *set, int node, unsigned int flags),
+	TP_ARGS(skip, tags, set, node, flags), 1);
+
+DECLARE_HOOK(android_vh_blk_mq_hctx_notify_dead,
+	TP_PROTO(bool *skip, struct blk_mq_hw_ctx *hctx),
+	TP_ARGS(skip, hctx));
+
+DECLARE_RESTRICTED_HOOK(android_rvh_blk_mq_init_allocated_queue,
+	TP_PROTO(struct request_queue *q),
+	TP_ARGS(q), 1);
+
+DECLARE_HOOK(android_vh_blk_mq_exit_queue,
+	TP_PROTO(struct request_queue *q),
+	TP_ARGS(q));
+
+DECLARE_HOOK(android_vh_blk_mq_alloc_tag_set,
+	TP_PROTO(struct blk_mq_tag_set *set),
+	TP_ARGS(set));
+
+DECLARE_HOOK(android_vh_blk_mq_update_nr_requests,
+	TP_PROTO(bool *skip, struct request_queue *q),
+	TP_ARGS(skip, q));
+
 #endif /* _TRACE_HOOK_BLOCK_H */
 
 /* This part must be outside protection */