path: root/android-mainline/ANDROID-mm-Memory-health-driver.patch

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Lee Jones <joneslee@google.com>
Date: Wed, 4 Oct 2023 15:26:58 +0100
Subject: ANDROID: mm: Memory health driver

Android has no-event driven way to know when certain memory conditions
occur, or memory management actions take place. This new memory health
driver will start capturing these cases, starting by keeping track of
out-of-memory (OOM) victims.

To access the list of OOM victims, the driver provides a proc file
named "oom_victim_list" in the "/proc/memhealth" directory. The
proc file supports sequential reading and polling.

Bug: 244232958
Change-Id: Ica5dcb4dc42229d85828f0e495a43d22390927f1
Signed-off-by: Carlos Galo <carlosgalo@google.com
Signed-off-by: Lee Jones <joneslee@google.com>
---
 .../ABI/testing/procfs-driver-memhealth       |  43 ++
 drivers/android/Kconfig                       |  12 +
 drivers/android/Makefile                      |   1 +
 drivers/android/memhealth.c                   | 423 ++++++++++++++++++
 mm/oom_kill.c                                 |   6 +
 mm/vmscan.c                                   |   7 +
 6 files changed, 492 insertions(+)
 create mode 100644 Documentation/ABI/testing/procfs-driver-memhealth
 create mode 100644 drivers/android/memhealth.c

diff --git a/Documentation/ABI/testing/procfs-driver-memhealth b/Documentation/ABI/testing/procfs-driver-memhealth
new file mode 100644
--- /dev/null
+++ b/Documentation/ABI/testing/procfs-driver-memhealth
@@ -0,0 +1,43 @@
+What:       /proc/memhealth/
+Date:       July 2023
+Contact:    carlosgalo@google.com
+Description:
+        This folder contains attributes that track different memory related events.
+
+What:       /proc/memhealth/oom_victim_list
+Date:       July 2023
+Contact:    carlosgalo@google.com
+Description:
+        This attribute shows information about out-of-memory (OOM)
+        victims. The format is
+        "<timestamp> <pid> <uid> <oom_score_adjusted> <process_name>\n",
+        each value is separated by an empty space, and the end of an
+        entry is determined by a line break.
+
+        The timestamp field is output in milliseconds. PID has already
+        been converted to the root namespace.
+
+        The file is read only, and requires CAP_SYS_PTRACE capability.
+        The max file size is calculated by page size and the size of
+        the `oom_victim` struct. In the case we hit the max size,
+        it will start removing the oldest OOM event entries.
+
+        Example of entries/values::
+        45439 618 1000 999 system_server
+        46866 1546 0 995 sh
+        73596 2401 10025 990 oid.documentsui
+
+What:       /proc/memhealth/direct_reclaim_state
+Date:       October 2023
+Contact:    carlosgalo@google.com
+Description:
+        This attribute shows information on whether or not we are
+        currently in direct reclaim. The format is "%d\n", we output
+        1 if we are currently in direct reclaim, otherwise 0.
+
+        The file is read only, and requires CAP_SYS_PTRACE capability.
+        The file can also be seeked to, but it will only accept seeks
+        to the start of the file.
+
+        Example of output::
+        1
diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig
--- a/drivers/android/Kconfig
+++ b/drivers/android/Kconfig
@@ -47,4 +47,16 @@ config ANDROID_BINDER_IPC_SELFTEST
 	  exhaustively with combinations of various buffer sizes and
 	  alignments.
 
+config ANDROID_MEMHEALTH
+	tristate "Android memory health module"
+	depends on MMU && TRACEPOINTS
+	help
+	  Android module to expose memory health-related events to userspace.
+	  Creates a proc/memhealth directory with pollable event files
+	  (currenty only oom_victim_list) to notify userspace of memory
+	  critical events.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called android_memhealth.
+
 endmenu
diff --git a/drivers/android/Makefile b/drivers/android/Makefile
--- a/drivers/android/Makefile
+++ b/drivers/android/Makefile
@@ -4,3 +4,4 @@ ccflags-y += -I$(src)			# needed for trace events
 obj-$(CONFIG_ANDROID_BINDERFS)		+= binderfs.o
 obj-$(CONFIG_ANDROID_BINDER_IPC)	+= binder.o binder_alloc.o
 obj-$(CONFIG_ANDROID_BINDER_IPC_SELFTEST) += binder_alloc_selftest.o
+obj-$(CONFIG_ANDROID_MEMHEALTH)		+= memhealth.o
diff --git a/drivers/android/memhealth.c b/drivers/android/memhealth.c
new file mode 100644
--- /dev/null
+++ b/drivers/android/memhealth.c
@@ -0,0 +1,423 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2023 Google, Inc.
+ */
+
+#include <linux/atomic.h>
+#include <linux/cred.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/poll.h>
+#include <linux/proc_fs.h>
+#include <linux/sched.h>
+#include <linux/seq_file.h>
+#include <linux/timekeeping.h>
+#include <linux/tracepoint.h>
+#include <linux/workqueue.h>
+
+#include <trace/events/oom.h>
+#include <trace/events/vmscan.h>
+
+#define MEMHEALTH_DIRECTORY "memhealth"
+#define OOM_VICTIM_LIST_ENTRY "oom_victim_list"
+#define PROCS_IN_DIRECT_RECLAIM_ENTRY "direct_reclaim_state"
+
+static wait_queue_head_t memhealth_wq;
+static struct proc_dir_entry *proc_memhealth_dir;
+
+/* List of oom victims */
+static struct list_head oom_victim_list;
+static size_t oom_victim_count;
+static size_t oom_victim_removed_count;
+/*
+ * Lock protecting oom_victim_list, oom_victim_count and
+ * oom_victim_removed_count
+ */
+static DEFINE_MUTEX(memhealth_mutex);
+/* List of new oom victims not yet added into oom_victim_list */
+static struct list_head new_oom_victims_list;
+static size_t new_oom_victims_count;
+
+/* Total processes in direct reclaim */
+static atomic_t procs_in_direct_reclaim = ATOMIC_INIT(0);
+
+/* Lock protecting new_oom_victims_list and new_oom_victims_count */
+static DEFINE_SPINLOCK(memhealth_spin_lock);
+
+struct oom_victim {
+	pid_t pid;
+	uid_t uid;
+	char process_name[TASK_COMM_LEN];
+	ktime_t timestamp;
+	short oom_score_adj;
+	struct list_head list;
+};
+
+#define OOM_VICTIM_LIST_MAX_SIZE ((PAGE_SIZE/sizeof(struct oom_victim)))
+
+
+/* Direct Reclaim state */
+static ssize_t direct_reclaim_state_read(struct file *file, char __user *buf,
+	size_t count, loff_t *offset)
+{
+	return simple_read_from_buffer(buf, count, offset,
+		atomic_read(&procs_in_direct_reclaim) > 0 ? "1\n" : "0\n", 2);
+}
+
+static __poll_t direct_reclaim_state_poll(struct file *file, poll_table *wait)
+{
+	__poll_t mask = DEFAULT_POLLMASK;
+	bool in_direct_reclaim, last_in_direct_reclaim;
+
+	last_in_direct_reclaim = (bool)file->private_data;
+	poll_wait(file, &memhealth_wq, wait);
+
+	in_direct_reclaim = atomic_read(&procs_in_direct_reclaim) > 0;
+	if (last_in_direct_reclaim != in_direct_reclaim) {
+		file->private_data = (void *)in_direct_reclaim;
+		mask |= EPOLLPRI;
+	}
+
+	return mask;
+}
+
+static int direct_reclaim_state_open(struct inode *inode, struct file *file)
+{
+	bool in_direct_reclaim;
+
+	if (!capable(CAP_SYS_PTRACE))
+		return -EPERM;
+
+	in_direct_reclaim = atomic_read(&procs_in_direct_reclaim) > 0;
+	file->private_data = (void *) in_direct_reclaim;
+
+	return 0;
+}
+
+static const struct proc_ops direct_reclaim_state_proc_ops = {
+	.proc_read	= direct_reclaim_state_read,
+	.proc_lseek	= default_llseek,
+	.proc_poll	= direct_reclaim_state_poll,
+	.proc_open	= direct_reclaim_state_open,
+};
+
+static void mm_vmscan_direct_reclaim_begin_probe(void *data, int order, gfp_t gfp_flags)
+{
+	if (atomic_inc_return(&procs_in_direct_reclaim) == 1)
+		wake_up_interruptible(&memhealth_wq);
+}
+
+static void mm_vmscan_direct_reclaim_end_probe(void *data, unsigned long nr_reclaimed)
+{
+	if (atomic_dec_return(&procs_in_direct_reclaim) == 0)
+		wake_up_interruptible(&memhealth_wq);
+}
+
+/* OOM */
+static void oom_list_move_victims(struct work_struct *work)
+{
+	struct oom_victim *head;
+	size_t total_new_nodes;
+
+	mutex_lock(&memhealth_mutex);
+	spin_lock(&memhealth_spin_lock);
+
+	if (list_empty(&new_oom_victims_list)) {
+		spin_unlock(&memhealth_spin_lock);
+		mutex_unlock(&memhealth_mutex);
+		return;
+	}
+
+	total_new_nodes = new_oom_victims_count;
+	list_splice_tail_init(&new_oom_victims_list, &oom_victim_list);
+	new_oom_victims_count = 0;
+
+	spin_unlock(&memhealth_spin_lock);
+
+	oom_victim_count += total_new_nodes;
+	while (oom_victim_count - oom_victim_removed_count >=
+			OOM_VICTIM_LIST_MAX_SIZE) {
+		head = list_first_entry(&oom_victim_list, struct oom_victim, list);
+
+		list_del(&head->list);
+
+		kfree(head);
+		oom_victim_removed_count++;
+	}
+	mutex_unlock(&memhealth_mutex);
+}
+static DECLARE_WORK(memhealth_oom_work, oom_list_move_victims);
+
+static int add_oom_victim_to_list(pid_t pid, ktime_t timestamp)
+{
+	struct oom_victim *new_node;
+	struct task_struct *task;
+	const struct cred *cred;
+	struct pid *pid_struct;
+	int ret = -EINVAL;
+
+	/*
+	 * Function could be called while a spinlock is being held, we want to
+	 * prevent blocking while allocating for the new node
+	 */
+	new_node = kmalloc(sizeof(*new_node), GFP_ATOMIC);
+	if (!new_node) {
+		pr_err("memhealth failed to create new oom node for pid %d\n", pid);
+		ret = -ENOMEM;
+		goto err_create_oom_node;
+	}
+
+	pid_struct = find_get_pid(pid);
+	if (!pid_struct) {
+		pr_err("memhealth failed to find pid %d\n", pid);
+		goto err_get_pid;
+	}
+
+	task = get_pid_task(pid_struct, PIDTYPE_PID);
+	put_pid(pid_struct);
+	if (!task) {
+		pr_err("memhealth failed to find task with pid %d\n", pid);
+		goto err_get_task;
+	}
+
+	cred = get_task_cred(task);
+	if (!cred) {
+		pr_err("memhealth failed to find credentials\n");
+		goto err_get_cred;
+	}
+
+	if (!task->signal) {
+		pr_err("memhealth failed to find signal in task\n");
+		goto err_no_task_signal;
+	}
+	new_node->oom_score_adj = task->signal->oom_score_adj;
+
+	ret = strscpy_pad(new_node->process_name,
+		task->comm, TASK_COMM_LEN);
+
+	if (ret < 0) {
+		pr_err("memhealth failed to copy process name to new oom victim node\n");
+		goto err_write_process_name;
+	}
+	put_task_struct(task);
+
+	new_node->pid = pid;
+	new_node->timestamp = timestamp;
+	new_node->uid = cred->uid.val;
+
+	put_cred(cred);
+
+	spin_lock(&memhealth_spin_lock);
+	/*
+	 * We add to `new_oom_victims_list` so the file read does not
+	 * block the caller of mark victim
+	 */
+	list_add_tail(&new_node->list, &new_oom_victims_list);
+	new_oom_victims_count++;
+	spin_unlock(&memhealth_spin_lock);
+
+	schedule_work(&memhealth_oom_work);
+	return 0;
+
+err_write_process_name:
+err_no_task_signal:
+	put_cred(cred);
+err_get_cred:
+	put_task_struct(task);
+err_get_task:
+err_get_pid:
+	kfree(new_node);
+err_create_oom_node:
+	return ret;
+}
+
+static void mark_victim_probe(void *data, pid_t pid)
+{
+	ktime_t timestamp;
+
+	timestamp = ktime_get();
+	if (add_oom_victim_to_list(pid, timestamp) < 0) {
+		pr_err("memhealth failed to add pid(%d) as new OOM killer victim\n", pid);
+		return;
+	}
+	wake_up_interruptible(&memhealth_wq);
+}
+
+static void *oom_victim_list_seq_start(struct seq_file *s, loff_t *pos)
+{
+	mutex_lock(&memhealth_mutex);
+	return seq_list_start(&oom_victim_list, *pos);
+}
+
+static void *oom_victim_list_seq_next(struct seq_file *s, void *v, loff_t *pos)
+{
+	return seq_list_next(v, &oom_victim_list, pos);
+}
+
+static void oom_victim_list_seq_stop(struct seq_file *s, void *v)
+{
+	mutex_unlock(&memhealth_mutex);
+}
+
+static int oom_victim_list_seq_show(struct seq_file *s, void *v)
+{
+	struct oom_victim *entry = list_entry(v, struct oom_victim, list);
+
+	seq_printf(s, "%llu %d %d %i %s\n",
+		ktime_to_ms(entry->timestamp), entry->pid,
+		entry->uid, entry->oom_score_adj, entry->process_name
+	);
+
+	return 0;
+}
+
+static const struct seq_operations oom_victim_list_seq_ops = {
+	.start = oom_victim_list_seq_start,
+	.next = oom_victim_list_seq_next,
+	.stop = oom_victim_list_seq_stop,
+	.show = oom_victim_list_seq_show,
+};
+
+static int oom_victim_list_seq_open(struct inode *inode, struct file *file)
+{
+	struct seq_file *seq;
+	int seq_open_result;
+
+	if (!capable(CAP_SYS_PTRACE))
+		return -EPERM;
+
+	seq_open_result = seq_open(file, &oom_victim_list_seq_ops);
+	if (seq_open_result) {
+		pr_err("memhealth failed opening OOM sequential file\n");
+		return seq_open_result;
+	}
+
+	seq = file->private_data;
+	seq->poll_event = 0;
+	return seq_open_result;
+}
+
+static __poll_t oom_victim_list_poll(struct file *filp, poll_table *wait)
+{
+	struct seq_file *seq = filp->private_data;
+	__poll_t mask = DEFAULT_POLLMASK;
+
+	poll_wait(filp, &memhealth_wq, wait);
+
+	mutex_lock(&memhealth_mutex);
+	if (seq->poll_event < oom_victim_count) {
+		seq->poll_event = oom_victim_count;
+		mask |= EPOLLPRI;
+	}
+	mutex_unlock(&memhealth_mutex);
+
+	return mask;
+}
+
+static const struct proc_ops oom_victims_list_proc_ops = {
+	.proc_read = seq_read,
+	.proc_lseek = seq_lseek,
+	.proc_release = seq_release,
+	.proc_open = oom_victim_list_seq_open,
+	.proc_poll = oom_victim_list_poll,
+};
+
+static int __init memhealthmod_start(void)
+{
+	struct proc_dir_entry *entry;
+	int ret = -ENOMEM;
+
+	proc_memhealth_dir = proc_mkdir(MEMHEALTH_DIRECTORY, NULL);
+	if (!proc_memhealth_dir) {
+		pr_err("memhealth failed to create directory (%s)\n", MEMHEALTH_DIRECTORY);
+		goto err_create_memhealth_dir;
+	}
+
+	entry = proc_create(OOM_VICTIM_LIST_ENTRY, 0444,
+		proc_memhealth_dir, &oom_victims_list_proc_ops);
+	if (!entry) {
+		pr_err("memhealth failed to create proc entry: %s\n",
+			OOM_VICTIM_LIST_ENTRY);
+		goto err_create_oom_entry;
+	}
+	entry = proc_create(PROCS_IN_DIRECT_RECLAIM_ENTRY, 0444,
+		proc_memhealth_dir, &direct_reclaim_state_proc_ops);
+	if (!entry) {
+		pr_err("memhealth failed to create proc entry: %s\n",
+			PROCS_IN_DIRECT_RECLAIM_ENTRY);
+		goto err_create_direct_reclaim_entry;
+	}
+
+	init_waitqueue_head(&memhealth_wq);
+	/* OOM */
+	INIT_LIST_HEAD(&oom_victim_list);
+	INIT_LIST_HEAD(&new_oom_victims_list);
+	oom_victim_count = 0;
+	oom_victim_removed_count = 0;
+	new_oom_victims_count = 0;
+
+	ret = register_trace_mark_victim(mark_victim_probe, NULL);
+	if (ret) {
+		pr_err("memhealth failed to hook a probe to the mark_victim tracepoint\n");
+		goto err_register_probe;
+	}
+	ret = register_trace_mm_vmscan_direct_reclaim_begin(
+		mm_vmscan_direct_reclaim_begin_probe, NULL);
+	if (ret) {
+		pr_err("memhealth failed to hook probe to the mm_vmscan_direct_reclaim_begin" \
+				" tracepoint\n");
+		goto err_register_probe;
+	}
+	ret = register_trace_mm_vmscan_direct_reclaim_end(mm_vmscan_direct_reclaim_end_probe, NULL);
+	if (ret) {
+		pr_err("memhealth failed to hook probe to the mm_vmscan_direct_reclaim_end" \
+				" tracepoint\n");
+		goto err_register_probe;
+	}
+
+	return 0;
+
+err_register_probe:
+	remove_proc_entry(PROCS_IN_DIRECT_RECLAIM_ENTRY, proc_memhealth_dir);
+err_create_direct_reclaim_entry:
+	remove_proc_entry(OOM_VICTIM_LIST_ENTRY, proc_memhealth_dir);
+err_create_oom_entry:
+	remove_proc_entry(MEMHEALTH_DIRECTORY, NULL);
+err_create_memhealth_dir:
+	return ret;
+}
+
+static void __exit memhealthmod_end(void)
+{
+	struct oom_victim *entry, *tmp;
+
+	/* OOM */
+	if (unregister_trace_mark_victim(mark_victim_probe, NULL))
+		pr_warn("memhealth failed to unhook a probe from the mark_victim tracepoint\n");
+
+	list_for_each_entry_safe(entry, tmp, &oom_victim_list, list) {
+		list_del(&entry->list);
+		kfree(entry);
+	}
+
+	/* Direct Reclaim state */
+	if (unregister_trace_mm_vmscan_direct_reclaim_begin(mm_vmscan_direct_reclaim_begin_probe,
+		NULL))
+		pr_warn("memhealth failed to unhook a probe from the" \
+				" mm_vmscan_direct_reclaim_begin tracepoint\n");
+	if (unregister_trace_mm_vmscan_direct_reclaim_end(mm_vmscan_direct_reclaim_end_probe, NULL))
+		pr_warn("memhealth failed to unhook a probe from the" \
+				" mm_vmscan_direct_reclaim_end tracepoint\n");
+
+	remove_proc_entry(PROCS_IN_DIRECT_RECLAIM_ENTRY, proc_memhealth_dir);
+	remove_proc_entry(OOM_VICTIM_LIST_ENTRY, proc_memhealth_dir);
+	remove_proc_entry(MEMHEALTH_DIRECTORY, NULL);
+}
+
+module_init(memhealthmod_start);
+module_exit(memhealthmod_end);
+
+MODULE_LICENSE("GPL");
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -775,6 +775,12 @@ static void mark_oom_victim(struct task_struct *tsk)
 	trace_mark_victim(tsk->pid);
 }
 
+/**
+ * ANDROID: Exporting tracepoint to assist in tracking OOM kills
+ * in `drivers/android/memhealth.c`.
+ */
+EXPORT_TRACEPOINT_SYMBOL_GPL(mark_victim);
+
 /**
  * exit_oom_victim - note the exit of an OOM victim
  */
diff --git a/mm/vmscan.c b/mm/vmscan.c
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -6388,6 +6388,13 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
 	return nr_reclaimed;
 }
 
+/**
+ * ANDROID: Exporting tracepoints to assist in tracking direct reclaim
+ * state in `drivers/android/memhealth.c`.
+ */
+EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_begin);
+EXPORT_TRACEPOINT_SYMBOL_GPL(mm_vmscan_direct_reclaim_end);
+
 #ifdef CONFIG_MEMCG
 
 /* Only used by soft limit reclaim. Do not reuse for anything else. */