path: root/mali_kbase/mali_kbase_vinstr.c

// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note
/*
 *
 * (C) COPYRIGHT 2011-2023 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU license.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, you can access it online at
 * http://www.gnu.org/licenses/gpl-2.0.html.
 *
 */

#include "mali_kbase_vinstr.h"
#include "hwcnt/mali_kbase_hwcnt_virtualizer.h"
#include "hwcnt/mali_kbase_hwcnt_types.h"
#include <uapi/gpu/arm/midgard/mali_kbase_hwcnt_reader.h>
#include "hwcnt/mali_kbase_hwcnt_gpu.h"
#include "hwcnt/mali_kbase_hwcnt_gpu_narrow.h"
#include <uapi/gpu/arm/midgard/mali_kbase_ioctl.h>
#include "mali_malisw.h"
#include "mali_kbase_debug.h"

#include <linux/anon_inodes.h>
#include <linux/fcntl.h>
#include <linux/fs.h>
#include <linux/hrtimer.h>
#include <linux/log2.h>
#include <linux/mm.h>
#include <linux/mutex.h>
#include <linux/poll.h>
#include <linux/slab.h>
#include <linux/version_compat_defs.h>
#include <linux/workqueue.h>

/* Explicitly include epoll header for old kernels. Not required from 4.16. */
#if KERNEL_VERSION(4, 16, 0) > LINUX_VERSION_CODE
#include <uapi/linux/eventpoll.h>
#endif

/* Hwcnt reader API version */
#define HWCNT_READER_API 1

/* The minimum allowed interval between dumps (equivalent to 10KHz) */
#define DUMP_INTERVAL_MIN_NS (100 * NSEC_PER_USEC)

/* The maximum allowed buffers per client */
#define MAX_BUFFER_COUNT 32

/**
 * struct kbase_vinstr_context - IOCTL interface for userspace hardware
 *                               counters.
 * @hvirt:         Hardware counter virtualizer used by vinstr.
 * @metadata:      Hardware counter metadata provided by virtualizer.
 * @metadata_user: API compatible hardware counter metadata provided by vinstr.
 *                 For compatibility with the user driver interface, this
 *                 contains a narrowed version of the HWCNT metadata limited
 *                 to 64 entries per block of 32 bits each.
 * @lock:          Lock protecting all vinstr state.
 * @suspend_count: Suspend reference count. If non-zero, timer and worker are
 *                 prevented from being re-scheduled.
 * @client_count:  Number of vinstr clients.
 * @clients:       List of vinstr clients.
 * @dump_timer:    Timer that enqueues dump_work to a workqueue.
 * @dump_work:     Worker for performing periodic counter dumps.
 */
struct kbase_vinstr_context {
	struct kbase_hwcnt_virtualizer *hvirt;
	const struct kbase_hwcnt_metadata *metadata;
	const struct kbase_hwcnt_metadata_narrow *metadata_user;
	struct mutex lock;
	size_t suspend_count;
	size_t client_count;
	struct list_head clients;
	struct hrtimer dump_timer;
	struct work_struct dump_work;
};

/**
 * struct kbase_vinstr_client - A vinstr client attached to a vinstr context.
 * @vctx:              Vinstr context client is attached to.
 * @hvcli:             Hardware counter virtualizer client.
 * @node:              Node used to attach this client to list in vinstr
 *                     context.
 * @dump_interval_ns:  Interval between periodic dumps. If 0, not a periodic
 *                     client.
 * @next_dump_time_ns: Time in ns when this client's next periodic dump must
 *                     occur. If 0, not a periodic client.
 * @enable_map:        Counters enable map.
 * @tmp_buf:           Temporary buffer to use before handing dump to client.
 * @dump_bufs:         Array of narrow dump buffers allocated by this client.
 * @dump_bufs_meta:    Metadata of hwcnt reader client buffers.
 * @meta_idx:          Index of metadata being accessed by userspace.
 * @read_idx:          Index of buffer read by userspace.
 * @write_idx:         Index of buffer being written by dump worker.
 * @waitq:             Client's notification queue.
 */
struct kbase_vinstr_client {
	struct kbase_vinstr_context *vctx;
	struct kbase_hwcnt_virtualizer_client *hvcli;
	struct list_head node;
	u64 next_dump_time_ns;
	u32 dump_interval_ns;
	struct kbase_hwcnt_enable_map enable_map;
	struct kbase_hwcnt_dump_buffer tmp_buf;
	struct kbase_hwcnt_dump_buffer_narrow_array dump_bufs;
	struct kbase_hwcnt_reader_metadata *dump_bufs_meta;
	atomic_t meta_idx;
	atomic_t read_idx;
	atomic_t write_idx;
	wait_queue_head_t waitq;
};

static __poll_t kbasep_vinstr_hwcnt_reader_poll(struct file *filp, poll_table *wait);

static long kbasep_vinstr_hwcnt_reader_ioctl(
	struct file *filp,
	unsigned int cmd,
	unsigned long arg);

static int kbasep_vinstr_hwcnt_reader_mmap(
	struct file *filp,
	struct vm_area_struct *vma);

static int kbasep_vinstr_hwcnt_reader_release(
	struct inode *inode,
	struct file *filp);

/* Vinstr client file operations */
static const struct file_operations vinstr_client_fops = {
	.owner = THIS_MODULE,
	.poll           = kbasep_vinstr_hwcnt_reader_poll,
	.unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl,
	.compat_ioctl   = kbasep_vinstr_hwcnt_reader_ioctl,
	.mmap           = kbasep_vinstr_hwcnt_reader_mmap,
	.release        = kbasep_vinstr_hwcnt_reader_release,
};

/**
 * kbasep_vinstr_timestamp_ns() - Get the current time in nanoseconds.
 *
 * Return: Current time in nanoseconds.
 */
static u64 kbasep_vinstr_timestamp_ns(void)
{
	return ktime_get_raw_ns();
}

/**
 * kbasep_vinstr_next_dump_time_ns() - Calculate the next periodic dump time.
 * @cur_ts_ns: Current time in nanoseconds.
 * @interval:  Interval between dumps in nanoseconds.
 *
 * Return: 0 if interval is 0 (i.e. a non-periodic client), or the next dump
 *         time that occurs after cur_ts_ns.
 */
static u64 kbasep_vinstr_next_dump_time_ns(u64 cur_ts_ns, u32 interval)
{
	/* Non-periodic client */
	if (interval == 0)
		return 0;

	/*
	 * Return the next interval after the current time relative to t=0.
	 * This means multiple clients with the same period will synchronise,
	 * regardless of when they were started, allowing the worker to be
	 * scheduled less frequently.
	 */
	do_div(cur_ts_ns, interval);
	return (cur_ts_ns + 1) * interval;
}

/**
 * kbasep_vinstr_client_dump() - Perform a dump for a client.
 * @vcli:     Non-NULL pointer to a vinstr client.
 * @event_id: Event type that triggered the dump.
 *
 * Return: 0 on success, else error code.
 */
static int kbasep_vinstr_client_dump(
	struct kbase_vinstr_client *vcli,
	enum base_hwcnt_reader_event event_id)
{
	int errcode;
	u64 ts_start_ns;
	u64 ts_end_ns;
	unsigned int write_idx;
	unsigned int read_idx;
	struct kbase_hwcnt_dump_buffer *tmp_buf;
	struct kbase_hwcnt_dump_buffer_narrow *dump_buf;
	struct kbase_hwcnt_reader_metadata *meta;
	u8 clk_cnt;

	WARN_ON(!vcli);
	lockdep_assert_held(&vcli->vctx->lock);

	write_idx = atomic_read(&vcli->write_idx);
	read_idx = atomic_read(&vcli->read_idx);

	/* Check if there is a place to copy HWC block into. */
	if (write_idx - read_idx == vcli->dump_bufs.buf_cnt)
		return -EBUSY;
	write_idx %= vcli->dump_bufs.buf_cnt;

	dump_buf = &vcli->dump_bufs.bufs[write_idx];
	meta = &vcli->dump_bufs_meta[write_idx];
	tmp_buf = &vcli->tmp_buf;

	errcode = kbase_hwcnt_virtualizer_client_dump(
		vcli->hvcli, &ts_start_ns, &ts_end_ns, tmp_buf);
	if (errcode)
		return errcode;

	/* Patch the dump buf headers, to hide the counters that other hwcnt
	 * clients are using.
	 */
	kbase_hwcnt_gpu_patch_dump_headers(tmp_buf, &vcli->enable_map);

	/* Copy the temp buffer to the userspace visible buffer. The strict
	 * variant will explicitly zero any non-enabled counters to ensure
	 * nothing except exactly what the user asked for is made visible.
	 *
	 * A narrow copy is required since virtualizer has a bigger buffer
	 * but user only needs part of it.
	 */
	kbase_hwcnt_dump_buffer_copy_strict_narrow(dump_buf, tmp_buf,
						   &vcli->enable_map);

	clk_cnt = vcli->vctx->metadata->clk_cnt;

	meta->timestamp = ts_end_ns;
	meta->event_id = event_id;
	meta->buffer_idx = write_idx;
	meta->cycles.top = (clk_cnt > 0) ? dump_buf->clk_cnt_buf[0] : 0;
	meta->cycles.shader_cores =
	    (clk_cnt > 1) ? dump_buf->clk_cnt_buf[1] : 0;

	/* Notify client. Make sure all changes to memory are visible. */
	wmb();
	atomic_inc(&vcli->write_idx);
	wake_up_interruptible(&vcli->waitq);
	return 0;
}

/**
 * kbasep_vinstr_client_clear() - Reset all the client's counters to zero.
 * @vcli: Non-NULL pointer to a vinstr client.
 *
 * Return: 0 on success, else error code.
 */
static int kbasep_vinstr_client_clear(struct kbase_vinstr_client *vcli)
{
	u64 ts_start_ns;
	u64 ts_end_ns;

	WARN_ON(!vcli);
	lockdep_assert_held(&vcli->vctx->lock);

	/* A virtualizer dump with a NULL buffer will just clear the virtualizer
	 * client's buffer.
	 */
	return kbase_hwcnt_virtualizer_client_dump(
		vcli->hvcli, &ts_start_ns, &ts_end_ns, NULL);
}

/**
 * kbasep_vinstr_reschedule_worker() - Update next dump times for all periodic
 *                                     vinstr clients, then reschedule the dump
 *                                     worker appropriately.
 * @vctx: Non-NULL pointer to the vinstr context.
 *
 * If there are no periodic clients, then the dump worker will not be
 * rescheduled. Else, the dump worker will be rescheduled for the next periodic
 * client dump.
 */
static void kbasep_vinstr_reschedule_worker(struct kbase_vinstr_context *vctx)
{
	u64 cur_ts_ns;
	u64 earliest_next_ns = U64_MAX;
	struct kbase_vinstr_client *pos;

	WARN_ON(!vctx);
	lockdep_assert_held(&vctx->lock);

	cur_ts_ns = kbasep_vinstr_timestamp_ns();

	/*
	 * Update each client's next dump time, and find the earliest next
	 * dump time if any of the clients have a non-zero interval.
	 */
	list_for_each_entry(pos, &vctx->clients, node) {
		const u64 cli_next_ns =
			kbasep_vinstr_next_dump_time_ns(
				cur_ts_ns, pos->dump_interval_ns);

		/* Non-zero next dump time implies a periodic client */
		if ((cli_next_ns != 0) && (cli_next_ns < earliest_next_ns))
			earliest_next_ns = cli_next_ns;

		pos->next_dump_time_ns = cli_next_ns;
	}

	/* Cancel the timer if it is already pending */
	hrtimer_cancel(&vctx->dump_timer);

	/* Start the timer if there are periodic clients and vinstr is not
	 * suspended.
	 */
	if ((earliest_next_ns != U64_MAX) &&
	    (vctx->suspend_count == 0) &&
	    !WARN_ON(earliest_next_ns < cur_ts_ns))
		hrtimer_start(
			&vctx->dump_timer,
			ns_to_ktime(earliest_next_ns - cur_ts_ns),
			HRTIMER_MODE_REL);
}

/**
 * kbasep_vinstr_dump_worker()- Dump worker, that dumps all periodic clients
 *                              that need to be dumped, then reschedules itself.
 * @work: Work structure.
 */
static void kbasep_vinstr_dump_worker(struct work_struct *work)
{
	struct kbase_vinstr_context *vctx =
		container_of(work, struct kbase_vinstr_context, dump_work);
	struct kbase_vinstr_client *pos;
	u64 cur_time_ns;

	mutex_lock(&vctx->lock);

	cur_time_ns = kbasep_vinstr_timestamp_ns();

	/* Dump all periodic clients whose next dump time is before the current
	 * time.
	 */
	list_for_each_entry(pos, &vctx->clients, node) {
		if ((pos->next_dump_time_ns != 0) &&
			(pos->next_dump_time_ns < cur_time_ns))
			kbasep_vinstr_client_dump(
				pos, BASE_HWCNT_READER_EVENT_PERIODIC);
	}

	/* Update the next dump times of all periodic clients, then reschedule
	 * this worker at the earliest next dump time.
	 */
	kbasep_vinstr_reschedule_worker(vctx);

	mutex_unlock(&vctx->lock);
}

/**
 * kbasep_vinstr_dump_timer() - Dump timer that schedules the dump worker for
 *                              execution as soon as possible.
 * @timer: Timer structure.
 *
 * Return: HRTIMER_NORESTART always.
 */
static enum hrtimer_restart kbasep_vinstr_dump_timer(struct hrtimer *timer)
{
	struct kbase_vinstr_context *vctx =
		container_of(timer, struct kbase_vinstr_context, dump_timer);

	/* We don't need to check vctx->suspend_count here, as the suspend
	 * function will ensure that any worker enqueued here is immediately
	 * cancelled, and the worker itself won't reschedule this timer if
	 * suspend_count != 0.
	 */
	kbase_hwcnt_virtualizer_queue_work(vctx->hvirt, &vctx->dump_work);
	return HRTIMER_NORESTART;
}

/**
 * kbasep_vinstr_client_destroy() - Destroy a vinstr client.
 * @vcli: vinstr client. Must not be attached to a vinstr context.
 */
static void kbasep_vinstr_client_destroy(struct kbase_vinstr_client *vcli)
{
	if (!vcli)
		return;

	kbase_hwcnt_virtualizer_client_destroy(vcli->hvcli);
	kfree(vcli->dump_bufs_meta);
	kbase_hwcnt_dump_buffer_narrow_array_free(&vcli->dump_bufs);
	kbase_hwcnt_dump_buffer_free(&vcli->tmp_buf);
	kbase_hwcnt_enable_map_free(&vcli->enable_map);
	kfree(vcli);
}

/**
 * kbasep_vinstr_client_create() - Create a vinstr client. Does not attach to
 *                                 the vinstr context.
 * @vctx:     Non-NULL pointer to vinstr context.
 * @setup:    Non-NULL pointer to hardware counter ioctl setup structure.
 *            setup->buffer_count must not be 0 and must be a power of 2.
 * @out_vcli: Non-NULL pointer to where created client will be stored on
 *            success.
 *
 * Return: 0 on success, else error code.
 */
static int kbasep_vinstr_client_create(
	struct kbase_vinstr_context *vctx,
	struct kbase_ioctl_hwcnt_reader_setup *setup,
	struct kbase_vinstr_client **out_vcli)
{
	int errcode;
	struct kbase_vinstr_client *vcli;
	struct kbase_hwcnt_physical_enable_map phys_em;

	WARN_ON(!vctx);
	WARN_ON(!setup);
	WARN_ON(setup->buffer_count == 0);
	WARN_ON(!is_power_of_2(setup->buffer_count));

	vcli = kzalloc(sizeof(*vcli), GFP_KERNEL);
	if (!vcli)
		return -ENOMEM;

	vcli->vctx = vctx;

	errcode = kbase_hwcnt_enable_map_alloc(
		vctx->metadata, &vcli->enable_map);
	if (errcode)
		goto error;

	phys_em.fe_bm = setup->fe_bm;
	phys_em.shader_bm = setup->shader_bm;
	phys_em.tiler_bm = setup->tiler_bm;
	phys_em.mmu_l2_bm = setup->mmu_l2_bm;
	kbase_hwcnt_gpu_enable_map_from_physical(&vcli->enable_map, &phys_em);

	/* Use virtualizer's metadata to alloc tmp buffer which interacts with
	 * the HWC virtualizer.
	 */
	errcode = kbase_hwcnt_dump_buffer_alloc(vctx->metadata, &vcli->tmp_buf);
	if (errcode)
		goto error;

	/* Enable all the available clk_enable_map. */
	vcli->enable_map.clk_enable_map = (1ull << vctx->metadata->clk_cnt) - 1;

	/* Use vinstr's narrowed metadata to alloc narrow dump buffers which
	 * interact with clients.
	 */
	errcode = kbase_hwcnt_dump_buffer_narrow_array_alloc(
		vctx->metadata_user, setup->buffer_count, &vcli->dump_bufs);
	if (errcode)
		goto error;

	errcode = -ENOMEM;
	vcli->dump_bufs_meta = kmalloc_array(
		setup->buffer_count, sizeof(*vcli->dump_bufs_meta), GFP_KERNEL | __GFP_ZERO);
	if (!vcli->dump_bufs_meta)
		goto error;

	errcode = kbase_hwcnt_virtualizer_client_create(
		vctx->hvirt, &vcli->enable_map, &vcli->hvcli);
	if (errcode)
		goto error;

	init_waitqueue_head(&vcli->waitq);

	*out_vcli = vcli;
	return 0;
error:
	kbasep_vinstr_client_destroy(vcli);
	return errcode;
}

int kbase_vinstr_init(
	struct kbase_hwcnt_virtualizer *hvirt,
	struct kbase_vinstr_context **out_vctx)
{
	int errcode;
	struct kbase_vinstr_context *vctx;
	const struct kbase_hwcnt_metadata *metadata;

	if (!hvirt || !out_vctx)
		return -EINVAL;

	metadata = kbase_hwcnt_virtualizer_metadata(hvirt);
	if (!metadata)
		return -EINVAL;

	vctx = kzalloc(sizeof(*vctx), GFP_KERNEL);
	if (!vctx)
		return -ENOMEM;

	vctx->hvirt = hvirt;
	vctx->metadata = metadata;
	errcode = kbase_hwcnt_gpu_metadata_narrow_create(&vctx->metadata_user,
							 metadata);
	if (errcode)
		goto err_metadata_create;

	mutex_init(&vctx->lock);
	INIT_LIST_HEAD(&vctx->clients);
	hrtimer_init(&vctx->dump_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	vctx->dump_timer.function = kbasep_vinstr_dump_timer;
	INIT_WORK(&vctx->dump_work, kbasep_vinstr_dump_worker);

	*out_vctx = vctx;
	return 0;

err_metadata_create:
	kfree(vctx);

	return errcode;
}

void kbase_vinstr_term(struct kbase_vinstr_context *vctx)
{
	if (!vctx)
		return;

	/* Non-zero client count implies client leak */
	if (WARN_ON(vctx->client_count != 0)) {
		struct kbase_vinstr_client *pos, *n;

		list_for_each_entry_safe(pos, n, &vctx->clients, node) {
			list_del(&pos->node);
			vctx->client_count--;
			kbasep_vinstr_client_destroy(pos);
		}
	}

	cancel_work_sync(&vctx->dump_work);
	kbase_hwcnt_gpu_metadata_narrow_destroy(vctx->metadata_user);

	WARN_ON(vctx->client_count != 0);
	kfree(vctx);
}

void kbase_vinstr_suspend(struct kbase_vinstr_context *vctx)
{
	if (!vctx) {
		pr_warn("%s: vctx is NULL\n", __func__);
		return;
	}

	mutex_lock(&vctx->lock);

	if (!WARN_ON(vctx->suspend_count == SIZE_MAX))
		vctx->suspend_count++;

	mutex_unlock(&vctx->lock);

	/* Always sync cancel the timer and then the worker, regardless of the
	 * new suspend count.
	 *
	 * This ensures concurrent calls to kbase_vinstr_suspend() always block
	 * until vinstr is fully suspended.
	 *
	 * The timer is cancelled before the worker, as the timer
	 * unconditionally re-enqueues the worker, but the worker checks the
	 * suspend_count that we just incremented before rescheduling the timer.
	 *
	 * Therefore if we cancel the worker first, the timer might re-enqueue
	 * the worker before we cancel the timer, but the opposite is not
	 * possible.
	 */
	hrtimer_cancel(&vctx->dump_timer);
	cancel_work_sync(&vctx->dump_work);
}

void kbase_vinstr_resume(struct kbase_vinstr_context *vctx)
{
	if (!vctx) {
		pr_warn("%s:vctx is NULL\n", __func__);
		return;
	}

	mutex_lock(&vctx->lock);

	if (!WARN_ON(vctx->suspend_count == 0)) {
		vctx->suspend_count--;

		/* Last resume, so re-enqueue the worker if we have any periodic
		 * clients.
		 */
		if (vctx->suspend_count == 0) {
			struct kbase_vinstr_client *pos;
			bool has_periodic_clients = false;

			list_for_each_entry(pos, &vctx->clients, node) {
				if (pos->dump_interval_ns != 0) {
					has_periodic_clients = true;
					break;
				}
			}

			if (has_periodic_clients)
				kbase_hwcnt_virtualizer_queue_work(
					vctx->hvirt, &vctx->dump_work);
		}
	}

	mutex_unlock(&vctx->lock);
}

int kbase_vinstr_hwcnt_reader_setup(
	struct kbase_vinstr_context *vctx,
	struct kbase_ioctl_hwcnt_reader_setup *setup)
{
	int errcode;
	int fd;
	struct kbase_vinstr_client *vcli = NULL;

	if (!vctx || !setup ||
	    (setup->buffer_count == 0) ||
	    (setup->buffer_count > MAX_BUFFER_COUNT) ||
	    !is_power_of_2(setup->buffer_count))
		return -EINVAL;

	errcode = kbasep_vinstr_client_create(vctx, setup, &vcli);
	if (errcode)
		goto error;

	/* Add the new client. No need to reschedule worker, as not periodic */
	mutex_lock(&vctx->lock);

	vctx->client_count++;
	list_add(&vcli->node, &vctx->clients);

	mutex_unlock(&vctx->lock);

	/* Expose to user-space only once the client is fully initialized */
	errcode = anon_inode_getfd(
		"[mali_vinstr_desc]",
		&vinstr_client_fops,
		vcli,
		O_RDONLY | O_CLOEXEC);
	if (errcode < 0)
		goto client_installed_error;

	fd = errcode;

	return fd;

client_installed_error:
	mutex_lock(&vctx->lock);

	vctx->client_count--;
	list_del(&vcli->node);

	mutex_unlock(&vctx->lock);
error:
	kbasep_vinstr_client_destroy(vcli);
	return errcode;
}

/**
 * kbasep_vinstr_hwcnt_reader_buffer_ready() - Check if client has ready
 *                                             buffers.
 * @cli: Non-NULL pointer to vinstr client.
 *
 * Return: Non-zero if client has at least one dumping buffer filled that was
 *         not notified to user yet.
 */
static int kbasep_vinstr_hwcnt_reader_buffer_ready(
	struct kbase_vinstr_client *cli)
{
	WARN_ON(!cli);
	return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx);
}

/**
 * kbasep_vinstr_hwcnt_reader_ioctl_dump() - Dump ioctl command.
 * @cli: Non-NULL pointer to vinstr client.
 *
 * Return: 0 on success, else error code.
 */
static long kbasep_vinstr_hwcnt_reader_ioctl_dump(
	struct kbase_vinstr_client *cli)
{
	int errcode;

	mutex_lock(&cli->vctx->lock);

	errcode = kbasep_vinstr_client_dump(
		cli, BASE_HWCNT_READER_EVENT_MANUAL);

	mutex_unlock(&cli->vctx->lock);
	return errcode;
}

/**
 * kbasep_vinstr_hwcnt_reader_ioctl_clear() - Clear ioctl command.
 * @cli: Non-NULL pointer to vinstr client.
 *
 * Return: 0 on success, else error code.
 */
static long kbasep_vinstr_hwcnt_reader_ioctl_clear(
	struct kbase_vinstr_client *cli)
{
	int errcode;

	mutex_lock(&cli->vctx->lock);

	errcode = kbasep_vinstr_client_clear(cli);

	mutex_unlock(&cli->vctx->lock);
	return errcode;
}

/**
 * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer() - Get buffer ioctl command.
 * @cli:    Non-NULL pointer to vinstr client.
 * @buffer: Non-NULL pointer to userspace buffer.
 * @size:   Size of buffer.
 *
 * Return: 0 on success, else error code.
 */
static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
	struct kbase_vinstr_client *cli,
	void __user *buffer,
	size_t size)
{
	unsigned int meta_idx = atomic_read(&cli->meta_idx);
	unsigned int idx = meta_idx % cli->dump_bufs.buf_cnt;

	struct kbase_hwcnt_reader_metadata *meta = &cli->dump_bufs_meta[idx];
	const size_t meta_size = sizeof(struct kbase_hwcnt_reader_metadata);
	const size_t min_size = min(size, meta_size);

	/* Metadata sanity check. */
	WARN_ON(idx != meta->buffer_idx);

	/* Check if there is any buffer available. */
	if (unlikely(atomic_read(&cli->write_idx) == meta_idx))
		return -EAGAIN;

	/* Check if previously taken buffer was put back. */
	if (unlikely(atomic_read(&cli->read_idx) != meta_idx))
		return -EBUSY;

	/* Clear user buffer to zero. */
	if (unlikely(meta_size < size && clear_user(buffer, size)))
		return -EFAULT;

	/* Copy next available buffer's metadata to user. */
	if (unlikely(copy_to_user(buffer, meta, min_size)))
		return -EFAULT;

	/* Compare exchange meta idx to protect against concurrent getters */
	if (meta_idx != atomic_cmpxchg(&cli->meta_idx, meta_idx, meta_idx + 1))
		return -EBUSY;

	return 0;
}

/**
 * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer() - Put buffer ioctl command.
 * @cli:    Non-NULL pointer to vinstr client.
 * @buffer: Non-NULL pointer to userspace buffer.
 * @size:   Size of buffer.
 *
 * Return: 0 on success, else error code.
 */
static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
	struct kbase_vinstr_client *cli,
	void __user *buffer,
	size_t size)
{
	unsigned int read_idx = atomic_read(&cli->read_idx);
	unsigned int idx = read_idx % cli->dump_bufs.buf_cnt;

	struct kbase_hwcnt_reader_metadata *meta;
	const size_t meta_size = sizeof(struct kbase_hwcnt_reader_metadata);
	const size_t max_size = max(size, meta_size);
	int ret = 0;
	u8 stack_kbuf[64];
	u8 *kbuf = NULL;
	size_t i;

	/* Check if any buffer was taken. */
	if (unlikely(atomic_read(&cli->meta_idx) == read_idx))
		return -EPERM;

	if (likely(max_size <= sizeof(stack_kbuf))) {
		/* Use stack buffer when the size is small enough. */
		if (unlikely(meta_size > size))
			memset(stack_kbuf, 0, sizeof(stack_kbuf));
		kbuf = stack_kbuf;
	} else {
		kbuf = kzalloc(max_size, GFP_KERNEL);
		if (unlikely(!kbuf))
			return -ENOMEM;
	}

	/*
	 * Copy user buffer to zero cleared kernel buffer which has enough
	 * space for both user buffer and kernel metadata.
	 */
	if (unlikely(copy_from_user(kbuf, buffer, size))) {
		ret = -EFAULT;
		goto out;
	}

	/*
	 * Make sure any "extra" data passed from userspace is zero.
	 * It's meaningful only in case meta_size < size.
	 */
	for (i = meta_size; i < size; i++) {
		/* Check if user data beyond meta size is zero. */
		if (unlikely(kbuf[i] != 0)) {
			ret = -EINVAL;
			goto out;
		}
	}

	/* Check if correct buffer is put back. */
	meta = (struct kbase_hwcnt_reader_metadata *)kbuf;
	if (unlikely(idx != meta->buffer_idx)) {
		ret = -EINVAL;
		goto out;
	}

	/* Compare exchange read idx to protect against concurrent putters */
	if (read_idx !=
	    atomic_cmpxchg(&cli->read_idx, read_idx, read_idx + 1)) {
		ret = -EPERM;
		goto out;
	}

out:
	if (unlikely(kbuf != stack_kbuf))
		kfree(kbuf);
	return ret;
}

/**
 * kbasep_vinstr_hwcnt_reader_ioctl_set_interval() - Set interval ioctl command.
 * @cli:      Non-NULL pointer to vinstr client.
 * @interval: Periodic dumping interval (disable periodic dumping if 0).
 *
 * Return: 0 always.
 */
static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
	struct kbase_vinstr_client *cli,
	u32 interval)
{
	mutex_lock(&cli->vctx->lock);

	if ((interval != 0) && (interval < DUMP_INTERVAL_MIN_NS))
		interval = DUMP_INTERVAL_MIN_NS;
	/* Update the interval, and put in a dummy next dump time */
	cli->dump_interval_ns = interval;
	cli->next_dump_time_ns = 0;

	/*
	 * If it's a periodic client, kick off the worker early to do a proper
	 * timer reschedule. Return value is ignored, as we don't care if the
	 * worker is already queued.
	 */
	if ((interval != 0) && (cli->vctx->suspend_count == 0))
		kbase_hwcnt_virtualizer_queue_work(cli->vctx->hvirt,
						   &cli->vctx->dump_work);

	mutex_unlock(&cli->vctx->lock);

	return 0;
}

/**
 * kbasep_vinstr_hwcnt_reader_ioctl_enable_event() - Enable event ioctl command.
 * @cli:      Non-NULL pointer to vinstr client.
 * @event_id: ID of event to enable.
 *
 * Return: 0 always.
 */
static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
		struct kbase_vinstr_client *cli,
		enum base_hwcnt_reader_event event_id)
{
	/* No-op, as events aren't supported */
	return 0;
}

/**
 * kbasep_vinstr_hwcnt_reader_ioctl_disable_event() - Disable event ioctl
 *                                                    command.
 * @cli:      Non-NULL pointer to vinstr client.
 * @event_id: ID of event to disable.
 *
 * Return: 0 always.
 */
static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
	struct kbase_vinstr_client *cli,
	enum base_hwcnt_reader_event event_id)
{
	/* No-op, as events aren't supported */
	return 0;
}

/**
 * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver() - Get HW version ioctl command.
 * @cli:   Non-NULL pointer to vinstr client.
 * @hwver: Non-NULL pointer to user buffer where HW version will be stored.
 *
 * Return: 0 on success, else error code.
 */
static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
	struct kbase_vinstr_client *cli,
	u32 __user *hwver)
{
	u32 ver = 5;
	const enum kbase_hwcnt_gpu_group_type type =
		kbase_hwcnt_metadata_group_type(cli->vctx->metadata, 0);

	if (WARN_ON(type != KBASE_HWCNT_GPU_GROUP_TYPE_V5))
		return -EINVAL;

	return put_user(ver, hwver);
}

/**
 * kbasep_vinstr_hwcnt_reader_ioctl_get_api_version() - get API version ioctl
 *                                                      command.
 * @cli:    The non-NULL pointer to the client
 * @arg:    Command's argument.
 * @size:   Size of arg.
 *
 * Return: 0 on success, else error code.
 */
static long kbasep_vinstr_hwcnt_reader_ioctl_get_api_version(
	struct kbase_vinstr_client *cli, unsigned long arg, size_t size)
{
	long ret = -EINVAL;

	if (size == sizeof(u32)) {
		ret = put_user(HWCNT_READER_API, (u32 __user *)arg);
	} else if (size == sizeof(struct kbase_hwcnt_reader_api_version)) {
		u8 clk_cnt = cli->vctx->metadata->clk_cnt;
		unsigned long bytes = 0;
		struct kbase_hwcnt_reader_api_version api_version = {
			.version = HWCNT_READER_API,
			.features = KBASE_HWCNT_READER_API_VERSION_NO_FEATURE,
		};

		if (clk_cnt > 0)
			api_version.features |=
			    KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_TOP;
		if (clk_cnt > 1)
			api_version.features |=
			    KBASE_HWCNT_READER_API_VERSION_FEATURE_CYCLES_SHADER_CORES;

		bytes = copy_to_user(
			(void __user *)arg, &api_version, sizeof(api_version));

		/* copy_to_user returns zero in case of success.
		 * If it fails, it returns the number of bytes that could NOT be copied
		 */
		if (bytes == 0)
			ret = 0;
		else
			ret = -EFAULT;
	}
	return ret;
}

/**
 * kbasep_vinstr_hwcnt_reader_ioctl() - hwcnt reader's ioctl.
 * @filp:   Non-NULL pointer to file structure.
 * @cmd:    User command.
 * @arg:    Command's argument.
 *
 * Return: 0 on success, else error code.
 */
static long kbasep_vinstr_hwcnt_reader_ioctl(
	struct file *filp,
	unsigned int cmd,
	unsigned long arg)
{
	long rcode;
	struct kbase_vinstr_client *cli;

	if (!filp || (_IOC_TYPE(cmd) != KBASE_HWCNT_READER))
		return -EINVAL;

	cli = filp->private_data;
	if (!cli)
		return -EINVAL;

	switch (_IOC_NR(cmd)) {
	case _IOC_NR(KBASE_HWCNT_READER_GET_API_VERSION):
		rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_api_version(
				cli, arg, _IOC_SIZE(cmd));
		break;
	case _IOC_NR(KBASE_HWCNT_READER_GET_HWVER):
		rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
			cli, (u32 __user *)arg);
		break;
	case _IOC_NR(KBASE_HWCNT_READER_GET_BUFFER_SIZE):
		rcode = put_user((u32)cli->vctx->metadata_user->dump_buf_bytes,
				 (u32 __user *)arg);
		break;
	case _IOC_NR(KBASE_HWCNT_READER_DUMP):
		rcode = kbasep_vinstr_hwcnt_reader_ioctl_dump(cli);
		break;
	case _IOC_NR(KBASE_HWCNT_READER_CLEAR):
		rcode = kbasep_vinstr_hwcnt_reader_ioctl_clear(cli);
		break;
	case _IOC_NR(KBASE_HWCNT_READER_GET_BUFFER):
		rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
			cli, (void __user *)arg, _IOC_SIZE(cmd));
		break;
	case _IOC_NR(KBASE_HWCNT_READER_PUT_BUFFER):
		rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
			cli, (void __user *)arg, _IOC_SIZE(cmd));
		break;
	case _IOC_NR(KBASE_HWCNT_READER_SET_INTERVAL):
		rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
			cli, (u32)arg);
		break;
	case _IOC_NR(KBASE_HWCNT_READER_ENABLE_EVENT):
		rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
			cli, (enum base_hwcnt_reader_event)arg);
		break;
	case _IOC_NR(KBASE_HWCNT_READER_DISABLE_EVENT):
		rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
			cli, (enum base_hwcnt_reader_event)arg);
		break;
	default:
		pr_warn("Unknown HWCNT ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd));
		rcode = -EINVAL;
		break;
	}

	return rcode;
}

/**
 * kbasep_vinstr_hwcnt_reader_poll() - hwcnt reader's poll.
 * @filp: Non-NULL pointer to file structure.
 * @wait: Non-NULL pointer to poll table.
 *
 * Return: EPOLLIN | EPOLLRDNORM if data can be read without blocking, 0 if
 *         data can not be read without blocking, else EPOLLHUP | EPOLLERR.
 */
static __poll_t kbasep_vinstr_hwcnt_reader_poll(struct file *filp, poll_table *wait)
{
	struct kbase_vinstr_client *cli;

	if (!filp || !wait)
		return EPOLLHUP | EPOLLERR;

	cli = filp->private_data;
	if (!cli)
		return EPOLLHUP | EPOLLERR;

	poll_wait(filp, &cli->waitq, wait);
	if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli))
		return EPOLLIN | EPOLLRDNORM;

	return (__poll_t)0;
}

/**
 * kbasep_vinstr_hwcnt_reader_mmap() - hwcnt reader's mmap.
 * @filp: Non-NULL pointer to file structure.
 * @vma:  Non-NULL pointer to vma structure.
 *
 * Return: 0 on success, else error code.
 */
static int kbasep_vinstr_hwcnt_reader_mmap(
	struct file *filp,
	struct vm_area_struct *vma)
{
	struct kbase_vinstr_client *cli;
	unsigned long vm_size, size, addr, pfn, offset;

	if (!filp || !vma)
		return -EINVAL;

	cli = filp->private_data;
	if (!cli)
		return -EINVAL;

	vm_size = vma->vm_end - vma->vm_start;

	/* The mapping is allowed to span the entirety of the page allocation,
	 * not just the chunk where the dump buffers are allocated.
	 * This accommodates the corner case where the combined size of the
	 * dump buffers is smaller than a single page.
	 * This does not pose a security risk as the pages are zeroed on
	 * allocation, and anything out of bounds of the dump buffers is never
	 * written to.
	 */
	size = (1ull << cli->dump_bufs.page_order) * PAGE_SIZE;

	if (vma->vm_pgoff > (size >> PAGE_SHIFT))
		return -EINVAL;

	offset = vma->vm_pgoff << PAGE_SHIFT;
	if (vm_size > size - offset)
		return -EINVAL;

	addr = __pa(cli->dump_bufs.page_addr + offset);
	pfn = addr >> PAGE_SHIFT;

	return remap_pfn_range(
		vma, vma->vm_start, pfn, vm_size, vma->vm_page_prot);
}

/**
 * kbasep_vinstr_hwcnt_reader_release() - hwcnt reader's release.
 * @inode: Non-NULL pointer to inode structure.
 * @filp:  Non-NULL pointer to file structure.
 *
 * Return: 0 always.
 */
static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode,
	struct file *filp)
{
	struct kbase_vinstr_client *vcli = filp->private_data;

	mutex_lock(&vcli->vctx->lock);

	vcli->vctx->client_count--;
	list_del(&vcli->node);

	mutex_unlock(&vcli->vctx->lock);

	kbasep_vinstr_client_destroy(vcli);

	return 0;
}