diff options
author | Jesse Hall <jessehall@google.com> | 2021-11-23 14:38:46 -0800 |
---|---|---|
committer | Jesse Hall <jessehall@google.com> | 2021-11-23 14:38:46 -0800 |
commit | 0c596dc70431fa2c70021fa1685e3efc969a852d (patch) | |
tree | 8c6cfe8da5d3bea214e991cc4438988f65d9081e /mali_kbase/mali_kbase_kinstr_prfcnt.c | |
parent | bbbb1cf6bb211bb2094dd66656966277c326867f (diff) | |
download | gpu-0c596dc70431fa2c70021fa1685e3efc969a852d.tar.gz |
Mali Valhall Android DDK r34p0-00dev1
Provenance:
046d23c969 (collaborate/google/android/v_r34p0-00dev1)
VX504X08X-BU-00000-r34p0-00dev1 - Valhall Android DDK
VX504X08X-SW-99006-r34p0-00dev1 - Valhall Android Renderscript AOSP parts
Documentation from VX504X08X-BU-00000 omitted.
Signed-off-by: Jesse Hall <jessehall@google.com>
Change-Id: I4ebbb3a3af709bd39f883eed3b35bf4657a95797
Diffstat (limited to 'mali_kbase/mali_kbase_kinstr_prfcnt.c')
-rw-r--r-- | mali_kbase/mali_kbase_kinstr_prfcnt.c | 1184 |
1 files changed, 1184 insertions, 0 deletions
diff --git a/mali_kbase/mali_kbase_kinstr_prfcnt.c b/mali_kbase/mali_kbase_kinstr_prfcnt.c new file mode 100644 index 0000000..ce996ca --- /dev/null +++ b/mali_kbase/mali_kbase_kinstr_prfcnt.c @@ -0,0 +1,1184 @@ +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +/* + * + * (C) COPYRIGHT 2021 ARM Limited. All rights reserved. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU license. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#include "mali_kbase_kinstr_prfcnt.h" +#include "mali_kbase_hwcnt_virtualizer.h" +#include "mali_kbase_hwcnt_types.h" +#include <uapi/gpu/arm/midgard/mali_kbase_hwcnt_reader.h> +#include "mali_kbase_hwcnt_gpu.h" +#include <uapi/gpu/arm/midgard/mali_kbase_ioctl.h> +#include "mali_malisw.h" +#include "mali_kbase_debug.h" + +#include <linux/anon_inodes.h> +#include <linux/fcntl.h> +#include <linux/fs.h> +#include <linux/hrtimer.h> +#include <linux/log2.h> +#include <linux/mm.h> +#include <linux/mutex.h> +#include <linux/poll.h> +#include <linux/slab.h> +#include <linux/workqueue.h> + +/* The minimum allowed interval between dumps, in nanoseconds + * (equivalent to 10KHz) + */ +#define DUMP_INTERVAL_MIN_NS (100 * NSEC_PER_USEC) + +/* The minimum allowed interval between dumps, in microseconds + * (equivalent to 10KHz) + */ +#define DUMP_INTERVAL_MIN_US (DUMP_INTERVAL_MIN_NS / 1000) + +/* The maximum allowed buffers per client */ +#define MAX_BUFFER_COUNT 32 + +/** + * struct kbase_kinstr_prfcnt_context - IOCTL interface for userspace hardware + * counters. + * @hvirt: Hardware counter virtualizer used by kinstr_prfcnt. + * @info_item_count: Number of metadata elements. + * @metadata: Hardware counter metadata provided by virtualizer. + * @lock: Lock protecting kinstr_prfcnt state. + * @suspend_count: Suspend reference count. If non-zero, timer and worker + * are prevented from being re-scheduled. + * @client_count: Number of kinstr_prfcnt clients. + * @clients: List of kinstr_prfcnt clients. + * @dump_timer: Timer that enqueues dump_work to a workqueue. + * @dump_work: Worker for performing periodic counter dumps. + */ +struct kbase_kinstr_prfcnt_context { + struct kbase_hwcnt_virtualizer *hvirt; + u32 info_item_count; + const struct kbase_hwcnt_metadata *metadata; + struct mutex lock; + size_t suspend_count; + size_t client_count; + struct list_head clients; + struct hrtimer dump_timer; + struct work_struct dump_work; +}; + +/** + * struct kbase_kinstr_prfcnt_sample - Buffer and descriptor for sample data. + * @sample_meta: Pointer to samle metadata. + * @dump_buf: Dump buffer containing sample data. + */ +struct kbase_kinstr_prfcnt_sample { + u64 *sample_meta; + struct kbase_hwcnt_dump_buffer dump_buf; +}; + +/** + * struct kbase_kinstr_prfcnt_sample_array - Array of sample data. + * @page_addr: Address of allocated pages. A single allocation is used + * for all Dump Buffers in the array. + * @page_order: The allocation order of the pages. + * @sample_count: Number of allocated samples. + * @samples: Non-NULL pointer to the array of Dump Buffers. + */ +struct kbase_kinstr_prfcnt_sample_array { + u64 page_addr; + unsigned int page_order; + size_t sample_count; + struct kbase_kinstr_prfcnt_sample *samples; +}; + +/** + * struct kbase_kinstr_prfcnt_client_config - Client session configuration. + * @prfcnt_mode: Sampling mode: either manual or periodic. + * @counter_set: Set of performance counter blocks. + * @buffer_count: Number of buffers used to store samples. + * @period_us: Sampling period, in microseconds, or 0 if manual mode. + * @phys_em: Enable map used by the GPU. + */ +struct kbase_kinstr_prfcnt_client_config { + u8 prfcnt_mode; + u8 counter_set; + u16 buffer_count; + u64 period_us; + struct kbase_hwcnt_physical_enable_map phys_em; +}; + +/** + * struct kbase_kinstr_prfcnt_client - A kinstr_prfcnt client attached + * to a kinstr_prfcnt context. + * @kinstr_ctx: kinstr_prfcnt context client is attached to. + * @hvcli: Hardware counter virtualizer client. + * @node: Node used to attach this client to list in kinstr_prfcnt + * context. + * @next_dump_time_ns: Time in ns when this client's next periodic dump must + * occur. If 0, not a periodic client. + * @dump_interval_ns: Interval between periodic dumps. If 0, not a periodic + * client. + * @config: Configuration of the client session. + * @enable_map: Counters enable map. + * @tmp_buf: Temporary buffer to use before handing over dump to + * client. + * @sample_arr: Array of dump buffers allocated by this client. + * @dump_bufs_meta: Metadata of dump buffers. + * @meta_idx: Index of metadata being accessed by userspace. + * @read_idx: Index of buffer read by userspace. + * @write_idx: Index of buffer being written by dump worker. + * @waitq: Client's notification queue. + * @sample_size: Size of the data required for one sample, in bytes. + * @sample_count: Number of samples the client is able to capture. + */ +struct kbase_kinstr_prfcnt_client { + struct kbase_kinstr_prfcnt_context *kinstr_ctx; + struct kbase_hwcnt_virtualizer_client *hvcli; + struct list_head node; + u64 next_dump_time_ns; + u32 dump_interval_ns; + struct kbase_kinstr_prfcnt_client_config config; + struct kbase_hwcnt_enable_map enable_map; + struct kbase_hwcnt_dump_buffer tmp_buf; + struct kbase_kinstr_prfcnt_sample_array sample_arr; + struct kbase_hwcnt_reader_metadata *dump_bufs_meta; + atomic_t meta_idx; + atomic_t read_idx; + atomic_t write_idx; + wait_queue_head_t waitq; + size_t sample_size; + size_t sample_count; +}; + +static struct prfcnt_enum_item kinstr_prfcnt_supported_requests[] = { + { + /* Request description for MODE request */ + .hdr = { + .item_type = PRFCNT_ENUM_TYPE_REQUEST, + .item_version = PRFCNT_READER_API_VERSION, + }, + .u.request = { + .request_item_type = PRFCNT_REQUEST_MODE, + .versions_mask = 0x1, + }, + }, + { + /* Request description for ENABLE request */ + .hdr = { + .item_type = PRFCNT_ENUM_TYPE_REQUEST, + .item_version = PRFCNT_READER_API_VERSION, + }, + .u.request = { + .request_item_type = PRFCNT_REQUEST_ENABLE, + .versions_mask = 0x1, + }, + }, +}; + +/** + * kbasep_kinstr_prfcnt_hwcnt_reader_buffer_ready() - Check if client has ready + * buffers. + * @cli: Non-NULL pointer to kinstr_prfcnt client. + * + * Return: Non-zero if client has at least one dumping buffer filled that was + * not notified to user yet. + */ +static int kbasep_kinstr_prfcnt_hwcnt_reader_buffer_ready( + struct kbase_kinstr_prfcnt_client *cli) +{ + WARN_ON(!cli); + return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx); +} + +/** + * kbasep_kinstr_prfcnt_hwcnt_reader_poll() - hwcnt reader's poll. + * @filp: Non-NULL pointer to file structure. + * @wait: Non-NULL pointer to poll table. + * + * Return: POLLIN if data can be read without blocking, 0 if data can not be + * read without blocking, else error code. + */ +static unsigned int kbasep_kinstr_prfcnt_hwcnt_reader_poll(struct file *filp, + poll_table *wait) +{ + struct kbase_kinstr_prfcnt_client *cli; + + if (!filp || !wait) + return -EINVAL; + + cli = filp->private_data; + + if (!cli) + return -EINVAL; + + poll_wait(filp, &cli->waitq, wait); + + if (kbasep_kinstr_prfcnt_hwcnt_reader_buffer_ready(cli)) + return POLLIN; + + return 0; +} + +/** + * kbasep_kinstr_prfcnt_hwcnt_reader_ioctl() - hwcnt reader's ioctl. + * @filp: Non-NULL pointer to file structure. + * @cmd: User command. + * @arg: Command's argument. + * + * Return: 0 on success, else error code. + */ +static long kbasep_kinstr_prfcnt_hwcnt_reader_ioctl(struct file *filp, + unsigned int cmd, + unsigned long arg) +{ + long rcode; + struct kbase_kinstr_prfcnt_client *cli; + + if (!filp || (_IOC_TYPE(cmd) != KBASE_HWCNT_READER)) + return -EINVAL; + + cli = filp->private_data; + + if (!cli) + return -EINVAL; + + switch (_IOC_NR(cmd)) { + default: + pr_warn("Unknown HWCNT ioctl 0x%x nr:%d", cmd, _IOC_NR(cmd)); + rcode = -EINVAL; + break; + } + + return rcode; +} + +/** + * kbasep_kinstr_prfcnt_hwcnt_reader_mmap() - hwcnt reader's mmap. + * @filp: Non-NULL pointer to file structure. + * @vma: Non-NULL pointer to vma structure. + * + * Return: 0 on success, else error code. + */ +static int kbasep_kinstr_prfcnt_hwcnt_reader_mmap(struct file *filp, + struct vm_area_struct *vma) +{ + struct kbase_kinstr_prfcnt_client *cli; + unsigned long vm_size, size, addr, pfn, offset; + + if (!filp || !vma) + return -EINVAL; + + cli = filp->private_data; + + if (!cli) + return -EINVAL; + + vm_size = vma->vm_end - vma->vm_start; + + /* The mapping is allowed to span the entirety of the page allocation, + * not just the chunk where the dump buffers are allocated. + * This accommodates the corner case where the combined size of the + * dump buffers is smaller than a single page. + * This does not pose a security risk as the pages are zeroed on + * allocation, and anything out of bounds of the dump buffers is never + * written to. + */ + size = (1ull << cli->sample_arr.page_order) * PAGE_SIZE; + + if (vma->vm_pgoff > (size >> PAGE_SHIFT)) + return -EINVAL; + + offset = vma->vm_pgoff << PAGE_SHIFT; + + if (vm_size > size - offset) + return -EINVAL; + + addr = __pa(cli->sample_arr.page_addr + offset); + pfn = addr >> PAGE_SHIFT; + + return remap_pfn_range(vma, vma->vm_start, pfn, vm_size, + vma->vm_page_prot); +} + +static void kbasep_kinstr_prfcnt_sample_array_free( + struct kbase_kinstr_prfcnt_sample_array *sample_arr) +{ + if (!sample_arr) + return; + + kfree((void *)sample_arr->samples); + kfree((void *)(size_t)sample_arr->page_addr); + memset(sample_arr, 0, sizeof(*sample_arr)); +} + +/** + * kbasep_kinstr_prfcnt_client_destroy() - Destroy a kinstr_prfcnt client. + * @cli: kinstr_prfcnt client. Must not be attached to a kinstr_prfcnt context. + */ +static void +kbasep_kinstr_prfcnt_client_destroy(struct kbase_kinstr_prfcnt_client *cli) +{ + if (!cli) + return; + + kbase_hwcnt_virtualizer_client_destroy(cli->hvcli); + kfree(cli->dump_bufs_meta); + kbasep_kinstr_prfcnt_sample_array_free(&cli->sample_arr); + kbase_hwcnt_dump_buffer_free(&cli->tmp_buf); + kbase_hwcnt_enable_map_free(&cli->enable_map); + kfree(cli); +} + +/** + * kbasep_kinstr_prfcnt_hwcnt_reader_release() - hwcnt reader's release. + * @inode: Non-NULL pointer to inode structure. + * @filp: Non-NULL pointer to file structure. + * + * Return: 0 always. + */ +static int kbasep_kinstr_prfcnt_hwcnt_reader_release(struct inode *inode, + struct file *filp) +{ + struct kbase_kinstr_prfcnt_client *cli = filp->private_data; + + mutex_lock(&cli->kinstr_ctx->lock); + + WARN_ON(cli->kinstr_ctx->client_count == 0); + if (cli->kinstr_ctx->client_count > 0) + cli->kinstr_ctx->client_count--; + list_del(&cli->node); + + mutex_unlock(&cli->kinstr_ctx->lock); + + kbasep_kinstr_prfcnt_client_destroy(cli); + + return 0; +} + +/* kinstr_prfcnt client file operations */ +static const struct file_operations kinstr_prfcnt_client_fops = { + .owner = THIS_MODULE, + .poll = kbasep_kinstr_prfcnt_hwcnt_reader_poll, + .unlocked_ioctl = kbasep_kinstr_prfcnt_hwcnt_reader_ioctl, + .compat_ioctl = kbasep_kinstr_prfcnt_hwcnt_reader_ioctl, + .mmap = kbasep_kinstr_prfcnt_hwcnt_reader_mmap, + .release = kbasep_kinstr_prfcnt_hwcnt_reader_release, +}; + +static size_t kbasep_kinstr_prfcnt_get_sample_size( + const struct kbase_hwcnt_metadata *metadata, + struct kbase_hwcnt_dump_buffer *dump_buf) +{ + size_t dump_buf_bytes; + size_t clk_cnt_buf_bytes; + size_t sample_meta_bytes; + size_t block_count = 0; + size_t grp, blk, blk_inst; + + if (!metadata) + return 0; + + kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) + block_count++; + + /* Reserve one for last sentinel item. */ + block_count++; + + sample_meta_bytes = sizeof(struct prfcnt_metadata) * block_count; + dump_buf_bytes = metadata->dump_buf_bytes; + clk_cnt_buf_bytes = sizeof(*dump_buf->clk_cnt_buf) * metadata->clk_cnt; + + return (sample_meta_bytes + dump_buf_bytes + clk_cnt_buf_bytes); +} + +/** + * kbasep_kinstr_prfcnt_dump_worker()- Dump worker, that dumps all periodic + * clients that need to be dumped, then + * reschedules itself. + * @work: Work structure. + */ +static void kbasep_kinstr_prfcnt_dump_worker(struct work_struct *work) +{ + /* Do nothing. */ +} + +/** + * kbasep_kinstr_prfcnt_dump_timer() - Dump timer that schedules the dump worker for + * execution as soon as possible. + * @timer: Timer structure. + */ +static enum hrtimer_restart +kbasep_kinstr_prfcnt_dump_timer(struct hrtimer *timer) +{ + return HRTIMER_NORESTART; +} + +int kbase_kinstr_prfcnt_init(struct kbase_hwcnt_virtualizer *hvirt, + struct kbase_kinstr_prfcnt_context **out_kinstr_ctx) +{ + struct kbase_kinstr_prfcnt_context *kinstr_ctx; + const struct kbase_hwcnt_metadata *metadata; + + if (!hvirt || !out_kinstr_ctx) + return -EINVAL; + + metadata = kbase_hwcnt_virtualizer_metadata(hvirt); + + if (!metadata) + return -EINVAL; + + kinstr_ctx = kzalloc(sizeof(*kinstr_ctx), GFP_KERNEL); + + if (!kinstr_ctx) + return -ENOMEM; + + kinstr_ctx->hvirt = hvirt; + kinstr_ctx->metadata = metadata; + + mutex_init(&kinstr_ctx->lock); + INIT_LIST_HEAD(&kinstr_ctx->clients); + hrtimer_init(&kinstr_ctx->dump_timer, CLOCK_MONOTONIC, + HRTIMER_MODE_REL); + kinstr_ctx->dump_timer.function = kbasep_kinstr_prfcnt_dump_timer; + INIT_WORK(&kinstr_ctx->dump_work, kbasep_kinstr_prfcnt_dump_worker); + + *out_kinstr_ctx = kinstr_ctx; + return 0; +} + +void kbase_kinstr_prfcnt_term(struct kbase_kinstr_prfcnt_context *kinstr_ctx) +{ + if (!kinstr_ctx) + return; + + cancel_work_sync(&kinstr_ctx->dump_work); + + /* Non-zero client count implies client leak */ + if (WARN_ON(kinstr_ctx->client_count > 0)) { + struct kbase_kinstr_prfcnt_client *pos, *n; + + list_for_each_entry_safe(pos, n, &kinstr_ctx->clients, node) { + list_del(&pos->node); + kinstr_ctx->client_count--; + kbasep_kinstr_prfcnt_client_destroy(pos); + } + } + + WARN_ON(kinstr_ctx->client_count > 0); + kfree(kinstr_ctx); +} + +void kbase_kinstr_prfcnt_suspend(struct kbase_kinstr_prfcnt_context *kinstr_ctx) +{ + if (WARN_ON(!kinstr_ctx)) + return; + + mutex_lock(&kinstr_ctx->lock); + + if (!WARN_ON(kinstr_ctx->suspend_count == SIZE_MAX)) + kinstr_ctx->suspend_count++; + + mutex_unlock(&kinstr_ctx->lock); + + /* Always sync cancel the timer and then the worker, regardless of the + * new suspend count. + * + * This ensures concurrent calls to kbase_kinstr_prfcnt_suspend() always block + * until kinstr_prfcnt is fully suspended. + * + * The timer is canceled before the worker, as the timer + * unconditionally re-enqueues the worker, but the worker checks the + * suspend_count that we just incremented before rescheduling the timer. + * + * Therefore if we cancel the worker first, the timer might re-enqueue + * the worker before we cancel the timer, but the opposite is not + * possible. + */ + hrtimer_cancel(&kinstr_ctx->dump_timer); + cancel_work_sync(&kinstr_ctx->dump_work); +} + +void kbase_kinstr_prfcnt_resume(struct kbase_kinstr_prfcnt_context *kinstr_ctx) +{ + if (WARN_ON(!kinstr_ctx)) + return; + + mutex_lock(&kinstr_ctx->lock); + + if (!WARN_ON(kinstr_ctx->suspend_count == 0)) { + kinstr_ctx->suspend_count--; + + /* Last resume, so re-enqueue the worker if we have any periodic + * clients. + */ + if (kinstr_ctx->suspend_count == 0) { + struct kbase_kinstr_prfcnt_client *pos; + bool has_periodic_clients = false; + + list_for_each_entry(pos, &kinstr_ctx->clients, node) { + if (pos->dump_interval_ns != 0) { + has_periodic_clients = true; + break; + } + } + + if (has_periodic_clients) + kbase_hwcnt_virtualizer_queue_work( + kinstr_ctx->hvirt, + &kinstr_ctx->dump_work); + } + } + + mutex_unlock(&kinstr_ctx->lock); +} + +static int kbasep_kinstr_prfcnt_sample_array_alloc( + const struct kbase_hwcnt_metadata *metadata, size_t n, + struct kbase_kinstr_prfcnt_sample_array *sample_arr) +{ + struct kbase_kinstr_prfcnt_sample *samples; + size_t sample_idx; + u64 addr; + unsigned int order; + size_t dump_buf_bytes; + size_t clk_cnt_buf_bytes; + size_t sample_meta_bytes; + size_t block_count = 0; + size_t sample_size; + size_t grp, blk, blk_inst; + + if (!metadata || !sample_arr) + return -EINVAL; + + kbase_hwcnt_metadata_for_each_block(metadata, grp, blk, blk_inst) + block_count++; + + /* Reserve one for last sentinel item. */ + block_count++; + + sample_meta_bytes = sizeof(struct prfcnt_metadata) * block_count; + dump_buf_bytes = metadata->dump_buf_bytes; + clk_cnt_buf_bytes = + sizeof(*samples->dump_buf.clk_cnt_buf) * metadata->clk_cnt; + sample_size = sample_meta_bytes + dump_buf_bytes + clk_cnt_buf_bytes; + + samples = kmalloc_array(n, sizeof(*samples), GFP_KERNEL); + + if (!samples) + return -ENOMEM; + + order = get_order(sample_size * n); + addr = (u64)(uintptr_t)kzalloc(sample_size * n, GFP_KERNEL); + + if (!addr) { + kfree((void *)samples); + return -ENOMEM; + } + + sample_arr->page_addr = addr; + sample_arr->page_order = order; + sample_arr->sample_count = n; + sample_arr->samples = samples; + + for (sample_idx = 0; sample_idx < n; sample_idx++) { + const size_t sample_meta_offset = sample_size * sample_idx; + const size_t dump_buf_offset = + sample_meta_offset + sample_meta_bytes; + const size_t clk_cnt_buf_offset = + dump_buf_offset + dump_buf_bytes; + + /* Internal layout in a sample buffer: [sample metadata, dump_buf, clk_cnt_buf]. */ + samples[sample_idx].dump_buf.metadata = metadata; + samples[sample_idx].sample_meta = + (u64 *)(uintptr_t)(addr + sample_meta_offset); + samples[sample_idx].dump_buf.dump_buf = + (u64 *)(uintptr_t)(addr + dump_buf_offset); + samples[sample_idx].dump_buf.clk_cnt_buf = + (u64 *)(uintptr_t)(addr + clk_cnt_buf_offset); + } + + return 0; +} + +static bool prfcnt_mode_supported(u8 mode) +{ + return (mode == PRFCNT_MODE_MANUAL) || (mode == PRFCNT_MODE_PERIODIC); +} + +static void +kbasep_kinstr_prfcnt_block_enable_to_physical(uint32_t *phys_em, + const uint64_t *enable_mask) +{ + *phys_em |= kbase_hwcnt_backend_gpu_block_map_to_physical( + enable_mask[0], enable_mask[1]); +} + +/** + * kbasep_kinstr_prfcnt_parse_request_enable - Parse an enable request + * @req_enable: Performance counters enable request to parse. + * @config: Client object the session configuration should be written to. + * + * This function parses a performance counters enable request. + * This type of request specifies a bitmask of HW counters to enable + * for one performance counters block type. In addition to that, + * a performance counters enable request may also set "global" + * configuration properties that affect the whole session, like the + * performance counters set, which shall be compatible with the same value + * set by other performance request items. + * + * Return: 0 on success, else error code. + */ +static int kbasep_kinstr_prfcnt_parse_request_enable( + const struct prfcnt_request_enable *req_enable, + struct kbase_kinstr_prfcnt_client_config *config) +{ + int err = 0; + u8 req_set = KBASE_HWCNT_SET_UNDEFINED, default_set; + + switch (req_enable->set) { + case PRFCNT_SET_PRIMARY: + req_set = KBASE_HWCNT_SET_PRIMARY; + break; + case PRFCNT_SET_SECONDARY: + req_set = KBASE_HWCNT_SET_SECONDARY; + break; + case PRFCNT_SET_TERTIARY: + req_set = KBASE_HWCNT_SET_TERTIARY; + break; + default: + err = -EINVAL; + break; + } + + /* The performance counter set is a "global" property that affects + * the whole session. Either this is the first request that sets + * the value, or it shall be identical to all previous requests. + */ + if (!err) { + if (config->counter_set == KBASE_HWCNT_SET_UNDEFINED) + config->counter_set = req_set; + else if (config->counter_set != req_set) + err = -EINVAL; + } + + /* Temporarily, the requested set cannot be different from the default + * set because it's the only one to be supported. This will change in + * the future. + */ +#if defined(CONFIG_MALI_PRFCNT_SET_SECONDARY) + default_set = KBASE_HWCNT_SET_SECONDARY; +#elif defined(CONFIG_MALI_PRFCNT_SET_TERTIARY) + default_set = KBASE_HWCNT_SET_TERTIARY; +#else + /* Default to primary */ + default_set = KBASE_HWCNT_SET_PRIMARY; +#endif + + if (req_set != default_set) + err = -EINVAL; + + if (err < 0) + return err; + + /* Enable the performance counters based on the bitmask provided + * by the user space client. + * It is possible to receive multiple requests for the same counter + * block, in which case the bitmask will be a logical OR of all the + * bitmasks given by the client. + */ + switch (req_enable->block_type) { + case PRFCNT_BLOCK_TYPE_FE: + kbasep_kinstr_prfcnt_block_enable_to_physical( + &config->phys_em.fe_bm, req_enable->enable_mask); + break; + case PRFCNT_BLOCK_TYPE_TILER: + kbasep_kinstr_prfcnt_block_enable_to_physical( + &config->phys_em.tiler_bm, req_enable->enable_mask); + break; + case PRFCNT_BLOCK_TYPE_MEMORY: + kbasep_kinstr_prfcnt_block_enable_to_physical( + &config->phys_em.mmu_l2_bm, req_enable->enable_mask); + break; + case PRFCNT_BLOCK_TYPE_SHADER_CORE: + kbasep_kinstr_prfcnt_block_enable_to_physical( + &config->phys_em.shader_bm, req_enable->enable_mask); + break; + default: + err = -EINVAL; + break; + } + + return err; +} + +/** + * kbasep_kinstr_prfcnt_parse_setup - Parse session setup + * @kinstr_ctx: Pointer to the kinstr_prfcnt context. + * @setup: Session setup information to parse. + * @config: Client object the session configuration should be written to. + * + * This function parses the list of "request" items sent by the user space + * client, and writes the configuration for the new client to be created + * for the session. + * + * Return: 0 on success, else error code. + */ +static int kbasep_kinstr_prfcnt_parse_setup( + struct kbase_kinstr_prfcnt_context *kinstr_ctx, + union kbase_ioctl_kinstr_prfcnt_setup *setup, + struct kbase_kinstr_prfcnt_client_config *config) +{ + uint32_t i; + struct prfcnt_request_item *req_arr; + int err = 0; + + if (!setup->in.requests_ptr || (setup->in.request_item_count == 0) || + (setup->in.request_item_size == 0)) { + return -EINVAL; + } + + req_arr = + (struct prfcnt_request_item *)(uintptr_t)setup->in.requests_ptr; + + if (req_arr[setup->in.request_item_count - 1].hdr.item_type != + FLEX_LIST_TYPE_NONE) { + return -EINVAL; + } + + if (req_arr[setup->in.request_item_count - 1].hdr.item_version != 0) + return -EINVAL; + + /* The session configuration can only feature one value for some + * properties (like capture mode and block counter set), but the client + * may potential issue multiple requests and try to set more than one + * value for those properties. While issuing multiple requests for the + * same property is allowed by the protocol, asking for different values + * is illegal. Leaving these properties as undefined is illegal, too. + */ + config->prfcnt_mode = PRFCNT_MODE_RESERVED; + config->counter_set = KBASE_HWCNT_SET_UNDEFINED; + + for (i = 0; i < setup->in.request_item_count - 1; i++) { + if (req_arr[i].hdr.item_version > PRFCNT_READER_API_VERSION) { + err = -EINVAL; + break; + } + + switch (req_arr[i].hdr.item_type) { + /* Capture mode is initialized as undefined. + * The first request of this type sets the capture mode. + * The protocol allows the client to send redundant requests, + * but only if they replicate the same value that has already + * been set by the first request. + */ + case PRFCNT_REQUEST_TYPE_MODE: + if (!prfcnt_mode_supported(req_arr[i].u.req_mode.mode)) + err = -EINVAL; + else if (config->prfcnt_mode == PRFCNT_MODE_RESERVED) + config->prfcnt_mode = + req_arr[i].u.req_mode.mode; + else if (req_arr[i].u.req_mode.mode != + config->prfcnt_mode) + err = -EINVAL; + + if (err < 0) + break; + + if (config->prfcnt_mode == PRFCNT_MODE_PERIODIC) { + config->period_us = + req_arr[i] + .u.req_mode.mode_config.periodic + .period_us; + + if ((config->period_us != 0) && + (config->period_us < + DUMP_INTERVAL_MIN_US)) { + config->period_us = + DUMP_INTERVAL_MIN_US; + } + } + break; + + case PRFCNT_REQUEST_TYPE_ENABLE: + err = kbasep_kinstr_prfcnt_parse_request_enable( + &req_arr[i].u.req_enable, config); + break; + + default: + err = -EINVAL; + break; + } + + if (err < 0) + break; + } + + /* Verify that properties (like capture mode and block counter set) + * have been defined by the user space client. + */ + if (config->prfcnt_mode == PRFCNT_MODE_RESERVED) + err = -EINVAL; + + if (config->counter_set == KBASE_HWCNT_SET_UNDEFINED) + err = -EINVAL; + + return err; +} + +/** + * kbasep_kinstr_prfcnt_client_create() - Create a kinstr_prfcnt client. + * Does not attach to the kinstr_prfcnt + * context. + * @kinstr_ctx: Non-NULL pointer to kinstr_prfcnt context. + * @setup: Non-NULL pointer to hardware counter ioctl setup structure. + * @out_vcli: Non-NULL pointer to where created client will be stored on + * success. + * + * Return: 0 on success, else error code. + */ +static int kbasep_kinstr_prfcnt_client_create( + struct kbase_kinstr_prfcnt_context *kinstr_ctx, + union kbase_ioctl_kinstr_prfcnt_setup *setup, + struct kbase_kinstr_prfcnt_client **out_vcli) +{ + int err; + struct kbase_kinstr_prfcnt_client *cli; + struct kbase_hwcnt_physical_enable_map phys_em; + + WARN_ON(!kinstr_ctx); + WARN_ON(!setup); + + cli = kzalloc(sizeof(*cli), GFP_KERNEL); + + if (!cli) + return -ENOMEM; + + cli->kinstr_ctx = kinstr_ctx; + err = kbasep_kinstr_prfcnt_parse_setup(kinstr_ctx, setup, &cli->config); + + if (err < 0) + goto error; + + cli->config.buffer_count = MAX_BUFFER_COUNT; + cli->dump_interval_ns = cli->config.period_us * NSEC_PER_USEC; + cli->next_dump_time_ns = 0; + err = kbase_hwcnt_enable_map_alloc(kinstr_ctx->metadata, + &cli->enable_map); + + if (err < 0) + goto error; + + phys_em.fe_bm = 0; + phys_em.shader_bm = 0; + phys_em.tiler_bm = 0; + phys_em.mmu_l2_bm = 0; + + kbase_hwcnt_gpu_enable_map_from_physical(&cli->enable_map, &phys_em); + + cli->sample_count = cli->config.buffer_count; + cli->sample_size = kbasep_kinstr_prfcnt_get_sample_size( + kinstr_ctx->metadata, &cli->tmp_buf); + + /* Use virtualizer's metadata to alloc tmp buffer which interacts with + * the HWC virtualizer. + */ + err = kbase_hwcnt_dump_buffer_alloc(kinstr_ctx->metadata, + &cli->tmp_buf); + + if (err < 0) + goto error; + + /* Enable all the available clk_enable_map. */ + cli->enable_map.clk_enable_map = + (1ull << kinstr_ctx->metadata->clk_cnt) - 1; + + /* Use metadata from virtualizer to allocate dump buffers if + * kinstr_prfcnt doesn't have the truncated metadata. + */ + err = kbasep_kinstr_prfcnt_sample_array_alloc(kinstr_ctx->metadata, + cli->config.buffer_count, + &cli->sample_arr); + + if (err < 0) + goto error; + + err = -ENOMEM; + + cli->dump_bufs_meta = + kmalloc_array(cli->config.buffer_count, + sizeof(*cli->dump_bufs_meta), GFP_KERNEL); + + if (!cli->dump_bufs_meta) + goto error; + + err = kbase_hwcnt_virtualizer_client_create( + kinstr_ctx->hvirt, &cli->enable_map, &cli->hvcli); + + if (err < 0) + goto error; + + init_waitqueue_head(&cli->waitq); + *out_vcli = cli; + + return 0; + +error: + kbasep_kinstr_prfcnt_client_destroy(cli); + return err; +} + +static size_t kbasep_kinstr_prfcnt_get_block_info_count( + const struct kbase_hwcnt_metadata *metadata) +{ + size_t grp; + size_t block_info_count = 0; + + if (!metadata) + return 0; + + for (grp = 0; grp < kbase_hwcnt_metadata_group_count(metadata); grp++) { + block_info_count += + kbase_hwcnt_metadata_block_count(metadata, grp); + } + + return block_info_count; +} + +static void kbasep_kinstr_prfcnt_get_request_info_list( + struct kbase_kinstr_prfcnt_context *kinstr_ctx, + struct prfcnt_enum_item *item_arr, size_t *arr_idx) +{ + memcpy(&item_arr[*arr_idx], kinstr_prfcnt_supported_requests, + sizeof(kinstr_prfcnt_supported_requests)); + *arr_idx += ARRAY_SIZE(kinstr_prfcnt_supported_requests); +} + +static enum prfcnt_block_type +kbase_hwcnt_metadata_block_type_to_prfcnt_block_type(u64 type) +{ + enum prfcnt_block_type block_type; + + switch (type) { + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE: + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE2: + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_FE3: + block_type = PRFCNT_BLOCK_TYPE_FE; + break; + + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_TILER: + block_type = PRFCNT_BLOCK_TYPE_TILER; + break; + + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC: + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC2: + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_SC3: + block_type = PRFCNT_BLOCK_TYPE_SHADER_CORE; + break; + + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS: + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_MEMSYS2: + block_type = PRFCNT_BLOCK_TYPE_MEMORY; + break; + + case KBASE_HWCNT_GPU_V5_BLOCK_TYPE_PERF_UNDEFINED: + default: + block_type = PRFCNT_BLOCK_TYPE_RESERVED; + break; + } + + return block_type; +} + +static int kbasep_kinstr_prfcnt_get_block_info_list( + const struct kbase_hwcnt_metadata *metadata, size_t block_set, + struct prfcnt_enum_item *item_arr, size_t *arr_idx) +{ + size_t grp; + size_t blk; + + if (!metadata || !item_arr || !arr_idx) + return -EINVAL; + + for (grp = 0; grp < kbase_hwcnt_metadata_group_count(metadata); grp++) { + for (blk = 0; + blk < kbase_hwcnt_metadata_block_count(metadata, grp); + blk++, (*arr_idx)++) { + item_arr[*arr_idx].hdr.item_type = + PRFCNT_ENUM_TYPE_BLOCK; + item_arr[*arr_idx].hdr.item_version = + PRFCNT_READER_API_VERSION; + item_arr[*arr_idx].u.block_counter.set = block_set; + + item_arr[*arr_idx].u.block_counter.block_type = + kbase_hwcnt_metadata_block_type_to_prfcnt_block_type( + kbase_hwcnt_metadata_block_type( + metadata, grp, blk)); + item_arr[*arr_idx].u.block_counter.num_instances = + kbase_hwcnt_metadata_block_instance_count( + metadata, grp, blk); + item_arr[*arr_idx].u.block_counter.num_values = + kbase_hwcnt_metadata_block_values_count( + metadata, grp, blk); + + /* The bitmask of available counters should be dynamic. + * Temporarily, it is set to U64_MAX, waiting for the + * required functionality to be available in the future. + */ + item_arr[*arr_idx].u.block_counter.counter_mask[0] = + U64_MAX; + item_arr[*arr_idx].u.block_counter.counter_mask[1] = + U64_MAX; + } + } + + return 0; +} + +static int kbasep_kinstr_prfcnt_enum_info_count( + struct kbase_kinstr_prfcnt_context *kinstr_ctx, + struct kbase_ioctl_kinstr_prfcnt_enum_info *enum_info) +{ + int err = 0; + uint32_t count = 0; + size_t block_info_count = 0; + const struct kbase_hwcnt_metadata *metadata; + + count = ARRAY_SIZE(kinstr_prfcnt_supported_requests); + metadata = kbase_hwcnt_virtualizer_metadata(kinstr_ctx->hvirt); + block_info_count = kbasep_kinstr_prfcnt_get_block_info_count(metadata); + count += block_info_count; + + /* Reserve one for the last sentinel item. */ + count++; + enum_info->info_item_count = count; + enum_info->info_item_size = sizeof(struct prfcnt_enum_item); + kinstr_ctx->info_item_count = count; + + return err; +} + +static int kbasep_kinstr_prfcnt_enum_info_list( + struct kbase_kinstr_prfcnt_context *kinstr_ctx, + struct kbase_ioctl_kinstr_prfcnt_enum_info *enum_info) +{ + struct prfcnt_enum_item *prfcnt_item_arr; + size_t arr_idx = 0; + int err = 0; + size_t block_info_count = 0; + const struct kbase_hwcnt_metadata *metadata; + + if ((enum_info->info_item_size == 0) || + (enum_info->info_item_count == 0) || !enum_info->info_list_ptr) + return -EINVAL; + + if (enum_info->info_item_count != kinstr_ctx->info_item_count) + return -EINVAL; + + prfcnt_item_arr = + (struct prfcnt_enum_item *)(uintptr_t)enum_info->info_list_ptr; + kbasep_kinstr_prfcnt_get_request_info_list(kinstr_ctx, prfcnt_item_arr, + &arr_idx); + metadata = kbase_hwcnt_virtualizer_metadata(kinstr_ctx->hvirt); + block_info_count = kbasep_kinstr_prfcnt_get_block_info_count(metadata); + + if (arr_idx + block_info_count >= enum_info->info_item_count) + err = -EINVAL; + + if (!err) { + size_t counter_set; + +#if defined(CONFIG_MALI_PRFCNT_SET_SECONDARY) + counter_set = KBASE_HWCNT_SET_SECONDARY; +#elif defined(CONFIG_MALI_PRFCNT_SET_TERTIARY) + counter_set = KBASE_HWCNT_SET_TERTIARY; +#else + /* Default to primary */ + counter_set = KBASE_HWCNT_SET_PRIMARY; +#endif + kbasep_kinstr_prfcnt_get_block_info_list( + metadata, counter_set, prfcnt_item_arr, &arr_idx); + if (arr_idx != enum_info->info_item_count - 1) + err = -EINVAL; + } + + /* The last sentinel item. */ + prfcnt_item_arr[enum_info->info_item_count - 1].hdr.item_type = + FLEX_LIST_TYPE_NONE; + prfcnt_item_arr[enum_info->info_item_count - 1].hdr.item_version = 0; + + return err; +} + +int kbase_kinstr_prfcnt_enum_info( + struct kbase_kinstr_prfcnt_context *kinstr_ctx, + struct kbase_ioctl_kinstr_prfcnt_enum_info *enum_info) +{ + int err; + + if (!kinstr_ctx || !enum_info) + return -EINVAL; + + if (!enum_info->info_list_ptr) + err = kbasep_kinstr_prfcnt_enum_info_count(kinstr_ctx, + enum_info); + else + err = kbasep_kinstr_prfcnt_enum_info_list(kinstr_ctx, + enum_info); + + return err; +} + +int kbase_kinstr_prfcnt_setup(struct kbase_kinstr_prfcnt_context *kinstr_ctx, + union kbase_ioctl_kinstr_prfcnt_setup *setup) +{ + int err; + struct kbase_kinstr_prfcnt_client *cli = NULL; + + if (!kinstr_ctx || !setup) + return -EINVAL; + + err = kbasep_kinstr_prfcnt_client_create(kinstr_ctx, setup, &cli); + + if (err < 0) + goto error; + + mutex_lock(&kinstr_ctx->lock); + kinstr_ctx->client_count++; + list_add(&cli->node, &kinstr_ctx->clients); + mutex_unlock(&kinstr_ctx->lock); + + setup->out.prfcnt_metadata_item_size = sizeof(struct prfcnt_metadata); + setup->out.prfcnt_mmap_size_bytes = + cli->sample_size * cli->sample_count; + + /* Expose to user-space only once the client is fully initialized */ + err = anon_inode_getfd("[mali_kinstr_prfcnt_desc]", + &kinstr_prfcnt_client_fops, cli, + O_RDONLY | O_CLOEXEC); + + if (err < 0) + goto client_installed_error; + + return err; + +client_installed_error: + mutex_lock(&kinstr_ctx->lock); + kinstr_ctx->client_count--; + list_del(&cli->node); + mutex_unlock(&kinstr_ctx->lock); +error: + kbasep_kinstr_prfcnt_client_destroy(cli); + return err; +} |