diff options
| author | Whi copybara merger <whitechapel-automerger@google.com> | 2021-10-13 06:43:36 +0000 |
|---|---|---|
| committer | Nrithya Kanakasabapathy <nrithya@google.com> | 2021-10-27 21:16:06 +0000 |
| commit | 1ac360cf5aca758039d87f67e87d8be451898509 (patch) | |
| tree | c709e45275396f16fa1aa2abdcfca0f9e13ee7a7 /drivers | |
| parent | 4c7500762b4905e1d245e76684fa1511108900d5 (diff) | |
| download | janeiro-1ac360cf5aca758039d87f67e87d8be451898509.tar.gz | |
Merge branch 'pro' into android13-gs-pixel-5.10
edgetpu: retry shutdown on GSA error response
Bug: 194954722
Revert "edgetpu: janeiro: temporarily disable external interface"
edgetpu: janeiro: update mbox manager to support aoc
Bug: 202489917
Bug: 201727491
edgetpu: janeiro: Add LPM recovery mechanism
Bug: 202402922
edgetpu: ensure TPU is completely off
Bug: 202744710
edgetpu: janeiro: temporarily disable external interface
edgetpu: add a mobile thermal module
Bug: 194241938
Bug: 201076651
edgetpu: support memory for f/w context switch
Bug: 200744848
edgetpu: abrolhos: add bus speed block KCI
Bug: 199166333
edgetpu: fix mobile pm variables scope
Bug: 199929284
edgetpu: Show mappings for every telemetry
Bug: 202089755
edgetpu: Support mmapping different buffers
Bug: 202089755
edgetpu: Introduce VMA_TYPE_WIDTH
Bug: 202089755
edgetpu: Set/unset event on every telemetry
Bug: 202089755
edgetpu: Check every telemetry when receiving irq
Bug: 202089755
edgetpu: Use telemetry[0] for telemetry_kci
Bug: 202089755
edgetpu: Initiate telemetry based on etdev->num_cores
Bug: 202089755
edgetpu: Add missing #if CONFIG_EDGETPU_TELEMETRY_TRACE
Bug: 202089755
edgetpu: Add num_cores to struct edgetpu_dev
Bug: 202089755
edgetpu: Add detail clk debugfs
Bug: 199929284
edgetpu: janeiro: fix frequency values
Bug: 199685518
edgetpu: change interop interface
Bug: 200658064
edgetpu: move mobile headers out from unittests/
Bug: 201267603
edgetpu: add include/ to the include path
Bug: 201267603
edgetpu: fix Kbuild inclusion path
Bug: 201267603
edgetpu: remove fault handlers for linux < 5.3
GitOrigin-RevId: f00278f72aefa16eec70384ec79d248fa52ca474
Change-Id: I364796b0dc127f83737f3afbaf0508ac35ac767d
Diffstat (limited to 'drivers')
30 files changed, 1412 insertions, 387 deletions
diff --git a/drivers/edgetpu/Kbuild b/drivers/edgetpu/Kbuild index 3479ad2..bb899dd 100644 --- a/drivers/edgetpu/Kbuild +++ b/drivers/edgetpu/Kbuild @@ -1,7 +1,7 @@ obj-m += janeiro.o -ccflags-y += -DCONFIG_EDGETPU_TELEMETRY_TRACE=1 -I$(src)/include # Use the absolute path of this Makefile to get the source directory. CURRENT_DIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +ccflags-y += -DCONFIG_EDGETPU_TELEMETRY_TRACE=1 -I$(CURRENT_DIR)/include GIT_PATH=$(CURRENT_DIR)/../../ ifeq ($(shell git --git-dir=$(GIT_PATH)/.git rev-parse --is-inside-work-tree),true) GIT_REPO_STATE=$(shell (git --git-dir=$(GIT_PATH)/.git --work-tree=$(GIT_PATH) status --porcelain | grep -q .) && echo -dirty) @@ -13,7 +13,7 @@ endif edgetpu-objs := edgetpu-mailbox.o edgetpu-kci.o edgetpu-telemetry.o edgetpu-mapping.o edgetpu-dmabuf.o edgetpu-async.o edgetpu-iremap-pool.o edgetpu-sw-watchdog.o edgetpu-firmware.o edgetpu-firmware-util.o -janeiro-y := janeiro-device.o janeiro-device-group.o janeiro-fs.o janeiro-core.o janeiro-platform.o janeiro-firmware.o janeiro-pm.o janeiro-debug-dump.o janeiro-usage-stats.o janeiro-iommu.o janeiro-wakelock.o janeiro-external.o $(edgetpu-objs) +janeiro-y := janeiro-device.o janeiro-device-group.o janeiro-fs.o janeiro-core.o janeiro-platform.o janeiro-firmware.o janeiro-thermal.o janeiro-pm.o janeiro-debug-dump.o janeiro-usage-stats.o janeiro-iommu.o janeiro-wakelock.o janeiro-external.o $(edgetpu-objs) CFLAGS_janeiro-fs.o := -DCONFIG_JANEIRO=1 diff --git a/drivers/edgetpu/Makefile b/drivers/edgetpu/Makefile index f21f1dc..4be6164 100644 --- a/drivers/edgetpu/Makefile +++ b/drivers/edgetpu/Makefile @@ -21,7 +21,8 @@ edgetpu-objs := edgetpu-async.o edgetpu-dmabuf.o edgetpu-iremap-pool.o \ janeiro-objs := janeiro-core.o janeiro-debug-dump.o janeiro-device-group.o \ janeiro-device.o janeiro-firmware.o janeiro-fs.o \ janeiro-iommu.o janeiro-platform.o janeiro-pm.o \ - janeiro-usage-stats.o janeiro-wakelock.o janeiro-external.o \ + janeiro-thermal.o janeiro-usage-stats.o janeiro-wakelock.o \ + janeiro-external.o \ $(edgetpu-objs) diff --git a/drivers/edgetpu/edgetpu-core.c b/drivers/edgetpu/edgetpu-core.c index 20e3372..f4b89ae 100644 --- a/drivers/edgetpu/edgetpu-core.c +++ b/drivers/edgetpu/edgetpu-core.c @@ -37,6 +37,12 @@ #include "edgetpu-wakelock.h" #include "edgetpu.h" +/* Bits higher than VMA_TYPE_WIDTH are used to carry type specific data, e.g., core id. */ +#define VMA_TYPE_WIDTH 16 +#define VMA_TYPE(x) ((x) & (BIT_MASK(VMA_TYPE_WIDTH) - 1)) +#define VMA_DATA_GET(x) ((x) >> VMA_TYPE_WIDTH) +#define VMA_DATA_SET(x, y) (VMA_TYPE(x) | ((y) << VMA_TYPE_WIDTH)) + enum edgetpu_vma_type { VMA_INVALID, @@ -44,14 +50,18 @@ enum edgetpu_vma_type { VMA_VII_CSR, VMA_VII_CMDQ, VMA_VII_RESPQ, + /* For VMA_LOG and VMA_TRACE, core id is stored in bits higher than VMA_TYPE_WIDTH. */ VMA_LOG, VMA_TRACE, }; +/* type that combines enum edgetpu_vma_type and data in higher bits. */ +typedef u32 edgetpu_vma_flags_t; + /* structure to be set to vma->vm_private_data on mmap */ struct edgetpu_vma_private { struct edgetpu_client *client; - enum edgetpu_vma_type type; + edgetpu_vma_flags_t flag; /* * vm_private_data is copied when a VMA is split, using this reference * counter to know when should this object be freed. @@ -80,7 +90,7 @@ static int edgetpu_mmap_full_csr(struct edgetpu_client *client, return ret; } -static enum edgetpu_vma_type mmap_vma_type(unsigned long pgoff) +static edgetpu_vma_flags_t mmap_vma_flag(unsigned long pgoff) { const unsigned long off = pgoff << PAGE_SHIFT; @@ -94,9 +104,15 @@ static enum edgetpu_vma_type mmap_vma_type(unsigned long pgoff) case EDGETPU_MMAP_RESP_QUEUE_OFFSET: return VMA_VII_RESPQ; case EDGETPU_MMAP_LOG_BUFFER_OFFSET: - return VMA_LOG; + return VMA_DATA_SET(VMA_LOG, 0); case EDGETPU_MMAP_TRACE_BUFFER_OFFSET: - return VMA_TRACE; + return VMA_DATA_SET(VMA_TRACE, 0); +#if EDGETPU_NUM_CORES > 1 + case EDGETPU_MMAP_LOG1_BUFFER_OFFSET: + return VMA_DATA_SET(VMA_LOG, 1); + case EDGETPU_MMAP_TRACE1_BUFFER_OFFSET: + return VMA_DATA_SET(VMA_TRACE, 1); +#endif /* EDGETPU_NUM_CORES > 1 */ default: return VMA_INVALID; } @@ -125,14 +141,14 @@ vma_type_to_wakelock_event(enum edgetpu_vma_type type) static struct edgetpu_vma_private * edgetpu_vma_private_alloc(struct edgetpu_client *client, - enum edgetpu_vma_type type) + edgetpu_vma_flags_t flag) { struct edgetpu_vma_private *pvt = kmalloc(sizeof(*pvt), GFP_KERNEL); if (!pvt) return NULL; pvt->client = edgetpu_client_get(client); - pvt->type = type; + pvt->flag = flag; refcount_set(&pvt->count, 1); return pvt; @@ -159,23 +175,25 @@ static void edgetpu_vma_open(struct vm_area_struct *vma) enum edgetpu_wakelock_event evt; struct edgetpu_client *client; struct edgetpu_dev *etdev; + enum edgetpu_vma_type type = VMA_TYPE(pvt->flag); edgetpu_vma_private_get(pvt); client = pvt->client; etdev = client->etdev; - evt = vma_type_to_wakelock_event(pvt->type); + evt = vma_type_to_wakelock_event(type); if (evt != EDGETPU_WAKELOCK_EVENT_END) edgetpu_wakelock_inc_event(client->wakelock, evt); /* handle telemetry types */ - switch (pvt->type) { + switch (type) { case VMA_LOG: - edgetpu_telemetry_inc_mmap_count(etdev, EDGETPU_TELEMETRY_LOG); + edgetpu_telemetry_inc_mmap_count(etdev, EDGETPU_TELEMETRY_LOG, + VMA_DATA_GET(pvt->flag)); break; case VMA_TRACE: - edgetpu_telemetry_inc_mmap_count(etdev, - EDGETPU_TELEMETRY_TRACE); + edgetpu_telemetry_inc_mmap_count(etdev, EDGETPU_TELEMETRY_TRACE, + VMA_DATA_GET(pvt->flag)); break; default: break; @@ -187,20 +205,22 @@ static void edgetpu_vma_close(struct vm_area_struct *vma) { struct edgetpu_vma_private *pvt = vma->vm_private_data; struct edgetpu_client *client = pvt->client; - enum edgetpu_wakelock_event evt = vma_type_to_wakelock_event(pvt->type); + enum edgetpu_vma_type type = VMA_TYPE(pvt->flag); + enum edgetpu_wakelock_event evt = vma_type_to_wakelock_event(type); struct edgetpu_dev *etdev = client->etdev; if (evt != EDGETPU_WAKELOCK_EVENT_END) edgetpu_wakelock_dec_event(client->wakelock, evt); /* handle telemetry types */ - switch (pvt->type) { + switch (type) { case VMA_LOG: - edgetpu_telemetry_dec_mmap_count(etdev, EDGETPU_TELEMETRY_LOG); + edgetpu_telemetry_dec_mmap_count(etdev, EDGETPU_TELEMETRY_LOG, + VMA_DATA_GET(pvt->flag)); break; case VMA_TRACE: - edgetpu_telemetry_dec_mmap_count(etdev, - EDGETPU_TELEMETRY_TRACE); + edgetpu_telemetry_dec_mmap_count(etdev, EDGETPU_TELEMETRY_TRACE, + VMA_DATA_GET(pvt->flag)); break; default: break; @@ -218,6 +238,7 @@ static const struct vm_operations_struct edgetpu_vma_ops = { int edgetpu_mmap(struct edgetpu_client *client, struct vm_area_struct *vma) { int ret = 0; + edgetpu_vma_flags_t flag; enum edgetpu_vma_type type; enum edgetpu_wakelock_event evt; struct edgetpu_vma_private *pvt; @@ -232,10 +253,11 @@ int edgetpu_mmap(struct edgetpu_client *client, struct vm_area_struct *vma) etdev_dbg(client->etdev, "%s: mmap pgoff = %#lX\n", __func__, vma->vm_pgoff); - type = mmap_vma_type(vma->vm_pgoff); + flag = mmap_vma_flag(vma->vm_pgoff); + type = VMA_TYPE(flag); if (type == VMA_INVALID) return -EINVAL; - pvt = edgetpu_vma_private_alloc(client, type); + pvt = edgetpu_vma_private_alloc(client, flag); if (!pvt) return -ENOMEM; @@ -259,19 +281,19 @@ int edgetpu_mmap(struct edgetpu_client *client, struct vm_area_struct *vma) /* Allow mapping log and telemetry buffers without a group */ if (type == VMA_LOG) { - ret = edgetpu_mmap_telemetry_buffer(client->etdev, - EDGETPU_TELEMETRY_LOG, vma); + ret = edgetpu_mmap_telemetry_buffer(client->etdev, EDGETPU_TELEMETRY_LOG, vma, + VMA_DATA_GET(flag)); goto out_set_op; } if (type == VMA_TRACE) { - ret = edgetpu_mmap_telemetry_buffer( - client->etdev, EDGETPU_TELEMETRY_TRACE, vma); + ret = edgetpu_mmap_telemetry_buffer(client->etdev, EDGETPU_TELEMETRY_TRACE, vma, + VMA_DATA_GET(flag)); goto out_set_op; } evt = vma_type_to_wakelock_event(type); /* - * @type should always correspond to a valid event since we handled + * VMA_TYPE(@flag) should always correspond to a valid event since we handled * telemetry mmaps above, still check evt != END in case new types are * added in the future. */ @@ -416,7 +438,7 @@ int edgetpu_device_add(struct edgetpu_dev *etdev, } etdev->telemetry = - devm_kzalloc(etdev->dev, sizeof(*etdev->telemetry), GFP_KERNEL); + devm_kcalloc(etdev->dev, etdev->num_cores, sizeof(*etdev->telemetry), GFP_KERNEL); if (!etdev->telemetry) { ret = -ENOMEM; goto remove_usage_stats; diff --git a/drivers/edgetpu/edgetpu-external.c b/drivers/edgetpu/edgetpu-external.c index bbbea8c..1552774 100644 --- a/drivers/edgetpu/edgetpu-external.c +++ b/drivers/edgetpu/edgetpu-external.c @@ -15,7 +15,20 @@ #include "edgetpu-internal.h" #include "edgetpu-mailbox.h" -static int edgetpu_external_mailbox_info_get(struct edgetpu_external_mailbox_info *info, +static enum edgetpu_ext_mailbox_type +edgetpu_external_client_to_mailbox_type(enum edgetpu_ext_client_type client_type) +{ + switch (client_type) { + case EDGETPU_EXTERNAL_CLIENT_TYPE_DSP: + return EDGETPU_EXTERNAL_MAILBOX_TYPE_DSP; + case EDGETPU_EXTERNAL_CLIENT_TYPE_AOC: + return EDGETPU_EXTERNAL_MAILBOX_TYPE_AOC; + default: + return -ENOENT; + } +} + +static int edgetpu_external_mailbox_info_get(struct edgetpu_ext_mailbox_info *info, struct edgetpu_external_mailbox *ext_mailbox) { int i; @@ -25,69 +38,54 @@ static int edgetpu_external_mailbox_info_get(struct edgetpu_external_mailbox_inf if (!info) return -EINVAL; - if (info->count < count) { - etdev_err(ext_mailbox->etdev, - "Insufficient space in provided buffer expected: %d received: %d\n", - count, info->count); - return -ENOMEM; - } - for (i = 0; i < count; i++) { desc = &ext_mailbox->descriptors[i]; info->mailboxes[i].cmdq_pa = desc->cmd_queue_mem.phys_addr; info->mailboxes[i].respq_pa = desc->resp_queue_mem.phys_addr; - info->mailboxes[i].mailbox_id = - desc->mailbox->mailbox_id - (EDGETPU_NUM_VII_MAILBOXES + 1); } info->cmdq_size = ext_mailbox->attr.cmd_queue_size; info->respq_size = ext_mailbox->attr.resp_queue_size; - info->count = count; return 0; } -static bool is_edgetpu_valid_client(struct edgetpu_external_mailbox *ext_mailbox, - enum edgetpu_external_client_type client_type) -{ - switch (client_type) { - case EDGETPU_EXTERNAL_CLIENT_TYPE_DSP: - return ext_mailbox->client_type == EDGETPU_EXT_MAILBOX_TYPE_DSP; - default: - return false; - } - - return false; -} - -static int edgetpu_mailbox_external_info_get_cmd(struct device *edgetpu_dev, - enum edgetpu_external_client_type client_type, - int *client_fd, - struct edgetpu_external_mailbox_info *info) +static int edgetpu_external_mailbox_alloc(struct device *edgetpu_dev, + struct edgetpu_ext_client_info *client_info, + struct edgetpu_ext_mailbox_info *info, + enum edgetpu_ext_client_type client_type) { struct edgetpu_client *client; struct edgetpu_device_group *group; struct edgetpu_external_mailbox *ext_mailbox; + struct edgetpu_external_mailbox_req req; int ret = 0; - struct fd f = fdget(*client_fd); + struct fd f = fdget(client_info->tpu_fd); struct file *file = f.file; if (!file) - return -ENOENT; + return -EBADF; if (!is_edgetpu_file(file)) { - ret = -ENOENT; + ret = -EINVAL; goto out; } client = file->private_data; - if (!client) { + if (!client || client->etdev->dev != edgetpu_dev) { ret = -EINVAL; goto out; } + req.mbox_type = edgetpu_external_client_to_mailbox_type(client_type); + req.mbox_map = client_info->mbox_map; + + ret = edgetpu_chip_get_ext_mailbox_index(req.mbox_type, &req.start, &req.end); + if (ret) + goto out; + mutex_lock(&client->group_lock); - if (!client->group || client->etdev->dev != edgetpu_dev) { + if (!client->group) { ret = -EINVAL; mutex_unlock(&client->group_lock); goto out; @@ -95,16 +93,24 @@ static int edgetpu_mailbox_external_info_get_cmd(struct device *edgetpu_dev, group = edgetpu_device_group_get(client->group); mutex_unlock(&client->group_lock); + if (copy_from_user(&req.attr, (void __user *)client_info->attr, sizeof(req.attr))) { + if (!client_info->attr) + etdev_warn(client->etdev, + "Illegal mailbox attributes, using VII mailbox attrs\n"); + req.attr = group->mbox_attr; + } + + ret = edgetpu_mailbox_enable_ext(client, EDGETPU_MAILBOX_ID_USE_ASSOC, &req); + if (ret) { + edgetpu_device_group_put(group); + goto out; + } mutex_lock(&group->lock); ext_mailbox = group->ext_mailbox; if (!ext_mailbox) { ret = -ENOENT; goto unlock; } - if (!is_edgetpu_valid_client(ext_mailbox, client_type)) { - ret = -EINVAL; - goto unlock; - } ret = edgetpu_external_mailbox_info_get(info, ext_mailbox); unlock: mutex_unlock(&group->lock); @@ -114,21 +120,45 @@ out: return ret; } -int edgetpu_ext_driver_cmd(struct device *edgetpu_dev, - enum edgetpu_external_client_type client_type, - enum edgetpu_external_commands cmd_id, void *in_data, void *out_data) +static int edgetpu_external_mailbox_free(struct device *edgetpu_dev, + struct edgetpu_ext_client_info *client_info) { + struct edgetpu_client *client; int ret = 0; + struct fd f = fdget(client_info->tpu_fd); + struct file *file = f.file; + + if (!file) + return -EBADF; + if (!is_edgetpu_file(file)) { + ret = -EINVAL; + goto out; + } + + client = file->private_data; + if (!client || client->etdev->dev != edgetpu_dev) { + ret = -EINVAL; + goto out; + } + + ret = edgetpu_mailbox_disable_ext(client, EDGETPU_MAILBOX_ID_USE_ASSOC); +out: + fdput(f); + return ret; +} + +int edgetpu_ext_driver_cmd(struct device *edgetpu_dev, + enum edgetpu_ext_client_type client_type, + enum edgetpu_ext_commands cmd_id, void *in_data, void *out_data) +{ switch (cmd_id) { - case MAILBOX_EXTERNAL_INFO_GET: - ret = edgetpu_mailbox_external_info_get_cmd(edgetpu_dev, client_type, in_data, - out_data); - break; + case ALLOCATE_EXTERNAL_MAILBOX: + return edgetpu_external_mailbox_alloc(edgetpu_dev, in_data, out_data, client_type); + case FREE_EXTERNAL_MAILBOX: + return edgetpu_external_mailbox_free(edgetpu_dev, in_data); default: return -ENOENT; } - - return ret; } EXPORT_SYMBOL_GPL(edgetpu_ext_driver_cmd); diff --git a/drivers/edgetpu/edgetpu-google-iommu.c b/drivers/edgetpu/edgetpu-google-iommu.c index eae1730..0bd68b8 100644 --- a/drivers/edgetpu/edgetpu-google-iommu.c +++ b/drivers/edgetpu/edgetpu-google-iommu.c @@ -12,7 +12,6 @@ #include <linux/scatterlist.h> #include <linux/slab.h> #include <linux/types.h> -#include <linux/version.h> #include "edgetpu-config.h" #include "edgetpu-internal.h" @@ -90,12 +89,6 @@ get_domain_by_context_id(struct edgetpu_dev *etdev, return domain; } -/* - * Kernel 5.3 introduced iommu_register_device_fault_handler - */ - -#if KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE - static int edgetpu_iommu_dev_fault_handler(struct iommu_fault *fault, void *token) { @@ -108,8 +101,7 @@ static int edgetpu_iommu_dev_fault_handler(struct iommu_fault *fault, etdev_warn(etdev, "pasid = %08X\n", fault->event.pasid); etdev_warn(etdev, "perms = %08X\n", fault->event.perm); etdev_warn(etdev, "addr = %llX\n", fault->event.addr); - etdev_warn(etdev, "fetch_addr = %llX\n", - fault->event.fetch_addr); + etdev_warn(etdev, "fetch_addr = %llX\n", fault->event.fetch_addr); } else if (fault->type == IOMMU_FAULT_PAGE_REQ) { etdev_dbg(etdev, "IOMMU page request fault!\n"); etdev_dbg(etdev, "flags = %08X\n", fault->prm.flags); @@ -122,37 +114,19 @@ static int edgetpu_iommu_dev_fault_handler(struct iommu_fault *fault, return -EAGAIN; } -static int -edgetpu_register_iommu_device_fault_handler(struct edgetpu_dev *etdev) +static int edgetpu_register_iommu_device_fault_handler(struct edgetpu_dev *etdev) { etdev_dbg(etdev, "Registering IOMMU device fault handler\n"); - return iommu_register_device_fault_handler( - etdev->dev, edgetpu_iommu_dev_fault_handler, etdev); + return iommu_register_device_fault_handler(etdev->dev, edgetpu_iommu_dev_fault_handler, + etdev); } -static int -edgetpu_unregister_iommu_device_fault_handler(struct edgetpu_dev *etdev) +static int edgetpu_unregister_iommu_device_fault_handler(struct edgetpu_dev *etdev) { etdev_dbg(etdev, "Unregistering IOMMU device fault handler\n"); return iommu_unregister_device_fault_handler(etdev->dev); } -#else /* kernel version before 5.3 */ - -static int -edgetpu_register_iommu_device_fault_handler(struct edgetpu_dev *etdev) -{ - return 0; -} - -static int -edgetpu_unregister_iommu_device_fault_handler(struct edgetpu_dev *etdev) -{ - return 0; -} - -#endif /* KERNEL_VERSION(5, 3, 0) <= LINUX_VERSION_CODE */ - /* A callback for idr_for_each to release the domains */ static int edgetpu_idr_free_domain_callback(int id, void *p, void *data) { diff --git a/drivers/edgetpu/edgetpu-internal.h b/drivers/edgetpu/edgetpu-internal.h index 6642e5f..122a3e7 100644 --- a/drivers/edgetpu/edgetpu-internal.h +++ b/drivers/edgetpu/edgetpu-internal.h @@ -175,6 +175,7 @@ enum edgetpu_dev_state { struct edgetpu_dev { struct device *dev; /* platform/pci bus device */ uint num_ifaces; /* Number of device interfaces */ + uint num_cores; /* Number of cores */ /* * Array of device interfaces * First element is the default interface @@ -476,4 +477,10 @@ int edgetpu_chip_acquire_ext_mailbox(struct edgetpu_client *client, int edgetpu_chip_release_ext_mailbox(struct edgetpu_client *client, struct edgetpu_ext_mailbox_ioctl *args); +/* + * Chip specific function to get indexes of external mailbox based on + * @mbox_type + */ +int edgetpu_chip_get_ext_mailbox_index(u32 mbox_type, u32 *start, u32 *end); + #endif /* __EDGETPU_INTERNAL_H__ */ diff --git a/drivers/edgetpu/edgetpu-kci.c b/drivers/edgetpu/edgetpu-kci.c index 17d92ab..52521d1 100644 --- a/drivers/edgetpu/edgetpu-kci.c +++ b/drivers/edgetpu/edgetpu-kci.c @@ -1055,3 +1055,20 @@ int edgetpu_kci_notify_throttling(struct edgetpu_dev *etdev, u32 level) return ret; } + +int edgetpu_kci_block_bus_speed_control(struct edgetpu_dev *etdev, bool block) +{ + struct edgetpu_command_element cmd = { + .code = KCI_CODE_BLOCK_BUS_SPEED_CONTROL, + .dma = { + .flags = (u32) block, + }, + }; + int ret; + + if (!etdev->kci) + return -ENODEV; + + ret = edgetpu_kci_send_cmd(etdev->kci, &cmd); + return ret; +} diff --git a/drivers/edgetpu/edgetpu-kci.h b/drivers/edgetpu/edgetpu-kci.h index 2893f20..acdf31a 100644 --- a/drivers/edgetpu/edgetpu-kci.h +++ b/drivers/edgetpu/edgetpu-kci.h @@ -109,6 +109,7 @@ enum edgetpu_kci_code { KCI_CODE_FIRMWARE_INFO = 11, KCI_CODE_GET_USAGE = 12, KCI_CODE_NOTIFY_THROTTLING = 13, + KCI_CODE_BLOCK_BUS_SPEED_CONTROL = 14, }; /* @@ -393,4 +394,11 @@ void edgetpu_kci_cancel_work_queues(struct edgetpu_kci *kci); */ int edgetpu_kci_notify_throttling(struct edgetpu_dev *etdev, u32 level); +/* + * Request the firmware to {un}block modulating bus clock speeds + * + * Used to prevent conflicts when sending a thermal policy request + */ +int edgetpu_kci_block_bus_speed_control(struct edgetpu_dev *etdev, bool block); + #endif /* __EDGETPU_KCI_H__ */ diff --git a/drivers/edgetpu/edgetpu-mailbox.c b/drivers/edgetpu/edgetpu-mailbox.c index f4a3e4e..a950e2c 100644 --- a/drivers/edgetpu/edgetpu-mailbox.c +++ b/drivers/edgetpu/edgetpu-mailbox.c @@ -6,6 +6,7 @@ */ #include <asm/page.h> +#include <linux/bitops.h> #include <linux/bits.h> #include <linux/dma-mapping.h> #include <linux/err.h> @@ -842,10 +843,10 @@ static bool edgetpu_mailbox_external_check_range(struct edgetpu_mailbox_manager static int edgetpu_mailbox_external_alloc(struct edgetpu_device_group *group, struct edgetpu_external_mailbox_req *ext_mailbox_req) { - u32 i, j = 0; + u32 i, j = 0, bmap, start, end; struct edgetpu_mailbox_manager *mgr = group->etdev->mailbox_manager; struct edgetpu_mailbox *mailbox; - int ret = 0, c = 0, count; + int ret = 0, count; struct edgetpu_external_mailbox *ext_mailbox; struct edgetpu_mailbox_attr attr; unsigned long flags; @@ -857,7 +858,6 @@ static int edgetpu_mailbox_external_alloc(struct edgetpu_device_group *group, if (ret) return ret; - count = ext_mailbox_req->count; attr = ext_mailbox_req->attr; if (!edgetpu_mailbox_external_check_range(mgr, ext_mailbox_req->start, @@ -868,6 +868,9 @@ static int edgetpu_mailbox_external_alloc(struct edgetpu_device_group *group, if (!ext_mailbox) return -ENOMEM; + bmap = ext_mailbox_req->mbox_map; + count = __sw_hweight32(bmap); + ext_mailbox->descriptors = kcalloc(count, sizeof(struct edgetpu_mailbox_descriptor), GFP_KERNEL); if (!ext_mailbox->descriptors) { @@ -876,33 +879,42 @@ static int edgetpu_mailbox_external_alloc(struct edgetpu_device_group *group, } ext_mailbox->attr = attr; - ext_mailbox->count = count; ext_mailbox->etdev = group->etdev; - ext_mailbox->client_type = ext_mailbox_req->client_type; + ext_mailbox->mbox_type = ext_mailbox_req->mbox_type; + + start = ext_mailbox_req->start; + end = ext_mailbox_req->end; write_lock_irqsave(&mgr->mailboxes_lock, flags); - for (i = ext_mailbox_req->start; i <= ext_mailbox_req->end; i++) { - if (!mgr->mailboxes[i]) - c++; - } - if (c < count) { - ret = -EBUSY; - goto unlock; + while (bmap) { + i = ffs(bmap) + start - 1; + if (i > end) { + ret = -EINVAL; + goto unlock; + } + if (mgr->mailboxes[i]) { + ret = -EBUSY; + goto unlock; + } + bmap = bmap & (bmap - 1); } - for (i = ext_mailbox_req->start; i <= ext_mailbox_req->end && j < count; i++) { - if (!mgr->mailboxes[i]) { - mailbox = edgetpu_mailbox_create_locked(mgr, i); - if (!IS_ERR(mailbox)) { - mgr->mailboxes[i] = mailbox; - ext_mailbox->descriptors[j++].mailbox = mailbox; - } else { - ret = PTR_ERR(mailbox); - goto release; - } + bmap = ext_mailbox_req->mbox_map; + while (bmap) { + i = ffs(bmap) + start - 1; + mailbox = edgetpu_mailbox_create_locked(mgr, i); + if (!IS_ERR(mailbox)) { + mgr->mailboxes[i] = mailbox; + ext_mailbox->descriptors[j++].mailbox = mailbox; + } else { + ret = PTR_ERR(mailbox); + goto release; } + bmap = bmap & (bmap - 1); } + ext_mailbox->count = j; + ret = edgetpu_mailbox_external_alloc_queue_batch(ext_mailbox); if (ret) goto release; diff --git a/drivers/edgetpu/edgetpu-mailbox.h b/drivers/edgetpu/edgetpu-mailbox.h index 3098760..1d284e6 100644 --- a/drivers/edgetpu/edgetpu-mailbox.h +++ b/drivers/edgetpu/edgetpu-mailbox.h @@ -88,15 +88,17 @@ struct edgetpu_mailbox_descriptor { edgetpu_queue_mem resp_queue_mem; }; +enum edgetpu_ext_mailbox_type { + EDGETPU_EXTERNAL_MAILBOX_TYPE_DSP, + EDGETPU_EXTERNAL_MAILBOX_TYPE_AOC, +}; + /* Structure to hold multiple external mailboxes allocated for a device group. */ struct edgetpu_external_mailbox { /* Number of external mailboxes allocated for a device group. */ u32 count; - /* - * Client type of external mailboxes requester, it belongs to - * EDGETPU_EXT_MAILBOX_TYPE_* - */ - u64 client_type; + /* Type of external mailbox */ + enum edgetpu_ext_mailbox_type mbox_type; /* Leader of device group. */ struct edgetpu_dev *etdev; /* Array of external mailboxes info with length @count. */ @@ -109,13 +111,8 @@ struct edgetpu_external_mailbox { struct edgetpu_external_mailbox_req { uint start; /* starting index of external mailbox in mailbox_manager */ uint end; /* end index of external mailbox in mailbox_manager */ - /* number of mailboxes to be allocated, should be less or equal to (end - start + 1) */ - uint count; - /* - * Client type of external mailboxes requester, it belongs to - * EDGETPU_EXT_MAILBOX_TYPE_* - */ - u64 client_type; + uint mbox_map; /* bitmap of mailbox indexes to be allocated */ + enum edgetpu_ext_mailbox_type mbox_type; /* Type of external mailbox */ struct edgetpu_mailbox_attr attr; /* mailbox attribute for allocation */ }; diff --git a/drivers/edgetpu/edgetpu-mobile-platform.c b/drivers/edgetpu/edgetpu-mobile-platform.c index 7b9abec..fe1c67c 100644 --- a/drivers/edgetpu/edgetpu-mobile-platform.c +++ b/drivers/edgetpu/edgetpu-mobile-platform.c @@ -26,24 +26,27 @@ * Log and trace buffers at the beginning of the remapped region, * pool memory afterwards. */ -#define EDGETPU_POOL_MEM_OFFSET (EDGETPU_TELEMETRY_BUFFER_SIZE * 2) +#define EDGETPU_POOL_MEM_OFFSET (EDGETPU_TELEMETRY_BUFFER_SIZE * 2 * EDGETPU_NUM_CORES) static void get_telemetry_mem(struct edgetpu_mobile_platform_dev *etmdev, enum edgetpu_telemetry_type type, struct edgetpu_coherent_mem *mem) { - int offset = type == EDGETPU_TELEMETRY_TRACE ? EDGETPU_TELEMETRY_BUFFER_SIZE : 0; - - mem->vaddr = etmdev->shared_mem_vaddr + offset; - mem->dma_addr = EDGETPU_REMAPPED_DATA_ADDR + offset; - mem->tpu_addr = EDGETPU_REMAPPED_DATA_ADDR + offset; - mem->host_addr = 0; - mem->size = EDGETPU_TELEMETRY_BUFFER_SIZE; + int i, offset = type == EDGETPU_TELEMETRY_TRACE ? EDGETPU_TELEMETRY_BUFFER_SIZE : 0; + + for (i = 0; i < etmdev->edgetpu_dev.num_cores; i++) { + mem[i].vaddr = etmdev->shared_mem_vaddr + offset; + mem[i].dma_addr = EDGETPU_REMAPPED_DATA_ADDR + offset; + mem[i].tpu_addr = EDGETPU_REMAPPED_DATA_ADDR + offset; + mem[i].host_addr = 0; + mem[i].size = EDGETPU_TELEMETRY_BUFFER_SIZE; + offset += EDGETPU_TELEMETRY_BUFFER_SIZE * 2; + } } static void edgetpu_mobile_get_telemetry_mem(struct edgetpu_mobile_platform_dev *etmdev) { - get_telemetry_mem(etmdev, EDGETPU_TELEMETRY_LOG, &etmdev->log_mem); - get_telemetry_mem(etmdev, EDGETPU_TELEMETRY_TRACE, &etmdev->trace_mem); + get_telemetry_mem(etmdev, EDGETPU_TELEMETRY_LOG, etmdev->log_mem); + get_telemetry_mem(etmdev, EDGETPU_TELEMETRY_TRACE, etmdev->trace_mem); } static int edgetpu_platform_setup_fw_region(struct edgetpu_mobile_platform_dev *etmdev) @@ -211,6 +214,7 @@ static int edgetpu_mobile_platform_probe(struct platform_device *pdev, platform_set_drvdata(pdev, etdev); etdev->dev = dev; + etdev->num_cores = EDGETPU_NUM_CORES; r = platform_get_resource(pdev, IORESOURCE_MEM, 0); if (IS_ERR_OR_NULL(r)) { @@ -277,8 +281,23 @@ static int edgetpu_mobile_platform_probe(struct platform_device *pdev, if (ret) dev_warn(dev, "SSMT setup failed (%d). Context isolation not enforced", ret); + etmdev->log_mem = devm_kcalloc(dev, etdev->num_cores, sizeof(*etmdev->log_mem), GFP_KERNEL); + if (!etmdev->log_mem) { + ret = -ENOMEM; + goto out_remove_irq; + } + +#if IS_ENABLED(CONFIG_EDGETPU_TELEMETRY_TRACE) + etmdev->trace_mem = + devm_kcalloc(dev, etdev->num_cores, sizeof(*etmdev->log_mem), GFP_KERNEL); + if (!etmdev->trace_mem) { + ret = -ENOMEM; + goto out_remove_irq; + } +#endif + edgetpu_mobile_get_telemetry_mem(etmdev); - ret = edgetpu_telemetry_init(etdev, &etmdev->log_mem, &etmdev->trace_mem); + ret = edgetpu_telemetry_init(etdev, etmdev->log_mem, etmdev->trace_mem); if (ret) goto out_remove_irq; @@ -288,6 +307,9 @@ static int edgetpu_mobile_platform_probe(struct platform_device *pdev, goto out_tel_exit; } + etdev_dbg(etdev, "Creating thermal device"); + etdev->thermal = devm_tpu_thermal_create(etdev->dev, etdev); + if (etmdev->after_probe) { ret = etmdev->after_probe(etmdev); if (ret) { diff --git a/drivers/edgetpu/edgetpu-mobile-platform.h b/drivers/edgetpu/edgetpu-mobile-platform.h index 8b3a25f..65184ae 100644 --- a/drivers/edgetpu/edgetpu-mobile-platform.h +++ b/drivers/edgetpu/edgetpu-mobile-platform.h @@ -75,6 +75,10 @@ struct edgetpu_mobile_platform_dev { phys_addr_t shared_mem_paddr; /* Size of the shared memory region size */ size_t shared_mem_size; + /* Physical address of the firmware context region */ + phys_addr_t fw_ctx_paddr; + /* Size of the firmware context region */ + size_t fw_ctx_size; /* * Pointer to GSA device for firmware authentication. * May be NULL if the chip does not support firmware authentication @@ -83,9 +87,9 @@ struct edgetpu_mobile_platform_dev { /* Virtual address of the SSMT block for this chip. */ void __iomem *ssmt_base; /* Coherent log buffer */ - struct edgetpu_coherent_mem log_mem; + struct edgetpu_coherent_mem *log_mem; /* Coherent trace buffer */ - struct edgetpu_coherent_mem trace_mem; + struct edgetpu_coherent_mem *trace_mem; #if IS_ENABLED(CONFIG_GOOGLE_BCL) struct bcl_device *bcl_dev; #endif diff --git a/drivers/edgetpu/edgetpu-pm.c b/drivers/edgetpu/edgetpu-pm.c index 872149f..e6bfa1d 100644 --- a/drivers/edgetpu/edgetpu-pm.c +++ b/drivers/edgetpu/edgetpu-pm.c @@ -8,6 +8,7 @@ #include <linux/iopoll.h> #include <linux/mutex.h> #include <linux/slab.h> +#include <linux/workqueue.h> #include "edgetpu-config.h" #include "edgetpu-internal.h" @@ -24,10 +25,19 @@ #define SIM_PCHANNEL(...) #endif +#define EDGETPU_ASYNC_POWER_DOWN_RETRY_DELAY 200 + struct edgetpu_pm_private { const struct edgetpu_pm_handlers *handlers; struct mutex lock; + /* Power up counter. Protected by @lock */ int power_up_count; + /* Flag indicating a deferred power down is pending. Protected by @lock */ + bool power_down_pending; + /* Worker to handle async power down retry */ + struct delayed_work power_down_work; + /* Back pointer to parent struct */ + struct edgetpu_pm *etpm; }; /* @@ -91,12 +101,49 @@ int edgetpu_pm_get(struct edgetpu_pm *etpm) if (!etpm || !etpm->p->handlers || !etpm->p->handlers->power_up) return 0; + mutex_lock(&etpm->p->lock); + etpm->p->power_down_pending = false; ret = edgetpu_pm_get_locked(etpm); mutex_unlock(&etpm->p->lock); + return ret; } +/* Caller must hold @etpm->p->lock */ +static void edgetpu_pm_try_power_down(struct edgetpu_pm *etpm) +{ + int ret = etpm->p->handlers->power_down(etpm); + + if (ret == -EAGAIN) { + etdev_warn(etpm->etdev, "Power down request denied. Retrying in %d ms\n", + EDGETPU_ASYNC_POWER_DOWN_RETRY_DELAY); + etpm->p->power_down_pending = true; + schedule_delayed_work(&etpm->p->power_down_work, + msecs_to_jiffies(EDGETPU_ASYNC_POWER_DOWN_RETRY_DELAY)); + } else { + if (ret) + etdev_warn(etpm->etdev, "Power down request failed (%d)\n", ret); + etpm->p->power_down_pending = false; + } +} + +/* Worker for async power down */ +static void edgetpu_pm_async_power_down_work(struct work_struct *work) +{ + struct delayed_work *dwork = container_of(work, struct delayed_work, work); + struct edgetpu_pm_private *p = + container_of(dwork, struct edgetpu_pm_private, power_down_work); + + mutex_lock(&p->lock); + etdev_info(p->etpm->etdev, "Delayed power down starting\n"); + if (p->power_down_pending) + edgetpu_pm_try_power_down(p->etpm); + else + etdev_info(p->etpm->etdev, "Delayed power down cancelled\n"); + mutex_unlock(&p->lock); +} + void edgetpu_pm_put(struct edgetpu_pm *etpm) { if (!etpm || !etpm->p->handlers || !etpm->p->handlers->power_down) @@ -110,7 +157,7 @@ void edgetpu_pm_put(struct edgetpu_pm *etpm) } if (!--etpm->p->power_up_count) { edgetpu_sw_wdt_stop(etpm->etdev); - etpm->p->handlers->power_down(etpm); + edgetpu_pm_try_power_down(etpm); } etdev_dbg(etpm->etdev, "%s: %d\n", __func__, etpm->p->power_up_count); mutex_unlock(&etpm->p->lock); @@ -138,6 +185,8 @@ int edgetpu_pm_create(struct edgetpu_dev *etdev, goto out_free_etpm; } + INIT_DELAYED_WORK(&etpm->p->power_down_work, edgetpu_pm_async_power_down_work); + etpm->p->etpm = etpm; etpm->p->handlers = handlers; etpm->etdev = etdev; @@ -167,6 +216,8 @@ void edgetpu_pm_destroy(struct edgetpu_dev *etdev) return; if (etdev->pm->p) { handlers = etdev->pm->p->handlers; + etdev->pm->p->power_down_pending = false; + cancel_delayed_work_sync(&etdev->pm->p->power_down_work); if (handlers && handlers->before_destroy) handlers->before_destroy(etdev->pm); kfree(etdev->pm->p); @@ -181,6 +232,7 @@ void edgetpu_pm_shutdown(struct edgetpu_dev *etdev, bool force) if (!etpm) return; + mutex_lock(&etpm->p->lock); /* someone is using the device */ diff --git a/drivers/edgetpu/edgetpu-pm.h b/drivers/edgetpu/edgetpu-pm.h index aef35f6..5f44ab4 100644 --- a/drivers/edgetpu/edgetpu-pm.h +++ b/drivers/edgetpu/edgetpu-pm.h @@ -28,7 +28,7 @@ struct edgetpu_pm_handlers { /* Platform-specific power up. Nesting is handled at generic layer */ int (*power_up)(struct edgetpu_pm *etpm); /* Platform-specific power down. Nesting is handled at generic layer */ - void (*power_down)(struct edgetpu_pm *etpm); + int (*power_down)(struct edgetpu_pm *etpm); }; struct edgetpu_pm { diff --git a/drivers/edgetpu/edgetpu-telemetry.c b/drivers/edgetpu/edgetpu-telemetry.c index 01c34a9..bad40ab 100644 --- a/drivers/edgetpu/edgetpu-telemetry.c +++ b/drivers/edgetpu/edgetpu-telemetry.c @@ -376,31 +376,43 @@ int edgetpu_telemetry_init(struct edgetpu_dev *etdev, struct edgetpu_coherent_mem *log_mem, struct edgetpu_coherent_mem *trace_mem) { - int ret; + int ret, i; if (!etdev->telemetry) return -ENODEV; - ret = telemetry_init(etdev, &etdev->telemetry->log, "telemetry_log", - log_mem, edgetpu_fw_log); - if (ret) - return ret; + + for (i = 0; i < etdev->num_cores; i++) { + ret = telemetry_init(etdev, &etdev->telemetry[i].log, "telemetry_log", + log_mem ? &log_mem[i] : NULL, edgetpu_fw_log); + if (ret) + break; #if IS_ENABLED(CONFIG_EDGETPU_TELEMETRY_TRACE) - ret = telemetry_init(etdev, &etdev->telemetry->trace, "telemetry_trace", - trace_mem, edgetpu_fw_trace); - if (ret) { - telemetry_exit(etdev, &etdev->telemetry->log); - return ret; - } + ret = telemetry_init(etdev, &etdev->telemetry[i].trace, "telemetry_trace", + trace_mem ? &trace_mem[i] : NULL, edgetpu_fw_trace); + if (ret) + break; #endif - return 0; + } + + if (ret) + edgetpu_telemetry_exit(etdev); + + return ret; } void edgetpu_telemetry_exit(struct edgetpu_dev *etdev) { + int i; + if (!etdev->telemetry) return; - telemetry_exit(etdev, &etdev->telemetry->trace); - telemetry_exit(etdev, &etdev->telemetry->log); + + for (i = 0; i < etdev->num_cores; i++) { +#if IS_ENABLED(CONFIG_EDGETPU_TELEMETRY_TRACE) + telemetry_exit(etdev, &etdev->telemetry[i].trace); +#endif + telemetry_exit(etdev, &etdev->telemetry[i].log); + } } int edgetpu_telemetry_kci(struct edgetpu_dev *etdev) @@ -409,14 +421,17 @@ int edgetpu_telemetry_kci(struct edgetpu_dev *etdev) if (!etdev->telemetry) return -ENODEV; - ret = telemetry_kci(etdev, &etdev->telemetry->log, - edgetpu_kci_map_log_buffer); + + /* Core 0 will notify other cores. */ + ret = telemetry_kci(etdev, &etdev->telemetry[0].log, edgetpu_kci_map_log_buffer); if (ret) return ret; - ret = telemetry_kci(etdev, &etdev->telemetry->trace, - edgetpu_kci_map_trace_buffer); + +#if IS_ENABLED(CONFIG_EDGETPU_TELEMETRY_TRACE) + ret = telemetry_kci(etdev, &etdev->telemetry[0].trace, edgetpu_kci_map_trace_buffer); if (ret) return ret; +#endif return 0; } @@ -424,59 +439,87 @@ int edgetpu_telemetry_kci(struct edgetpu_dev *etdev) int edgetpu_telemetry_set_event(struct edgetpu_dev *etdev, enum edgetpu_telemetry_type type, u32 eventfd) { + int i, ret; + if (!etdev->telemetry) return -ENODEV; - return telemetry_set_event( - etdev, select_telemetry(etdev->telemetry, type), eventfd); + + for (i = 0; i < etdev->num_cores; i++) { + ret = telemetry_set_event(etdev, select_telemetry(&etdev->telemetry[i], type), + eventfd); + if (ret) { + edgetpu_telemetry_unset_event(etdev, type); + return ret; + } + } + + return 0; } void edgetpu_telemetry_unset_event(struct edgetpu_dev *etdev, enum edgetpu_telemetry_type type) { + int i; + if (!etdev->telemetry) return; - telemetry_unset_event(etdev, select_telemetry(etdev->telemetry, type)); + + for (i = 0; i < etdev->num_cores; i++) + telemetry_unset_event(etdev, select_telemetry(&etdev->telemetry[i], type)); } void edgetpu_telemetry_irq_handler(struct edgetpu_dev *etdev) { + int i; + if (!etdev->telemetry) return; - telemetry_irq_handler(etdev, &etdev->telemetry->log); - telemetry_irq_handler(etdev, &etdev->telemetry->trace); + + for (i = 0; i < etdev->num_cores; i++) { + telemetry_irq_handler(etdev, &etdev->telemetry[i].log); +#if IS_ENABLED(CONFIG_EDGETPU_TELEMETRY_TRACE) + telemetry_irq_handler(etdev, &etdev->telemetry[i].trace); +#endif + } } void edgetpu_telemetry_mappings_show(struct edgetpu_dev *etdev, struct seq_file *s) { + int i; + if (!etdev->telemetry) return; - telemetry_mappings_show(&etdev->telemetry->log, s); - telemetry_mappings_show(&etdev->telemetry->trace, s); + + for (i = 0; i < etdev->num_cores; i++) { + telemetry_mappings_show(&etdev->telemetry[i].log, s); +#if IS_ENABLED(CONFIG_EDGETPU_TELEMETRY_TRACE) + telemetry_mappings_show(&etdev->telemetry[i].trace, s); +#endif + } } -int edgetpu_mmap_telemetry_buffer(struct edgetpu_dev *etdev, - enum edgetpu_telemetry_type type, - struct vm_area_struct *vma) +int edgetpu_mmap_telemetry_buffer(struct edgetpu_dev *etdev, enum edgetpu_telemetry_type type, + struct vm_area_struct *vma, int core_id) { if (!etdev->telemetry) return -ENODEV; - return telemetry_mmap_buffer( - etdev, select_telemetry(etdev->telemetry, type), vma); + return telemetry_mmap_buffer(etdev, select_telemetry(&etdev->telemetry[core_id], type), + vma); } -void edgetpu_telemetry_inc_mmap_count(struct edgetpu_dev *etdev, - enum edgetpu_telemetry_type type) +void edgetpu_telemetry_inc_mmap_count(struct edgetpu_dev *etdev, enum edgetpu_telemetry_type type, + int core_id) { if (!etdev->telemetry) return; - telemetry_inc_mmap_count(select_telemetry(etdev->telemetry, type), 1); + telemetry_inc_mmap_count(select_telemetry(&etdev->telemetry[core_id], type), 1); } -void edgetpu_telemetry_dec_mmap_count(struct edgetpu_dev *etdev, - enum edgetpu_telemetry_type type) +void edgetpu_telemetry_dec_mmap_count(struct edgetpu_dev *etdev, enum edgetpu_telemetry_type type, + int core_id) { if (!etdev->telemetry) return; - telemetry_inc_mmap_count(select_telemetry(etdev->telemetry, type), -1); + telemetry_inc_mmap_count(select_telemetry(&etdev->telemetry[core_id], type), -1); } diff --git a/drivers/edgetpu/edgetpu-telemetry.h b/drivers/edgetpu/edgetpu-telemetry.h index a25d002..6d7b3b1 100644 --- a/drivers/edgetpu/edgetpu-telemetry.h +++ b/drivers/edgetpu/edgetpu-telemetry.h @@ -99,7 +99,7 @@ struct edgetpu_telemetry_ctx { /* * Allocates resources needed for @etdev->telemetry. * - * Optionally provide coherent_mem buffers for log and trace. + * Optionally provide arrays of etdev->num_cores coherent_mem buffers for log and trace. * If any of these are NULL, they will be allocated and freed by telemetry code. * * Returns 0 on success, or a negative errno on error. @@ -139,12 +139,11 @@ void edgetpu_telemetry_mappings_show(struct edgetpu_dev *etdev, struct seq_file *s); /* Map telemetry buffer into user space. */ -int edgetpu_mmap_telemetry_buffer(struct edgetpu_dev *etdev, - enum edgetpu_telemetry_type type, - struct vm_area_struct *vma); -void edgetpu_telemetry_inc_mmap_count(struct edgetpu_dev *etdev, - enum edgetpu_telemetry_type type); -void edgetpu_telemetry_dec_mmap_count(struct edgetpu_dev *etdev, - enum edgetpu_telemetry_type type); +int edgetpu_mmap_telemetry_buffer(struct edgetpu_dev *etdev, enum edgetpu_telemetry_type type, + struct vm_area_struct *vma, int core_id); +void edgetpu_telemetry_inc_mmap_count(struct edgetpu_dev *etdev, enum edgetpu_telemetry_type type, + int core_id); +void edgetpu_telemetry_dec_mmap_count(struct edgetpu_dev *etdev, enum edgetpu_telemetry_type type, + int core_id); #endif /* __EDGETPU_TELEMETRY_H__ */ diff --git a/drivers/edgetpu/edgetpu.h b/drivers/edgetpu/edgetpu.h index 5a630a4..6a210fb 100644 --- a/drivers/edgetpu/edgetpu.h +++ b/drivers/edgetpu/edgetpu.h @@ -17,6 +17,8 @@ /* mmap offsets for logging and tracing buffers */ #define EDGETPU_MMAP_LOG_BUFFER_OFFSET 0x1B00000 #define EDGETPU_MMAP_TRACE_BUFFER_OFFSET 0x1C00000 +#define EDGETPU_MMAP_LOG1_BUFFER_OFFSET 0x1D00000 +#define EDGETPU_MMAP_TRACE1_BUFFER_OFFSET 0x1E00000 /* EdgeTPU map flag macros */ @@ -46,7 +48,6 @@ typedef __u32 edgetpu_map_flag_t; /* External mailbox types */ #define EDGETPU_EXT_MAILBOX_TYPE_TZ 1 #define EDGETPU_EXT_MAILBOX_TYPE_GSA 2 -#define EDGETPU_EXT_MAILBOX_TYPE_DSP 3 struct edgetpu_map_ioctl { __u64 host_address; /* user-space address to be mapped */ diff --git a/drivers/edgetpu/include/linux/acpm_dvfs.h b/drivers/edgetpu/include/linux/acpm_dvfs.h new file mode 100644 index 0000000..6872abe --- /dev/null +++ b/drivers/edgetpu/include/linux/acpm_dvfs.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Fallback header for systems without Exynos ACPM support. + * + * Copyright (C) 2021 Google, Inc. + */ + +#ifndef __ACPM_DVFS_H__ +#define __ACPM_DVFS_H__ + +static inline int exynos_acpm_set_init_freq(unsigned int dfs_id, unsigned long freq) +{ + return 0; +} + +static inline int exynos_acpm_set_policy(unsigned int id, unsigned long policy) +{ + return 0; +} + +#if IS_ENABLED(CONFIG_EDGETPU_TEST) + +int exynos_acpm_set_rate(unsigned int id, unsigned long rate); +unsigned long exynos_acpm_get_rate(unsigned int id, unsigned long dbg_val); + +#else /* IS_ENABLED(CONFIG_EDGETPU_TEST) */ + +static inline int exynos_acpm_set_rate(unsigned int id, unsigned long rate) +{ + return 0; +} + +static inline unsigned long exynos_acpm_get_rate(unsigned int id, unsigned long dbg_val) +{ + return 0; +} + +#endif /* IS_ENABLED(CONFIG_EDGETPU_TEST) */ + +#endif /* __ACPM_DVFS_H__ */ diff --git a/drivers/edgetpu/include/linux/gsa/gsa_tpu.h b/drivers/edgetpu/include/linux/gsa/gsa_tpu.h new file mode 100644 index 0000000..03dc951 --- /dev/null +++ b/drivers/edgetpu/include/linux/gsa/gsa_tpu.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Fallback header for systems without GSA support. + * + * Copyright (C) 2020 Google, Inc. + */ + +#ifndef __LINUX_GSA_TPU_H +#define __LINUX_GSA_TPU_H + +#include <linux/device.h> +#include <linux/types.h> + +#if IS_ENABLED(CONFIG_EDGETPU_TEST) + +int gsa_load_tpu_fw_image(struct device *gsa, dma_addr_t img_meta, phys_addr_t img_body); + +#else /* IS_ENABLED(CONFIG_EDGETPU_TEST) */ + +static inline int gsa_load_tpu_fw_image(struct device *gsa, dma_addr_t img_meta, + phys_addr_t img_body) +{ + return 0; +} + +#endif /* IS_ENABLED(CONFIG_EDGETPU_TEST) */ + +static inline int gsa_unload_tpu_fw_image(struct device *gsa) +{ + return 0; +} + +enum gsa_tpu_state { + GSA_TPU_STATE_INACTIVE = 0, + GSA_TPU_STATE_LOADED, + GSA_TPU_STATE_RUNNING, + GSA_TPU_STATE_SUSPENDED, +}; + +enum gsa_tpu_cmd { + GSA_TPU_GET_STATE = 0, + GSA_TPU_START, + GSA_TPU_SUSPEND, + GSA_TPU_RESUME, + GSA_TPU_SHUTDOWN, +}; + +static inline int gsa_send_tpu_cmd(struct device *gsa, enum gsa_tpu_cmd cmd) +{ + return 0; +} + +#endif /* __LINUX_GSA_TPU_H */ diff --git a/drivers/edgetpu/janeiro-device.c b/drivers/edgetpu/janeiro-device.c index cc4c975..5f8ae04 100644 --- a/drivers/edgetpu/janeiro-device.c +++ b/drivers/edgetpu/janeiro-device.c @@ -111,30 +111,31 @@ void edgetpu_chip_handle_reverse_kci(struct edgetpu_dev *etdev, } } +int edgetpu_chip_get_ext_mailbox_index(u32 mbox_type, u32 *start, u32 *end) +{ + switch (mbox_type) { + case EDGETPU_EXTERNAL_MAILBOX_TYPE_DSP: + *start = JANEIRO_EXT_DSP_MAILBOX_START; + *end = JANEIRO_EXT_DSP_MAILBOX_END; + return 0; + case EDGETPU_EXTERNAL_MAILBOX_TYPE_AOC: + *start = JANEIRO_EXT_AOC_MAILBOX_START; + *end = JANEIRO_EXT_AOC_MAILBOX_END; + return 0; + default: + return -ENOENT; + } +} + int edgetpu_chip_acquire_ext_mailbox(struct edgetpu_client *client, struct edgetpu_ext_mailbox_ioctl *args) { - struct edgetpu_external_mailbox_req req; - - if (args->type == EDGETPU_EXT_MAILBOX_TYPE_DSP) { - if (!args->count || args->count > EDGETPU_NUM_EXT_MAILBOXES) - return -EINVAL; - if (copy_from_user(&req.attr, (void __user *)args->attrs, sizeof(req.attr))) - return -EFAULT; - req.count = args->count; - req.start = JANEIRO_EXT_DSP_MAILBOX_START; - req.end = JANEIRO_EXT_DSP_MAILBOX_END; - req.client_type = EDGETPU_EXT_MAILBOX_TYPE_DSP; - return edgetpu_mailbox_enable_ext(client, EDGETPU_MAILBOX_ID_USE_ASSOC, &req); - } return -ENODEV; } int edgetpu_chip_release_ext_mailbox(struct edgetpu_client *client, struct edgetpu_ext_mailbox_ioctl *args) { - if (args->type == EDGETPU_EXT_MAILBOX_TYPE_DSP) - return edgetpu_mailbox_disable_ext(client, EDGETPU_MAILBOX_ID_USE_ASSOC); return -ENODEV; } diff --git a/drivers/edgetpu/janeiro-platform.c b/drivers/edgetpu/janeiro-platform.c index 9ca3ff3..dea49aa 100644 --- a/drivers/edgetpu/janeiro-platform.c +++ b/drivers/edgetpu/janeiro-platform.c @@ -47,31 +47,66 @@ static int janeiro_mmu_set_shareability(struct device *dev, u32 reg_base) return 0; } -static int janeiro_parse_set_dt_property(struct device *dev) +static int janeiro_set_fw_ctx_memory(struct edgetpu_mobile_platform_dev *etmdev) { + struct edgetpu_dev *etdev = &etmdev->edgetpu_dev; + struct device *dev = etdev->dev; + struct resource r; + struct device_node *np; int ret; - u32 reg; - if (!of_find_property(dev->of_node, "edgetpu,shareability", NULL)) + np = of_parse_phandle(dev->of_node, "memory-region", 1); + if (!np) { + etdev_warn(etdev, "No memory for firmware contexts"); return -ENODEV; - ret = of_property_read_u32_index(dev->of_node, "edgetpu,shareability", 0, ®); - if (ret) + } + + ret = of_address_to_resource(np, 0, &r); + of_node_put(np); + if (ret) { + etdev_warn(etdev, "No memory address for firmware contexts"); return ret; - return janeiro_mmu_set_shareability(dev, reg); + } + + etmdev->fw_ctx_paddr = r.start; + etmdev->fw_ctx_size = resource_size(&r); + return 0; } -static int janeiro_platform_after_probe(struct edgetpu_mobile_platform_dev *etmdev) +static int janeiro_parse_set_dt_property(struct edgetpu_mobile_platform_dev *etmdev) { int ret; + u32 reg; struct edgetpu_dev *etdev = &etmdev->edgetpu_dev; - - ret = janeiro_parse_set_dt_property(etdev->dev); + struct device *dev = etdev->dev; + + ret = janeiro_set_fw_ctx_memory(etmdev); + /* + * TODO(b/202262532): + * ignore return value till ctx switching support is added on + * firmware. + */ + + if (!of_find_property(dev->of_node, "edgetpu,shareability", NULL)) { + ret = -ENODEV; + goto err; + } + ret = of_property_read_u32_index(dev->of_node, "edgetpu,shareability", 0, ®); + if (ret) + goto err; + ret = janeiro_mmu_set_shareability(dev, reg); +err: if (ret) - dev_warn(etdev->dev, "failed to enable shareability: %d", ret); + etdev_warn(etdev, "failed to enable shareability: %d", ret); return 0; } +static int janeiro_platform_after_probe(struct edgetpu_mobile_platform_dev *etmdev) +{ + return janeiro_parse_set_dt_property(etmdev); +} + static int edgetpu_platform_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; diff --git a/drivers/edgetpu/janeiro-pm.c b/drivers/edgetpu/janeiro-pm.c index 681f58d..eec9340 100644 --- a/drivers/edgetpu/janeiro-pm.c +++ b/drivers/edgetpu/janeiro-pm.c @@ -30,11 +30,15 @@ #define EDGETPU_PSM1_START 0x1c2884 #define EDGETPU_PSM1_STATUS 0x1c2888 #define EDGETPU_LPM_CHANGE_TIMEOUT 30000 +#define EDGETPU_PSM0_GPOUT_WRT_LO 0x1c18a8 +#define EDGETPU_PSM0_GPOUT_WRT_HI 0x1c18ac +#define EDGETPU_PSM0_DEBUGCFG 0x1c188c +#define EDGETPU_PSM0_RECOVER_VAL 125888 static void janeiro_lpm_down(struct edgetpu_dev *etdev) { int timeout_cnt = 0; - u32 val; + u32 val, val1; do { /* Manually delay 20us per retry till LPM shutdown finished */ @@ -44,9 +48,22 @@ static void janeiro_lpm_down(struct edgetpu_dev *etdev) break; timeout_cnt++; } while (timeout_cnt < SHUTDOWN_MAX_DELAY_COUNT); - if (timeout_cnt == SHUTDOWN_MAX_DELAY_COUNT) + if (timeout_cnt == SHUTDOWN_MAX_DELAY_COUNT) { // Log the issue then continue to perform the shutdown forcefully. - etdev_warn(etdev, "LPM shutdown failure, continuing BLK shutdown\n"); + etdev_err(etdev, "LPM shutdown failure, attempt to recover\n"); + val = edgetpu_dev_read_32_sync(etdev, EDGETPU_PSM0_STATUS); + val1 = edgetpu_dev_read_32_sync(etdev, EDGETPU_PSM1_STATUS); + etdev_err(etdev, "PSM0_STATUS: 0x%x, PSM1_STATUS: 0x%x\n", val, val1); + + edgetpu_dev_write_32_sync(etdev, EDGETPU_PSM0_GPOUT_WRT_LO, + EDGETPU_PSM0_RECOVER_VAL); + edgetpu_dev_write_32_sync(etdev, EDGETPU_PSM0_GPOUT_WRT_HI, 0); + val = edgetpu_dev_read_32_sync(etdev, EDGETPU_PSM0_DEBUGCFG); + // Write then clear EDGETPU_PSM0_DEBUGCFG[1] to update the value. + edgetpu_dev_write_32_sync(etdev, EDGETPU_PSM0_DEBUGCFG, 0x2 | val); + edgetpu_dev_write_32_sync(etdev, EDGETPU_PSM0_DEBUGCFG, 0x1 & val); + etdev_err(etdev, "Recovery attempt finish\n"); + } } static int janeiro_lpm_up(struct edgetpu_dev *etdev) diff --git a/drivers/edgetpu/janeiro-thermal.c b/drivers/edgetpu/janeiro-thermal.c new file mode 100644 index 0000000..deb763f --- /dev/null +++ b/drivers/edgetpu/janeiro-thermal.c @@ -0,0 +1,2 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "mobile-thermal.c" diff --git a/drivers/edgetpu/janeiro/config-mailbox.h b/drivers/edgetpu/janeiro/config-mailbox.h index 6d7bce9..a491e15 100644 --- a/drivers/edgetpu/janeiro/config-mailbox.h +++ b/drivers/edgetpu/janeiro/config-mailbox.h @@ -12,19 +12,28 @@ #define EDGETPU_NUM_VII_MAILBOXES 7 #define EDGETPU_NUM_P2P_MAILBOXES 0 -#define EDGETPU_NUM_EXT_MAILBOXES 4 +#define EDGETPU_NUM_EXT_DSP_MAILBOXES 4 +#define EDGETPU_NUM_EXT_AOC_MAILBOXES 1 +#define EDGETPU_NUM_EXT_MAILBOXES (EDGETPU_NUM_EXT_DSP_MAILBOXES + EDGETPU_NUM_EXT_AOC_MAILBOXES) #define EDGETPU_NUM_MAILBOXES (EDGETPU_NUM_VII_MAILBOXES + EDGETPU_NUM_EXT_MAILBOXES + 1) /* * Mailbox index layout in mailbox manager is like: - * --------------------------------------------- - * | KCI X 1 | VII(s) X 7 | EXT_DSP(s) X 4 | - * --------------------------------------------- + * --------------------------------------------------------------- + * | KCI X 1 | VII(s) X 7 | EXT_DSP(s) X 4 | EXT_AOC(s) X 1 | + * --------------------------------------------------------------- */ #define JANEIRO_EXT_DSP_MAILBOX_START (EDGETPU_NUM_VII_MAILBOXES + 1) -#define JANEIRO_EXT_DSP_MAILBOX_END (EDGETPU_NUM_EXT_MAILBOXES + JANEIRO_EXT_DSP_MAILBOX_START - 1) +#define JANEIRO_EXT_DSP_MAILBOX_END \ + (EDGETPU_NUM_EXT_DSP_MAILBOXES + JANEIRO_EXT_DSP_MAILBOX_START - 1) + +#define JANEIRO_EXT_AOC_MAILBOX_START (JANEIRO_EXT_DSP_MAILBOX_END + 1) +#define JANEIRO_EXT_AOC_MAILBOX_END \ + (EDGETPU_NUM_EXT_AOC_MAILBOXES + JANEIRO_EXT_AOC_MAILBOX_START - 1) + #define JANEIRO_CSR_MBOX2_CONTEXT_ENABLE 0xa0000 /* starting kernel mb*/ -#define JANEIRO_CSR_MBOX11_CONTEXT_ENABLE 0xc0000 /* DSP mailbox */ +#define JANEIRO_CSR_AOC_MBOX_CONTEXT_ENABLE 0xb0000 /* AOC mailbox */ +#define JANEIRO_CSR_DSP_MBOX_CONTEXT_ENABLE 0xc0000 /* DSP mailbox */ #define EDGETPU_MBOX_CSRS_SIZE 0x2000 /* CSR size of each mailbox */ #define JANEIRO_CSR_MBOX_CMD_QUEUE_DOORBELL_SET_OFFSET 0x1000 @@ -35,35 +44,27 @@ static inline u32 edgetpu_mailbox_get_context_csr_base(u32 index) { u32 base; - if (index >= 0 && index <= EDGETPU_NUM_VII_MAILBOXES) + if (index <= EDGETPU_NUM_VII_MAILBOXES) { base = JANEIRO_CSR_MBOX2_CONTEXT_ENABLE; - else - base = JANEIRO_CSR_MBOX11_CONTEXT_ENABLE; - return base + (index % JANEIRO_EXT_DSP_MAILBOX_START) * EDGETPU_MBOX_CSRS_SIZE; + return base + index * EDGETPU_MBOX_CSRS_SIZE; + } else if (index <= JANEIRO_EXT_DSP_MAILBOX_END) { + base = JANEIRO_CSR_DSP_MBOX_CONTEXT_ENABLE; + return base + (index - JANEIRO_EXT_DSP_MAILBOX_START) * EDGETPU_MBOX_CSRS_SIZE; + } + + return JANEIRO_CSR_AOC_MBOX_CONTEXT_ENABLE; } static inline u32 edgetpu_mailbox_get_cmd_queue_csr_base(u32 index) { - u32 base; - - if (index >= 0 && index <= EDGETPU_NUM_VII_MAILBOXES) - base = JANEIRO_CSR_MBOX2_CONTEXT_ENABLE; - else - base = JANEIRO_CSR_MBOX11_CONTEXT_ENABLE; - return base + JANEIRO_CSR_MBOX_CMD_QUEUE_DOORBELL_SET_OFFSET + - ((index % JANEIRO_EXT_DSP_MAILBOX_START) * EDGETPU_MBOX_CSRS_SIZE); + return edgetpu_mailbox_get_context_csr_base(index) + + JANEIRO_CSR_MBOX_CMD_QUEUE_DOORBELL_SET_OFFSET; } static inline u32 edgetpu_mailbox_get_resp_queue_csr_base(u32 index) { - u32 base; - - if (index >= 0 && index <= EDGETPU_NUM_VII_MAILBOXES) - base = JANEIRO_CSR_MBOX2_CONTEXT_ENABLE; - else - base = JANEIRO_CSR_MBOX11_CONTEXT_ENABLE; - return base + JANEIRO_CSR_MBOX_RESP_QUEUE_DOORBELL_SET_OFFSET + - ((index % JANEIRO_EXT_DSP_MAILBOX_START) * EDGETPU_MBOX_CSRS_SIZE); + return edgetpu_mailbox_get_context_csr_base(index) + + JANEIRO_CSR_MBOX_RESP_QUEUE_DOORBELL_SET_OFFSET; } #endif /* __JANEIRO_CONFIG_MAILBOX_H__ */ diff --git a/drivers/edgetpu/janeiro/config-pwr-state.h b/drivers/edgetpu/janeiro/config-pwr-state.h index f30d82e..20b1b57 100644 --- a/drivers/edgetpu/janeiro/config-pwr-state.h +++ b/drivers/edgetpu/janeiro/config-pwr-state.h @@ -11,15 +11,15 @@ /* * TPU Power States: * 0: Off - * 227000 Ultra Underdrive @227MHz - * 625000: Super Underdrive @625MHz + * 226000 Ultra Underdrive @226MHz + * 627000: Super Underdrive @627MHz * 845000: Underdrive @845MHz * 1066000: Nominal @1066MHz */ enum edgetpu_pwr_state { TPU_OFF = 0, - TPU_ACTIVE_UUD = 227000, - TPU_ACTIVE_SUD = 625000, + TPU_ACTIVE_UUD = 226000, + TPU_ACTIVE_SUD = 627000, TPU_ACTIVE_UD = 845000, TPU_ACTIVE_NOM = 1066000, }; diff --git a/drivers/edgetpu/mobile-firmware.c b/drivers/edgetpu/mobile-firmware.c index 601ecfc..40ac077 100644 --- a/drivers/edgetpu/mobile-firmware.c +++ b/drivers/edgetpu/mobile-firmware.c @@ -24,6 +24,119 @@ #include "edgetpu-mobile-platform.h" #include "mobile-firmware.h" +static struct mobile_image_config *mobile_firmware_get_image_config(struct edgetpu_dev *etdev) +{ + return (struct mobile_image_config *) edgetpu_firmware_get_data(etdev->firmware); +} + +static void mobile_firmware_clear_mappings(struct edgetpu_dev *etdev, + struct mobile_image_config *image_config) +{ + tpu_addr_t tpu_addr; + size_t size; + int i; + + for (i = 0; i < image_config->num_iommu_mapping; i++) { + tpu_addr = image_config->mappings[i].virt_address; + size = CONFIG_TO_SIZE(image_config->mappings[i].image_config_value); + edgetpu_mmu_remove_translation(etdev, tpu_addr, size, EDGETPU_CONTEXT_KCI); + } +} + +static int mobile_firmware_setup_mappings(struct edgetpu_dev *etdev, + struct mobile_image_config *image_config) +{ + int i, ret; + tpu_addr_t tpu_addr; + size_t size; + phys_addr_t phys_addr; + + for (i = 0; i < image_config->num_iommu_mapping; i++) { + tpu_addr = image_config->mappings[i].virt_address; + if (!tpu_addr) { + etdev_warn(etdev, "Invalid firmware header\n"); + goto err; + } + size = CONFIG_TO_SIZE(image_config->mappings[i].image_config_value); + phys_addr = image_config->mappings[i].image_config_value & ~(0xFFF); + + etdev_dbg(etdev, "Adding IOMMU mapping for firmware : %llu -> %08llX", tpu_addr, + phys_addr); + + ret = edgetpu_mmu_add_translation(etdev, tpu_addr, phys_addr, size, + IOMMU_READ | IOMMU_WRITE, EDGETPU_CONTEXT_KCI); + if (ret) { + etdev_err(etdev, + "Unable to Map: %d tpu_addr: %#llx phys_addr: %#llx size: %#lx\n", + ret, tpu_addr, phys_addr, size); + goto err; + } + } + + return 0; + +err: + while (i--) { + tpu_addr = image_config->mappings[i].virt_address; + size = CONFIG_TO_SIZE(image_config->mappings[i].image_config_value); + edgetpu_mmu_remove_translation(etdev, tpu_addr, size, EDGETPU_CONTEXT_KCI); + } + return ret; +} + +static void mobile_firmware_clear_ns_mappings(struct edgetpu_dev *etdev, + struct mobile_image_config *image_config) +{ + tpu_addr_t tpu_addr; + size_t size; + int i; + + for (i = 0; i < image_config->num_ns_iommu_mappings; i++) { + tpu_addr = image_config->ns_iommu_mappings[i] & ~(0xFFF); + size = CONFIG_TO_MBSIZE(image_config->ns_iommu_mappings[i]); + edgetpu_mmu_remove_translation(etdev, tpu_addr, size, EDGETPU_CONTEXT_KCI); + } +} + +static int mobile_firmware_setup_ns_mappings(struct edgetpu_dev *etdev, + struct mobile_image_config *image_config) +{ + tpu_addr_t tpu_addr; + size_t size = 0; + int ret = 0, i; + struct edgetpu_mobile_platform_dev *etmdev = to_mobile_dev(etdev); + phys_addr_t phys_addr = etmdev->fw_ctx_paddr; + + for (i = 0; i < image_config->num_ns_iommu_mappings; i++) + size += CONFIG_TO_MBSIZE(image_config->ns_iommu_mappings[i]); + + if (size > etmdev->fw_ctx_size) { + etdev_err(etdev, "Insufficient firmware context memory"); + return -ENOSPC; + } + + for (i = 0; i < image_config->num_ns_iommu_mappings; i++) { + size = CONFIG_TO_MBSIZE(image_config->ns_iommu_mappings[i]); + tpu_addr = image_config->ns_iommu_mappings[i] & ~(0xFFF); + ret = edgetpu_mmu_add_translation(etdev, tpu_addr, phys_addr, + size, IOMMU_READ | IOMMU_WRITE, + EDGETPU_CONTEXT_KCI); + if (ret) + goto err; + phys_addr += size; + } + + return 0; + +err: + while (i--) { + tpu_addr = image_config->ns_iommu_mappings[i] & ~(0xFFF); + size = CONFIG_TO_MBSIZE(image_config->ns_iommu_mappings[i]); + edgetpu_mmu_remove_translation(etdev, tpu_addr, size, EDGETPU_CONTEXT_KCI); + } + return ret; +} + static int mobile_firmware_after_create(struct edgetpu_firmware *et_fw) { /* @@ -43,22 +156,15 @@ static int mobile_firmware_after_create(struct edgetpu_firmware *et_fw) static void mobile_firmware_before_destroy(struct edgetpu_firmware *et_fw) { struct mobile_image_config *image_config; - u32 i, tpu_addr, size; struct edgetpu_dev *etdev = et_fw->etdev; - image_config = edgetpu_firmware_get_data(et_fw); + image_config = mobile_firmware_get_image_config(etdev); - if (image_config && image_config->privilege_level == FW_PRIV_LEVEL_NS) { - for (i = 0; i < image_config->num_iommu_mapping; i++) { - tpu_addr = image_config->mappings[i].virt_address; - if (!tpu_addr) - continue; - size = CONFIG_TO_SIZE(image_config->mappings[i].image_config_value); - edgetpu_mmu_remove_translation(etdev, tpu_addr, size, EDGETPU_CONTEXT_KCI); - } - edgetpu_firmware_set_data(et_fw, NULL); - kfree(image_config); - } + mobile_firmware_clear_ns_mappings(etdev, image_config); + if (image_config->privilege_level == FW_PRIV_LEVEL_NS) + mobile_firmware_clear_mappings(etdev, image_config); + edgetpu_firmware_set_data(et_fw, NULL); + kfree(image_config); } static int mobile_firmware_alloc_buffer( @@ -95,11 +201,6 @@ static void mobile_firmware_free_buffer( fw_buf->used_size_align = 0; } -static struct mobile_image_config *mobile_firmware_get_image_config(struct edgetpu_dev *etdev) -{ - return (struct mobile_image_config *) edgetpu_firmware_get_data(etdev->firmware); -} - static void mobile_firmware_save_image_config(struct edgetpu_dev *etdev, struct mobile_image_config *image_config) { @@ -108,98 +209,16 @@ static void mobile_firmware_save_image_config(struct edgetpu_dev *etdev, memcpy(saved_image_config, image_config, sizeof(*saved_image_config)); } -static void mobile_firmware_clear_image_config(struct edgetpu_dev *etdev) -{ - struct mobile_image_config *saved_image_config = mobile_firmware_get_image_config(etdev); - - memset(saved_image_config, 0, sizeof(*saved_image_config)); -} - -static void mobile_firmware_clear_ns_mappings(struct edgetpu_dev *etdev, - struct mobile_image_config *image_config) -{ - unsigned int tpu_addr, size; - int i; - - etdev_dbg(etdev, "Firmware image config changed, removing previous mappings\n"); - for (i = 0; i < image_config->num_iommu_mapping; i++) { - tpu_addr = image_config->mappings[i].virt_address; - size = CONFIG_TO_SIZE(image_config->mappings[i].image_config_value); - edgetpu_mmu_remove_translation(etdev, tpu_addr, size, EDGETPU_CONTEXT_KCI); - } -} - -static int mobile_firmware_setup_ns_mappings(struct edgetpu_firmware *et_fw, - struct mobile_image_config *image_config) -{ - struct edgetpu_dev *etdev = et_fw->etdev; - struct mobile_image_config *last_image_config = mobile_firmware_get_image_config(etdev); - int i, ret; - unsigned int tpu_addr, size; - phys_addr_t phys_addr; - - if (memcmp(image_config, last_image_config, sizeof(*image_config))) { - mobile_firmware_clear_ns_mappings(etdev, last_image_config); - } else { - return 0; - } - - /* Clear saved image config. It will be updated once the new config is confirmed valid */ - mobile_firmware_clear_image_config(etdev); - - for (i = 0; i < image_config->num_iommu_mapping; i++) { - tpu_addr = image_config->mappings[i].virt_address; - if (!tpu_addr) { - etdev_warn(etdev, "Invalid firmware header\n"); - goto err; - } - size = CONFIG_TO_SIZE(image_config->mappings[i].image_config_value); - phys_addr = (image_config->mappings[i].image_config_value & ~(0xFFF)); - - etdev_dbg(etdev, "Adding IOMMU mapping for firmware : %08X -> %08llX", tpu_addr, - phys_addr); - - ret = edgetpu_mmu_add_translation(etdev, tpu_addr, phys_addr, size, - IOMMU_READ | IOMMU_WRITE, EDGETPU_CONTEXT_KCI); - if (ret) { - etdev_err(etdev, - "Unable to Map: %d tpu_addr: %#x phys_addr: %#llx size: %#x\n", - ret, tpu_addr, phys_addr, size); - goto err; - } - } - - mobile_firmware_save_image_config(etdev, image_config); - return 0; - -err: - while (i--) { - tpu_addr = image_config->mappings[i].virt_address; - size = CONFIG_TO_SIZE(image_config->mappings[i].image_config_value); - edgetpu_mmu_remove_translation(etdev, tpu_addr, size, EDGETPU_CONTEXT_KCI); - } - return ret; -} - static int mobile_firmware_gsa_authenticate(struct edgetpu_mobile_platform_dev *etmdev, struct edgetpu_firmware_buffer *fw_buf, struct mobile_image_config *image_config, void *image_vaddr) { struct edgetpu_dev *etdev = &etmdev->edgetpu_dev; - struct mobile_image_config *saved_image_config = mobile_firmware_get_image_config(etdev); - struct mobile_image_config last_image_config; void *header_vaddr; dma_addr_t header_dma_addr; int tpu_state; int ret = 0; - bool image_config_changed = memcmp(image_config, saved_image_config, sizeof(*image_config)); - - /* Need a local copy of the saved image in case we need to undo NS mappings */ - memcpy(&last_image_config, saved_image_config, sizeof(last_image_config)); - - /* Clear saved image config. It will be updated once the new config is confirmed valid */ - mobile_firmware_clear_image_config(etdev); tpu_state = gsa_send_tpu_cmd(etmdev->gsa_dev, GSA_TPU_GET_STATE); @@ -238,24 +257,8 @@ static int mobile_firmware_gsa_authenticate(struct edgetpu_mobile_platform_dev * ret = gsa_load_tpu_fw_image(etmdev->gsa_dev, header_dma_addr, etmdev->fw_region_paddr); - if (ret) { + if (ret) etdev_err(etdev, "GSA authentication failed: %d\n", ret); - } else { - /* Transitioning from NS to something else, clear NS mappings */ - if (last_image_config.privilege_level == FW_PRIV_LEVEL_NS && - image_config->privilege_level != FW_PRIV_LEVEL_NS) - mobile_firmware_clear_ns_mappings(etdev, &last_image_config); - - /* - * No change in image config, save it here so mobile_firmware_setup_ns_mappings() - * can re-use existing mappings - */ - if (!image_config_changed) - mobile_firmware_save_image_config(etdev, image_config); - - if (image_config->privilege_level == FW_PRIV_LEVEL_NS) - ret = mobile_firmware_setup_ns_mappings(etdev->firmware, image_config); - } dma_free_coherent(etmdev->gsa_dev, MOBILE_FW_HEADER_SIZE, header_vaddr, header_dma_addr); @@ -324,10 +327,12 @@ static int mobile_firmware_setup_buffer(struct edgetpu_firmware *et_fw, { int ret = 0; void *image_vaddr; - struct mobile_image_config *image_config; struct edgetpu_dev *etdev = et_fw->etdev; + struct mobile_image_config *image_config; + struct mobile_image_config *last_image_config = mobile_firmware_get_image_config(etdev); struct edgetpu_mobile_platform_dev *etmdev = to_mobile_dev(etdev); phys_addr_t image_start, image_end, carveout_start, carveout_end; + bool image_config_changed; if (fw_buf->used_size < MOBILE_FW_HEADER_SIZE) { etdev_err(etdev, "Invalid buffer size: %zu < %d\n", @@ -347,6 +352,8 @@ static int mobile_firmware_setup_buffer(struct edgetpu_firmware *et_fw, memcpy(&etdev->fw_version, &image_config->firmware_versions, sizeof(etdev->fw_version)); + image_config_changed = memcmp(image_config, last_image_config, sizeof(*image_config)); + if (etmdev->gsa_dev) { ret = mobile_firmware_gsa_authenticate(etmdev, fw_buf, image_config, image_vaddr); } else if (image_config->privilege_level == FW_PRIV_LEVEL_NS) { @@ -354,7 +361,6 @@ static int mobile_firmware_setup_buffer(struct edgetpu_firmware *et_fw, /* Copy the firmware image to the target location, skipping the header */ memcpy(image_vaddr, fw_buf->vaddr + MOBILE_FW_HEADER_SIZE, fw_buf->used_size - MOBILE_FW_HEADER_SIZE); - ret = mobile_firmware_setup_ns_mappings(et_fw, image_config); } else { etdev_err(etdev, "Cannot load firmware at privilege level %d with no authentication\n", @@ -376,8 +382,26 @@ static int mobile_firmware_setup_buffer(struct edgetpu_firmware *et_fw, etdev_err(etdev, "Image config: %pap - %pap\n", &image_start, &image_end); etdev_err(etdev, "Carveout: %pap - %pap\n", &carveout_start, &carveout_end); ret = -ERANGE; + goto out; } + if (image_config_changed) { + /* clear last image mappings */ + if (last_image_config->privilege_level == FW_PRIV_LEVEL_NS) + mobile_firmware_clear_mappings(etdev, last_image_config); + + if (image_config->privilege_level == FW_PRIV_LEVEL_NS) + ret = mobile_firmware_setup_mappings(etdev, image_config); + if (ret) + goto out; + mobile_firmware_clear_ns_mappings(etdev, last_image_config); + ret = mobile_firmware_setup_ns_mappings(etdev, image_config); + if (ret) { + mobile_firmware_clear_mappings(etdev, image_config); + goto out; + } + mobile_firmware_save_image_config(etdev, image_config); + } out: memunmap(image_vaddr); return ret; diff --git a/drivers/edgetpu/mobile-firmware.h b/drivers/edgetpu/mobile-firmware.h index 1250d7e..d17d0c0 100644 --- a/drivers/edgetpu/mobile-firmware.h +++ b/drivers/edgetpu/mobile-firmware.h @@ -12,7 +12,8 @@ #include "edgetpu-internal.h" #include "edgetpu.h" -#define MAX_IOMMU_MAPPINGS 26 +#define MAX_IOMMU_MAPPINGS 23 +#define MAX_NS_IOMMU_MAPPINGS 5 #define FW_PRIV_LEVEL_GSA (0) #define FW_PRIV_LEVEL_TZ (1) @@ -27,6 +28,8 @@ #define CONFIG_TO_SIZE(a) ((1 << ((a) & 0xFFF)) << 12) +#define CONFIG_TO_MBSIZE(a) (((a) & 0xFFF) << 20) + struct iommu_mapping { /* TPU virt address */ __u32 virt_address; @@ -51,6 +54,8 @@ struct mobile_image_config { __u32 remapped_region_end; __u32 num_iommu_mapping; struct iommu_mapping mappings[MAX_IOMMU_MAPPINGS]; + __u32 num_ns_iommu_mappings; + __u32 ns_iommu_mappings[MAX_NS_IOMMU_MAPPINGS]; } __packed; /* diff --git a/drivers/edgetpu/mobile-pm.c b/drivers/edgetpu/mobile-pm.c index de21bda..186b425 100644 --- a/drivers/edgetpu/mobile-pm.c +++ b/drivers/edgetpu/mobile-pm.c @@ -79,6 +79,147 @@ static int mobile_pwr_state_init(struct device *dev) return ret; } +static int edgetpu_core_rate_get(void *data, u64 *val) +{ + *val = exynos_acpm_get_rate(TPU_ACPM_DOMAIN, + TPU_DEBUG_REQ | TPU_CLK_CORE_DEBUG); + return 0; +} + +static int edgetpu_core_rate_set(void *data, u64 val) +{ + unsigned long dbg_rate_req; + + dbg_rate_req = TPU_DEBUG_REQ | TPU_CLK_CORE_DEBUG; + dbg_rate_req |= val; + + return exynos_acpm_set_rate(TPU_ACPM_DOMAIN, dbg_rate_req); +} + +static int edgetpu_ctl_rate_get(void *data, u64 *val) +{ + *val = exynos_acpm_get_rate(TPU_ACPM_DOMAIN, + TPU_DEBUG_REQ | TPU_CLK_CTL_DEBUG); + return 0; +} + +static int edgetpu_ctl_rate_set(void *data, u64 val) +{ + unsigned long dbg_rate_req; + + dbg_rate_req = TPU_DEBUG_REQ | TPU_CLK_CTL_DEBUG; + dbg_rate_req |= 1000; + + return exynos_acpm_set_rate(TPU_ACPM_DOMAIN, dbg_rate_req); +} + +static int edgetpu_axi_rate_get(void *data, u64 *val) +{ + *val = exynos_acpm_get_rate(TPU_ACPM_DOMAIN, + TPU_DEBUG_REQ | TPU_CLK_AXI_DEBUG); + return 0; +} + +static int edgetpu_axi_rate_set(void *data, u64 val) +{ + unsigned long dbg_rate_req; + + dbg_rate_req = TPU_DEBUG_REQ | TPU_CLK_AXI_DEBUG; + dbg_rate_req |= 1000; + + return exynos_acpm_set_rate(TPU_ACPM_DOMAIN, dbg_rate_req); +} + +static int edgetpu_apb_rate_get(void *data, u64 *val) +{ + *val = exynos_acpm_get_rate(TPU_ACPM_DOMAIN, + TPU_DEBUG_REQ | TPU_CLK_APB_DEBUG); + return 0; +} + +static int edgetpu_uart_rate_get(void *data, u64 *val) +{ + *val = exynos_acpm_get_rate(TPU_ACPM_DOMAIN, + TPU_DEBUG_REQ | TPU_CLK_UART_DEBUG); + return 0; +} + +static int edgetpu_vdd_int_m_set(void *data, u64 val) +{ + struct device *dev = (struct device *)data; + unsigned long dbg_rate_req; + + if (val > MAX_VOLTAGE_VAL) { + dev_err(dev, "Preventing INT_M voltage > %duV", + MAX_VOLTAGE_VAL); + return -EINVAL; + } + + dbg_rate_req = TPU_DEBUG_REQ | TPU_VDD_INT_M_DEBUG; + dbg_rate_req |= val; + + return exynos_acpm_set_rate(TPU_ACPM_DOMAIN, dbg_rate_req); +} + +static int edgetpu_vdd_int_m_get(void *data, u64 *val) +{ + *val = exynos_acpm_get_rate(TPU_ACPM_DOMAIN, + TPU_DEBUG_REQ | TPU_VDD_INT_M_DEBUG); + return 0; +} + +static int edgetpu_vdd_tpu_set(void *data, u64 val) +{ + int ret; + struct device *dev = (struct device *)data; + unsigned long dbg_rate_req; + + if (val > MAX_VOLTAGE_VAL) { + dev_err(dev, "Preventing VDD_TPU voltage > %duV", + MAX_VOLTAGE_VAL); + return -EINVAL; + } + + dbg_rate_req = TPU_DEBUG_REQ | TPU_VDD_TPU_DEBUG; + dbg_rate_req |= val; + + ret = exynos_acpm_set_rate(TPU_ACPM_DOMAIN, dbg_rate_req); + return ret; +} + +static int edgetpu_vdd_tpu_get(void *data, u64 *val) +{ + *val = exynos_acpm_get_rate(TPU_ACPM_DOMAIN, + TPU_DEBUG_REQ | TPU_VDD_TPU_DEBUG); + return 0; +} + +static int edgetpu_vdd_tpu_m_set(void *data, u64 val) +{ + int ret; + struct device *dev = (struct device *)data; + unsigned long dbg_rate_req; + + if (val > MAX_VOLTAGE_VAL) { + dev_err(dev, "Preventing VDD_TPU voltage > %duV", + MAX_VOLTAGE_VAL); + return -EINVAL; + } + + dbg_rate_req = TPU_DEBUG_REQ | TPU_VDD_TPU_M_DEBUG; + dbg_rate_req |= val; + + ret = exynos_acpm_set_rate(TPU_ACPM_DOMAIN, dbg_rate_req); + return ret; +} + +static int edgetpu_vdd_tpu_m_get(void *data, u64 *val) +{ + *val = exynos_acpm_get_rate(TPU_ACPM_DOMAIN, + TPU_DEBUG_REQ | TPU_VDD_TPU_M_DEBUG); + return 0; +} + static int mobile_pwr_state_set_locked(struct edgetpu_mobile_platform_dev *etmdev, u64 val) { int ret; @@ -221,13 +362,36 @@ static int mobile_pwr_policy_get(void *data, u64 *val) } DEFINE_DEBUGFS_ATTRIBUTE(fops_tpu_pwr_policy, mobile_pwr_policy_get, mobile_pwr_policy_set, - "%llu\n"); + "%llu\n"); DEFINE_DEBUGFS_ATTRIBUTE(fops_tpu_pwr_state, mobile_pwr_state_get, mobile_pwr_state_set, "%llu\n"); DEFINE_DEBUGFS_ATTRIBUTE(fops_tpu_min_pwr_state, mobile_min_pwr_state_get, mobile_min_pwr_state_set, + "%llu\n"); + +DEFINE_DEBUGFS_ATTRIBUTE(fops_tpu_core_rate, edgetpu_core_rate_get, + edgetpu_core_rate_set, "%llu\n"); + +DEFINE_DEBUGFS_ATTRIBUTE(fops_tpu_ctl_rate, edgetpu_ctl_rate_get, + edgetpu_ctl_rate_set, "%llu\n"); + +DEFINE_DEBUGFS_ATTRIBUTE(fops_tpu_axi_rate, edgetpu_axi_rate_get, + edgetpu_axi_rate_set, "%llu\n"); + +DEFINE_DEBUGFS_ATTRIBUTE(fops_tpu_apb_rate, edgetpu_apb_rate_get, NULL, + "%llu\n"); + +DEFINE_DEBUGFS_ATTRIBUTE(fops_tpu_uart_rate, edgetpu_uart_rate_get, NULL, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_tpu_vdd_int_m, edgetpu_vdd_int_m_get, + edgetpu_vdd_int_m_set, "%llu\n"); + +DEFINE_DEBUGFS_ATTRIBUTE(fops_tpu_vdd_tpu, edgetpu_vdd_tpu_get, + edgetpu_vdd_tpu_set, "%llu\n"); +DEFINE_DEBUGFS_ATTRIBUTE(fops_tpu_vdd_tpu_m, edgetpu_vdd_tpu_m_get, + edgetpu_vdd_tpu_m_set, "%llu\n"); + static int mobile_get_initial_pwr_state(struct device *dev) { switch (power_state) { @@ -259,7 +423,7 @@ static int mobile_get_initial_pwr_state(struct device *dev) return power_state; } -static void mobile_power_down(struct edgetpu_pm *etpm); +static int mobile_power_down(struct edgetpu_pm *etpm); static int mobile_power_up(struct edgetpu_pm *etpm) { @@ -357,7 +521,7 @@ static void mobile_pm_cleanup_bts_scenario(struct edgetpu_dev *etdev) mutex_unlock(&platform_pwr->scenario_lock); } -static void mobile_power_down(struct edgetpu_pm *etpm) +static int mobile_power_down(struct edgetpu_pm *etpm) { struct edgetpu_dev *etdev = etpm->etdev; struct edgetpu_mobile_platform_dev *etmdev = to_mobile_dev(etdev); @@ -370,7 +534,7 @@ static void mobile_power_down(struct edgetpu_pm *etpm) if (min_state >= MIN_ACTIVE_STATE) { etdev_info(etdev, "Power down skipped due to min state = %d\n", min_state); - return; + return 0; } if (mobile_pwr_state_get(etdev, &val)) { @@ -379,22 +543,31 @@ static void mobile_power_down(struct edgetpu_pm *etpm) } if (val == TPU_OFF) { etdev_dbg(etdev, "Device already off, skipping shutdown\n"); - return; + return 0; } if (edgetpu_firmware_status_locked(etdev) == FW_VALID) { - /* Update usage stats before we power off fw. */ - edgetpu_kci_update_usage_locked(etdev); - platform_pwr->firmware_down(etdev); + etdev_dbg(etdev, "Power down with valid firmware, device state = %d\n", + etdev->state); + if (etdev->state == ETDEV_STATE_GOOD) { + /* Update usage stats before we power off fw. */ + edgetpu_kci_update_usage_locked(etdev); + platform_pwr->firmware_down(etdev); + /* Ensure firmware is completely off */ + if (platform_pwr->lpm_down) + platform_pwr->lpm_down(etdev); + /* Indicate firmware is no longer running */ + etdev->state = ETDEV_STATE_NOFW; + } edgetpu_kci_cancel_work_queues(etdev->kci); res = edgetpu_mobile_firmware_reset_cpu(etdev, true); + /* TODO(b/198181290): remove -EIO once gsaproxy wakelock is implemented */ + if (res == -EAGAIN || res == -EIO) + return -EAGAIN; if (res < 0) etdev_warn(etdev, "CPU reset request failed (%d)\n", res); } - if (platform_pwr->lpm_down) - platform_pwr->lpm_down(etdev); - mobile_pwr_state_set(etdev, TPU_OFF); /* Remove our vote for INT/MIF state (if any) */ @@ -410,6 +583,8 @@ static void mobile_power_down(struct edgetpu_pm *etpm) * Clear the state here just in case. */ etmdev->secure_client = NULL; + + return 0; } static int mobile_pm_after_create(struct edgetpu_pm *etpm) @@ -449,6 +624,14 @@ static int mobile_pm_after_create(struct edgetpu_pm *etpm) debugfs_create_file("min_state", 0660, platform_pwr->debugfs_dir, etdev, &fops_tpu_min_pwr_state); debugfs_create_file("policy", 0660, platform_pwr->debugfs_dir, etdev, &fops_tpu_pwr_policy); + debugfs_create_file("vdd_tpu", 0660, platform_pwr->debugfs_dir, dev, &fops_tpu_vdd_tpu); + debugfs_create_file("vdd_tpu_m", 0660, platform_pwr->debugfs_dir, dev, &fops_tpu_vdd_tpu_m); + debugfs_create_file("vdd_int_m", 0660, platform_pwr->debugfs_dir, dev, &fops_tpu_vdd_int_m); + debugfs_create_file("core_rate", 0660, platform_pwr->debugfs_dir, dev, &fops_tpu_core_rate); + debugfs_create_file("ctl_rate", 0660, platform_pwr->debugfs_dir, dev, &fops_tpu_ctl_rate); + debugfs_create_file("axi_rate", 0660, platform_pwr->debugfs_dir, dev, &fops_tpu_axi_rate); + debugfs_create_file("apb_rate", 0440, platform_pwr->debugfs_dir, dev, &fops_tpu_apb_rate); + debugfs_create_file("uart_rate", 0440, platform_pwr->debugfs_dir, dev, &fops_tpu_uart_rate); if (platform_pwr->after_create) ret = platform_pwr->after_create(etdev); diff --git a/drivers/edgetpu/mobile-pm.h b/drivers/edgetpu/mobile-pm.h index e5fafb6..0cb4b77 100644 --- a/drivers/edgetpu/mobile-pm.h +++ b/drivers/edgetpu/mobile-pm.h @@ -49,6 +49,19 @@ enum mobile_reverse_kci_code { RKCI_CODE_BTS = RKCI_CHIP_CODE_FIRST + 2, }; +#define MAX_VOLTAGE_VAL 1250000 +#define TPU_DEBUG_REQ (1 << 31) +#define TPU_VDD_TPU_DEBUG (0 << 27) +#define TPU_VDD_TPU_M_DEBUG (1 << 27) +#define TPU_VDD_INT_M_DEBUG (2 << 27) +#define TPU_CLK_CORE_DEBUG (3 << 27) +#define TPU_CLK_CTL_DEBUG (4 << 27) +#define TPU_CLK_AXI_DEBUG (5 << 27) +#define TPU_CLK_APB_DEBUG (6 << 27) +#define TPU_CLK_UART_DEBUG (7 << 27) +#define TPU_CORE_PWR_DEBUG (8 << 27) +#define TPU_DEBUG_VALUE_MASK ((1 << 27) - 1) + /* * Initialize a power management interface for an edgetpu device on mobile * chipsets. diff --git a/drivers/edgetpu/mobile-thermal.c b/drivers/edgetpu/mobile-thermal.c new file mode 100644 index 0000000..cf7dadf --- /dev/null +++ b/drivers/edgetpu/mobile-thermal.c @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Common EdgeTPU mobile thermal management support + * + * Copyright (C) 2021 Google, Inc. + */ + +#include <linux/debugfs.h> +#include <linux/device.h> +#include <linux/gfp.h> +#include <linux/kernel.h> +#include <linux/mutex.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/slab.h> +#include <linux/thermal.h> +#include <linux/version.h> +#include <soc/google/gs101_tmu.h> + +#include "edgetpu-config.h" +#include "edgetpu-internal.h" +#include "edgetpu-kci.h" +#include "edgetpu-mmu.h" +#include "edgetpu-pm.h" +#include "edgetpu-thermal.h" +#include "mobile-pm.h" + +#define MAX_NUM_TPU_STATES 10 +#define OF_DATA_NUM_MAX (MAX_NUM_TPU_STATES * 2) +static struct edgetpu_state_pwr state_pwr_map[MAX_NUM_TPU_STATES] = {0}; + +/* + * Sends the thermal throttling KCI if the device is powered. + * + * Returns the return value of KCI if the device is powered, otherwise 0. + */ +static int edgetpu_thermal_kci_if_powered(struct edgetpu_dev *etdev, enum edgetpu_pwr_state state) +{ + int ret = 0; + + if (edgetpu_pm_get_if_powered(etdev->pm)) { + ret = edgetpu_kci_notify_throttling(etdev, state); + if (ret) + etdev_err_ratelimited(etdev, + "Failed to notify FW about power state %u, error:%d", + state, ret); + edgetpu_pm_put(etdev->pm); + } + return ret; +} + +static int edgetpu_get_max_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + struct edgetpu_thermal *thermal = cdev->devdata; + + if (thermal->tpu_num_states <= 0) + return -ENODEV; + + *state = thermal->tpu_num_states - 1; + return 0; +} + +/* + * Set cooling state. + */ +static int edgetpu_set_cur_state(struct thermal_cooling_device *cdev, + unsigned long state_original) +{ + int ret; + struct edgetpu_thermal *cooling = cdev->devdata; + struct device *dev = cooling->dev; + unsigned long pwr_state; + + if (state_original >= cooling->tpu_num_states) { + dev_err(dev, "%s: invalid cooling state %lu\n", __func__, + state_original); + return -EINVAL; + } + + state_original = max(cooling->sysfs_req, state_original); + + mutex_lock(&cooling->lock); + pwr_state = state_pwr_map[state_original].state; + if (state_original != cooling->cooling_state) { + /* + * Set the thermal policy through ACPM to allow cooling by DVFS. Any states lower + * than UUD should be handled by firmware when it gets the throttling notification + * KCI + */ + if (pwr_state < TPU_ACTIVE_UUD) { + dev_warn_ratelimited( + dev, "Setting lowest DVFS state, waiting for FW to shutdown TPU"); + ret = exynos_acpm_set_policy(TPU_ACPM_DOMAIN, TPU_ACTIVE_UUD); + } else { + ret = exynos_acpm_set_policy(TPU_ACPM_DOMAIN, pwr_state); + } + + if (ret) { + dev_err(dev, "error setting tpu policy: %d\n", ret); + goto out; + } + cooling->cooling_state = state_original; + } else { + ret = -EALREADY; + } + +out: + mutex_unlock(&cooling->lock); + return ret; +} + +static int edgetpu_get_cur_state(struct thermal_cooling_device *cdev, + unsigned long *state) +{ + int ret = 0; + struct edgetpu_thermal *cooling = cdev->devdata; + + *state = cooling->cooling_state; + if (*state >= cooling->tpu_num_states) { + dev_warn(cooling->dev, + "Unknown cooling state: %lu, resetting\n", *state); + mutex_lock(&cooling->lock); + +#if IS_ENABLED(CONFIG_ABROLHOS) + ret = exynos_acpm_set_policy(TPU_ACPM_DOMAIN, TPU_ACTIVE_OD); +#else + ret = exynos_acpm_set_policy(TPU_ACPM_DOMAIN, TPU_ACTIVE_NOM); +#endif /* IS_ENABLED(CONFIG_ABROLHOS) */ + if (ret) { + dev_err(cooling->dev, "error setting tpu policy: %d\n", + ret); + mutex_unlock(&cooling->lock); + return ret; + } + + //setting back to "no cooling" + cooling->cooling_state = 0; + mutex_unlock(&cooling->lock); + } + + return 0; +} + +static int edgetpu_state2power_internal(unsigned long state, u32 *power, + struct edgetpu_thermal *thermal) +{ + int i; + + for (i = 0; i < thermal->tpu_num_states; ++i) { + if (state == state_pwr_map[i].state) { + *power = state_pwr_map[i].power; + return 0; + } + } + dev_err(thermal->dev, "Unknown state req for: %lu\n", state); + *power = 0; + return -EINVAL; +} + +static int edgetpu_get_requested_power(struct thermal_cooling_device *cdev, +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) + struct thermal_zone_device *tz, +#endif + u32 *power) +{ + unsigned long state_original; + struct edgetpu_thermal *cooling = cdev->devdata; + + state_original = exynos_acpm_get_rate(TPU_ACPM_DOMAIN, 0); + return edgetpu_state2power_internal(state_original, power, + cooling); +} + +static int edgetpu_state2power(struct thermal_cooling_device *cdev, +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) + struct thermal_zone_device *tz, +#endif + unsigned long state, u32 *power) +{ + struct edgetpu_thermal *cooling = cdev->devdata; + + if (state >= cooling->tpu_num_states) { + dev_err(cooling->dev, "%s: invalid state: %lu\n", __func__, + state); + return -EINVAL; + } + + return edgetpu_state2power_internal(state_pwr_map[state].state, power, + cooling); +} + +static int edgetpu_power2state(struct thermal_cooling_device *cdev, +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0) + struct thermal_zone_device *tz, +#endif + u32 power, unsigned long *state) +{ + int i, penultimate_throttle_state; + struct edgetpu_thermal *thermal = cdev->devdata; + + *state = 0; + if (thermal->tpu_num_states < 2) + return thermal->tpu_num_states == 1 ? 0 : -ENODEV; + + penultimate_throttle_state = thermal->tpu_num_states - 2; + /* + * argument "power" is the maximum allowed power consumption in mW as + * defined by the PID control loop. Check for the first state that is + * less than or equal to the current allowed power. state_pwr_map is + * descending, so lowest power consumption is last value in the array + * return lowest state even if it consumes more power than allowed as + * not all platforms can handle throttling below an active state + */ + for (i = penultimate_throttle_state; i >= 0; --i) { + if (power < state_pwr_map[i].power) { + *state = i + 1; + break; + } + } + return 0; +} + +static struct thermal_cooling_device_ops edgetpu_cooling_ops = { + .get_max_state = edgetpu_get_max_state, + .get_cur_state = edgetpu_get_cur_state, + .set_cur_state = edgetpu_set_cur_state, + .get_requested_power = edgetpu_get_requested_power, + .state2power = edgetpu_state2power, + .power2state = edgetpu_power2state, +}; + +static void tpu_thermal_exit_cooling(struct edgetpu_thermal *thermal) +{ + if (!IS_ERR_OR_NULL(thermal->cdev)) + thermal_cooling_device_unregister(thermal->cdev); +} + +static void tpu_thermal_exit(struct edgetpu_thermal *thermal) +{ + tpu_thermal_exit_cooling(thermal); + debugfs_remove_recursive(thermal->cooling_root); +} + +static void devm_tpu_thermal_release(struct device *dev, void *res) +{ + struct edgetpu_thermal *thermal = res; + + tpu_thermal_exit(thermal); +} + +static int tpu_thermal_parse_dvfs_table(struct edgetpu_thermal *thermal) +{ + int row_size, col_size, tbl_size, i; + int of_data_int_array[OF_DATA_NUM_MAX]; + + if (of_property_read_u32_array(thermal->dev->of_node, + "tpu_dvfs_table_size", of_data_int_array, 2)) + goto error; + + row_size = of_data_int_array[0]; + col_size = of_data_int_array[1]; + tbl_size = row_size * col_size; + if (row_size > MAX_NUM_TPU_STATES) { + dev_err(thermal->dev, "too many TPU states\n"); + goto error; + } + + if (tbl_size > OF_DATA_NUM_MAX) + goto error; + + if (of_property_read_u32_array(thermal->dev->of_node, + "tpu_dvfs_table", of_data_int_array, tbl_size)) + goto error; + + thermal->tpu_num_states = row_size; + for (i = 0; i < row_size; ++i) { + int idx = col_size * i; + + state_pwr_map[i].state = of_data_int_array[idx]; + state_pwr_map[i].power = of_data_int_array[idx + 1]; + } + + return 0; + +error: + dev_err(thermal->dev, "failed to parse DVFS table\n"); + return -EINVAL; +} + +static ssize_t +user_vote_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct thermal_cooling_device *cdev = + container_of(dev, struct thermal_cooling_device, device); + struct edgetpu_thermal *cooling = cdev->devdata; + + if (!cooling) + return -ENODEV; + + return sysfs_emit(buf, "%lu\n", cooling->sysfs_req); +} + +static ssize_t user_vote_store(struct device *dev, struct device_attribute *attr, const char *buf, + size_t count) +{ + struct thermal_cooling_device *cdev = + container_of(dev, struct thermal_cooling_device, device); + struct edgetpu_thermal *cooling = cdev->devdata; + int ret; + unsigned long state; + + if (!cooling) + return -ENODEV; + + ret = kstrtoul(buf, 0, &state); + if (ret) + return ret; + + if (state >= cooling->tpu_num_states) + return -EINVAL; + + mutex_lock(&cdev->lock); + cooling->sysfs_req = state; + cdev->updated = false; + mutex_unlock(&cdev->lock); + thermal_cdev_update(cdev); + + return count; +} + +static DEVICE_ATTR_RW(user_vote); + +static int tpu_pause_callback(enum thermal_pause_state action, void *dev) +{ + int ret = -EINVAL; + + if (!dev) + return ret; + + if (action == THERMAL_SUSPEND) + ret = edgetpu_thermal_suspend(dev); + else if (action == THERMAL_RESUME) + ret = edgetpu_thermal_resume(dev); + + return ret; +} + +static int +tpu_thermal_cooling_register(struct edgetpu_thermal *thermal, char *type) +{ + struct device_node *cooling_node = NULL; + int err = 0; + + thermal->op_data = NULL; + thermal->tpu_num_states = 0; + + err = tpu_thermal_parse_dvfs_table(thermal); + if (err) + return err; + + mutex_init(&thermal->lock); + cooling_node = of_find_node_by_name(NULL, "tpu-cooling"); + if (!cooling_node) + dev_warn(thermal->dev, "failed to find cooling node\n"); + // Initialize the cooling state as 0, means "no cooling" + thermal->cooling_state = 0; + thermal->cdev = thermal_of_cooling_device_register( + cooling_node, type, thermal, &edgetpu_cooling_ops); + if (IS_ERR(thermal->cdev)) + return PTR_ERR(thermal->cdev); + + return device_create_file(&thermal->cdev->device, &dev_attr_user_vote); +} + +static int tpu_thermal_init(struct edgetpu_thermal *thermal, struct device *dev) +{ + int err; + struct dentry *d; + + d = debugfs_create_dir("cooling", edgetpu_fs_debugfs_dir()); + /* don't let debugfs creation failure abort the init procedure */ + if (IS_ERR_OR_NULL(d)) + dev_warn(dev, "failed to create debug fs for cooling"); + thermal->dev = dev; + thermal->cooling_root = d; + + err = tpu_thermal_cooling_register(thermal, EDGETPU_COOLING_NAME); + if (err) { + dev_err(dev, "failed to initialize external cooling\n"); + tpu_thermal_exit(thermal); + return err; + } + + register_tpu_thermal_pause_cb(tpu_pause_callback, dev); + + return 0; +} + +struct edgetpu_thermal +*devm_tpu_thermal_create(struct device *dev, struct edgetpu_dev *etdev) +{ + struct edgetpu_thermal *thermal; + int err; + + thermal = devres_alloc(devm_tpu_thermal_release, sizeof(*thermal), + GFP_KERNEL); + if (!thermal) + return ERR_PTR(-ENOMEM); + + thermal->etdev = etdev; + err = tpu_thermal_init(thermal, dev); + if (err) { + devres_free(thermal); + return ERR_PTR(err); + } + + devres_add(dev, thermal); + return thermal; +} + +int edgetpu_thermal_suspend(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct edgetpu_dev *etdev = platform_get_drvdata(pdev); + struct edgetpu_thermal *cooling = etdev->thermal; + int ret = 0; + + if (IS_ERR(cooling)) + return PTR_ERR(cooling); + mutex_lock(&cooling->lock); + /* + * Always set as suspended even when the FW cannot handle the KCI (it's dead for some + * unknown reasons) because we still want to prevent the runtime from using TPU. + */ + cooling->thermal_suspended = true; + ret = edgetpu_thermal_kci_if_powered(etdev, TPU_OFF); + mutex_unlock(&cooling->lock); + return ret; +} + +int edgetpu_thermal_resume(struct device *dev) +{ + struct platform_device *pdev = to_platform_device(dev); + struct edgetpu_dev *etdev = platform_get_drvdata(pdev); + struct edgetpu_thermal *cooling = etdev->thermal; + int ret = 0; + + if (IS_ERR(cooling)) + return PTR_ERR(cooling); + mutex_lock(&cooling->lock); + ret = edgetpu_thermal_kci_if_powered(etdev, state_pwr_map[0].state); + /* + * Unlike edgetpu_thermal_suspend(), only set the device is resumed if the FW handled the + * KCI request. + */ + if (!ret) + cooling->thermal_suspended = false; + mutex_unlock(&cooling->lock); + return ret; +} |