diff options
author | Victor Hsu <hsuvictor@google.com> | 2021-11-10 14:01:16 +0800 |
---|---|---|
committer | Victor Hsu <hsuvictor@google.com> | 2021-12-13 13:34:17 +0800 |
commit | c972edc5cac0526943caaa390ddd0faf2ecb18db (patch) | |
tree | 653bbcbe5021b5cac251e2da2ac41583c6b0868d /mhi | |
parent | 11b8045bbc3a20844fd6abeb460df31dcd70bbb4 (diff) | |
download | cnss2-c972edc5cac0526943caaa390ddd0faf2ecb18db.tar.gz |
wcn6740: Build mhi/qmi/qrtr out of kernel tree
Bug: 203725513
Signed-off-by: Victor Hsu <hsuvictor@google.com>
Change-Id: Ie484ffaf1bde124d141e8e87adf9e3beb5cf7174
Diffstat (limited to 'mhi')
-rw-r--r-- | mhi/Kconfig | 31 | ||||
-rw-r--r-- | mhi/Makefile | 3 | ||||
-rw-r--r-- | mhi/core/Makefile | 7 | ||||
-rw-r--r-- | mhi/core/boot.c | 570 | ||||
-rw-r--r-- | mhi/core/debugfs.c | 413 | ||||
-rw-r--r-- | mhi/core/init.c | 1447 | ||||
-rw-r--r-- | mhi/core/internal.h | 720 | ||||
-rw-r--r-- | mhi/core/main.c | 1858 | ||||
-rw-r--r-- | mhi/core/misc.c | 1674 | ||||
-rw-r--r-- | mhi/core/misc.h | 303 | ||||
-rw-r--r-- | mhi/core/pm.c | 1255 |
11 files changed, 8281 insertions, 0 deletions
diff --git a/mhi/Kconfig b/mhi/Kconfig new file mode 100644 index 0000000..7eb139c --- /dev/null +++ b/mhi/Kconfig @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# MHI bus +# +# Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. +# + +config MHI_BUS + tristate "Modem Host Interface (MHI) bus" + help + Bus driver for MHI protocol. Modem Host Interface (MHI) is a + communication protocol used by the host processors to control + and communicate with modem devices over a high speed peripheral + bus or shared memory. + +config MHI_BUS_MISC + bool "Support for miscellaneous MHI features" + depends on MHI_BUS + help + Miscellaneous features support for MHI Bus driver includes IPC logs, + introduction of a list of controllers for debug using ramdumps and + other features not present upstream such as Dynamic Resource Vote, + SFR parsing using RDDM dumps, scanning for the RDDM cookie and more. + +config MHI_BUS_DEBUG + bool "Debugfs support for the MHI bus" + depends on MHI_BUS && DEBUG_FS + help + Enable debugfs support for use with the MHI transport. Allows + reading and/or modifying some values within the MHI controller + for debug and test purposes. diff --git a/mhi/Makefile b/mhi/Makefile new file mode 100644 index 0000000..edd806f --- /dev/null +++ b/mhi/Makefile @@ -0,0 +1,3 @@ +ccflags-y += -I$(WLAN_PLATFORM_ROOT)/inc +# core layer +obj-y += core/ diff --git a/mhi/core/Makefile b/mhi/core/Makefile new file mode 100644 index 0000000..17be4ee --- /dev/null +++ b/mhi/core/Makefile @@ -0,0 +1,7 @@ +obj-$(CONFIG_MHI_BUS) += mhi.o + +mhi-y := init.o main.o pm.o boot.o +mhi-$(CONFIG_MHI_BUS_MISC) += misc.o +mhi-$(CONFIG_MHI_BUS_DEBUG) += debugfs.o + +ccflags-y += -DDEBUG -DCONFIG_WCN_GOOGLE diff --git a/mhi/core/boot.c b/mhi/core/boot.c new file mode 100644 index 0000000..39961b8 --- /dev/null +++ b/mhi/core/boot.c @@ -0,0 +1,570 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * + */ + +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/dma-direction.h> +#include <linux/dma-mapping.h> +#include <linux/firmware.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/mhi.h> +#include <linux/module.h> +#include <linux/random.h> +#include <linux/slab.h> +#include <linux/wait.h> +#include "internal.h" + +/* Setup RDDM vector table for RDDM transfer and program RXVEC */ +void mhi_rddm_prepare(struct mhi_controller *mhi_cntrl, + struct image_info *img_info) +{ + struct mhi_buf *mhi_buf = img_info->mhi_buf; + struct bhi_vec_entry *bhi_vec = img_info->bhi_vec; + void __iomem *base = mhi_cntrl->bhie; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + u32 sequence_id; + unsigned int i; + + for (i = 0; i < img_info->entries - 1; i++, mhi_buf++, bhi_vec++) { + bhi_vec->dma_addr = mhi_buf->dma_addr; + bhi_vec->size = mhi_buf->len; + } + + MHI_VERB("BHIe programming for RDDM\n"); + + mhi_write_reg(mhi_cntrl, base, BHIE_RXVECADDR_HIGH_OFFS, + upper_32_bits(mhi_buf->dma_addr)); + + mhi_write_reg(mhi_cntrl, base, BHIE_RXVECADDR_LOW_OFFS, + lower_32_bits(mhi_buf->dma_addr)); + + mhi_write_reg(mhi_cntrl, base, BHIE_RXVECSIZE_OFFS, mhi_buf->len); + sequence_id = MHI_RANDOM_U32_NONZERO(BHIE_RXVECSTATUS_SEQNUM_BMSK); + + mhi_write_reg_field(mhi_cntrl, base, BHIE_RXVECDB_OFFS, + BHIE_RXVECDB_SEQNUM_BMSK, BHIE_RXVECDB_SEQNUM_SHFT, + sequence_id); + + MHI_VERB("Address: %p and len: 0x%zx sequence: %u\n", + &mhi_buf->dma_addr, mhi_buf->len, sequence_id); +} + +/* Collect RDDM buffer during kernel panic */ +static int __mhi_download_rddm_in_panic(struct mhi_controller *mhi_cntrl) +{ + int ret; + u32 rx_status; + enum mhi_ee_type ee; + const u32 delayus = 2000; + u32 retry = (mhi_cntrl->timeout_ms * 1000) / delayus; + const u32 rddm_timeout_us = 200000; + int rddm_retry = rddm_timeout_us / delayus; + void __iomem *base = mhi_cntrl->bhie; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + + MHI_VERB("Entered with pm_state:%s dev_state:%s ee:%s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state), + TO_MHI_STATE_STR(mhi_cntrl->dev_state), + TO_MHI_EXEC_STR(mhi_cntrl->ee)); + + /* + * This should only be executing during a kernel panic, we expect all + * other cores to shutdown while we're collecting RDDM buffer. After + * returning from this function, we expect the device to reset. + * + * Normaly, we read/write pm_state only after grabbing the + * pm_lock, since we're in a panic, skipping it. Also there is no + * gurantee that this state change would take effect since + * we're setting it w/o grabbing pm_lock + */ + mhi_cntrl->pm_state = MHI_PM_LD_ERR_FATAL_DETECT; + /* update should take the effect immediately */ + smp_wmb(); + + /* + * Make sure device is not already in RDDM. In case the device asserts + * and a kernel panic follows, device will already be in RDDM. + * Do not trigger SYS ERR again and proceed with waiting for + * image download completion. + */ + ee = mhi_get_exec_env(mhi_cntrl); + if (ee == MHI_EE_MAX) + goto error_exit_rddm; + + if (ee != MHI_EE_RDDM) { + MHI_VERB("Trigger device into RDDM mode using SYS ERR\n"); + mhi_set_mhi_state(mhi_cntrl, MHI_STATE_SYS_ERR); + + MHI_VERB("Waiting for device to enter RDDM\n"); + while (rddm_retry--) { + ee = mhi_get_exec_env(mhi_cntrl); + if (ee == MHI_EE_RDDM) + break; + + udelay(delayus); + } + + if (rddm_retry <= 0) { + /* Hardware reset so force device to enter RDDM */ + MHI_VERB( + "Did not enter RDDM, do a host req reset\n"); + mhi_soc_reset(mhi_cntrl); + udelay(delayus); + } + + ee = mhi_get_exec_env(mhi_cntrl); + } + + MHI_VERB( + "Waiting for RDDM image download via BHIe, current EE:%s\n", + TO_MHI_EXEC_STR(ee)); + + while (retry--) { + ret = mhi_read_reg_field(mhi_cntrl, base, BHIE_RXVECSTATUS_OFFS, + BHIE_RXVECSTATUS_STATUS_BMSK, + BHIE_RXVECSTATUS_STATUS_SHFT, + &rx_status); + if (ret) + return -EIO; + + if (rx_status == BHIE_RXVECSTATUS_STATUS_XFER_COMPL) + return 0; + + udelay(delayus); + } + + ee = mhi_get_exec_env(mhi_cntrl); + ret = mhi_read_reg(mhi_cntrl, base, BHIE_RXVECSTATUS_OFFS, &rx_status); + + MHI_ERR("RXVEC_STATUS: 0x%x\n", rx_status); + +error_exit_rddm: + MHI_ERR("RDDM transfer failed. Current EE: %s\n", + TO_MHI_EXEC_STR(ee)); + + return -EIO; +} + +/* Download RDDM image from device */ +int mhi_download_rddm_image(struct mhi_controller *mhi_cntrl, bool in_panic) +{ + void __iomem *base = mhi_cntrl->bhie; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + u32 rx_status; + + if (in_panic) + return __mhi_download_rddm_in_panic(mhi_cntrl); + + MHI_VERB("Waiting for RDDM image download via BHIe\n"); + + /* Wait for the image download to complete */ + wait_event_timeout(mhi_cntrl->state_event, + mhi_read_reg_field(mhi_cntrl, base, + BHIE_RXVECSTATUS_OFFS, + BHIE_RXVECSTATUS_STATUS_BMSK, + BHIE_RXVECSTATUS_STATUS_SHFT, + &rx_status) || rx_status, + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + + return (rx_status == BHIE_RXVECSTATUS_STATUS_XFER_COMPL) ? 0 : -EIO; +} +EXPORT_SYMBOL_GPL(mhi_download_rddm_image); + +static int mhi_fw_load_bhie(struct mhi_controller *mhi_cntrl, + const struct mhi_buf *mhi_buf) +{ + void __iomem *base = mhi_cntrl->bhie; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + rwlock_t *pm_lock = &mhi_cntrl->pm_lock; + u32 tx_status, sequence_id, val; + int ret, rd; + + read_lock_bh(pm_lock); + if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) { + read_unlock_bh(pm_lock); + return -EIO; + } + + sequence_id = MHI_RANDOM_U32_NONZERO(BHIE_TXVECSTATUS_SEQNUM_BMSK); + MHI_VERB("Starting image download via BHIe. Sequence ID: %u\n", + sequence_id); + mhi_write_reg(mhi_cntrl, base, BHIE_TXVECADDR_HIGH_OFFS, + upper_32_bits(mhi_buf->dma_addr)); + + mhi_write_reg(mhi_cntrl, base, BHIE_TXVECADDR_LOW_OFFS, + lower_32_bits(mhi_buf->dma_addr)); + + mhi_write_reg(mhi_cntrl, base, BHIE_TXVECSIZE_OFFS, mhi_buf->len); + + mhi_write_reg_field(mhi_cntrl, base, BHIE_TXVECDB_OFFS, + BHIE_TXVECDB_SEQNUM_BMSK, BHIE_TXVECDB_SEQNUM_SHFT, + sequence_id); + read_unlock_bh(pm_lock); + + /* Wait for the image download to complete */ + ret = wait_event_timeout(mhi_cntrl->state_event, + MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) || + mhi_read_reg_field(mhi_cntrl, base, + BHIE_TXVECSTATUS_OFFS, + BHIE_TXVECSTATUS_STATUS_BMSK, + BHIE_TXVECSTATUS_STATUS_SHFT, + &tx_status) || tx_status, + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) || + tx_status != BHIE_TXVECSTATUS_STATUS_XFER_COMPL) { + rd = mhi_read_reg(mhi_cntrl, base, BHIE_TXVECSTATUS_OFFS, &val); + MHI_ERR("BHIE_TXVECSTATUS: 0x%x, reg read: %d, tx_status: %u\n", + val, rd, tx_status); + return -EIO; + } + + return (!ret) ? -ETIMEDOUT : 0; +} + +static int mhi_fw_load_bhi(struct mhi_controller *mhi_cntrl, + dma_addr_t dma_addr, + size_t size) +{ + u32 tx_status, val; + int i, ret; + void __iomem *base = mhi_cntrl->bhi; + rwlock_t *pm_lock = &mhi_cntrl->pm_lock; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct { + char *name; + u32 offset; + } error_reg[] = { + { "ERROR_CODE", BHI_ERRCODE }, + { "ERROR_DBG1", BHI_ERRDBG1 }, + { "ERROR_DBG2", BHI_ERRDBG2 }, + { "ERROR_DBG3", BHI_ERRDBG3 }, + { NULL }, + }; + + read_lock_bh(pm_lock); + if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) { + read_unlock_bh(pm_lock); + goto invalid_pm_state; + } + + mhi_cntrl->session_id = MHI_RANDOM_U32_NONZERO(BHI_TXDB_SEQNUM_BMSK); + MHI_VERB("Starting image download via BHI. Session ID: %u\n", + mhi_cntrl->session_id); + mhi_write_reg(mhi_cntrl, base, BHI_STATUS, 0); + mhi_write_reg(mhi_cntrl, base, BHI_IMGADDR_HIGH, + upper_32_bits(dma_addr)); + mhi_write_reg(mhi_cntrl, base, BHI_IMGADDR_LOW, + lower_32_bits(dma_addr)); + mhi_write_reg(mhi_cntrl, base, BHI_IMGSIZE, size); + mhi_write_reg(mhi_cntrl, base, BHI_IMGTXDB, mhi_cntrl->session_id); + read_unlock_bh(pm_lock); + + /* Wait for the image download to complete */ + ret = wait_event_timeout(mhi_cntrl->state_event, + MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) || + mhi_read_reg_field(mhi_cntrl, base, BHI_STATUS, + BHI_STATUS_MASK, BHI_STATUS_SHIFT, + &tx_status) || tx_status, + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) + { + MHI_ERR("Image transfer failed: MHI_PM_IN_ERROR_STATE\n"); + goto invalid_pm_state; + } + + if (tx_status == BHI_STATUS_ERROR) { + MHI_ERR("Image transfer failed\n"); + read_lock_bh(pm_lock); + if (MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) { + for (i = 0; error_reg[i].name; i++) { + ret = mhi_read_reg(mhi_cntrl, base, + error_reg[i].offset, &val); + if (ret) + break; + MHI_ERR("Reg: %s value: 0x%x\n", + error_reg[i].name, val); + } + } + read_unlock_bh(pm_lock); + goto invalid_pm_state; + } + + return (!ret) ? -ETIMEDOUT : 0; + +invalid_pm_state: + + return -EIO; +} + +void mhi_free_bhie_table(struct mhi_controller *mhi_cntrl, + struct image_info **image_info) +{ + int i; + struct mhi_buf *mhi_buf = (*image_info)->mhi_buf; + + if (mhi_cntrl->img_pre_alloc) + return; + + for (i = 0; i < (*image_info)->entries; i++, mhi_buf++) + dma_free_attrs(mhi_cntrl->cntrl_dev, mhi_buf->len, mhi_buf->buf, + mhi_buf->dma_addr, DMA_ATTR_FORCE_CONTIGUOUS); + + kfree((*image_info)->mhi_buf); + kfree(*image_info); + + *image_info = NULL; +} + +int mhi_alloc_bhie_table(struct mhi_controller *mhi_cntrl, + struct image_info **image_info, + size_t alloc_size) +{ + size_t seg_size = mhi_cntrl->seg_len; + int segments = DIV_ROUND_UP(alloc_size, seg_size) + 1; + int i; + struct image_info *img_info; + struct mhi_buf *mhi_buf; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + + if (mhi_cntrl->img_pre_alloc) + return 0; + + MHI_LOG("Allocating bytes: %zu seg_size: %zu total_seg: %u\n", + alloc_size, seg_size, segments); + + img_info = kzalloc(sizeof(*img_info), GFP_KERNEL); + if (!img_info) + return -ENOMEM; + + /* Allocate memory for entries */ + img_info->mhi_buf = kcalloc(segments, sizeof(*img_info->mhi_buf), + GFP_KERNEL); + if (!img_info->mhi_buf) + goto error_alloc_mhi_buf; + + /* Allocate and populate vector table */ + mhi_buf = img_info->mhi_buf; + for (i = 0; i < segments; i++, mhi_buf++) { + size_t vec_size = seg_size; + + /* Vector table is the last entry */ + if (i == segments - 1) + vec_size = sizeof(struct bhi_vec_entry) * i; + + mhi_buf->len = vec_size; + mhi_buf->buf = dma_alloc_attrs(mhi_cntrl->cntrl_dev, vec_size, + &mhi_buf->dma_addr, GFP_KERNEL, + DMA_ATTR_FORCE_CONTIGUOUS); + if (!mhi_buf->buf) + goto error_alloc_segment; + + MHI_LOG("Entry: %d Address: 0x%llx size: %lu\n", i, + mhi_buf->dma_addr, mhi_buf->len); + } + + img_info->bhi_vec = img_info->mhi_buf[segments - 1].buf; + img_info->entries = segments; + *image_info = img_info; + + MHI_LOG("Successfully allocated BHIe vector table\n"); + + return 0; + +error_alloc_segment: + for (--i, --mhi_buf; i >= 0; i--, mhi_buf--) + dma_free_attrs(mhi_cntrl->cntrl_dev, mhi_buf->len, mhi_buf->buf, + mhi_buf->dma_addr, DMA_ATTR_FORCE_CONTIGUOUS); + +error_alloc_mhi_buf: + kfree(img_info); + + return -ENOMEM; +} + +static void mhi_firmware_copy(struct mhi_controller *mhi_cntrl, + const struct firmware *firmware, + struct image_info *img_info) +{ + size_t remainder = firmware->size; + size_t to_cpy; + const u8 *buf = firmware->data; + struct mhi_buf *mhi_buf = img_info->mhi_buf; + struct bhi_vec_entry *bhi_vec = img_info->bhi_vec; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + + while (remainder) { + to_cpy = min(remainder, mhi_buf->len); + memcpy(mhi_buf->buf, buf, to_cpy); + bhi_vec->dma_addr = mhi_buf->dma_addr; + bhi_vec->size = to_cpy; + + MHI_VERB("Setting Vector: 0x%llx size: %llu\n", + bhi_vec->dma_addr, bhi_vec->size); + + buf += to_cpy; + remainder -= to_cpy; + bhi_vec++; + mhi_buf++; + } +} + +void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl) +{ + const struct firmware *firmware = NULL; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + const char *fw_name; + void *buf; + dma_addr_t dma_addr; + size_t size; + int i, ret; + + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) { + MHI_ERR("Device MHI is not in valid state\n"); + return; + } + + /* save hardware info from BHI */ + ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_SERIALNU, + &mhi_cntrl->serial_number); + if (ret) + MHI_ERR("Could not capture serial number via BHI\n"); + + for (i = 0; i < ARRAY_SIZE(mhi_cntrl->oem_pk_hash); i++) { + ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_OEMPKHASH(i), + &mhi_cntrl->oem_pk_hash[i]); + if (ret) { + MHI_ERR("Could not capture OEM PK HASH via BHI\n"); + break; + } + } + + /* If device is in pass through, do reset to ready state transition */ + if (mhi_cntrl->ee == MHI_EE_PTHRU) + goto fw_load_ee_pthru; + + fw_name = (mhi_cntrl->ee == MHI_EE_EDL) ? + mhi_cntrl->edl_image : mhi_cntrl->fw_image; + + if (!fw_name || (mhi_cntrl->fbc_download && (!mhi_cntrl->sbl_size || + !mhi_cntrl->seg_len))) { + MHI_ERR( + "No firmware image defined or !sbl_size || !seg_len\n"); + goto error_fw_load; + } + + ret = request_firmware(&firmware, fw_name, dev->parent); + if (ret) { + if (!mhi_cntrl->fallback_fw_image) { + MHI_ERR("Error loading firmware: %d\n", ret); + goto error_fw_load; + } + + ret = request_firmware(&firmware, + mhi_cntrl->fallback_fw_image, + dev->parent); + if (ret) { + MHI_ERR("Error loading fallback firmware: %d\n", + ret); + goto error_fw_load; + } + mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_FALLBACK_IMG); + } + + size = (mhi_cntrl->fbc_download) ? mhi_cntrl->sbl_size : firmware->size; + + /* SBL size provided is maximum size, not necessarily the image size */ + if (size > firmware->size) + size = firmware->size; + + buf = dma_alloc_coherent(mhi_cntrl->cntrl_dev, size, &dma_addr, GFP_KERNEL); + if (!buf) { + release_firmware(firmware); + goto error_fw_load; + } + + /* Download image using BHI */ + memcpy(buf, firmware->data, size); + ret = mhi_fw_load_bhi(mhi_cntrl, dma_addr, size); + dma_free_coherent(mhi_cntrl->cntrl_dev, size, buf, dma_addr); + + /* Error or in EDL mode, we're done */ + if (ret) { + MHI_ERR("MHI did not load image over BHI, ret: %d\n", ret); + release_firmware(firmware); + goto error_fw_load; + } + + if (mhi_cntrl->ee == MHI_EE_EDL) { + release_firmware(firmware); + return; + } + + write_lock_irq(&mhi_cntrl->pm_lock); + mhi_cntrl->dev_state = MHI_STATE_RESET; + write_unlock_irq(&mhi_cntrl->pm_lock); + + /* + * If we're doing fbc, populate vector tables while + * device transitioning into MHI READY state + */ + if (mhi_cntrl->fbc_download) { + ret = mhi_alloc_bhie_table(mhi_cntrl, &mhi_cntrl->fbc_image, + firmware->size); + if (ret) { + release_firmware(firmware); + goto error_fw_load; + } + + /* Load the firmware into BHIE vec table */ + mhi_firmware_copy(mhi_cntrl, firmware, mhi_cntrl->fbc_image); + } + + release_firmware(firmware); + +fw_load_ee_pthru: + /* Transitioning into MHI RESET->READY state */ + ret = mhi_ready_state_transition(mhi_cntrl); + + if (!mhi_cntrl->fbc_download) + return; + + if (ret) { + MHI_ERR("MHI did not enter READY state\n"); + goto error_ready_state; + } + + return; + +error_ready_state: + mhi_free_bhie_table(mhi_cntrl, &mhi_cntrl->fbc_image); + +error_fw_load: + mhi_cntrl->pm_state = MHI_PM_FW_DL_ERR; + wake_up_all(&mhi_cntrl->state_event); +} + +int mhi_download_amss_image(struct mhi_controller *mhi_cntrl) +{ + struct image_info *image_info = mhi_cntrl->fbc_image; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + int ret; + + if (!image_info) + return -EIO; + + ret = mhi_fw_load_bhie(mhi_cntrl, + /* Vector table is the last entry */ + &image_info->mhi_buf[image_info->entries - 1]); + if (ret) { + MHI_ERR("MHI did not load AMSS, ret:%d\n", ret); + mhi_cntrl->pm_state = MHI_PM_FW_DL_ERR; + wake_up_all(&mhi_cntrl->state_event); + } + + return ret; +} diff --git a/mhi/core/debugfs.c b/mhi/core/debugfs.c new file mode 100644 index 0000000..704fcf5 --- /dev/null +++ b/mhi/core/debugfs.c @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2020, The Linux Foundation. All rights reserved. + * + */ + +#include <linux/debugfs.h> +#include <linux/device.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/mhi.h> +#include <linux/module.h> +#include "internal.h" + +static int mhi_debugfs_states_show(struct seq_file *m, void *d) +{ + struct mhi_controller *mhi_cntrl = m->private; + + /* states */ + seq_printf(m, "PM state: %s Device: %s MHI state: %s EE: %s wake: %s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state), + mhi_is_active(mhi_cntrl) ? "Active" : "Inactive", + TO_MHI_STATE_STR(mhi_cntrl->dev_state), + TO_MHI_EXEC_STR(mhi_cntrl->ee), + mhi_cntrl->wake_set ? "true" : "false"); + + /* counters */ + seq_printf(m, "M0: %u M2: %u M3: %u, M3_fast: %u", mhi_cntrl->M0, + mhi_cntrl->M2, mhi_cntrl->M3, mhi_cntrl->M3_fast); + + seq_printf(m, " device wake: %u pending packets: %u\n", + atomic_read(&mhi_cntrl->dev_wake), + atomic_read(&mhi_cntrl->pending_pkts)); + + return 0; +} + +static int mhi_debugfs_events_show(struct seq_file *m, void *d) +{ + struct mhi_controller *mhi_cntrl = m->private; + struct mhi_event *mhi_event; + struct mhi_event_ctxt *er_ctxt; + int i; + + if (!mhi_is_active(mhi_cntrl)) { + seq_puts(m, "Device not ready\n"); + return -ENODEV; + } + + er_ctxt = mhi_cntrl->mhi_ctxt->er_ctxt; + mhi_event = mhi_cntrl->mhi_event; + for (i = 0; i < mhi_cntrl->total_ev_rings; + i++, er_ctxt++, mhi_event++) { + struct mhi_ring *ring = &mhi_event->ring; + + if (mhi_event->offload_ev) { + seq_printf(m, "Index: %d is an offload event ring\n", + i); + continue; + } + + seq_printf(m, "Index: %d intmod count: %lu time: %lu", + i, (er_ctxt->intmod & EV_CTX_INTMODC_MASK) >> + EV_CTX_INTMODC_SHIFT, + (er_ctxt->intmod & EV_CTX_INTMODT_MASK) >> + EV_CTX_INTMODT_SHIFT); + + seq_printf(m, " base: 0x%0llx len: 0x%llx", er_ctxt->rbase, + er_ctxt->rlen); + + seq_printf(m, " rp: 0x%llx wp: 0x%llx", er_ctxt->rp, + er_ctxt->wp); + + seq_printf(m, " local rp: 0x%pK db: 0x%pad\n", ring->rp, + &mhi_event->db_cfg.db_val); + } + + return 0; +} + +static int mhi_debugfs_channels_show(struct seq_file *m, void *d) +{ + struct mhi_controller *mhi_cntrl = m->private; + struct mhi_chan *mhi_chan; + struct mhi_chan_ctxt *chan_ctxt; + int i; + + if (!mhi_is_active(mhi_cntrl)) { + seq_puts(m, "Device not ready\n"); + return -ENODEV; + } + + mhi_chan = mhi_cntrl->mhi_chan; + chan_ctxt = mhi_cntrl->mhi_ctxt->chan_ctxt; + for (i = 0; i < mhi_cntrl->max_chan; i++, chan_ctxt++, mhi_chan++) { + struct mhi_ring *ring = &mhi_chan->tre_ring; + + if (mhi_chan->offload_ch) { + seq_printf(m, "%s(%u) is an offload channel\n", + mhi_chan->name, mhi_chan->chan); + continue; + } + + if (!mhi_chan->mhi_dev) + continue; + + seq_printf(m, + "%s(%u) state: 0x%lx brstmode: 0x%lx pollcfg: 0x%lx", + mhi_chan->name, mhi_chan->chan, (chan_ctxt->chcfg & + CHAN_CTX_CHSTATE_MASK) >> CHAN_CTX_CHSTATE_SHIFT, + (chan_ctxt->chcfg & CHAN_CTX_BRSTMODE_MASK) >> + CHAN_CTX_BRSTMODE_SHIFT, (chan_ctxt->chcfg & + CHAN_CTX_POLLCFG_MASK) >> CHAN_CTX_POLLCFG_SHIFT); + + seq_printf(m, " type: 0x%x event ring: %u", chan_ctxt->chtype, + chan_ctxt->erindex); + + seq_printf(m, " base: 0x%llx len: 0x%llx rp: 0x%llx wp: 0x%llx", + chan_ctxt->rbase, chan_ctxt->rlen, chan_ctxt->rp, + chan_ctxt->wp); + + seq_printf(m, " local rp: 0x%pK local wp: 0x%pK db: 0x%pad\n", + ring->rp, ring->wp, + &mhi_chan->db_cfg.db_val); + } + + return 0; +} + +static int mhi_device_info_show(struct device *dev, void *data) +{ + struct mhi_device *mhi_dev; + + if (dev->bus != &mhi_bus_type) + return 0; + + mhi_dev = to_mhi_device(dev); + + seq_printf((struct seq_file *)data, "%s: type: %s dev_wake: %u", + mhi_dev->name, mhi_dev->dev_type ? "Controller" : "Transfer", + mhi_dev->dev_wake); + + /* for transfer device types only */ + if (mhi_dev->dev_type == MHI_DEVICE_XFER) + seq_printf((struct seq_file *)data, " channels: %u(UL)/%u(DL)", + mhi_dev->ul_chan_id, mhi_dev->dl_chan_id); + + seq_puts((struct seq_file *)data, "\n"); + + return 0; +} + +static int mhi_debugfs_devices_show(struct seq_file *m, void *d) +{ + struct mhi_controller *mhi_cntrl = m->private; + + if (!mhi_is_active(mhi_cntrl)) { + seq_puts(m, "Device not ready\n"); + return -ENODEV; + } + + /* Show controller and client(s) info */ + mhi_device_info_show(&mhi_cntrl->mhi_dev->dev, m); + device_for_each_child(&mhi_cntrl->mhi_dev->dev, m, mhi_device_info_show); + + return 0; +} + +static int mhi_debugfs_regdump_show(struct seq_file *m, void *d) +{ + struct mhi_controller *mhi_cntrl = m->private; + enum mhi_state state; + enum mhi_ee_type ee; + int i, ret = -EIO; + u32 val; + void __iomem *mhi_base = mhi_cntrl->regs; + void __iomem *bhi_base = mhi_cntrl->bhi; + void __iomem *bhie_base = mhi_cntrl->bhie; + void __iomem *wake_db = mhi_cntrl->wake_db; + struct { + const char *name; + int offset; + void __iomem *base; + } regs[] = { + { "MHI_REGLEN", MHIREGLEN, mhi_base}, + { "MHI_VER", MHIVER, mhi_base}, + { "MHI_CFG", MHICFG, mhi_base}, + { "MHI_CTRL", MHICTRL, mhi_base}, + { "MHI_STATUS", MHISTATUS, mhi_base}, + { "MHI_WAKE_DB", 0, wake_db}, + { "BHI_EXECENV", BHI_EXECENV, bhi_base}, + { "BHI_STATUS", BHI_STATUS, bhi_base}, + { "BHI_ERRCODE", BHI_ERRCODE, bhi_base}, + { "BHI_ERRDBG1", BHI_ERRDBG1, bhi_base}, + { "BHI_ERRDBG2", BHI_ERRDBG2, bhi_base}, + { "BHI_ERRDBG3", BHI_ERRDBG3, bhi_base}, + { "BHIE_TXVEC_DB", BHIE_TXVECDB_OFFS, bhie_base}, + { "BHIE_TXVEC_STATUS", BHIE_TXVECSTATUS_OFFS, bhie_base}, + { "BHIE_RXVEC_DB", BHIE_RXVECDB_OFFS, bhie_base}, + { "BHIE_RXVEC_STATUS", BHIE_RXVECSTATUS_OFFS, bhie_base}, + { NULL }, + }; + + if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) + return ret; + + seq_printf(m, "Host PM state: %s Device state: %s EE: %s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state), + TO_MHI_STATE_STR(mhi_cntrl->dev_state), + TO_MHI_EXEC_STR(mhi_cntrl->ee)); + + state = mhi_get_mhi_state(mhi_cntrl); + ee = mhi_get_exec_env(mhi_cntrl); + seq_printf(m, "Device EE: %s state: %s\n", TO_MHI_EXEC_STR(ee), + TO_MHI_STATE_STR(state)); + + for (i = 0; regs[i].name; i++) { + if (!regs[i].base) + continue; + ret = mhi_read_reg(mhi_cntrl, regs[i].base, regs[i].offset, + &val); + if (ret) + continue; + + seq_printf(m, "%s: 0x%x\n", regs[i].name, val); + } + + return 0; +} + +static int mhi_debugfs_device_wake_show(struct seq_file *m, void *d) +{ + struct mhi_controller *mhi_cntrl = m->private; + struct mhi_device *mhi_dev = mhi_cntrl->mhi_dev; + + if (!mhi_is_active(mhi_cntrl)) { + seq_puts(m, "Device not ready\n"); + return -ENODEV; + } + + seq_printf(m, + "Wake count: %d\n%s\n", mhi_dev->dev_wake, + "Usage: echo get/put > device_wake to vote/unvote for M0"); + + return 0; +} + +static ssize_t mhi_debugfs_device_wake_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct seq_file *m = file->private_data; + struct mhi_controller *mhi_cntrl = m->private; + struct mhi_device *mhi_dev = mhi_cntrl->mhi_dev; + char buf[16]; + int ret = -EINVAL; + + if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count))) + return -EFAULT; + + if (!strncmp(buf, "get", 3)) { + ret = mhi_device_get_sync(mhi_dev); + } else if (!strncmp(buf, "put", 3)) { + mhi_device_put(mhi_dev); + ret = 0; + } + + return ret ? ret : count; +} + +static int mhi_debugfs_timeout_ms_show(struct seq_file *m, void *d) +{ + struct mhi_controller *mhi_cntrl = m->private; + + seq_printf(m, "%u ms\n", mhi_cntrl->timeout_ms); + + return 0; +} + +static ssize_t mhi_debugfs_timeout_ms_write(struct file *file, + const char __user *ubuf, + size_t count, loff_t *ppos) +{ + struct seq_file *m = file->private_data; + struct mhi_controller *mhi_cntrl = m->private; + u32 timeout_ms; + + if (kstrtou32_from_user(ubuf, count, 0, &timeout_ms)) + return -EINVAL; + + mhi_cntrl->timeout_ms = timeout_ms; + + return count; +} + +static int mhi_debugfs_states_open(struct inode *inode, struct file *fp) +{ + return single_open(fp, mhi_debugfs_states_show, inode->i_private); +} + +static int mhi_debugfs_events_open(struct inode *inode, struct file *fp) +{ + return single_open(fp, mhi_debugfs_events_show, inode->i_private); +} + +static int mhi_debugfs_channels_open(struct inode *inode, struct file *fp) +{ + return single_open(fp, mhi_debugfs_channels_show, inode->i_private); +} + +static int mhi_debugfs_devices_open(struct inode *inode, struct file *fp) +{ + return single_open(fp, mhi_debugfs_devices_show, inode->i_private); +} + +static int mhi_debugfs_regdump_open(struct inode *inode, struct file *fp) +{ + return single_open(fp, mhi_debugfs_regdump_show, inode->i_private); +} + +static int mhi_debugfs_device_wake_open(struct inode *inode, struct file *fp) +{ + return single_open(fp, mhi_debugfs_device_wake_show, inode->i_private); +} + +static int mhi_debugfs_timeout_ms_open(struct inode *inode, struct file *fp) +{ + return single_open(fp, mhi_debugfs_timeout_ms_show, inode->i_private); +} + +static const struct file_operations debugfs_states_fops = { + .open = mhi_debugfs_states_open, + .release = single_release, + .read = seq_read, +}; + +static const struct file_operations debugfs_events_fops = { + .open = mhi_debugfs_events_open, + .release = single_release, + .read = seq_read, +}; + +static const struct file_operations debugfs_channels_fops = { + .open = mhi_debugfs_channels_open, + .release = single_release, + .read = seq_read, +}; + +static const struct file_operations debugfs_devices_fops = { + .open = mhi_debugfs_devices_open, + .release = single_release, + .read = seq_read, +}; + +static const struct file_operations debugfs_regdump_fops = { + .open = mhi_debugfs_regdump_open, + .release = single_release, + .read = seq_read, +}; + +static const struct file_operations debugfs_device_wake_fops = { + .open = mhi_debugfs_device_wake_open, + .write = mhi_debugfs_device_wake_write, + .release = single_release, + .read = seq_read, +}; + +static const struct file_operations debugfs_timeout_ms_fops = { + .open = mhi_debugfs_timeout_ms_open, + .write = mhi_debugfs_timeout_ms_write, + .release = single_release, + .read = seq_read, +}; + +static struct dentry *mhi_debugfs_root; + +void mhi_create_debugfs(struct mhi_controller *mhi_cntrl) +{ + mhi_cntrl->debugfs_dentry = + debugfs_create_dir(dev_name(mhi_cntrl->cntrl_dev), + mhi_debugfs_root); + + debugfs_create_file("states", 0444, mhi_cntrl->debugfs_dentry, + mhi_cntrl, &debugfs_states_fops); + debugfs_create_file("events", 0444, mhi_cntrl->debugfs_dentry, + mhi_cntrl, &debugfs_events_fops); + debugfs_create_file("channels", 0444, mhi_cntrl->debugfs_dentry, + mhi_cntrl, &debugfs_channels_fops); + debugfs_create_file("devices", 0444, mhi_cntrl->debugfs_dentry, + mhi_cntrl, &debugfs_devices_fops); + debugfs_create_file("regdump", 0444, mhi_cntrl->debugfs_dentry, + mhi_cntrl, &debugfs_regdump_fops); + debugfs_create_file("device_wake", 0644, mhi_cntrl->debugfs_dentry, + mhi_cntrl, &debugfs_device_wake_fops); + debugfs_create_file("timeout_ms", 0644, mhi_cntrl->debugfs_dentry, + mhi_cntrl, &debugfs_timeout_ms_fops); +} + +void mhi_destroy_debugfs(struct mhi_controller *mhi_cntrl) +{ + debugfs_remove_recursive(mhi_cntrl->debugfs_dentry); + mhi_cntrl->debugfs_dentry = NULL; +} + +void mhi_debugfs_init(void) +{ + mhi_debugfs_root = debugfs_create_dir(mhi_bus_type.name, NULL); +} + +void mhi_debugfs_exit(void) +{ + debugfs_remove_recursive(mhi_debugfs_root); +} diff --git a/mhi/core/init.c b/mhi/core/init.c new file mode 100644 index 0000000..94d1c6b --- /dev/null +++ b/mhi/core/init.c @@ -0,0 +1,1447 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * + */ + +#include <linux/debugfs.h> +#include <linux/device.h> +#include <linux/dma-direction.h> +#include <linux/dma-mapping.h> +#include <linux/idr.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/mhi.h> +#include <linux/mod_devicetable.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/wait.h> +#include "internal.h" + +static DEFINE_IDA(mhi_controller_ida); + +const char * const mhi_ee_str[MHI_EE_MAX] = { + [MHI_EE_PBL] = "PBL", + [MHI_EE_SBL] = "SBL", + [MHI_EE_AMSS] = "AMSS", + [MHI_EE_RDDM] = "RDDM", + [MHI_EE_WFW] = "WFW", + [MHI_EE_PTHRU] = "PASS THRU", + [MHI_EE_EDL] = "EDL", + [MHI_EE_DISABLE_TRANSITION] = "DISABLE", + [MHI_EE_NOT_SUPPORTED] = "NOT SUPPORTED", +}; + +const char * const dev_state_tran_str[DEV_ST_TRANSITION_MAX] = { + [DEV_ST_TRANSITION_PBL] = "PBL", + [DEV_ST_TRANSITION_READY] = "READY", + [DEV_ST_TRANSITION_SBL] = "SBL", + [DEV_ST_TRANSITION_MISSION_MODE] = "MISSION_MODE", + [DEV_ST_TRANSITION_SYS_ERR] = "SYS_ERR", + [DEV_ST_TRANSITION_DISABLE] = "DISABLE", +}; + +const char * const mhi_state_str[MHI_STATE_MAX] = { + [MHI_STATE_RESET] = "RESET", + [MHI_STATE_READY] = "READY", + [MHI_STATE_M0] = "M0", + [MHI_STATE_M1] = "M1", + [MHI_STATE_M2] = "M2", + [MHI_STATE_M3] = "M3", + [MHI_STATE_M3_FAST] = "M3_FAST", + [MHI_STATE_BHI] = "BHI", + [MHI_STATE_SYS_ERR] = "SYS_ERR", +}; + +const char * const mhi_ch_state_type_str[MHI_CH_STATE_TYPE_MAX] = { + [MHI_CH_STATE_TYPE_RESET] = "RESET", + [MHI_CH_STATE_TYPE_STOP] = "STOP", + [MHI_CH_STATE_TYPE_START] = "START", +}; + +static const char * const mhi_pm_state_str[] = { + [MHI_PM_STATE_DISABLE] = "DISABLE", + [MHI_PM_STATE_POR] = "POR", + [MHI_PM_STATE_M0] = "M0", + [MHI_PM_STATE_M2] = "M2", + [MHI_PM_STATE_M3_ENTER] = "M?->M3", + [MHI_PM_STATE_M3] = "M3", + [MHI_PM_STATE_M3_EXIT] = "M3->M0", + [MHI_PM_STATE_FW_DL_ERR] = "FW DL Error", + [MHI_PM_STATE_SYS_ERR_DETECT] = "SYS_ERR Detect", + [MHI_PM_STATE_SYS_ERR_PROCESS] = "SYS_ERR Process", + [MHI_PM_STATE_SHUTDOWN_PROCESS] = "SHUTDOWN Process", + [MHI_PM_STATE_LD_ERR_FATAL_DETECT] = "LD or Error Fatal Detect", +}; + +const char *to_mhi_pm_state_str(enum mhi_pm_state state) +{ + int index; + + if (state) + index = __fls(state); + + if (!state || index >= ARRAY_SIZE(mhi_pm_state_str)) + return "Invalid State"; + + return mhi_pm_state_str[index]; +} + +static ssize_t serial_number_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mhi_device *mhi_dev = to_mhi_device(dev); + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + + return snprintf(buf, PAGE_SIZE, "Serial Number: %u\n", + mhi_cntrl->serial_number); +} +static DEVICE_ATTR_RO(serial_number); + +static ssize_t oem_pk_hash_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mhi_device *mhi_dev = to_mhi_device(dev); + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + int i, cnt = 0; + + for (i = 0; i < ARRAY_SIZE(mhi_cntrl->oem_pk_hash); i++) + cnt += snprintf(buf + cnt, PAGE_SIZE - cnt, + "OEMPKHASH[%d]: 0x%x\n", i, + mhi_cntrl->oem_pk_hash[i]); + + return cnt; +} +static DEVICE_ATTR_RO(oem_pk_hash); + +static struct attribute *mhi_dev_attrs[] = { + &dev_attr_serial_number.attr, + &dev_attr_oem_pk_hash.attr, + NULL, +}; +ATTRIBUTE_GROUPS(mhi_dev); + +/* MHI protocol requires the transfer ring to be aligned with ring length */ +static int mhi_alloc_aligned_ring(struct mhi_controller *mhi_cntrl, + struct mhi_ring *ring, + u64 len) +{ + ring->alloc_size = len + (len - 1); + ring->pre_aligned = dma_alloc_coherent(mhi_cntrl->cntrl_dev, + ring->alloc_size, + &ring->dma_handle, + GFP_KERNEL); + if (!ring->pre_aligned) + return -ENOMEM; + + ring->iommu_base = (ring->dma_handle + (len - 1)) & ~(len - 1); + ring->base = ring->pre_aligned + (ring->iommu_base - ring->dma_handle); + + return 0; +} + +void mhi_deinit_free_irq(struct mhi_controller *mhi_cntrl) +{ + int i; + struct mhi_event *mhi_event = mhi_cntrl->mhi_event; + + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) { + if (mhi_event->offload_ev) + continue; + + free_irq(mhi_cntrl->irq[mhi_event->irq], mhi_event); + } + + free_irq(mhi_cntrl->irq[0], mhi_cntrl); +} + +int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl) +{ + struct mhi_event *mhi_event = mhi_cntrl->mhi_event; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + int i, ret; + + /* Setup BHI_INTVEC IRQ */ + ret = request_threaded_irq(mhi_cntrl->irq[0], mhi_intvec_handler, + mhi_intvec_threaded_handler, + IRQF_SHARED | IRQF_NO_SUSPEND, + "bhi", mhi_cntrl); + if (ret) + return ret; + + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) { + if (mhi_event->offload_ev) + continue; + + if (mhi_event->irq >= mhi_cntrl->nr_irqs) { + MHI_ERR("irq %d not available for event ring\n", + mhi_event->irq); + ret = -EINVAL; + goto error_request; + } + + ret = request_irq(mhi_cntrl->irq[mhi_event->irq], + mhi_irq_handler, + IRQF_SHARED | IRQF_NO_SUSPEND, + "mhi", mhi_event); + if (ret) { + MHI_ERR("Error requesting irq:%d for ev:%d\n", + mhi_cntrl->irq[mhi_event->irq], i); + goto error_request; + } + } + + return 0; + +error_request: + for (--i, --mhi_event; i >= 0; i--, mhi_event--) { + if (mhi_event->offload_ev) + continue; + + free_irq(mhi_cntrl->irq[mhi_event->irq], mhi_event); + } + free_irq(mhi_cntrl->irq[0], mhi_cntrl); + + return ret; +} + +void mhi_deinit_dev_ctxt(struct mhi_controller *mhi_cntrl) +{ + int i; + struct mhi_ctxt *mhi_ctxt = mhi_cntrl->mhi_ctxt; + struct mhi_cmd *mhi_cmd; + struct mhi_event *mhi_event; + struct mhi_ring *ring; + + mhi_cmd = mhi_cntrl->mhi_cmd; + for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++) { + ring = &mhi_cmd->ring; + dma_free_coherent(mhi_cntrl->cntrl_dev, ring->alloc_size, + ring->pre_aligned, ring->dma_handle); + ring->base = NULL; + ring->iommu_base = 0; + } + + dma_free_coherent(mhi_cntrl->cntrl_dev, + sizeof(*mhi_ctxt->cmd_ctxt) * NR_OF_CMD_RINGS, + mhi_ctxt->cmd_ctxt, mhi_ctxt->cmd_ctxt_addr); + + mhi_event = mhi_cntrl->mhi_event; + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) { + if (mhi_event->offload_ev) + continue; + + ring = &mhi_event->ring; + dma_free_coherent(mhi_cntrl->cntrl_dev, ring->alloc_size, + ring->pre_aligned, ring->dma_handle); + ring->base = NULL; + ring->iommu_base = 0; + } + + dma_free_coherent(mhi_cntrl->cntrl_dev, sizeof(*mhi_ctxt->er_ctxt) * + mhi_cntrl->total_ev_rings, mhi_ctxt->er_ctxt, + mhi_ctxt->er_ctxt_addr); + + dma_free_coherent(mhi_cntrl->cntrl_dev, sizeof(*mhi_ctxt->chan_ctxt) * + mhi_cntrl->max_chan, mhi_ctxt->chan_ctxt, + mhi_ctxt->chan_ctxt_addr); + + kfree(mhi_ctxt); + mhi_cntrl->mhi_ctxt = NULL; +} + +int mhi_init_dev_ctxt(struct mhi_controller *mhi_cntrl) +{ + struct mhi_ctxt *mhi_ctxt; + struct mhi_chan_ctxt *chan_ctxt; + struct mhi_event_ctxt *er_ctxt; + struct mhi_cmd_ctxt *cmd_ctxt; + struct mhi_chan *mhi_chan; + struct mhi_event *mhi_event; + struct mhi_cmd *mhi_cmd; + u32 tmp; + int ret = -ENOMEM, i; + + atomic_set(&mhi_cntrl->dev_wake, 0); + atomic_set(&mhi_cntrl->pending_pkts, 0); + + mhi_ctxt = kzalloc(sizeof(*mhi_ctxt), GFP_KERNEL); + if (!mhi_ctxt) + return -ENOMEM; + + /* Setup channel ctxt */ + mhi_ctxt->chan_ctxt = dma_alloc_coherent(mhi_cntrl->cntrl_dev, + sizeof(*mhi_ctxt->chan_ctxt) * + mhi_cntrl->max_chan, + &mhi_ctxt->chan_ctxt_addr, + GFP_KERNEL); + if (!mhi_ctxt->chan_ctxt) + goto error_alloc_chan_ctxt; + + mhi_chan = mhi_cntrl->mhi_chan; + chan_ctxt = mhi_ctxt->chan_ctxt; + for (i = 0; i < mhi_cntrl->max_chan; i++, chan_ctxt++, mhi_chan++) { + /* Skip if it is an offload channel */ + if (mhi_chan->offload_ch) + continue; + + tmp = chan_ctxt->chcfg; + tmp &= ~CHAN_CTX_CHSTATE_MASK; + tmp |= (MHI_CH_STATE_DISABLED << CHAN_CTX_CHSTATE_SHIFT); + tmp &= ~CHAN_CTX_BRSTMODE_MASK; + tmp |= (mhi_chan->db_cfg.brstmode << CHAN_CTX_BRSTMODE_SHIFT); + tmp &= ~CHAN_CTX_POLLCFG_MASK; + tmp |= (mhi_chan->db_cfg.pollcfg << CHAN_CTX_POLLCFG_SHIFT); + chan_ctxt->chcfg = tmp; + + chan_ctxt->chtype = mhi_chan->type; + chan_ctxt->erindex = mhi_chan->er_index; + + mhi_chan->ch_state = MHI_CH_STATE_DISABLED; + mhi_chan->tre_ring.db_addr = (void __iomem *)&chan_ctxt->wp; + } + + /* Setup event context */ + mhi_ctxt->er_ctxt = dma_alloc_coherent(mhi_cntrl->cntrl_dev, + sizeof(*mhi_ctxt->er_ctxt) * + mhi_cntrl->total_ev_rings, + &mhi_ctxt->er_ctxt_addr, + GFP_KERNEL); + if (!mhi_ctxt->er_ctxt) + goto error_alloc_er_ctxt; + + er_ctxt = mhi_ctxt->er_ctxt; + mhi_event = mhi_cntrl->mhi_event; + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, er_ctxt++, + mhi_event++) { + struct mhi_ring *ring = &mhi_event->ring; + + /* Skip if it is an offload event */ + if (mhi_event->offload_ev) + continue; + + tmp = er_ctxt->intmod; + tmp &= ~EV_CTX_INTMODC_MASK; + tmp &= ~EV_CTX_INTMODT_MASK; + tmp |= (mhi_event->intmod << EV_CTX_INTMODT_SHIFT); + er_ctxt->intmod = tmp; + + er_ctxt->ertype = MHI_ER_TYPE_VALID; + er_ctxt->msivec = mhi_event->irq; + mhi_event->db_cfg.db_mode = true; + + ring->el_size = sizeof(struct mhi_tre); + ring->len = ring->el_size * ring->elements; + ret = mhi_alloc_aligned_ring(mhi_cntrl, ring, ring->len); + if (ret) + goto error_alloc_er; + + /* + * If the read pointer equals to the write pointer, then the + * ring is empty + */ + ring->rp = ring->wp = ring->base; + er_ctxt->rbase = ring->iommu_base; + er_ctxt->rp = er_ctxt->wp = er_ctxt->rbase; + er_ctxt->rlen = ring->len; + ring->ctxt_wp = &er_ctxt->wp; + } + + /* Setup cmd context */ + ret = -ENOMEM; + mhi_ctxt->cmd_ctxt = dma_alloc_coherent(mhi_cntrl->cntrl_dev, + sizeof(*mhi_ctxt->cmd_ctxt) * + NR_OF_CMD_RINGS, + &mhi_ctxt->cmd_ctxt_addr, + GFP_KERNEL); + if (!mhi_ctxt->cmd_ctxt) + goto error_alloc_er; + + mhi_cmd = mhi_cntrl->mhi_cmd; + cmd_ctxt = mhi_ctxt->cmd_ctxt; + for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++, cmd_ctxt++) { + struct mhi_ring *ring = &mhi_cmd->ring; + + ring->el_size = sizeof(struct mhi_tre); + ring->elements = CMD_EL_PER_RING; + ring->len = ring->el_size * ring->elements; + ret = mhi_alloc_aligned_ring(mhi_cntrl, ring, ring->len); + if (ret) + goto error_alloc_cmd; + + ring->rp = ring->wp = ring->base; + cmd_ctxt->rbase = ring->iommu_base; + cmd_ctxt->rp = cmd_ctxt->wp = cmd_ctxt->rbase; + cmd_ctxt->rlen = ring->len; + ring->ctxt_wp = &cmd_ctxt->wp; + } + + mhi_cntrl->mhi_ctxt = mhi_ctxt; + + return 0; + +error_alloc_cmd: + for (--i, --mhi_cmd; i >= 0; i--, mhi_cmd--) { + struct mhi_ring *ring = &mhi_cmd->ring; + + dma_free_coherent(mhi_cntrl->cntrl_dev, ring->alloc_size, + ring->pre_aligned, ring->dma_handle); + } + dma_free_coherent(mhi_cntrl->cntrl_dev, + sizeof(*mhi_ctxt->cmd_ctxt) * NR_OF_CMD_RINGS, + mhi_ctxt->cmd_ctxt, mhi_ctxt->cmd_ctxt_addr); + i = mhi_cntrl->total_ev_rings; + mhi_event = mhi_cntrl->mhi_event + i; + +error_alloc_er: + for (--i, --mhi_event; i >= 0; i--, mhi_event--) { + struct mhi_ring *ring = &mhi_event->ring; + + if (mhi_event->offload_ev) + continue; + + dma_free_coherent(mhi_cntrl->cntrl_dev, ring->alloc_size, + ring->pre_aligned, ring->dma_handle); + } + dma_free_coherent(mhi_cntrl->cntrl_dev, sizeof(*mhi_ctxt->er_ctxt) * + mhi_cntrl->total_ev_rings, mhi_ctxt->er_ctxt, + mhi_ctxt->er_ctxt_addr); + +error_alloc_er_ctxt: + dma_free_coherent(mhi_cntrl->cntrl_dev, sizeof(*mhi_ctxt->chan_ctxt) * + mhi_cntrl->max_chan, mhi_ctxt->chan_ctxt, + mhi_ctxt->chan_ctxt_addr); + +error_alloc_chan_ctxt: + kfree(mhi_ctxt); + + return ret; +} + +int mhi_init_mmio(struct mhi_controller *mhi_cntrl) +{ + u32 val; + int i, ret; + struct mhi_chan *mhi_chan; + struct mhi_event *mhi_event; + void __iomem *base = mhi_cntrl->regs; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct { + u32 offset; + u32 mask; + u32 shift; + u32 val; + } reg_info[] = { + { + CCABAP_HIGHER, U32_MAX, 0, + upper_32_bits(mhi_cntrl->mhi_ctxt->chan_ctxt_addr), + }, + { + CCABAP_LOWER, U32_MAX, 0, + lower_32_bits(mhi_cntrl->mhi_ctxt->chan_ctxt_addr), + }, + { + ECABAP_HIGHER, U32_MAX, 0, + upper_32_bits(mhi_cntrl->mhi_ctxt->er_ctxt_addr), + }, + { + ECABAP_LOWER, U32_MAX, 0, + lower_32_bits(mhi_cntrl->mhi_ctxt->er_ctxt_addr), + }, + { + CRCBAP_HIGHER, U32_MAX, 0, + upper_32_bits(mhi_cntrl->mhi_ctxt->cmd_ctxt_addr), + }, + { + CRCBAP_LOWER, U32_MAX, 0, + lower_32_bits(mhi_cntrl->mhi_ctxt->cmd_ctxt_addr), + }, + { + MHICFG, MHICFG_NER_MASK, MHICFG_NER_SHIFT, + mhi_cntrl->total_ev_rings, + }, + { + MHICFG, MHICFG_NHWER_MASK, MHICFG_NHWER_SHIFT, + mhi_cntrl->hw_ev_rings, + }, + { + MHICTRLBASE_HIGHER, U32_MAX, 0, + upper_32_bits(mhi_cntrl->iova_start), + }, + { + MHICTRLBASE_LOWER, U32_MAX, 0, + lower_32_bits(mhi_cntrl->iova_start), + }, + { + MHIDATABASE_HIGHER, U32_MAX, 0, + upper_32_bits(mhi_cntrl->iova_start), + }, + { + MHIDATABASE_LOWER, U32_MAX, 0, + lower_32_bits(mhi_cntrl->iova_start), + }, + { + MHICTRLLIMIT_HIGHER, U32_MAX, 0, + upper_32_bits(mhi_cntrl->iova_stop), + }, + { + MHICTRLLIMIT_LOWER, U32_MAX, 0, + lower_32_bits(mhi_cntrl->iova_stop), + }, + { + MHIDATALIMIT_HIGHER, U32_MAX, 0, + upper_32_bits(mhi_cntrl->iova_stop), + }, + { + MHIDATALIMIT_LOWER, U32_MAX, 0, + lower_32_bits(mhi_cntrl->iova_stop), + }, + { 0, 0, 0 } + }; + + MHI_VERB("Initializing MHI registers\n"); + + /* Read channel db offset */ + ret = mhi_read_reg_field(mhi_cntrl, base, CHDBOFF, CHDBOFF_CHDBOFF_MASK, + CHDBOFF_CHDBOFF_SHIFT, &val); + if (ret) { + MHI_ERR("Unable to read CHDBOFF register\n"); + return -EIO; + } + + /* Setup wake db */ + mhi_cntrl->wake_db = base + val + (8 * MHI_DEV_WAKE_DB); + mhi_write_reg(mhi_cntrl, mhi_cntrl->wake_db, 4, 0); + mhi_write_reg(mhi_cntrl, mhi_cntrl->wake_db, 0, 0); + mhi_cntrl->wake_set = false; + + /* Setup channel db address for each channel in tre_ring */ + mhi_chan = mhi_cntrl->mhi_chan; + for (i = 0; i < mhi_cntrl->max_chan; i++, val += 8, mhi_chan++) + mhi_chan->tre_ring.db_addr = base + val; + + /* Read event ring db offset */ + ret = mhi_read_reg_field(mhi_cntrl, base, ERDBOFF, ERDBOFF_ERDBOFF_MASK, + ERDBOFF_ERDBOFF_SHIFT, &val); + if (ret) { + MHI_ERR("Unable to read ERDBOFF register\n"); + return -EIO; + } + + /* Setup event db address for each ev_ring */ + mhi_event = mhi_cntrl->mhi_event; + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, val += 8, mhi_event++) { + if (mhi_event->offload_ev) + continue; + + mhi_event->ring.db_addr = base + val; + } + + /* Setup DB register for primary CMD rings */ + mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING].ring.db_addr = base + CRDB_LOWER; + + /* Write to MMIO registers */ + for (i = 0; reg_info[i].offset; i++) + mhi_write_reg_field(mhi_cntrl, base, reg_info[i].offset, + reg_info[i].mask, reg_info[i].shift, + reg_info[i].val); + + mhi_misc_init_mmio(mhi_cntrl); + + return 0; +} + +void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan) +{ + struct mhi_ring *buf_ring; + struct mhi_ring *tre_ring; + struct mhi_chan_ctxt *chan_ctxt; + u32 tmp; + + buf_ring = &mhi_chan->buf_ring; + tre_ring = &mhi_chan->tre_ring; + chan_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[mhi_chan->chan]; + + dma_free_coherent(mhi_cntrl->cntrl_dev, tre_ring->alloc_size, + tre_ring->pre_aligned, tre_ring->dma_handle); + vfree(buf_ring->base); + + buf_ring->base = tre_ring->base = NULL; + tre_ring->ctxt_wp = NULL; + chan_ctxt->rbase = 0; + chan_ctxt->rlen = 0; + chan_ctxt->rp = 0; + chan_ctxt->wp = 0; + + tmp = chan_ctxt->chcfg; + tmp &= ~CHAN_CTX_CHSTATE_MASK; + tmp |= (MHI_CH_STATE_DISABLED << CHAN_CTX_CHSTATE_SHIFT); + chan_ctxt->chcfg = tmp; + + /* Update to all cores */ + smp_wmb(); +} + +int mhi_init_chan_ctxt(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan) +{ + struct mhi_ring *buf_ring; + struct mhi_ring *tre_ring; + struct mhi_chan_ctxt *chan_ctxt; + u32 tmp; + int ret; + + buf_ring = &mhi_chan->buf_ring; + tre_ring = &mhi_chan->tre_ring; + tre_ring->el_size = sizeof(struct mhi_tre); + tre_ring->len = tre_ring->el_size * tre_ring->elements; + chan_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[mhi_chan->chan]; + ret = mhi_alloc_aligned_ring(mhi_cntrl, tre_ring, tre_ring->len); + if (ret) + return -ENOMEM; + + buf_ring->el_size = sizeof(struct mhi_buf_info); + buf_ring->len = buf_ring->el_size * buf_ring->elements; + buf_ring->base = vzalloc(buf_ring->len); + + if (!buf_ring->base) { + dma_free_coherent(mhi_cntrl->cntrl_dev, tre_ring->alloc_size, + tre_ring->pre_aligned, tre_ring->dma_handle); + return -ENOMEM; + } + + tmp = chan_ctxt->chcfg; + tmp &= ~CHAN_CTX_CHSTATE_MASK; + tmp |= (MHI_CH_STATE_ENABLED << CHAN_CTX_CHSTATE_SHIFT); + chan_ctxt->chcfg = tmp; + + chan_ctxt->rbase = tre_ring->iommu_base; + chan_ctxt->rp = chan_ctxt->wp = chan_ctxt->rbase; + chan_ctxt->rlen = tre_ring->len; + tre_ring->ctxt_wp = &chan_ctxt->wp; + + tre_ring->rp = tre_ring->wp = tre_ring->base; + buf_ring->rp = buf_ring->wp = buf_ring->base; + mhi_chan->db_cfg.db_mode = 1; + + /* Update to all cores */ + smp_wmb(); + + return 0; +} + +static int parse_ev_cfg(struct mhi_controller *mhi_cntrl, + const struct mhi_controller_config *config) +{ + struct mhi_event *mhi_event; + const struct mhi_event_config *event_cfg; + struct device *dev = mhi_cntrl->cntrl_dev; + int i, num; + + num = config->num_events; + mhi_cntrl->total_ev_rings = num; + mhi_cntrl->mhi_event = kcalloc(num, sizeof(*mhi_cntrl->mhi_event), + GFP_KERNEL); + if (!mhi_cntrl->mhi_event) + return -ENOMEM; + + /* Populate event ring */ + mhi_event = mhi_cntrl->mhi_event; + for (i = 0; i < num; i++) { + event_cfg = &config->event_cfg[i]; + + mhi_event->er_index = i; + mhi_event->ring.elements = event_cfg->num_elements; + mhi_event->intmod = event_cfg->irq_moderation_ms; + mhi_event->irq = event_cfg->irq; + + if (event_cfg->channel != U32_MAX) { + /* This event ring has a dedicated channel */ + mhi_event->chan = event_cfg->channel; + if (mhi_event->chan >= mhi_cntrl->max_chan) { + MHI_ERR( + "Event Ring channel not available\n"); + goto error_ev_cfg; + } + + mhi_event->mhi_chan = + &mhi_cntrl->mhi_chan[mhi_event->chan]; + } + + mhi_event->priority = event_cfg->priority; + + mhi_event->db_cfg.brstmode = event_cfg->mode; + if (MHI_INVALID_BRSTMODE(mhi_event->db_cfg.brstmode)) + goto error_ev_cfg; + + if (mhi_event->db_cfg.brstmode == MHI_DB_BRST_ENABLE) + mhi_event->db_cfg.process_db = mhi_db_brstmode; + else + mhi_event->db_cfg.process_db = mhi_db_brstmode_disable; + + mhi_event->data_type = event_cfg->data_type; + + switch (mhi_event->data_type) { + case MHI_ER_DATA: + mhi_event->process_event = mhi_process_data_event_ring; + break; + case MHI_ER_CTRL: + mhi_event->process_event = mhi_process_ctrl_ev_ring; + break; + case MHI_ER_BW_SCALE: + mhi_event->process_event = mhi_process_misc_bw_ev_ring; + break; + case MHI_ER_TIMESYNC: + mhi_event->process_event = + mhi_process_misc_tsync_ev_ring; + break; + default: + MHI_ERR("Event Ring type not supported\n"); + goto error_ev_cfg; + } + + mhi_event->hw_ring = event_cfg->hardware_event; + if (mhi_event->hw_ring) + mhi_cntrl->hw_ev_rings++; + else + mhi_cntrl->sw_ev_rings++; + + mhi_event->cl_manage = event_cfg->client_managed; + mhi_event->offload_ev = event_cfg->offload_channel; + mhi_event++; + } + + return 0; + +error_ev_cfg: + + kfree(mhi_cntrl->mhi_event); + return -EINVAL; +} + +static int parse_ch_cfg(struct mhi_controller *mhi_cntrl, + const struct mhi_controller_config *config) +{ + const struct mhi_channel_config *ch_cfg; + struct device *dev = mhi_cntrl->cntrl_dev; + int i; + u32 chan; + + mhi_cntrl->max_chan = config->max_channels; + + /* + * The allocation of MHI channels can exceed 32KB in some scenarios, + * so to avoid any memory possible allocation failures, vzalloc is + * used here + */ + mhi_cntrl->mhi_chan = vzalloc(mhi_cntrl->max_chan * + sizeof(*mhi_cntrl->mhi_chan)); + if (!mhi_cntrl->mhi_chan) + return -ENOMEM; + + INIT_LIST_HEAD(&mhi_cntrl->lpm_chans); + + /* Populate channel configurations */ + for (i = 0; i < config->num_channels; i++) { + struct mhi_chan *mhi_chan; + + ch_cfg = &config->ch_cfg[i]; + + chan = ch_cfg->num; + if (chan >= mhi_cntrl->max_chan) { + MHI_ERR("Channel %d not available\n", chan); + goto error_chan_cfg; + } + + mhi_chan = &mhi_cntrl->mhi_chan[chan]; + mhi_chan->name = ch_cfg->name; + mhi_chan->chan = chan; + + mhi_chan->tre_ring.elements = ch_cfg->num_elements; + if (!mhi_chan->tre_ring.elements) + goto error_chan_cfg; + + /* + * For some channels, local ring length should be bigger than + * the transfer ring length due to internal logical channels + * in device. So host can queue much more buffers than transfer + * ring length. Example, RSC channels should have a larger local + * channel length than transfer ring length. + */ + mhi_chan->buf_ring.elements = ch_cfg->local_elements; + if (!mhi_chan->buf_ring.elements) + mhi_chan->buf_ring.elements = mhi_chan->tre_ring.elements; + mhi_chan->er_index = ch_cfg->event_ring; + mhi_chan->dir = ch_cfg->dir; + + /* + * For most channels, chtype is identical to channel directions. + * So, if it is not defined then assign channel direction to + * chtype + */ + mhi_chan->type = ch_cfg->type; + if (!mhi_chan->type) + mhi_chan->type = (enum mhi_ch_type)mhi_chan->dir; + + mhi_chan->ee_mask = ch_cfg->ee_mask; + mhi_chan->db_cfg.pollcfg = ch_cfg->pollcfg; + mhi_chan->lpm_notify = ch_cfg->lpm_notify; + mhi_chan->offload_ch = ch_cfg->offload_channel; + mhi_chan->db_cfg.reset_req = ch_cfg->doorbell_mode_switch; + mhi_chan->pre_alloc = ch_cfg->auto_queue; + + /* + * If MHI host allocates buffers, then the channel direction + * should be DMA_FROM_DEVICE + */ + if (mhi_chan->pre_alloc && mhi_chan->dir != DMA_FROM_DEVICE) { + MHI_ERR("Invalid channel configuration\n"); + goto error_chan_cfg; + } + + /* + * Bi-directional and direction less channel must be an + * offload channel + */ + if ((mhi_chan->dir == DMA_BIDIRECTIONAL || + mhi_chan->dir == DMA_NONE) && !mhi_chan->offload_ch) { + MHI_ERR("Invalid channel configuration\n"); + goto error_chan_cfg; + } + + if (!mhi_chan->offload_ch) { + mhi_chan->db_cfg.brstmode = ch_cfg->doorbell; + if (MHI_INVALID_BRSTMODE(mhi_chan->db_cfg.brstmode)) { + MHI_ERR("Invalid Door bell mode\n"); + goto error_chan_cfg; + } + } + + if (mhi_chan->db_cfg.brstmode == MHI_DB_BRST_ENABLE) + mhi_chan->db_cfg.process_db = mhi_db_brstmode; + else + mhi_chan->db_cfg.process_db = mhi_db_brstmode_disable; + + mhi_chan->configured = true; + + if (mhi_chan->lpm_notify) + list_add_tail(&mhi_chan->node, &mhi_cntrl->lpm_chans); + } + + return 0; + +error_chan_cfg: + vfree(mhi_cntrl->mhi_chan); + + return -EINVAL; +} + +static int parse_config(struct mhi_controller *mhi_cntrl, + const struct mhi_controller_config *config) +{ + int ret; + + /* Parse MHI channel configuration */ + ret = parse_ch_cfg(mhi_cntrl, config); + if (ret) + return ret; + + /* Parse MHI event configuration */ + ret = parse_ev_cfg(mhi_cntrl, config); + if (ret) + goto error_ev_cfg; + + mhi_cntrl->timeout_ms = config->timeout_ms; + if (!mhi_cntrl->timeout_ms) + mhi_cntrl->timeout_ms = MHI_TIMEOUT_MS; + + mhi_cntrl->bounce_buf = config->use_bounce_buf; + mhi_cntrl->buffer_len = config->buf_len; + if (!mhi_cntrl->buffer_len) + mhi_cntrl->buffer_len = MHI_MAX_MTU; + + /* By default, host is allowed to ring DB in both M0 and M2 states */ + mhi_cntrl->db_access = MHI_PM_M0 | MHI_PM_M2; + if (config->m2_no_db) + mhi_cntrl->db_access &= ~MHI_PM_M2; + + return 0; + +error_ev_cfg: + vfree(mhi_cntrl->mhi_chan); + + return ret; +} + +int mhi_register_controller(struct mhi_controller *mhi_cntrl, + const struct mhi_controller_config *config) +{ + struct mhi_event *mhi_event; + struct mhi_chan *mhi_chan; + struct mhi_cmd *mhi_cmd; + struct mhi_device *mhi_dev; + u32 soc_info; + int ret, i; + + if (!mhi_cntrl) + return -EINVAL; + + if (!mhi_cntrl->runtime_get || !mhi_cntrl->runtime_put || + !mhi_cntrl->status_cb || !mhi_cntrl->read_reg || + !mhi_cntrl->write_reg || !mhi_cntrl->nr_irqs) + return -EINVAL; + + ret = parse_config(mhi_cntrl, config); + if (ret) + return -EINVAL; + + mhi_cntrl->mhi_cmd = kcalloc(NR_OF_CMD_RINGS, + sizeof(*mhi_cntrl->mhi_cmd), GFP_KERNEL); + if (!mhi_cntrl->mhi_cmd) { + ret = -ENOMEM; + goto err_free_event; + } + + INIT_LIST_HEAD(&mhi_cntrl->transition_list); + mutex_init(&mhi_cntrl->pm_mutex); + rwlock_init(&mhi_cntrl->pm_lock); + spin_lock_init(&mhi_cntrl->transition_lock); + spin_lock_init(&mhi_cntrl->wlock); + INIT_WORK(&mhi_cntrl->st_worker, mhi_pm_st_worker); + init_waitqueue_head(&mhi_cntrl->state_event); + + mhi_cntrl->hiprio_wq = alloc_ordered_workqueue("mhi_hiprio_wq", + WQ_HIGHPRI); + if (!mhi_cntrl->hiprio_wq) { + dev_err(mhi_cntrl->cntrl_dev, "Failed to allocate workqueue\n"); + ret = -ENOMEM; + goto err_free_cmd; + } + + mhi_cmd = mhi_cntrl->mhi_cmd; + for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++) + spin_lock_init(&mhi_cmd->lock); + + mhi_event = mhi_cntrl->mhi_event; + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) { + /* Skip for offload events */ + if (mhi_event->offload_ev) + continue; + + mhi_event->mhi_cntrl = mhi_cntrl; + spin_lock_init(&mhi_event->lock); + + if (mhi_event->priority == MHI_ER_PRIORITY_HI_SLEEP) + INIT_WORK(&mhi_event->work, mhi_process_ev_work); + else + tasklet_init(&mhi_event->task, + (mhi_event->data_type == MHI_ER_CTRL) ? + mhi_ctrl_ev_task : mhi_ev_task, + (ulong)mhi_event); + } + + mhi_chan = mhi_cntrl->mhi_chan; + for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) { + mutex_init(&mhi_chan->mutex); + init_completion(&mhi_chan->completion); + rwlock_init(&mhi_chan->lock); + + /* used in setting bei field of TRE */ + mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index]; + mhi_chan->intmod = mhi_event->intmod; + } + + if (mhi_cntrl->bounce_buf) { + mhi_cntrl->map_single = mhi_map_single_use_bb; + mhi_cntrl->unmap_single = mhi_unmap_single_use_bb; + } else { + mhi_cntrl->map_single = mhi_map_single_no_bb; + mhi_cntrl->unmap_single = mhi_unmap_single_no_bb; + } + + /* Read the MHI device info */ + ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, + SOC_HW_VERSION_OFFS, &soc_info); + if (ret) + goto err_destroy_wq; + + mhi_cntrl->family_number = (soc_info & SOC_HW_VERSION_FAM_NUM_BMSK) >> + SOC_HW_VERSION_FAM_NUM_SHFT; + mhi_cntrl->device_number = (soc_info & SOC_HW_VERSION_DEV_NUM_BMSK) >> + SOC_HW_VERSION_DEV_NUM_SHFT; + mhi_cntrl->major_version = (soc_info & SOC_HW_VERSION_MAJOR_VER_BMSK) >> + SOC_HW_VERSION_MAJOR_VER_SHFT; + mhi_cntrl->minor_version = (soc_info & SOC_HW_VERSION_MINOR_VER_BMSK) >> + SOC_HW_VERSION_MINOR_VER_SHFT; + + mhi_cntrl->index = ida_alloc(&mhi_controller_ida, GFP_KERNEL); + if (mhi_cntrl->index < 0) { + ret = mhi_cntrl->index; + goto err_destroy_wq; + } + + /* Register controller with MHI bus */ + mhi_dev = mhi_alloc_device(mhi_cntrl); + if (IS_ERR(mhi_dev)) { + dev_err(mhi_cntrl->cntrl_dev, "Failed to allocate MHI device\n"); + ret = PTR_ERR(mhi_dev); + goto err_ida_free; + } + + mhi_dev->dev_type = MHI_DEVICE_CONTROLLER; + mhi_dev->mhi_cntrl = mhi_cntrl; + dev_set_name(&mhi_dev->dev, "mhi%d", mhi_cntrl->index); + mhi_dev->name = dev_name(&mhi_dev->dev); + + /* Init wakeup source */ + device_init_wakeup(&mhi_dev->dev, true); + + ret = device_add(&mhi_dev->dev); + if (ret) + goto err_release_dev; + + mhi_cntrl->mhi_dev = mhi_dev; + + ret = mhi_misc_register_controller(mhi_cntrl); + if (ret) { + dev_err(mhi_cntrl->cntrl_dev, + "Could not enable miscellaneous features\n"); + mhi_cntrl->mhi_dev = NULL; + goto err_release_dev; + } + + mhi_create_debugfs(mhi_cntrl); + + return 0; + +err_release_dev: + put_device(&mhi_dev->dev); +err_ida_free: + ida_free(&mhi_controller_ida, mhi_cntrl->index); +err_destroy_wq: + destroy_workqueue(mhi_cntrl->hiprio_wq); +err_free_cmd: + kfree(mhi_cntrl->mhi_cmd); +err_free_event: + kfree(mhi_cntrl->mhi_event); + vfree(mhi_cntrl->mhi_chan); + + return ret; +} +EXPORT_SYMBOL_GPL(mhi_register_controller); + +void mhi_unregister_controller(struct mhi_controller *mhi_cntrl) +{ + struct mhi_device *mhi_dev = mhi_cntrl->mhi_dev; + struct mhi_chan *mhi_chan = mhi_cntrl->mhi_chan; + unsigned int i; + + mhi_misc_unregister_controller(mhi_cntrl); + + /* Free the memory controller wanted to preserve for BHIe images */ + if (mhi_cntrl->img_pre_alloc) { + mhi_cntrl->img_pre_alloc = false; + if (mhi_cntrl->fbc_image) + mhi_free_bhie_table(mhi_cntrl, &mhi_cntrl->fbc_image); + if (mhi_cntrl->rddm_image) + mhi_free_bhie_table(mhi_cntrl, &mhi_cntrl->rddm_image); + } + + mhi_destroy_debugfs(mhi_cntrl); + + destroy_workqueue(mhi_cntrl->hiprio_wq); + kfree(mhi_cntrl->mhi_cmd); + kfree(mhi_cntrl->mhi_event); + + /* Drop the references to MHI devices created for channels */ + for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) { + if (!mhi_chan->mhi_dev) + continue; + + put_device(&mhi_chan->mhi_dev->dev); + } + vfree(mhi_cntrl->mhi_chan); + + device_del(&mhi_dev->dev); + put_device(&mhi_dev->dev); + + ida_free(&mhi_controller_ida, mhi_cntrl->index); +} +EXPORT_SYMBOL_GPL(mhi_unregister_controller); + +struct mhi_controller *mhi_alloc_controller(void) +{ + struct mhi_controller *mhi_cntrl; + + mhi_cntrl = kzalloc(sizeof(*mhi_cntrl), GFP_KERNEL); + + return mhi_cntrl; +} +EXPORT_SYMBOL_GPL(mhi_alloc_controller); + +void mhi_free_controller(struct mhi_controller *mhi_cntrl) +{ + kfree(mhi_cntrl); +} +EXPORT_SYMBOL_GPL(mhi_free_controller); + +int mhi_prepare_for_power_up(struct mhi_controller *mhi_cntrl) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + u32 bhi_off, bhie_off; + int ret; + + mutex_lock(&mhi_cntrl->pm_mutex); + + ret = mhi_init_dev_ctxt(mhi_cntrl); + if (ret) + goto error_dev_ctxt; + + ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, BHIOFF, + &bhi_off); + if (ret) { + MHI_ERR("Error getting BHI offset\n"); + goto error_reg_offset; + } + if (bhi_off >= mhi_cntrl->reg_len) { + MHI_ERR("BHI offset is out of range\n"); + ret = -EINVAL; + goto error_reg_offset; + } + mhi_cntrl->bhi = mhi_cntrl->regs + bhi_off; + + if (mhi_cntrl->fbc_download || mhi_cntrl->rddm_size) { + ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->regs, BHIEOFF, + &bhie_off); + if (ret) { + MHI_ERR("Error getting BHIE offset\n"); + goto error_reg_offset; + } + if (bhie_off >= mhi_cntrl->reg_len) { + MHI_ERR("BHIe offset is out of range\n"); + ret = -EINVAL; + goto error_reg_offset; + } + mhi_cntrl->bhie = mhi_cntrl->regs + bhie_off; + } + + if (mhi_cntrl->rddm_size) { + /* + * This controller supports RDDM, so we need to manually clear + * BHIE RX registers since POR values are undefined. + */ + memset_io(mhi_cntrl->bhie + BHIE_RXVECADDR_LOW_OFFS, + 0, BHIE_RXVECSTATUS_OFFS - BHIE_RXVECADDR_LOW_OFFS + + 4); + /* + * Allocate RDDM table for debugging purpose if specified + */ + mhi_alloc_bhie_table(mhi_cntrl, &mhi_cntrl->rddm_image, + mhi_cntrl->rddm_size); + if (mhi_cntrl->rddm_image) + mhi_rddm_prepare(mhi_cntrl, mhi_cntrl->rddm_image); + } + + mutex_unlock(&mhi_cntrl->pm_mutex); + + return 0; + +error_reg_offset: + mhi_deinit_dev_ctxt(mhi_cntrl); + +error_dev_ctxt: + mutex_unlock(&mhi_cntrl->pm_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(mhi_prepare_for_power_up); + +void mhi_unprepare_after_power_down(struct mhi_controller *mhi_cntrl) +{ + if (mhi_cntrl->rddm_image) + mhi_free_bhie_table(mhi_cntrl, &mhi_cntrl->rddm_image); + + mhi_cntrl->bhi = NULL; + mhi_cntrl->bhie = NULL; + + mhi_deinit_dev_ctxt(mhi_cntrl); +} +EXPORT_SYMBOL_GPL(mhi_unprepare_after_power_down); + +static void mhi_release_device(struct device *dev) +{ + struct mhi_device *mhi_dev = to_mhi_device(dev); + + /* + * We need to set the mhi_chan->mhi_dev to NULL here since the MHI + * devices for the channels will only get created if the mhi_dev + * associated with it is NULL. This scenario will happen during the + * controller suspend and resume. + */ + if (mhi_dev->ul_chan) + mhi_dev->ul_chan->mhi_dev = NULL; + + if (mhi_dev->dl_chan) + mhi_dev->dl_chan->mhi_dev = NULL; + + kfree(mhi_dev); +} + +struct mhi_device *mhi_alloc_device(struct mhi_controller *mhi_cntrl) +{ + struct mhi_device *mhi_dev; + struct device *dev; + + mhi_dev = kzalloc(sizeof(*mhi_dev), GFP_KERNEL); + if (!mhi_dev) + return ERR_PTR(-ENOMEM); + + dev = &mhi_dev->dev; + device_initialize(dev); + dev->bus = &mhi_bus_type; + dev->release = mhi_release_device; + + if (mhi_cntrl->mhi_dev) { + /* for MHI client devices, parent is the MHI controller device */ + dev->parent = &mhi_cntrl->mhi_dev->dev; + } else { + /* for MHI controller device, parent is the bus device (e.g. pci device) */ + dev->parent = mhi_cntrl->cntrl_dev; + } + + mhi_dev->mhi_cntrl = mhi_cntrl; + mhi_dev->dev_wake = 0; + + return mhi_dev; +} + +static int mhi_driver_probe(struct device *dev) +{ + struct mhi_device *mhi_dev = to_mhi_device(dev); + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct device_driver *drv = dev->driver; + struct mhi_driver *mhi_drv = to_mhi_driver(drv); + struct mhi_event *mhi_event; + struct mhi_chan *ul_chan = mhi_dev->ul_chan; + struct mhi_chan *dl_chan = mhi_dev->dl_chan; + int ret; + + /* Bring device out of LPM */ + ret = mhi_device_get_sync(mhi_dev); + if (ret) + return ret; + + ret = -EINVAL; + + if (ul_chan) { + /* + * If channel supports LPM notifications then status_cb should + * be provided + */ + if (ul_chan->lpm_notify && !mhi_drv->status_cb) + goto exit_probe; + + /* For non-offload channels then xfer_cb should be provided */ + if (!ul_chan->offload_ch && !mhi_drv->ul_xfer_cb) + goto exit_probe; + + ul_chan->xfer_cb = mhi_drv->ul_xfer_cb; + } + + ret = -EINVAL; + if (dl_chan) { + /* + * If channel supports LPM notifications then status_cb should + * be provided + */ + if (dl_chan->lpm_notify && !mhi_drv->status_cb) + goto exit_probe; + + /* For non-offload channels then xfer_cb should be provided */ + if (!dl_chan->offload_ch && !mhi_drv->dl_xfer_cb) + goto exit_probe; + + mhi_event = &mhi_cntrl->mhi_event[dl_chan->er_index]; + + /* + * If the channel event ring is managed by client, then + * status_cb must be provided so that the framework can + * notify pending data + */ + if (mhi_event->cl_manage && !mhi_drv->status_cb) + goto exit_probe; + + dl_chan->xfer_cb = mhi_drv->dl_xfer_cb; + } + + /* Call the user provided probe function */ + ret = mhi_drv->probe(mhi_dev, mhi_dev->id); + if (ret) + goto exit_probe; + + mhi_device_put(mhi_dev); + + return ret; + +exit_probe: + mhi_unprepare_from_transfer(mhi_dev); + + mhi_device_put(mhi_dev); + + return ret; +} + +static int mhi_driver_remove(struct device *dev) +{ + struct mhi_device *mhi_dev = to_mhi_device(dev); + struct mhi_driver *mhi_drv = to_mhi_driver(dev->driver); + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct mhi_chan *mhi_chan; + enum mhi_ch_state ch_state[] = { + MHI_CH_STATE_DISABLED, + MHI_CH_STATE_DISABLED + }; + int dir; + + /* Skip if it is a controller device */ + if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER) + return 0; + + /* Reset both channels */ + for (dir = 0; dir < 2; dir++) { + mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan; + + if (!mhi_chan) + continue; + + /* Wake all threads waiting for completion */ + write_lock_irq(&mhi_chan->lock); + mhi_chan->ccs = MHI_EV_CC_INVALID; + complete_all(&mhi_chan->completion); + write_unlock_irq(&mhi_chan->lock); + + /* Set the channel state to disabled */ + mutex_lock(&mhi_chan->mutex); + write_lock_irq(&mhi_chan->lock); + ch_state[dir] = mhi_chan->ch_state; + mhi_chan->ch_state = MHI_CH_STATE_SUSPENDED; + write_unlock_irq(&mhi_chan->lock); + + /* Reset the non-offload channel */ + if (!mhi_chan->offload_ch) + mhi_reset_chan(mhi_cntrl, mhi_chan); + + mutex_unlock(&mhi_chan->mutex); + } + + mhi_drv->remove(mhi_dev); + + /* De-init channel if it was enabled */ + for (dir = 0; dir < 2; dir++) { + mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan; + + if (!mhi_chan) + continue; + + mutex_lock(&mhi_chan->mutex); + + if ((ch_state[dir] == MHI_CH_STATE_ENABLED || + ch_state[dir] == MHI_CH_STATE_STOP) && + mhi_chan->ch_state != MHI_CH_STATE_DISABLED && + !mhi_chan->offload_ch) + mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan); + + mhi_chan->ch_state = MHI_CH_STATE_DISABLED; + + mutex_unlock(&mhi_chan->mutex); + } + + while (mhi_dev->dev_wake) + mhi_device_put(mhi_dev); + + return 0; +} + +int __mhi_driver_register(struct mhi_driver *mhi_drv, struct module *owner) +{ + struct device_driver *driver = &mhi_drv->driver; + + if (!mhi_drv->probe || !mhi_drv->remove) + return -EINVAL; + + driver->bus = &mhi_bus_type; + driver->owner = owner; + driver->probe = mhi_driver_probe; + driver->remove = mhi_driver_remove; + + return driver_register(driver); +} +EXPORT_SYMBOL_GPL(__mhi_driver_register); + +void mhi_driver_unregister(struct mhi_driver *mhi_drv) +{ + driver_unregister(&mhi_drv->driver); +} +EXPORT_SYMBOL_GPL(mhi_driver_unregister); + +static int mhi_uevent(struct device *dev, struct kobj_uevent_env *env) +{ + struct mhi_device *mhi_dev = to_mhi_device(dev); + + return add_uevent_var(env, "MODALIAS=" MHI_DEVICE_MODALIAS_FMT, + mhi_dev->name); +} + +static int mhi_match(struct device *dev, struct device_driver *drv) +{ + struct mhi_device *mhi_dev = to_mhi_device(dev); + struct mhi_driver *mhi_drv = to_mhi_driver(drv); + const struct mhi_device_id *id; + + /* + * If the device is a controller type then there is no client driver + * associated with it + */ + if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER) + return 0; + + for (id = mhi_drv->id_table; id->chan[0]; id++) + if (!strcmp(mhi_dev->name, id->chan)) { + mhi_dev->id = id; + return 1; + } + + return 0; +}; + +struct bus_type mhi_bus_type = { + .name = "mhi", + .dev_name = "mhi", + .match = mhi_match, + .uevent = mhi_uevent, + .dev_groups = mhi_dev_groups, +}; + +static int __init mhi_init(void) +{ + mhi_misc_init(); + mhi_debugfs_init(); + return bus_register(&mhi_bus_type); +} + +static void __exit mhi_exit(void) +{ + mhi_misc_exit(); + mhi_debugfs_exit(); + bus_unregister(&mhi_bus_type); +} + +postcore_initcall(mhi_init); +module_exit(mhi_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("MHI Host Interface"); diff --git a/mhi/core/internal.h b/mhi/core/internal.h new file mode 100644 index 0000000..1b14dc6 --- /dev/null +++ b/mhi/core/internal.h @@ -0,0 +1,720 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * + */ + +#ifndef _MHI_INT_H +#define _MHI_INT_H + +#include <linux/mhi.h> +#include "misc.h" + +extern struct bus_type mhi_bus_type; + +#define MHIREGLEN (0x0) +#define MHIREGLEN_MHIREGLEN_MASK (0xFFFFFFFF) +#define MHIREGLEN_MHIREGLEN_SHIFT (0) + +#define MHIVER (0x8) +#define MHIVER_MHIVER_MASK (0xFFFFFFFF) +#define MHIVER_MHIVER_SHIFT (0) + +#define MHICFG (0x10) +#define MHICFG_NHWER_MASK (0xFF000000) +#define MHICFG_NHWER_SHIFT (24) +#define MHICFG_NER_MASK (0xFF0000) +#define MHICFG_NER_SHIFT (16) +#define MHICFG_NHWCH_MASK (0xFF00) +#define MHICFG_NHWCH_SHIFT (8) +#define MHICFG_NCH_MASK (0xFF) +#define MHICFG_NCH_SHIFT (0) + +#define CHDBOFF (0x18) +#define CHDBOFF_CHDBOFF_MASK (0xFFFFFFFF) +#define CHDBOFF_CHDBOFF_SHIFT (0) + +#define ERDBOFF (0x20) +#define ERDBOFF_ERDBOFF_MASK (0xFFFFFFFF) +#define ERDBOFF_ERDBOFF_SHIFT (0) + +#define BHIOFF (0x28) +#define BHIOFF_BHIOFF_MASK (0xFFFFFFFF) +#define BHIOFF_BHIOFF_SHIFT (0) + +#define BHIEOFF (0x2C) +#define BHIEOFF_BHIEOFF_MASK (0xFFFFFFFF) +#define BHIEOFF_BHIEOFF_SHIFT (0) + +#define DEBUGOFF (0x30) +#define DEBUGOFF_DEBUGOFF_MASK (0xFFFFFFFF) +#define DEBUGOFF_DEBUGOFF_SHIFT (0) + +#define MHICTRL (0x38) +#define MHICTRL_MHISTATE_MASK (0x0000FF00) +#define MHICTRL_MHISTATE_SHIFT (8) +#define MHICTRL_RESET_MASK (0x2) +#define MHICTRL_RESET_SHIFT (1) + +#define MHISTATUS (0x48) +#define MHISTATUS_MHISTATE_MASK (0x0000FF00) +#define MHISTATUS_MHISTATE_SHIFT (8) +#define MHISTATUS_SYSERR_MASK (0x4) +#define MHISTATUS_SYSERR_SHIFT (2) +#define MHISTATUS_READY_MASK (0x1) +#define MHISTATUS_READY_SHIFT (0) + +#define CCABAP_LOWER (0x58) +#define CCABAP_LOWER_CCABAP_LOWER_MASK (0xFFFFFFFF) +#define CCABAP_LOWER_CCABAP_LOWER_SHIFT (0) + +#define CCABAP_HIGHER (0x5C) +#define CCABAP_HIGHER_CCABAP_HIGHER_MASK (0xFFFFFFFF) +#define CCABAP_HIGHER_CCABAP_HIGHER_SHIFT (0) + +#define ECABAP_LOWER (0x60) +#define ECABAP_LOWER_ECABAP_LOWER_MASK (0xFFFFFFFF) +#define ECABAP_LOWER_ECABAP_LOWER_SHIFT (0) + +#define ECABAP_HIGHER (0x64) +#define ECABAP_HIGHER_ECABAP_HIGHER_MASK (0xFFFFFFFF) +#define ECABAP_HIGHER_ECABAP_HIGHER_SHIFT (0) + +#define CRCBAP_LOWER (0x68) +#define CRCBAP_LOWER_CRCBAP_LOWER_MASK (0xFFFFFFFF) +#define CRCBAP_LOWER_CRCBAP_LOWER_SHIFT (0) + +#define CRCBAP_HIGHER (0x6C) +#define CRCBAP_HIGHER_CRCBAP_HIGHER_MASK (0xFFFFFFFF) +#define CRCBAP_HIGHER_CRCBAP_HIGHER_SHIFT (0) + +#define CRDB_LOWER (0x70) +#define CRDB_LOWER_CRDB_LOWER_MASK (0xFFFFFFFF) +#define CRDB_LOWER_CRDB_LOWER_SHIFT (0) + +#define CRDB_HIGHER (0x74) +#define CRDB_HIGHER_CRDB_HIGHER_MASK (0xFFFFFFFF) +#define CRDB_HIGHER_CRDB_HIGHER_SHIFT (0) + +#define MHICTRLBASE_LOWER (0x80) +#define MHICTRLBASE_LOWER_MHICTRLBASE_LOWER_MASK (0xFFFFFFFF) +#define MHICTRLBASE_LOWER_MHICTRLBASE_LOWER_SHIFT (0) + +#define MHICTRLBASE_HIGHER (0x84) +#define MHICTRLBASE_HIGHER_MHICTRLBASE_HIGHER_MASK (0xFFFFFFFF) +#define MHICTRLBASE_HIGHER_MHICTRLBASE_HIGHER_SHIFT (0) + +#define MHICTRLLIMIT_LOWER (0x88) +#define MHICTRLLIMIT_LOWER_MHICTRLLIMIT_LOWER_MASK (0xFFFFFFFF) +#define MHICTRLLIMIT_LOWER_MHICTRLLIMIT_LOWER_SHIFT (0) + +#define MHICTRLLIMIT_HIGHER (0x8C) +#define MHICTRLLIMIT_HIGHER_MHICTRLLIMIT_HIGHER_MASK (0xFFFFFFFF) +#define MHICTRLLIMIT_HIGHER_MHICTRLLIMIT_HIGHER_SHIFT (0) + +#define MHIDATABASE_LOWER (0x98) +#define MHIDATABASE_LOWER_MHIDATABASE_LOWER_MASK (0xFFFFFFFF) +#define MHIDATABASE_LOWER_MHIDATABASE_LOWER_SHIFT (0) + +#define MHIDATABASE_HIGHER (0x9C) +#define MHIDATABASE_HIGHER_MHIDATABASE_HIGHER_MASK (0xFFFFFFFF) +#define MHIDATABASE_HIGHER_MHIDATABASE_HIGHER_SHIFT (0) + +#define MHIDATALIMIT_LOWER (0xA0) +#define MHIDATALIMIT_LOWER_MHIDATALIMIT_LOWER_MASK (0xFFFFFFFF) +#define MHIDATALIMIT_LOWER_MHIDATALIMIT_LOWER_SHIFT (0) + +#define MHIDATALIMIT_HIGHER (0xA4) +#define MHIDATALIMIT_HIGHER_MHIDATALIMIT_HIGHER_MASK (0xFFFFFFFF) +#define MHIDATALIMIT_HIGHER_MHIDATALIMIT_HIGHER_SHIFT (0) + +/* Host request register */ +#define MHI_SOC_RESET_REQ_OFFSET (0xB0) +#define MHI_SOC_RESET_REQ BIT(0) + +/* MHI BHI offfsets */ +#define BHI_BHIVERSION_MINOR (0x00) +#define BHI_BHIVERSION_MAJOR (0x04) +#define BHI_IMGADDR_LOW (0x08) +#define BHI_IMGADDR_HIGH (0x0C) +#define BHI_IMGSIZE (0x10) +#define BHI_RSVD1 (0x14) +#define BHI_IMGTXDB (0x18) +#define BHI_TXDB_SEQNUM_BMSK (0x3FFFFFFF) +#define BHI_TXDB_SEQNUM_SHFT (0) +#define BHI_RSVD2 (0x1C) +#define BHI_INTVEC (0x20) +#define BHI_RSVD3 (0x24) +#define BHI_EXECENV (0x28) +#define BHI_STATUS (0x2C) +#define BHI_ERRCODE (0x30) +#define BHI_ERRDBG1 (0x34) +#define BHI_ERRDBG2 (0x38) +#define BHI_ERRDBG3 (0x3C) +#define BHI_SERIALNU (0x40) +#define BHI_SBLANTIROLLVER (0x44) +#define BHI_NUMSEG (0x48) +#define BHI_MSMHWID(n) (0x4C + (0x4 * (n))) +#define BHI_OEMPKHASH(n) (0x64 + (0x4 * (n))) +#define BHI_RSVD5 (0xC4) +#define BHI_STATUS_MASK (0xC0000000) +#define BHI_STATUS_SHIFT (30) +#define BHI_STATUS_ERROR (3) +#define BHI_STATUS_SUCCESS (2) +#define BHI_STATUS_RESET (0) + +/* MHI BHIE offsets */ +#define BHIE_MSMSOCID_OFFS (0x0000) +#define BHIE_TXVECADDR_LOW_OFFS (0x002C) +#define BHIE_TXVECADDR_HIGH_OFFS (0x0030) +#define BHIE_TXVECSIZE_OFFS (0x0034) +#define BHIE_TXVECDB_OFFS (0x003C) +#define BHIE_TXVECDB_SEQNUM_BMSK (0x3FFFFFFF) +#define BHIE_TXVECDB_SEQNUM_SHFT (0) +#define BHIE_TXVECSTATUS_OFFS (0x0044) +#define BHIE_TXVECSTATUS_SEQNUM_BMSK (0x3FFFFFFF) +#define BHIE_TXVECSTATUS_SEQNUM_SHFT (0) +#define BHIE_TXVECSTATUS_STATUS_BMSK (0xC0000000) +#define BHIE_TXVECSTATUS_STATUS_SHFT (30) +#define BHIE_TXVECSTATUS_STATUS_RESET (0x00) +#define BHIE_TXVECSTATUS_STATUS_XFER_COMPL (0x02) +#define BHIE_TXVECSTATUS_STATUS_ERROR (0x03) +#define BHIE_RXVECADDR_LOW_OFFS (0x0060) +#define BHIE_RXVECADDR_HIGH_OFFS (0x0064) +#define BHIE_RXVECSIZE_OFFS (0x0068) +#define BHIE_RXVECDB_OFFS (0x0070) +#define BHIE_RXVECDB_SEQNUM_BMSK (0x3FFFFFFF) +#define BHIE_RXVECDB_SEQNUM_SHFT (0) +#define BHIE_RXVECSTATUS_OFFS (0x0078) +#define BHIE_RXVECSTATUS_SEQNUM_BMSK (0x3FFFFFFF) +#define BHIE_RXVECSTATUS_SEQNUM_SHFT (0) +#define BHIE_RXVECSTATUS_STATUS_BMSK (0xC0000000) +#define BHIE_RXVECSTATUS_STATUS_SHFT (30) +#define BHIE_RXVECSTATUS_STATUS_RESET (0x00) +#define BHIE_RXVECSTATUS_STATUS_XFER_COMPL (0x02) +#define BHIE_RXVECSTATUS_STATUS_ERROR (0x03) + +#define SOC_HW_VERSION_OFFS (0x224) +#define SOC_HW_VERSION_FAM_NUM_BMSK (0xF0000000) +#define SOC_HW_VERSION_FAM_NUM_SHFT (28) +#define SOC_HW_VERSION_DEV_NUM_BMSK (0x0FFF0000) +#define SOC_HW_VERSION_DEV_NUM_SHFT (16) +#define SOC_HW_VERSION_MAJOR_VER_BMSK (0x0000FF00) +#define SOC_HW_VERSION_MAJOR_VER_SHFT (8) +#define SOC_HW_VERSION_MINOR_VER_BMSK (0x000000FF) +#define SOC_HW_VERSION_MINOR_VER_SHFT (0) + +#define EV_CTX_RESERVED_MASK GENMASK(7, 0) +#define EV_CTX_INTMODC_MASK GENMASK(15, 8) +#define EV_CTX_INTMODC_SHIFT 8 +#define EV_CTX_INTMODT_MASK GENMASK(31, 16) +#define EV_CTX_INTMODT_SHIFT 16 +struct mhi_event_ctxt { + __u32 intmod; + __u32 ertype; + __u32 msivec; + + __u64 rbase __packed __aligned(4); + __u64 rlen __packed __aligned(4); + __u64 rp __packed __aligned(4); + __u64 wp __packed __aligned(4); +}; + +#define CHAN_CTX_CHSTATE_MASK GENMASK(7, 0) +#define CHAN_CTX_CHSTATE_SHIFT 0 +#define CHAN_CTX_BRSTMODE_MASK GENMASK(9, 8) +#define CHAN_CTX_BRSTMODE_SHIFT 8 +#define CHAN_CTX_POLLCFG_MASK GENMASK(15, 10) +#define CHAN_CTX_POLLCFG_SHIFT 10 +#define CHAN_CTX_RESERVED_MASK GENMASK(31, 16) +struct mhi_chan_ctxt { + __u32 chcfg; + __u32 chtype; + __u32 erindex; + + __u64 rbase __packed __aligned(4); + __u64 rlen __packed __aligned(4); + __u64 rp __packed __aligned(4); + __u64 wp __packed __aligned(4); +}; + +struct mhi_cmd_ctxt { + __u32 reserved0; + __u32 reserved1; + __u32 reserved2; + + __u64 rbase __packed __aligned(4); + __u64 rlen __packed __aligned(4); + __u64 rp __packed __aligned(4); + __u64 wp __packed __aligned(4); +}; + +struct mhi_ctxt { + struct mhi_event_ctxt *er_ctxt; + struct mhi_chan_ctxt *chan_ctxt; + struct mhi_cmd_ctxt *cmd_ctxt; + dma_addr_t er_ctxt_addr; + dma_addr_t chan_ctxt_addr; + dma_addr_t cmd_ctxt_addr; +}; + +struct mhi_tre { + u64 ptr; + u32 dword[2]; +}; + +struct bhi_vec_entry { + u64 dma_addr; + u64 size; +}; + +enum mhi_cmd_type { + MHI_CMD_NOP = 1, + MHI_CMD_RESET_CHAN = 16, + MHI_CMD_STOP_CHAN = 17, + MHI_CMD_START_CHAN = 18, + MHI_CMD_SFR_CFG = 73, +}; + +/* No operation command */ +#define MHI_TRE_CMD_NOOP_PTR (0) +#define MHI_TRE_CMD_NOOP_DWORD0 (0) +#define MHI_TRE_CMD_NOOP_DWORD1 (MHI_CMD_NOP << 16) + +/* Channel reset command */ +#define MHI_TRE_CMD_RESET_PTR (0) +#define MHI_TRE_CMD_RESET_DWORD0 (0) +#define MHI_TRE_CMD_RESET_DWORD1(chid) ((chid << 24) | \ + (MHI_CMD_RESET_CHAN << 16)) + +/* Channel stop command */ +#define MHI_TRE_CMD_STOP_PTR (0) +#define MHI_TRE_CMD_STOP_DWORD0 (0) +#define MHI_TRE_CMD_STOP_DWORD1(chid) ((chid << 24) | \ + (MHI_CMD_STOP_CHAN << 16)) + +/* Channel start command */ +#define MHI_TRE_CMD_START_PTR (0) +#define MHI_TRE_CMD_START_DWORD0 (0) +#define MHI_TRE_CMD_START_DWORD1(chid) ((chid << 24) | \ + (MHI_CMD_START_CHAN << 16)) + +#define MHI_TRE_GET_CMD_CHID(tre) (((tre)->dword[1] >> 24) & 0xFF) +#define MHI_TRE_GET_CMD_TYPE(tre) (((tre)->dword[1] >> 16) & 0xFF) + +/* Event descriptor macros */ +#define MHI_TRE_EV_PTR(ptr) (ptr) +#define MHI_TRE_EV_DWORD0(code, len) ((code << 24) | len) +#define MHI_TRE_EV_DWORD1(chid, type) ((chid << 24) | (type << 16)) +#define MHI_TRE_GET_EV_PTR(tre) ((tre)->ptr) +#define MHI_TRE_GET_EV_CODE(tre) (((tre)->dword[0] >> 24) & 0xFF) +#define MHI_TRE_GET_EV_LEN(tre) ((tre)->dword[0] & 0xFFFF) +#define MHI_TRE_GET_EV_CHID(tre) (((tre)->dword[1] >> 24) & 0xFF) +#define MHI_TRE_GET_EV_TYPE(tre) (((tre)->dword[1] >> 16) & 0xFF) +#define MHI_TRE_GET_EV_STATE(tre) (((tre)->dword[0] >> 24) & 0xFF) +#define MHI_TRE_GET_EV_EXECENV(tre) (((tre)->dword[0] >> 24) & 0xFF) +#define MHI_TRE_GET_EV_SEQ(tre) ((tre)->dword[0]) +#define MHI_TRE_GET_EV_TIME(tre) ((tre)->ptr) +#define MHI_TRE_GET_EV_COOKIE(tre) lower_32_bits((tre)->ptr) +#define MHI_TRE_GET_EV_VEID(tre) (((tre)->dword[0] >> 16) & 0xFF) +#define MHI_TRE_GET_EV_LINKSPEED(tre) (((tre)->dword[1] >> 24) & 0xFF) +#define MHI_TRE_GET_EV_LINKWIDTH(tre) ((tre)->dword[0] & 0xFF) + +/* Transfer descriptor macros */ +#define MHI_TRE_DATA_PTR(ptr) (ptr) +#define MHI_TRE_DATA_DWORD0(len) (len & MHI_MAX_MTU) +#define MHI_TRE_DATA_DWORD1(bei, ieot, ieob, chain) ((2 << 16) | (bei << 10) \ + | (ieot << 9) | (ieob << 8) | chain) + +/* RSC transfer descriptor macros */ +#define MHI_RSCTRE_DATA_PTR(ptr, len) (((u64)len << 48) | ptr) +#define MHI_RSCTRE_DATA_DWORD0(cookie) (cookie) +#define MHI_RSCTRE_DATA_DWORD1 (MHI_PKT_TYPE_COALESCING << 16) + +enum mhi_pkt_type { + MHI_PKT_TYPE_INVALID = 0x0, + MHI_PKT_TYPE_NOOP_CMD = 0x1, + MHI_PKT_TYPE_TRANSFER = 0x2, + MHI_PKT_TYPE_COALESCING = 0x8, + MHI_PKT_TYPE_RESET_CHAN_CMD = 0x10, + MHI_PKT_TYPE_STOP_CHAN_CMD = 0x11, + MHI_PKT_TYPE_START_CHAN_CMD = 0x12, + MHI_PKT_TYPE_STATE_CHANGE_EVENT = 0x20, + MHI_PKT_TYPE_CMD_COMPLETION_EVENT = 0x21, + MHI_PKT_TYPE_TX_EVENT = 0x22, + MHI_PKT_TYPE_RSC_TX_EVENT = 0x28, + MHI_PKT_TYPE_EE_EVENT = 0x40, + MHI_PKT_TYPE_TSYNC_EVENT = 0x48, + MHI_PKT_TYPE_BW_REQ_EVENT = 0x50, + MHI_PKT_TYPE_STALE_EVENT, /* internal event */ +}; + +/* MHI transfer completion events */ +enum mhi_ev_ccs { + MHI_EV_CC_INVALID = 0x0, + MHI_EV_CC_SUCCESS = 0x1, + MHI_EV_CC_EOT = 0x2, /* End of transfer event */ + MHI_EV_CC_OVERFLOW = 0x3, + MHI_EV_CC_EOB = 0x4, /* End of block event */ + MHI_EV_CC_OOB = 0x5, /* Out of block event */ + MHI_EV_CC_DB_MODE = 0x6, + MHI_EV_CC_UNDEFINED_ERR = 0x10, + MHI_EV_CC_BAD_TRE = 0x11, +}; + +enum mhi_ch_state { + MHI_CH_STATE_DISABLED = 0x0, + MHI_CH_STATE_ENABLED = 0x1, + MHI_CH_STATE_RUNNING = 0x2, + MHI_CH_STATE_SUSPENDED = 0x3, + MHI_CH_STATE_STOP = 0x4, + MHI_CH_STATE_ERROR = 0x5, +}; + +enum mhi_ch_state_type { + MHI_CH_STATE_TYPE_RESET, + MHI_CH_STATE_TYPE_STOP, + MHI_CH_STATE_TYPE_START, + MHI_CH_STATE_TYPE_MAX, +}; + +extern const char * const mhi_ch_state_type_str[MHI_CH_STATE_TYPE_MAX]; +#define TO_CH_STATE_TYPE_STR(state) (((state) >= MHI_CH_STATE_TYPE_MAX) ? \ + "INVALID_STATE" : \ + mhi_ch_state_type_str[(state)]) + +#define MHI_INVALID_BRSTMODE(mode) (mode != MHI_DB_BRST_DISABLE && \ + mode != MHI_DB_BRST_ENABLE) + +extern const char * const mhi_ee_str[MHI_EE_MAX]; +#define TO_MHI_EXEC_STR(ee) (((ee) >= MHI_EE_MAX) ? \ + "INVALID_EE" : mhi_ee_str[ee]) + +#define MHI_IN_PBL(ee) (ee == MHI_EE_PBL || ee == MHI_EE_PTHRU || \ + ee == MHI_EE_EDL) + +#define MHI_IN_MISSION_MODE(ee) (ee == MHI_EE_AMSS || ee == MHI_EE_WFW) + +enum dev_st_transition { + DEV_ST_TRANSITION_PBL, + DEV_ST_TRANSITION_READY, + DEV_ST_TRANSITION_SBL, + DEV_ST_TRANSITION_MISSION_MODE, + DEV_ST_TRANSITION_SYS_ERR, + DEV_ST_TRANSITION_DISABLE, + DEV_ST_TRANSITION_MAX, +}; + +extern const char * const dev_state_tran_str[DEV_ST_TRANSITION_MAX]; +#define TO_DEV_STATE_TRANS_STR(state) (((state) >= DEV_ST_TRANSITION_MAX) ? \ + "INVALID_STATE" : dev_state_tran_str[state]) + +extern const char * const mhi_state_str[MHI_STATE_MAX]; +#define TO_MHI_STATE_STR(state) ((state >= MHI_STATE_MAX || \ + !mhi_state_str[state]) ? \ + "INVALID_STATE" : mhi_state_str[state]) + +/* internal power states */ +enum mhi_pm_state { + MHI_PM_STATE_DISABLE, + MHI_PM_STATE_POR, + MHI_PM_STATE_M0, + MHI_PM_STATE_M2, + MHI_PM_STATE_M3_ENTER, + MHI_PM_STATE_M3, + MHI_PM_STATE_M3_EXIT, + MHI_PM_STATE_FW_DL_ERR, + MHI_PM_STATE_SYS_ERR_DETECT, + MHI_PM_STATE_SYS_ERR_PROCESS, + MHI_PM_STATE_SHUTDOWN_PROCESS, + MHI_PM_STATE_LD_ERR_FATAL_DETECT, + MHI_PM_STATE_MAX +}; + +#define MHI_PM_DISABLE BIT(0) +#define MHI_PM_POR BIT(1) +#define MHI_PM_M0 BIT(2) +#define MHI_PM_M2 BIT(3) +#define MHI_PM_M3_ENTER BIT(4) +#define MHI_PM_M3 BIT(5) +#define MHI_PM_M3_EXIT BIT(6) +/* firmware download failure state */ +#define MHI_PM_FW_DL_ERR BIT(7) +#define MHI_PM_SYS_ERR_DETECT BIT(8) +#define MHI_PM_SYS_ERR_PROCESS BIT(9) +#define MHI_PM_SHUTDOWN_PROCESS BIT(10) +/* link not accessible */ +#define MHI_PM_LD_ERR_FATAL_DETECT BIT(11) + +#define MHI_REG_ACCESS_VALID(pm_state) ((pm_state & (MHI_PM_POR | MHI_PM_M0 | \ + MHI_PM_M2 | MHI_PM_M3_ENTER | MHI_PM_M3_EXIT | \ + MHI_PM_SYS_ERR_DETECT | MHI_PM_SYS_ERR_PROCESS | \ + MHI_PM_SHUTDOWN_PROCESS | MHI_PM_FW_DL_ERR))) +#define MHI_PM_IN_ERROR_STATE(pm_state) (pm_state >= MHI_PM_FW_DL_ERR) +#define MHI_PM_IN_FATAL_STATE(pm_state) (pm_state == MHI_PM_LD_ERR_FATAL_DETECT) +#define MHI_DB_ACCESS_VALID(mhi_cntrl) (mhi_cntrl->pm_state & \ + mhi_cntrl->db_access) +#define MHI_WAKE_DB_CLEAR_VALID(pm_state) (pm_state & (MHI_PM_M0 | \ + MHI_PM_M2 | MHI_PM_M3_EXIT)) +#define MHI_WAKE_DB_SET_VALID(pm_state) (pm_state & MHI_PM_M2) +#define MHI_WAKE_DB_FORCE_SET_VALID(pm_state) MHI_WAKE_DB_CLEAR_VALID(pm_state) +#define MHI_EVENT_ACCESS_INVALID(pm_state) (pm_state == MHI_PM_DISABLE || \ + MHI_PM_IN_ERROR_STATE(pm_state)) +#define MHI_PM_IN_SUSPEND_STATE(pm_state) (pm_state & \ + (MHI_PM_M3_ENTER | MHI_PM_M3)) + +#define NR_OF_CMD_RINGS 1 +#define CMD_EL_PER_RING 128 +#define PRIMARY_CMD_RING 0 +#define MHI_DEV_WAKE_DB 127 +#define MHI_RANDOM_U32_NONZERO(bmsk) (prandom_u32_max(bmsk) + 1) + +enum mhi_er_type { + MHI_ER_TYPE_INVALID = 0x0, + MHI_ER_TYPE_VALID = 0x1, +}; + +struct db_cfg { + bool reset_req; + bool db_mode; + u32 pollcfg; + enum mhi_db_brst_mode brstmode; + dma_addr_t db_val; + void (*process_db)(struct mhi_controller *mhi_cntrl, + struct db_cfg *db_cfg, void __iomem *io_addr, + dma_addr_t db_val); +}; + +struct mhi_pm_transitions { + enum mhi_pm_state from_state; + u32 to_states; +}; + +struct state_transition { + struct list_head node; + enum dev_st_transition state; +}; + +struct mhi_ring { + dma_addr_t dma_handle; + dma_addr_t iommu_base; + u64 *ctxt_wp; /* point to ctxt wp */ + void *pre_aligned; + void *base; + void *rp; + void *wp; + size_t el_size; + size_t len; + size_t elements; + size_t alloc_size; + void __iomem *db_addr; +}; + +struct mhi_cmd { + struct mhi_ring ring; + spinlock_t lock; +}; + +struct mhi_buf_info { + void *v_addr; + void *bb_addr; + void *wp; + void *cb_buf; + dma_addr_t p_addr; + size_t len; + enum dma_data_direction dir; + bool used; /* Indicates whether the buffer is used or not */ + bool pre_mapped; /* Already pre-mapped by client */ +}; + +struct mhi_event { + struct mhi_controller *mhi_cntrl; + struct mhi_chan *mhi_chan; /* dedicated to channel */ + u32 er_index; + u32 intmod; + u32 irq; + int chan; /* this event ring is dedicated to a channel (optional) */ + u32 priority; + enum mhi_er_data_type data_type; + struct mhi_ring ring; + struct db_cfg db_cfg; + struct tasklet_struct task; + struct work_struct work; + spinlock_t lock; + int (*process_event)(struct mhi_controller *mhi_cntrl, + struct mhi_event *mhi_event, + u32 event_quota); + bool hw_ring; + bool cl_manage; + bool offload_ev; /* managed by a device driver */ +}; + +struct mhi_chan { + const char *name; + /* + * Important: When consuming, increment tre_ring first and when + * releasing, decrement buf_ring first. If tre_ring has space, buf_ring + * is guranteed to have space so we do not need to check both rings. + */ + struct mhi_ring buf_ring; + struct mhi_ring tre_ring; + u32 chan; + u32 er_index; + u32 intmod; + enum mhi_ch_type type; + enum dma_data_direction dir; + struct db_cfg db_cfg; + enum mhi_ch_ee_mask ee_mask; + enum mhi_ch_state ch_state; + enum mhi_ev_ccs ccs; + struct mhi_device *mhi_dev; + void (*xfer_cb)(struct mhi_device *mhi_dev, struct mhi_result *result); + struct mutex mutex; + struct completion completion; + rwlock_t lock; + struct list_head node; + bool lpm_notify; + bool configured; + bool offload_ch; + bool pre_alloc; + bool wake_capable; +}; + +/* Default MHI timeout */ +#define MHI_TIMEOUT_MS (1000) + +/* debugfs related functions */ +#ifdef CONFIG_MHI_BUS_DEBUG +void mhi_create_debugfs(struct mhi_controller *mhi_cntrl); +void mhi_destroy_debugfs(struct mhi_controller *mhi_cntrl); +void mhi_debugfs_init(void); +void mhi_debugfs_exit(void); +#else +static inline void mhi_create_debugfs(struct mhi_controller *mhi_cntrl) +{ +} + +static inline void mhi_destroy_debugfs(struct mhi_controller *mhi_cntrl) +{ +} + +static inline void mhi_debugfs_init(void) +{ +} + +static inline void mhi_debugfs_exit(void) +{ +} +#endif + +struct mhi_device *mhi_alloc_device(struct mhi_controller *mhi_cntrl); + +int mhi_destroy_device(struct device *dev, void *data); +void mhi_create_devices(struct mhi_controller *mhi_cntrl); + +int mhi_alloc_bhie_table(struct mhi_controller *mhi_cntrl, + struct image_info **image_info, size_t alloc_size); +void mhi_free_bhie_table(struct mhi_controller *mhi_cntrl, + struct image_info **image_info); + +/* Power management APIs */ +enum mhi_pm_state __must_check mhi_tryset_pm_state( + struct mhi_controller *mhi_cntrl, + enum mhi_pm_state state); +const char *to_mhi_pm_state_str(enum mhi_pm_state state); +int mhi_queue_state_transition(struct mhi_controller *mhi_cntrl, + enum dev_st_transition state); +void mhi_pm_st_worker(struct work_struct *work); +void mhi_pm_sys_err_handler(struct mhi_controller *mhi_cntrl); +int mhi_ready_state_transition(struct mhi_controller *mhi_cntrl); +int mhi_pm_m0_transition(struct mhi_controller *mhi_cntrl); +void mhi_pm_m1_transition(struct mhi_controller *mhi_cntrl); +int mhi_pm_m3_transition(struct mhi_controller *mhi_cntrl); +int __mhi_device_get_sync(struct mhi_controller *mhi_cntrl); +int mhi_send_cmd(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan, + enum mhi_cmd_type cmd); +int mhi_download_amss_image(struct mhi_controller *mhi_cntrl); +static inline bool mhi_is_active(struct mhi_controller *mhi_cntrl) +{ + return (mhi_cntrl->dev_state >= MHI_STATE_M0 && + mhi_cntrl->dev_state <= MHI_STATE_M3_FAST); +} + +static inline void mhi_trigger_resume(struct mhi_controller *mhi_cntrl) +{ + pm_wakeup_event(&mhi_cntrl->mhi_dev->dev, 0); + mhi_cntrl->runtime_get(mhi_cntrl); + mhi_cntrl->runtime_put(mhi_cntrl); +} + +/* Register access methods */ +void mhi_db_brstmode(struct mhi_controller *mhi_cntrl, struct db_cfg *db_cfg, + void __iomem *db_addr, dma_addr_t db_val); +void mhi_db_brstmode_disable(struct mhi_controller *mhi_cntrl, + struct db_cfg *db_mode, void __iomem *db_addr, + dma_addr_t db_val); +int __must_check mhi_read_reg(struct mhi_controller *mhi_cntrl, + void __iomem *base, u32 offset, u32 *out); +int __must_check mhi_read_reg_field(struct mhi_controller *mhi_cntrl, + void __iomem *base, u32 offset, u32 mask, + u32 shift, u32 *out); +int __must_check mhi_poll_reg_field(struct mhi_controller *mhi_cntrl, + void __iomem *base, u32 offset, u32 mask, + u32 shift, u32 val, u32 delayus); +void mhi_write_reg(struct mhi_controller *mhi_cntrl, void __iomem *base, + u32 offset, u32 val); +void mhi_write_reg_field(struct mhi_controller *mhi_cntrl, void __iomem *base, + u32 offset, u32 mask, u32 shift, u32 val); +void mhi_ring_er_db(struct mhi_event *mhi_event); +void mhi_write_db(struct mhi_controller *mhi_cntrl, void __iomem *db_addr, + dma_addr_t db_val); +void mhi_ring_cmd_db(struct mhi_controller *mhi_cntrl, struct mhi_cmd *mhi_cmd); +void mhi_ring_chan_db(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan); +void *mhi_to_virtual(struct mhi_ring *ring, dma_addr_t addr); + +/* Initialization methods */ +int mhi_init_mmio(struct mhi_controller *mhi_cntrl); +int mhi_init_dev_ctxt(struct mhi_controller *mhi_cntrl); +void mhi_deinit_dev_ctxt(struct mhi_controller *mhi_cntrl); +int mhi_init_irq_setup(struct mhi_controller *mhi_cntrl); +void mhi_deinit_free_irq(struct mhi_controller *mhi_cntrl); +void mhi_rddm_prepare(struct mhi_controller *mhi_cntrl, + struct image_info *img_info); +void mhi_fw_load_handler(struct mhi_controller *mhi_cntrl); +int mhi_prepare_channel(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan); +int mhi_init_chan_ctxt(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan); +void mhi_deinit_chan_ctxt(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan); +void mhi_reset_chan(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan); + +/* Event processing methods */ +void mhi_ctrl_ev_task(unsigned long data); +void mhi_ev_task(unsigned long data); +void mhi_process_ev_work(struct work_struct *work); +void mhi_process_sleeping_events(struct mhi_controller *mhi_cntrl); +int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl, + struct mhi_event *mhi_event, u32 event_quota); +int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl, + struct mhi_event *mhi_event, u32 event_quota); + +/* ISR handlers */ +irqreturn_t mhi_irq_handler(int irq_number, void *dev); +irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *dev); +irqreturn_t mhi_intvec_handler(int irq_number, void *dev); + +int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan, + struct mhi_buf_info *info, enum mhi_flags flags); +int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl, + struct mhi_buf_info *buf_info); +int mhi_map_single_use_bb(struct mhi_controller *mhi_cntrl, + struct mhi_buf_info *buf_info); +void mhi_unmap_single_no_bb(struct mhi_controller *mhi_cntrl, + struct mhi_buf_info *buf_info); +void mhi_unmap_single_use_bb(struct mhi_controller *mhi_cntrl, + struct mhi_buf_info *buf_info); + +#endif /* _MHI_INT_H */ diff --git a/mhi/core/main.c b/mhi/core/main.c new file mode 100644 index 0000000..925b468 --- /dev/null +++ b/mhi/core/main.c @@ -0,0 +1,1858 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * + */ + +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/dma-direction.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/mhi.h> +#include <linux/module.h> +#include <linux/skbuff.h> +#include <linux/slab.h> +#include "internal.h" + +int __must_check mhi_read_reg(struct mhi_controller *mhi_cntrl, + void __iomem *base, u32 offset, u32 *out) +{ + return mhi_cntrl->read_reg(mhi_cntrl, base + offset, out); +} + +int __must_check mhi_read_reg_field(struct mhi_controller *mhi_cntrl, + void __iomem *base, u32 offset, + u32 mask, u32 shift, u32 *out) +{ + u32 tmp; + int ret; + + ret = mhi_read_reg(mhi_cntrl, base, offset, &tmp); + if (ret) + return ret; + + *out = (tmp & mask) >> shift; + + return 0; +} + +int __must_check mhi_poll_reg_field(struct mhi_controller *mhi_cntrl, + void __iomem *base, u32 offset, + u32 mask, u32 shift, u32 val, u32 delayus) +{ + int ret; + u32 out, retry = (mhi_cntrl->timeout_ms * 1000) / delayus; + + while (retry--) { + ret = mhi_read_reg_field(mhi_cntrl, base, offset, mask, shift, + &out); + if (ret) + return ret; + + if (out == val) + return 0; + + fsleep(delayus); + } + + return -ETIMEDOUT; +} + +void mhi_write_reg(struct mhi_controller *mhi_cntrl, void __iomem *base, + u32 offset, u32 val) +{ + mhi_cntrl->write_reg(mhi_cntrl, base + offset, val); +} + +void mhi_write_reg_field(struct mhi_controller *mhi_cntrl, void __iomem *base, + u32 offset, u32 mask, u32 shift, u32 val) +{ + int ret; + u32 tmp; + + ret = mhi_read_reg(mhi_cntrl, base, offset, &tmp); + if (ret) + return; + + tmp &= ~mask; + tmp |= (val << shift); + mhi_write_reg(mhi_cntrl, base, offset, tmp); +} + +void mhi_write_db(struct mhi_controller *mhi_cntrl, void __iomem *db_addr, + dma_addr_t db_val) +{ + mhi_write_reg(mhi_cntrl, db_addr, 4, upper_32_bits(db_val)); + mhi_write_reg(mhi_cntrl, db_addr, 0, lower_32_bits(db_val)); +} + +void mhi_db_brstmode(struct mhi_controller *mhi_cntrl, + struct db_cfg *db_cfg, + void __iomem *db_addr, + dma_addr_t db_val) +{ + if (db_cfg->db_mode) { + db_cfg->db_val = db_val; + mhi_write_db(mhi_cntrl, db_addr, db_val); + db_cfg->db_mode = 0; + } +} + +void mhi_db_brstmode_disable(struct mhi_controller *mhi_cntrl, + struct db_cfg *db_cfg, + void __iomem *db_addr, + dma_addr_t db_val) +{ + db_cfg->db_val = db_val; + mhi_write_db(mhi_cntrl, db_addr, db_val); +} + +void mhi_ring_er_db(struct mhi_event *mhi_event) +{ + struct mhi_ring *ring = &mhi_event->ring; + + mhi_event->db_cfg.process_db(mhi_event->mhi_cntrl, &mhi_event->db_cfg, + ring->db_addr, *ring->ctxt_wp); +} + +void mhi_ring_cmd_db(struct mhi_controller *mhi_cntrl, struct mhi_cmd *mhi_cmd) +{ + dma_addr_t db; + struct mhi_ring *ring = &mhi_cmd->ring; + + db = ring->iommu_base + (ring->wp - ring->base); + *ring->ctxt_wp = db; + mhi_write_db(mhi_cntrl, ring->db_addr, db); +} + +void mhi_ring_chan_db(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan) +{ + struct mhi_ring *ring = &mhi_chan->tre_ring; + dma_addr_t db; + + db = ring->iommu_base + (ring->wp - ring->base); + *ring->ctxt_wp = db; + mhi_chan->db_cfg.process_db(mhi_cntrl, &mhi_chan->db_cfg, + ring->db_addr, db); +} + +enum mhi_ee_type mhi_get_exec_env(struct mhi_controller *mhi_cntrl) +{ + u32 exec; + int ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_EXECENV, &exec); + + return (ret) ? MHI_EE_MAX : exec; +} +EXPORT_SYMBOL_GPL(mhi_get_exec_env); + +enum mhi_state mhi_get_mhi_state(struct mhi_controller *mhi_cntrl) +{ + u32 state; + int ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, MHISTATUS, + MHISTATUS_MHISTATE_MASK, + MHISTATUS_MHISTATE_SHIFT, &state); + return ret ? MHI_STATE_MAX : state; +} +EXPORT_SYMBOL_GPL(mhi_get_mhi_state); + +void mhi_soc_reset(struct mhi_controller *mhi_cntrl) +{ + if (mhi_cntrl->reset) { + mhi_cntrl->reset(mhi_cntrl); + return; + } + + /* Generic MHI SoC reset */ + mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, MHI_SOC_RESET_REQ_OFFSET, + MHI_SOC_RESET_REQ); +} +EXPORT_SYMBOL_GPL(mhi_soc_reset); + +int mhi_map_single_no_bb(struct mhi_controller *mhi_cntrl, + struct mhi_buf_info *buf_info) +{ + buf_info->p_addr = dma_map_single(mhi_cntrl->cntrl_dev, + buf_info->v_addr, buf_info->len, + buf_info->dir); + if (dma_mapping_error(mhi_cntrl->cntrl_dev, buf_info->p_addr)) + return -ENOMEM; + + return 0; +} + +int mhi_map_single_use_bb(struct mhi_controller *mhi_cntrl, + struct mhi_buf_info *buf_info) +{ + void *buf = dma_alloc_coherent(mhi_cntrl->cntrl_dev, buf_info->len, + &buf_info->p_addr, GFP_ATOMIC); + + if (!buf) + return -ENOMEM; + + if (buf_info->dir == DMA_TO_DEVICE) + memcpy(buf, buf_info->v_addr, buf_info->len); + + buf_info->bb_addr = buf; + + return 0; +} + +void mhi_unmap_single_no_bb(struct mhi_controller *mhi_cntrl, + struct mhi_buf_info *buf_info) +{ + dma_unmap_single(mhi_cntrl->cntrl_dev, buf_info->p_addr, buf_info->len, + buf_info->dir); +} + +void mhi_unmap_single_use_bb(struct mhi_controller *mhi_cntrl, + struct mhi_buf_info *buf_info) +{ + if (buf_info->dir == DMA_FROM_DEVICE) + memcpy(buf_info->v_addr, buf_info->bb_addr, buf_info->len); + + dma_free_coherent(mhi_cntrl->cntrl_dev, buf_info->len, buf_info->bb_addr, + buf_info->p_addr); +} + +static int get_nr_avail_ring_elements(struct mhi_controller *mhi_cntrl, + struct mhi_ring *ring) +{ + int nr_el; + + if (ring->wp < ring->rp) { + nr_el = ((ring->rp - ring->wp) / ring->el_size) - 1; + } else { + nr_el = (ring->rp - ring->base) / ring->el_size; + nr_el += ((ring->base + ring->len - ring->wp) / + ring->el_size) - 1; + } + + return nr_el; +} + +void *mhi_to_virtual(struct mhi_ring *ring, dma_addr_t addr) +{ + return (addr - ring->iommu_base) + ring->base; +} + +static void mhi_add_ring_element(struct mhi_controller *mhi_cntrl, + struct mhi_ring *ring) +{ + ring->wp += ring->el_size; + if (ring->wp >= (ring->base + ring->len)) + ring->wp = ring->base; + /* smp update */ + smp_wmb(); +} + +static void mhi_del_ring_element(struct mhi_controller *mhi_cntrl, + struct mhi_ring *ring) +{ + ring->rp += ring->el_size; + if (ring->rp >= (ring->base + ring->len)) + ring->rp = ring->base; + /* smp update */ + smp_wmb(); +} + +static bool is_valid_ring_ptr(struct mhi_ring *ring, dma_addr_t addr) +{ + return addr >= ring->iommu_base && addr < ring->iommu_base + ring->len; +} + +int mhi_destroy_device(struct device *dev, void *data) +{ + struct mhi_chan *ul_chan, *dl_chan; + struct mhi_device *mhi_dev; + struct mhi_controller *mhi_cntrl; + enum mhi_ee_type ee = MHI_EE_MAX; + + if (dev->bus != &mhi_bus_type) + return 0; + + mhi_dev = to_mhi_device(dev); + mhi_cntrl = mhi_dev->mhi_cntrl; + + /* Only destroy virtual devices thats attached to bus */ + if (mhi_dev->dev_type == MHI_DEVICE_CONTROLLER) + return 0; + + ul_chan = mhi_dev->ul_chan; + dl_chan = mhi_dev->dl_chan; + + /* + * If execution environment is specified, remove only those devices that + * started in them based on ee_mask for the channels as we move on to a + * different execution environment + */ + if (data) + ee = *(enum mhi_ee_type *)data; + + /* + * For the suspend and resume case, this function will get called + * without mhi_unregister_controller(). Hence, we need to drop the + * references to mhi_dev created for ul and dl channels. We can + * be sure that there will be no instances of mhi_dev left after + * this. + */ + if (ul_chan) { + if (ee != MHI_EE_MAX && !(ul_chan->ee_mask & BIT(ee))) + return 0; + + put_device(&ul_chan->mhi_dev->dev); + } + + if (dl_chan) { + if (ee != MHI_EE_MAX && !(dl_chan->ee_mask & BIT(ee))) + return 0; + + put_device(&dl_chan->mhi_dev->dev); + } + + dev_dbg(&mhi_cntrl->mhi_dev->dev, "destroy device for chan:%s\n", + mhi_dev->name); + + /* Notify the client and remove the device from MHI bus */ + device_del(dev); + put_device(dev); + + return 0; +} + +int mhi_get_free_desc_count(struct mhi_device *mhi_dev, + enum dma_data_direction dir) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? + mhi_dev->ul_chan : mhi_dev->dl_chan; + struct mhi_ring *tre_ring = &mhi_chan->tre_ring; + + return get_nr_avail_ring_elements(mhi_cntrl, tre_ring); +} +EXPORT_SYMBOL(mhi_get_free_desc_count); + +void mhi_notify(struct mhi_device *mhi_dev, enum mhi_callback cb_reason) +{ + struct mhi_driver *mhi_drv; + + if (!mhi_dev->dev.driver) + return; + + mhi_drv = to_mhi_driver(mhi_dev->dev.driver); + + if (mhi_drv->status_cb) + mhi_drv->status_cb(mhi_dev, cb_reason); +} +EXPORT_SYMBOL_GPL(mhi_notify); + +/* Bind MHI channels to MHI devices */ +void mhi_create_devices(struct mhi_controller *mhi_cntrl) +{ + struct mhi_chan *mhi_chan; + struct mhi_device *mhi_dev; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + int i, ret; + + mhi_chan = mhi_cntrl->mhi_chan; + for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) { + if (!mhi_chan->configured || mhi_chan->mhi_dev || + !(mhi_chan->ee_mask & BIT(mhi_cntrl->ee))) + continue; + mhi_dev = mhi_alloc_device(mhi_cntrl); + if (IS_ERR(mhi_dev)) + return; + + mhi_dev->dev_type = MHI_DEVICE_XFER; + switch (mhi_chan->dir) { + case DMA_TO_DEVICE: + mhi_dev->ul_chan = mhi_chan; + mhi_dev->ul_chan_id = mhi_chan->chan; + mhi_dev->ul_event_id = mhi_chan->er_index; + break; + case DMA_FROM_DEVICE: + /* We use dl_chan as offload channels */ + mhi_dev->dl_chan = mhi_chan; + mhi_dev->dl_chan_id = mhi_chan->chan; + mhi_dev->dl_event_id = mhi_chan->er_index; + break; + default: + MHI_ERR("Direction not supported\n"); + put_device(&mhi_dev->dev); + return; + } + + get_device(&mhi_dev->dev); + mhi_chan->mhi_dev = mhi_dev; + + /* Check next channel if it matches */ + if ((i + 1) < mhi_cntrl->max_chan && mhi_chan[1].configured) { + if (!strcmp(mhi_chan[1].name, mhi_chan->name)) { + i++; + mhi_chan++; + if (mhi_chan->dir == DMA_TO_DEVICE) { + mhi_dev->ul_chan = mhi_chan; + mhi_dev->ul_chan_id = mhi_chan->chan; + mhi_dev->ul_event_id = mhi_chan->er_index; + } else { + mhi_dev->dl_chan = mhi_chan; + mhi_dev->dl_chan_id = mhi_chan->chan; + mhi_dev->dl_event_id = mhi_chan->er_index; + } + get_device(&mhi_dev->dev); + mhi_chan->mhi_dev = mhi_dev; + } + } + + /* Channel name is same for both UL and DL */ + mhi_dev->name = mhi_chan->name; + dev_set_name(&mhi_dev->dev, "%s_%s", dev_name(dev), + mhi_dev->name); + + /* Init wakeup source if available */ + if (mhi_dev->dl_chan && mhi_dev->dl_chan->wake_capable) + device_init_wakeup(&mhi_dev->dev, true); + + ret = device_add(&mhi_dev->dev); + if (ret) + put_device(&mhi_dev->dev); + } +} + +void mhi_process_sleeping_events(struct mhi_controller *mhi_cntrl) +{ + struct mhi_event *mhi_event; + struct mhi_event_ctxt *er_ctxt; + struct mhi_ring *ev_ring; + int i; + + mhi_event = mhi_cntrl->mhi_event; + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) { + if (mhi_event->offload_ev || mhi_event->priority != + MHI_ER_PRIORITY_HI_SLEEP) + continue; + + er_ctxt = &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index]; + ev_ring = &mhi_event->ring; + + /* Only proceed if event ring has pending events */ + if (ev_ring->rp == mhi_to_virtual(ev_ring, er_ctxt->rp)) + continue; + + queue_work(mhi_cntrl->hiprio_wq, &mhi_event->work); + } +} + +irqreturn_t mhi_irq_handler(int irq_number, void *priv) +{ + struct mhi_event *mhi_event = priv; + struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_event_ctxt *er_ctxt = + &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index]; + struct mhi_ring *ev_ring = &mhi_event->ring; + dma_addr_t ptr = er_ctxt->rp; + void *dev_rp; + + if (!is_valid_ring_ptr(ev_ring, ptr)) { + MHI_ERR("Event ring rp points outside of the event ring\n"); + return IRQ_HANDLED; + } + + dev_rp = mhi_to_virtual(ev_ring, ptr); + + /* Only proceed if event ring has pending events */ + if (ev_ring->rp == dev_rp) + return IRQ_HANDLED; + + /* For client managed event ring, notify pending data */ + if (mhi_event->cl_manage) { + struct mhi_chan *mhi_chan = mhi_event->mhi_chan; + struct mhi_device *mhi_dev = mhi_chan->mhi_dev; + + if (mhi_dev) + mhi_notify(mhi_dev, MHI_CB_PENDING_DATA); + + return IRQ_HANDLED; + } + + switch (mhi_event->priority) { + case MHI_ER_PRIORITY_HI_NOSLEEP: + tasklet_hi_schedule(&mhi_event->task); + break; + case MHI_ER_PRIORITY_DEFAULT_NOSLEEP: + tasklet_schedule(&mhi_event->task); + break; + case MHI_ER_PRIORITY_HI_SLEEP: + queue_work(mhi_cntrl->hiprio_wq, &mhi_event->work); + break; + default: + MHI_VERB("skip unknown priority event\n"); + break; + } + + return IRQ_HANDLED; +} + +irqreturn_t mhi_intvec_threaded_handler(int irq_number, void *priv) +{ + struct mhi_controller *mhi_cntrl = priv; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + enum mhi_state state = MHI_STATE_MAX; + enum mhi_pm_state pm_state = 0; + enum mhi_ee_type ee = MHI_EE_MAX; + + write_lock_irq(&mhi_cntrl->pm_lock); + if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) { + write_unlock_irq(&mhi_cntrl->pm_lock); + goto exit_intvec; + } + + state = mhi_get_mhi_state(mhi_cntrl); + ee = mhi_get_exec_env(mhi_cntrl); + MHI_VERB("local ee:%s state:%s device ee:%s state:%s\n", + TO_MHI_EXEC_STR(mhi_cntrl->ee), + TO_MHI_STATE_STR(mhi_cntrl->dev_state), + TO_MHI_EXEC_STR(ee), TO_MHI_STATE_STR(state)); + + if (state == MHI_STATE_SYS_ERR) { + MHI_VERB("System error detected\n"); + pm_state = mhi_tryset_pm_state(mhi_cntrl, + MHI_PM_SYS_ERR_DETECT); + } + write_unlock_irq(&mhi_cntrl->pm_lock); + + /* If device supports RDDM don't bother processing SYS error */ + if (mhi_cntrl->rddm_image) { + /* host may be performing a device power down already */ + if (!mhi_is_active(mhi_cntrl)) + goto exit_intvec; + + if (ee == MHI_EE_RDDM && mhi_cntrl->ee != MHI_EE_RDDM) { + mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_RDDM); + mhi_cntrl->ee = ee; + wake_up_all(&mhi_cntrl->state_event); + } + goto exit_intvec; + } + + if (pm_state == MHI_PM_SYS_ERR_DETECT) { + wake_up_all(&mhi_cntrl->state_event); + + /* For fatal errors, we let controller decide next step */ + if (MHI_IN_PBL(ee)) { + mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_FATAL_ERROR); + mhi_cntrl->ee = ee; + } else { + mhi_pm_sys_err_handler(mhi_cntrl); + } + } + +exit_intvec: + + return IRQ_HANDLED; +} + +irqreturn_t mhi_intvec_handler(int irq_number, void *dev) +{ + struct mhi_controller *mhi_cntrl = dev; + + /* Wake up events waiting for state change */ + wake_up_all(&mhi_cntrl->state_event); + + return IRQ_WAKE_THREAD; +} + +static void mhi_recycle_ev_ring_element(struct mhi_controller *mhi_cntrl, + struct mhi_ring *ring) +{ + dma_addr_t ctxt_wp; + + /* Update the WP */ + ring->wp += ring->el_size; + ctxt_wp = *ring->ctxt_wp + ring->el_size; + + if (ring->wp >= (ring->base + ring->len)) { + ring->wp = ring->base; + ctxt_wp = ring->iommu_base; + } + + *ring->ctxt_wp = ctxt_wp; + + /* Update the RP */ + ring->rp += ring->el_size; + if (ring->rp >= (ring->base + ring->len)) + ring->rp = ring->base; + + /* Update to all cores */ + smp_wmb(); +} + +static int parse_xfer_event(struct mhi_controller *mhi_cntrl, + struct mhi_tre *event, + struct mhi_chan *mhi_chan) +{ + struct mhi_ring *buf_ring, *tre_ring; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_result result; + unsigned long flags = 0; + u32 ev_code; + + ev_code = MHI_TRE_GET_EV_CODE(event); + buf_ring = &mhi_chan->buf_ring; + tre_ring = &mhi_chan->tre_ring; + + result.transaction_status = (ev_code == MHI_EV_CC_OVERFLOW) ? + -EOVERFLOW : 0; + + /* + * If it's a DB Event then we need to grab the lock + * with preemption disabled and as a write because we + * have to update db register and there are chances that + * another thread could be doing the same. + */ + if (ev_code >= MHI_EV_CC_OOB) + write_lock_irqsave(&mhi_chan->lock, flags); + else + read_lock_bh(&mhi_chan->lock); + + if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED) + goto end_process_tx_event; + + switch (ev_code) { + case MHI_EV_CC_OVERFLOW: + case MHI_EV_CC_EOB: + case MHI_EV_CC_EOT: + { + dma_addr_t ptr = MHI_TRE_GET_EV_PTR(event); + struct mhi_tre *local_rp, *ev_tre; + void *dev_rp; + struct mhi_buf_info *buf_info; + u16 xfer_len; + + if (!is_valid_ring_ptr(tre_ring, ptr)) { + MHI_ERR("Event element points outside of the tre ring\n"); + break; + } + /* Get the TRB this event points to */ + ev_tre = mhi_to_virtual(tre_ring, ptr); + + dev_rp = ev_tre + 1; + if (dev_rp >= (tre_ring->base + tre_ring->len)) + dev_rp = tre_ring->base; + + result.dir = mhi_chan->dir; + + local_rp = tre_ring->rp; + while (local_rp != dev_rp) { + buf_info = buf_ring->rp; + /* If it's the last TRE, get length from the event */ + if (local_rp == ev_tre) + xfer_len = MHI_TRE_GET_EV_LEN(event); + else + xfer_len = buf_info->len; + + /* Unmap if it's not pre-mapped by client */ + if (likely(!buf_info->pre_mapped)) + mhi_cntrl->unmap_single(mhi_cntrl, buf_info); + + result.buf_addr = buf_info->cb_buf; + + /* truncate to buf len if xfer_len is larger */ + result.bytes_xferd = + min_t(u16, xfer_len, buf_info->len); + mhi_del_ring_element(mhi_cntrl, buf_ring); + mhi_del_ring_element(mhi_cntrl, tre_ring); + local_rp = tre_ring->rp; + + /* notify client */ + mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result); + + if (mhi_chan->dir == DMA_TO_DEVICE) + atomic_dec(&mhi_cntrl->pending_pkts); + + /* + * Recycle the buffer if buffer is pre-allocated, + * if there is an error, not much we can do apart + * from dropping the packet + */ + if (mhi_chan->pre_alloc) { + if (mhi_queue_buf(mhi_chan->mhi_dev, + mhi_chan->dir, + buf_info->cb_buf, + buf_info->len, MHI_EOT)) { + MHI_ERR( + "Error recycling buffer for chan:%d\n", + mhi_chan->chan); + kfree(buf_info->cb_buf); + } + } + } + break; + } /* CC_EOT */ + case MHI_EV_CC_OOB: + case MHI_EV_CC_DB_MODE: + { + unsigned long flags; + + mhi_chan->db_cfg.db_mode = 1; + read_lock_irqsave(&mhi_cntrl->pm_lock, flags); + if (tre_ring->wp != tre_ring->rp && + MHI_DB_ACCESS_VALID(mhi_cntrl)) { + mhi_ring_chan_db(mhi_cntrl, mhi_chan); + } + read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags); + break; + } + case MHI_EV_CC_BAD_TRE: + default: + MHI_ERR("Unknown event 0x%x\n", ev_code); + panic("Unknown event 0x%x\n", ev_code); + break; + } /* switch(MHI_EV_READ_CODE(EV_TRB_CODE,event)) */ + +end_process_tx_event: + if (ev_code >= MHI_EV_CC_OOB) + write_unlock_irqrestore(&mhi_chan->lock, flags); + else + read_unlock_bh(&mhi_chan->lock); + + return 0; +} + +static int parse_rsc_event(struct mhi_controller *mhi_cntrl, + struct mhi_tre *event, + struct mhi_chan *mhi_chan) +{ + struct mhi_ring *buf_ring, *tre_ring; + struct mhi_buf_info *buf_info; + struct mhi_result result; + int ev_code; + u32 cookie; /* offset to local descriptor */ + u16 xfer_len; + + buf_ring = &mhi_chan->buf_ring; + tre_ring = &mhi_chan->tre_ring; + + ev_code = MHI_TRE_GET_EV_CODE(event); + cookie = MHI_TRE_GET_EV_COOKIE(event); + xfer_len = MHI_TRE_GET_EV_LEN(event); + + /* Received out of bound cookie */ + WARN_ON(cookie >= buf_ring->len); + + buf_info = buf_ring->base + cookie; + + result.transaction_status = (ev_code == MHI_EV_CC_OVERFLOW) ? + -EOVERFLOW : 0; + + /* truncate to buf len if xfer_len is larger */ + result.bytes_xferd = min_t(u16, xfer_len, buf_info->len); + result.buf_addr = buf_info->cb_buf; + result.dir = mhi_chan->dir; + + read_lock_bh(&mhi_chan->lock); + + if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED) + goto end_process_rsc_event; + + WARN_ON(!buf_info->used); + + /* notify the client */ + mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result); + + /* + * Note: We're arbitrarily incrementing RP even though, completion + * packet we processed might not be the same one, reason we can do this + * is because device guaranteed to cache descriptors in order it + * receive, so even though completion event is different we can re-use + * all descriptors in between. + * Example: + * Transfer Ring has descriptors: A, B, C, D + * Last descriptor host queue is D (WP) and first descriptor + * host queue is A (RP). + * The completion event we just serviced is descriptor C. + * Then we can safely queue descriptors to replace A, B, and C + * even though host did not receive any completions. + */ + mhi_del_ring_element(mhi_cntrl, tre_ring); + buf_info->used = false; + +end_process_rsc_event: + read_unlock_bh(&mhi_chan->lock); + + return 0; +} + +static void mhi_process_cmd_completion(struct mhi_controller *mhi_cntrl, + struct mhi_tre *tre) +{ + dma_addr_t ptr = MHI_TRE_GET_EV_PTR(tre); + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_cmd *cmd_ring = &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING]; + struct mhi_ring *mhi_ring = &cmd_ring->ring; + struct mhi_tre *cmd_pkt; + struct mhi_chan *mhi_chan; + u32 chan; + + if (!is_valid_ring_ptr(mhi_ring, ptr)) { + MHI_ERR("Event element points outside of the cmd ring\n"); + return; + } + + cmd_pkt = mhi_to_virtual(mhi_ring, ptr); + + if (cmd_pkt != mhi_ring->rp) + panic("Out of order cmd completion: 0x%llx. Expected: 0x%llx\n", + cmd_pkt, mhi_ring->rp); + + if (MHI_TRE_GET_CMD_TYPE(cmd_pkt) == MHI_CMD_SFR_CFG) { + mhi_misc_cmd_completion(mhi_cntrl, MHI_CMD_SFR_CFG, + MHI_TRE_GET_EV_CODE(tre)); + goto exit_cmd_completion; + } + + chan = MHI_TRE_GET_CMD_CHID(cmd_pkt); + if (chan >= mhi_cntrl->max_chan) { + MHI_ERR("Invalid channel id: %u\n", chan); + goto exit_cmd_completion; + } + mhi_chan = &mhi_cntrl->mhi_chan[chan]; + write_lock_bh(&mhi_chan->lock); + mhi_chan->ccs = MHI_TRE_GET_EV_CODE(tre); + complete(&mhi_chan->completion); + write_unlock_bh(&mhi_chan->lock); + +exit_cmd_completion: + mhi_del_ring_element(mhi_cntrl, mhi_ring); +} + +int mhi_process_ctrl_ev_ring(struct mhi_controller *mhi_cntrl, + struct mhi_event *mhi_event, + u32 event_quota) +{ + struct mhi_tre *dev_rp, *local_rp; + struct mhi_ring *ev_ring = &mhi_event->ring; + struct mhi_event_ctxt *er_ctxt = + &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index]; + struct mhi_chan *mhi_chan; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + u32 chan; + int count = 0; + dma_addr_t ptr = er_ctxt->rp; + + /* + * This is a quick check to avoid unnecessary event processing + * in case MHI is already in error state, but it's still possible + * to transition to error state while processing events + */ + if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state))) + return -EIO; + + if (!is_valid_ring_ptr(ev_ring, ptr)) { + MHI_ERR("Event ring rp points outside of the event ring\n"); + return -EIO; + } + + dev_rp = mhi_to_virtual(ev_ring, ptr); + local_rp = ev_ring->rp; + + while (dev_rp != local_rp) { + enum mhi_pkt_type type = MHI_TRE_GET_EV_TYPE(local_rp); + + MHI_VERB("Processing Event:0x%llx 0x%08x 0x%08x\n", + local_rp->ptr, local_rp->dword[0], local_rp->dword[1]); + + switch (type) { + case MHI_PKT_TYPE_BW_REQ_EVENT: + { + struct mhi_link_info *link_info; + + link_info = &mhi_cntrl->mhi_link_info; + write_lock_irq(&mhi_cntrl->pm_lock); + link_info->target_link_speed = + MHI_TRE_GET_EV_LINKSPEED(local_rp); + link_info->target_link_width = + MHI_TRE_GET_EV_LINKWIDTH(local_rp); + write_unlock_irq(&mhi_cntrl->pm_lock); + MHI_VERB("Received BW_REQ event\n"); + mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_BW_REQ); + break; + } + case MHI_PKT_TYPE_STATE_CHANGE_EVENT: + { + enum mhi_state new_state; + + new_state = MHI_TRE_GET_EV_STATE(local_rp); + + MHI_VERB("State change event to state: %s\n", + TO_MHI_STATE_STR(new_state)); + + switch (new_state) { + case MHI_STATE_M0: + mhi_pm_m0_transition(mhi_cntrl); + break; + case MHI_STATE_M1: + mhi_pm_m1_transition(mhi_cntrl); + break; + case MHI_STATE_M3: + mhi_pm_m3_transition(mhi_cntrl); + break; + case MHI_STATE_SYS_ERR: + { + enum mhi_pm_state new_state; + + MHI_VERB("System error detected\n"); + write_lock_irq(&mhi_cntrl->pm_lock); + new_state = mhi_tryset_pm_state(mhi_cntrl, + MHI_PM_SYS_ERR_DETECT); + write_unlock_irq(&mhi_cntrl->pm_lock); + if (new_state == MHI_PM_SYS_ERR_DETECT) + mhi_pm_sys_err_handler(mhi_cntrl); + break; + } + default: + MHI_ERR("Invalid state: %s\n", + TO_MHI_STATE_STR(new_state)); + } + + break; + } + case MHI_PKT_TYPE_CMD_COMPLETION_EVENT: + mhi_process_cmd_completion(mhi_cntrl, local_rp); + break; + case MHI_PKT_TYPE_EE_EVENT: + { + enum dev_st_transition st = DEV_ST_TRANSITION_MAX; + enum mhi_ee_type event = MHI_TRE_GET_EV_EXECENV(local_rp); + + MHI_VERB("Received EE event: %s\n", + TO_MHI_EXEC_STR(event)); + switch (event) { + case MHI_EE_SBL: + st = DEV_ST_TRANSITION_SBL; + break; + case MHI_EE_WFW: + case MHI_EE_AMSS: + st = DEV_ST_TRANSITION_MISSION_MODE; + break; + case MHI_EE_RDDM: + mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_RDDM); + write_lock_irq(&mhi_cntrl->pm_lock); + mhi_cntrl->ee = event; + write_unlock_irq(&mhi_cntrl->pm_lock); + wake_up_all(&mhi_cntrl->state_event); + break; + default: + MHI_ERR( + "Unhandled EE event: 0x%x\n", type); + } + if (st != DEV_ST_TRANSITION_MAX) + mhi_queue_state_transition(mhi_cntrl, st); + + break; + } + case MHI_PKT_TYPE_TX_EVENT: + chan = MHI_TRE_GET_EV_CHID(local_rp); + + WARN_ON(chan >= mhi_cntrl->max_chan); + + /* + * Only process the event ring elements whose channel + * ID is within the maximum supported range. + */ + if (chan < mhi_cntrl->max_chan) { + mhi_chan = &mhi_cntrl->mhi_chan[chan]; + if (!mhi_chan->configured) + break; + parse_xfer_event(mhi_cntrl, local_rp, mhi_chan); + event_quota--; + } + break; + default: + MHI_ERR("Unhandled event type: %d\n", type); + break; + } + + mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring); + local_rp = ev_ring->rp; + + ptr = er_ctxt->rp; + if (!is_valid_ring_ptr(ev_ring, ptr)) { + MHI_ERR("Event ring rp points outside of the event ring\n"); + return -EIO; + } + + dev_rp = mhi_to_virtual(ev_ring, ptr); + count++; + } + + read_lock_bh(&mhi_cntrl->pm_lock); + if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) + mhi_ring_er_db(mhi_event); + read_unlock_bh(&mhi_cntrl->pm_lock); + + return count; +} + +int mhi_process_data_event_ring(struct mhi_controller *mhi_cntrl, + struct mhi_event *mhi_event, + u32 event_quota) +{ + struct mhi_tre *dev_rp, *local_rp; + struct mhi_ring *ev_ring = &mhi_event->ring; + struct mhi_event_ctxt *er_ctxt = + &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index]; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + int count = 0; + u32 chan; + struct mhi_chan *mhi_chan; + dma_addr_t ptr = er_ctxt->rp; + + if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state))) + return -EIO; + + if (!is_valid_ring_ptr(ev_ring, ptr)) { + MHI_ERR("Event ring rp points outside of the event ring\n"); + return -EIO; + } + + dev_rp = mhi_to_virtual(ev_ring, ptr); + local_rp = ev_ring->rp; + + while (dev_rp != local_rp && event_quota > 0) { + enum mhi_pkt_type type = MHI_TRE_GET_EV_TYPE(local_rp); + + MHI_VERB("Processing Event:0x%llx 0x%08x 0x%08x\n", + local_rp->ptr, local_rp->dword[0], local_rp->dword[1]); + + chan = MHI_TRE_GET_EV_CHID(local_rp); + + WARN_ON(chan >= mhi_cntrl->max_chan); + + /* + * Only process the event ring elements whose channel + * ID is within the maximum supported range. + */ + if (chan < mhi_cntrl->max_chan && + mhi_cntrl->mhi_chan[chan].configured) { + mhi_chan = &mhi_cntrl->mhi_chan[chan]; + + if (likely(type == MHI_PKT_TYPE_TX_EVENT)) { + parse_xfer_event(mhi_cntrl, local_rp, mhi_chan); + event_quota--; + } else if (type == MHI_PKT_TYPE_RSC_TX_EVENT) { + parse_rsc_event(mhi_cntrl, local_rp, mhi_chan); + event_quota--; + } + } + + mhi_recycle_ev_ring_element(mhi_cntrl, ev_ring); + local_rp = ev_ring->rp; + + ptr = er_ctxt->rp; + if (!is_valid_ring_ptr(ev_ring, ptr)) { + MHI_ERR("Event ring rp points outside of the event ring\n"); + return -EIO; + } + + dev_rp = mhi_to_virtual(ev_ring, ptr); + count++; + } + read_lock_bh(&mhi_cntrl->pm_lock); + if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) + mhi_ring_er_db(mhi_event); + read_unlock_bh(&mhi_cntrl->pm_lock); + + return count; +} + +void mhi_ev_task(unsigned long data) +{ + unsigned long flags; + struct mhi_event *mhi_event = (struct mhi_event *)data; + struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl; + + /* process all pending events */ + spin_lock_irqsave(&mhi_event->lock, flags); + mhi_event->process_event(mhi_cntrl, mhi_event, U32_MAX); + spin_unlock_irqrestore(&mhi_event->lock, flags); +} + +void mhi_ctrl_ev_task(unsigned long data) +{ + struct mhi_event *mhi_event = (struct mhi_event *)data; + struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + enum mhi_state state; + enum mhi_pm_state pm_state = 0; + int ret; + + /* + * We can check PM state w/o a lock here because there is no way + * PM state can change from reg access valid to no access while this + * thread being executed. + */ + if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) { + /* + * We may have a pending event but not allowed to + * process it since we are probably in a suspended state, + * so trigger a resume. + */ + mhi_trigger_resume(mhi_cntrl); + + return; + } + + /* Process ctrl events events */ + ret = mhi_event->process_event(mhi_cntrl, mhi_event, U32_MAX); + + /* + * We received an IRQ but no events to process, maybe device went to + * SYS_ERR state? Check the state to confirm. + */ + if (!ret) { + write_lock_irq(&mhi_cntrl->pm_lock); + state = mhi_get_mhi_state(mhi_cntrl); + if (state == MHI_STATE_SYS_ERR) { + MHI_VERB("System error detected\n"); + pm_state = mhi_tryset_pm_state(mhi_cntrl, + MHI_PM_SYS_ERR_DETECT); + } + write_unlock_irq(&mhi_cntrl->pm_lock); + if (pm_state == MHI_PM_SYS_ERR_DETECT) + mhi_pm_sys_err_handler(mhi_cntrl); + } +} + +void mhi_process_ev_work(struct work_struct *work) +{ + struct mhi_event *mhi_event = container_of(work, struct mhi_event, + work); + struct mhi_controller *mhi_cntrl = mhi_event->mhi_cntrl; + struct device *dev = mhi_cntrl->cntrl_dev; + + MHI_VERB("Enter with pm_state:%s MHI_STATE:%s ee:%s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state), + TO_MHI_STATE_STR(mhi_cntrl->dev_state), + TO_MHI_EXEC_STR(mhi_cntrl->ee)); + + if (unlikely(MHI_EVENT_ACCESS_INVALID(mhi_cntrl->pm_state))) + return; + + mhi_event->process_event(mhi_cntrl, mhi_event, U32_MAX); +} + +static bool mhi_is_ring_full(struct mhi_controller *mhi_cntrl, + struct mhi_ring *ring) +{ + void *tmp = ring->wp + ring->el_size; + + if (tmp >= (ring->base + ring->len)) + tmp = ring->base; + + return (tmp == ring->rp); +} + +int mhi_queue_skb(struct mhi_device *mhi_dev, enum dma_data_direction dir, + struct sk_buff *skb, size_t len, enum mhi_flags mflags) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : + mhi_dev->dl_chan; + struct mhi_ring *tre_ring = &mhi_chan->tre_ring; + struct mhi_buf_info buf_info = { }; + int ret; + + /* If MHI host pre-allocates buffers then client drivers cannot queue */ + if (mhi_chan->pre_alloc) + return -EINVAL; + + if (mhi_is_ring_full(mhi_cntrl, tre_ring)) + return -ENOMEM; + + read_lock_bh(&mhi_cntrl->pm_lock); + if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) { + read_unlock_bh(&mhi_cntrl->pm_lock); + return -EIO; + } + + /* we're in M3 or transitioning to M3 */ + if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) + mhi_trigger_resume(mhi_cntrl); + + /* Toggle wake to exit out of M2 */ + mhi_cntrl->wake_toggle(mhi_cntrl); + + buf_info.v_addr = skb->data; + buf_info.cb_buf = skb; + buf_info.len = len; + + ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); + if (unlikely(ret)) { + read_unlock_bh(&mhi_cntrl->pm_lock); + return ret; + } + + if (mhi_chan->dir == DMA_TO_DEVICE) + atomic_inc(&mhi_cntrl->pending_pkts); + + if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { + read_lock_bh(&mhi_chan->lock); + mhi_ring_chan_db(mhi_cntrl, mhi_chan); + read_unlock_bh(&mhi_chan->lock); + } + + read_unlock_bh(&mhi_cntrl->pm_lock); + + return 0; +} +EXPORT_SYMBOL_GPL(mhi_queue_skb); + +int mhi_queue_dma(struct mhi_device *mhi_dev, enum dma_data_direction dir, + struct mhi_buf *mhi_buf, size_t len, enum mhi_flags mflags) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : + mhi_dev->dl_chan; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_ring *tre_ring = &mhi_chan->tre_ring; + struct mhi_buf_info buf_info = { }; + int ret; + + /* If MHI host pre-allocates buffers then client drivers cannot queue */ + if (mhi_chan->pre_alloc) + return -EINVAL; + + if (mhi_is_ring_full(mhi_cntrl, tre_ring)) + return -ENOMEM; + + read_lock_bh(&mhi_cntrl->pm_lock); + if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) { + MHI_ERR("MHI is not in activate state, PM state: %s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + read_unlock_bh(&mhi_cntrl->pm_lock); + + return -EIO; + } + + /* we're in M3 or transitioning to M3 */ + if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) + mhi_trigger_resume(mhi_cntrl); + + /* Toggle wake to exit out of M2 */ + mhi_cntrl->wake_toggle(mhi_cntrl); + + buf_info.p_addr = mhi_buf->dma_addr; + buf_info.cb_buf = mhi_buf; + buf_info.pre_mapped = true; + buf_info.len = len; + + ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); + if (unlikely(ret)) { + read_unlock_bh(&mhi_cntrl->pm_lock); + return ret; + } + + if (mhi_chan->dir == DMA_TO_DEVICE) + atomic_inc(&mhi_cntrl->pending_pkts); + + if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { + read_lock_bh(&mhi_chan->lock); + mhi_ring_chan_db(mhi_cntrl, mhi_chan); + read_unlock_bh(&mhi_chan->lock); + } + + read_unlock_bh(&mhi_cntrl->pm_lock); + + return 0; +} +EXPORT_SYMBOL_GPL(mhi_queue_dma); + +int mhi_gen_tre(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan, + struct mhi_buf_info *info, enum mhi_flags flags) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_ring *buf_ring, *tre_ring; + struct mhi_tre *mhi_tre; + struct mhi_buf_info *buf_info; + int eot, eob, chain, bei; + int ret; + + buf_ring = &mhi_chan->buf_ring; + tre_ring = &mhi_chan->tre_ring; + + buf_info = buf_ring->wp; + WARN_ON(buf_info->used); + buf_info->pre_mapped = info->pre_mapped; + if (info->pre_mapped) + buf_info->p_addr = info->p_addr; + else + buf_info->v_addr = info->v_addr; + buf_info->cb_buf = info->cb_buf; + buf_info->wp = tre_ring->wp; + buf_info->dir = mhi_chan->dir; + buf_info->len = info->len; + + if (!info->pre_mapped) { + ret = mhi_cntrl->map_single(mhi_cntrl, buf_info); + if (ret) + return ret; + } + + eob = !!(flags & MHI_EOB); + eot = !!(flags & MHI_EOT); + chain = !!(flags & MHI_CHAIN); + bei = !!(mhi_chan->intmod); + + mhi_tre = tre_ring->wp; + mhi_tre->ptr = MHI_TRE_DATA_PTR(buf_info->p_addr); + mhi_tre->dword[0] = MHI_TRE_DATA_DWORD0(info->len); + mhi_tre->dword[1] = MHI_TRE_DATA_DWORD1(bei, eot, eob, chain); + + MHI_VERB("Channel: %d WP: 0x%llx TRE: 0x%llx 0x%08x 0x%08x\n", + mhi_chan->chan, mhi_tre, mhi_tre->ptr, mhi_tre->dword[0], + mhi_tre->dword[1]); + + /* increment WP */ + mhi_add_ring_element(mhi_cntrl, tre_ring); + mhi_add_ring_element(mhi_cntrl, buf_ring); + + return 0; +} + +int mhi_queue_buf(struct mhi_device *mhi_dev, enum dma_data_direction dir, + void *buf, size_t len, enum mhi_flags mflags) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct mhi_chan *mhi_chan = (dir == DMA_TO_DEVICE) ? mhi_dev->ul_chan : + mhi_dev->dl_chan; + struct mhi_ring *tre_ring; + struct mhi_buf_info buf_info = { }; + unsigned long flags; + int ret; + + /* + * this check here only as a guard, it's always + * possible mhi can enter error while executing rest of function, + * which is not fatal so we do not need to hold pm_lock + */ + if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) + return -EIO; + + tre_ring = &mhi_chan->tre_ring; + if (mhi_is_ring_full(mhi_cntrl, tre_ring)) + return -ENOMEM; + + buf_info.v_addr = buf; + buf_info.cb_buf = buf; + buf_info.len = len; + + ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &buf_info, mflags); + if (unlikely(ret)) + return ret; + + read_lock_irqsave(&mhi_cntrl->pm_lock, flags); + + /* we're in M3 or transitioning to M3 */ + if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) + mhi_trigger_resume(mhi_cntrl); + + /* Toggle wake to exit out of M2 */ + mhi_cntrl->wake_toggle(mhi_cntrl); + + if (mhi_chan->dir == DMA_TO_DEVICE) + atomic_inc(&mhi_cntrl->pending_pkts); + + if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { + unsigned long flags; + + read_lock_irqsave(&mhi_chan->lock, flags); + mhi_ring_chan_db(mhi_cntrl, mhi_chan); + read_unlock_irqrestore(&mhi_chan->lock, flags); + } + + read_unlock_irqrestore(&mhi_cntrl->pm_lock, flags); + + return 0; +} +EXPORT_SYMBOL_GPL(mhi_queue_buf); + +int mhi_send_cmd(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan, + enum mhi_cmd_type cmd) +{ + struct mhi_tre *cmd_tre = NULL; + struct mhi_cmd *mhi_cmd = &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING]; + struct mhi_ring *ring = &mhi_cmd->ring; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + int chan = 0; + + if (mhi_chan) + chan = mhi_chan->chan; + + spin_lock_bh(&mhi_cmd->lock); + if (!get_nr_avail_ring_elements(mhi_cntrl, ring)) { + spin_unlock_bh(&mhi_cmd->lock); + return -ENOMEM; + } + + /* prepare the cmd tre */ + cmd_tre = ring->wp; + switch (cmd) { + case MHI_CMD_RESET_CHAN: + cmd_tre->ptr = MHI_TRE_CMD_RESET_PTR; + cmd_tre->dword[0] = MHI_TRE_CMD_RESET_DWORD0; + cmd_tre->dword[1] = MHI_TRE_CMD_RESET_DWORD1(chan); + break; + case MHI_CMD_STOP_CHAN: + cmd_tre->ptr = MHI_TRE_CMD_STOP_PTR; + cmd_tre->dword[0] = MHI_TRE_CMD_STOP_DWORD0; + cmd_tre->dword[1] = MHI_TRE_CMD_STOP_DWORD1(chan); + break; + case MHI_CMD_START_CHAN: + cmd_tre->ptr = MHI_TRE_CMD_START_PTR; + cmd_tre->dword[0] = MHI_TRE_CMD_START_DWORD0; + cmd_tre->dword[1] = MHI_TRE_CMD_START_DWORD1(chan); + break; + case MHI_CMD_SFR_CFG: + mhi_misc_cmd_configure(mhi_cntrl, MHI_CMD_SFR_CFG, + &cmd_tre->ptr, &cmd_tre->dword[0], + &cmd_tre->dword[1]); + break; + default: + MHI_ERR("Command not supported\n"); + break; + } + + /* queue to hardware */ + mhi_add_ring_element(mhi_cntrl, ring); + read_lock_bh(&mhi_cntrl->pm_lock); + if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) + mhi_ring_cmd_db(mhi_cntrl, mhi_cmd); + read_unlock_bh(&mhi_cntrl->pm_lock); + spin_unlock_bh(&mhi_cmd->lock); + + return 0; +} + +static int mhi_update_channel_state(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan, + enum mhi_ch_state_type to_state) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + enum mhi_cmd_type cmd = MHI_CMD_NOP; + int ret = -EIO; + + MHI_VERB("Updating channel %s(%d) state to: %s\n", mhi_chan->name, + mhi_chan->chan, TO_CH_STATE_TYPE_STR(to_state)); + + switch (to_state) { + case MHI_CH_STATE_TYPE_RESET: + write_lock_irq(&mhi_chan->lock); + if (mhi_chan->ch_state != MHI_CH_STATE_STOP && + mhi_chan->ch_state != MHI_CH_STATE_ENABLED && + mhi_chan->ch_state != MHI_CH_STATE_SUSPENDED) { + write_unlock_irq(&mhi_chan->lock); + goto exit_invalid_state; + } + mhi_chan->ch_state = MHI_CH_STATE_DISABLED; + write_unlock_irq(&mhi_chan->lock); + + cmd = MHI_CMD_RESET_CHAN; + break; + case MHI_CH_STATE_TYPE_STOP: + if (mhi_chan->ch_state != MHI_CH_STATE_ENABLED) + goto exit_invalid_state; + + cmd = MHI_CMD_STOP_CHAN; + break; + case MHI_CH_STATE_TYPE_START: + if (mhi_chan->ch_state != MHI_CH_STATE_STOP && + mhi_chan->ch_state != MHI_CH_STATE_DISABLED) + goto exit_invalid_state; + + cmd = MHI_CMD_START_CHAN; + break; + default: + goto exit_invalid_state; + } + + /* bring host and device out of suspended states */ + ret = mhi_device_get_sync(mhi_cntrl->mhi_dev); + if (ret) + return ret; + mhi_cntrl->runtime_get(mhi_cntrl); + + reinit_completion(&mhi_chan->completion); + ret = mhi_send_cmd(mhi_cntrl, mhi_chan, cmd); + if (ret) { + MHI_ERR("Failed to send %s(%d) %s command\n", + mhi_chan->name, mhi_chan->chan, + TO_CH_STATE_TYPE_STR(to_state)); + goto exit_command_failure; + } + + ret = wait_for_completion_timeout(&mhi_chan->completion, + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + if (!ret || mhi_chan->ccs != MHI_EV_CC_SUCCESS) { + MHI_ERR("Failed to receive %s(%d) %s command completion\n", + mhi_chan->name, mhi_chan->chan, + TO_CH_STATE_TYPE_STR(to_state)); + ret = -EIO; + goto exit_command_failure; + } + + ret = 0; + + if (to_state != MHI_CH_STATE_TYPE_RESET) { + write_lock_irq(&mhi_chan->lock); + mhi_chan->ch_state = (to_state == MHI_CH_STATE_TYPE_START) ? + MHI_CH_STATE_ENABLED : MHI_CH_STATE_STOP; + write_unlock_irq(&mhi_chan->lock); + } + + MHI_VERB("Channel %s(%d) state change to %s successful\n", + mhi_chan->name, mhi_chan->chan, TO_CH_STATE_TYPE_STR(to_state)); + +exit_command_failure: + mhi_cntrl->runtime_put(mhi_cntrl); + mhi_device_put(mhi_cntrl->mhi_dev); + + return ret; + +exit_invalid_state: + MHI_ERR("Channel %s(%d) update to %s not allowed\n", + mhi_chan->name, mhi_chan->chan, TO_CH_STATE_TYPE_STR(to_state)); + + return -EINVAL; +} + +static void mhi_unprepare_channel(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan) +{ + int ret; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + + /* no more processing events for this channel */ + mutex_lock(&mhi_chan->mutex); + + if (!(BIT(mhi_cntrl->ee) & mhi_chan->ee_mask)) { + MHI_ERR( + "Current EE: %s Required EE Mask: 0x%x for chan: %s\n", + TO_MHI_EXEC_STR(mhi_cntrl->ee), mhi_chan->ee_mask, + mhi_chan->name); + goto exit_unprepare_channel; + } + + ret = mhi_update_channel_state(mhi_cntrl, mhi_chan, + MHI_CH_STATE_TYPE_RESET); + if (ret) + MHI_ERR("Failed to reset channel, still resetting\n"); + +exit_unprepare_channel: + write_lock_irq(&mhi_chan->lock); + mhi_chan->ch_state = MHI_CH_STATE_DISABLED; + write_unlock_irq(&mhi_chan->lock); + + if (!mhi_chan->offload_ch) { + mhi_reset_chan(mhi_cntrl, mhi_chan); + mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan); + } + MHI_VERB("chan:%d successfully reset\n", mhi_chan->chan); + + mutex_unlock(&mhi_chan->mutex); +} + +int mhi_prepare_channel(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan) +{ + int ret = 0; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + + if (!(BIT(mhi_cntrl->ee) & mhi_chan->ee_mask)) { + MHI_ERR( + "Current EE: %s Required EE Mask: 0x%x for chan: %s\n", + TO_MHI_EXEC_STR(mhi_cntrl->ee), mhi_chan->ee_mask, + mhi_chan->name); + return -ENOTCONN; + } + + mutex_lock(&mhi_chan->mutex); + + /* Check of client manages channel context for offload channels */ + if (!mhi_chan->offload_ch) { + ret = mhi_init_chan_ctxt(mhi_cntrl, mhi_chan); + if (ret) + goto error_init_chan; + } + + ret = mhi_update_channel_state(mhi_cntrl, mhi_chan, + MHI_CH_STATE_TYPE_START); + if (ret) + goto error_pm_state; + + /* Pre-allocate buffer for xfer ring */ + if (mhi_chan->pre_alloc) { + int nr_el = get_nr_avail_ring_elements(mhi_cntrl, + &mhi_chan->tre_ring); + size_t len = mhi_cntrl->buffer_len; + + while (nr_el--) { + void *buf; + struct mhi_buf_info info = { }; + buf = kmalloc(len, GFP_KERNEL); + if (!buf) { + ret = -ENOMEM; + goto error_pre_alloc; + } + + /* Prepare transfer descriptors */ + info.v_addr = buf; + info.cb_buf = buf; + info.len = len; + ret = mhi_gen_tre(mhi_cntrl, mhi_chan, &info, MHI_EOT); + if (ret) { + kfree(buf); + goto error_pre_alloc; + } + } + + read_lock_bh(&mhi_cntrl->pm_lock); + if (MHI_DB_ACCESS_VALID(mhi_cntrl)) { + read_lock_irq(&mhi_chan->lock); + mhi_ring_chan_db(mhi_cntrl, mhi_chan); + read_unlock_irq(&mhi_chan->lock); + } + read_unlock_bh(&mhi_cntrl->pm_lock); + } + + mutex_unlock(&mhi_chan->mutex); + + return 0; + +error_pm_state: + if (!mhi_chan->offload_ch) + mhi_deinit_chan_ctxt(mhi_cntrl, mhi_chan); + +error_init_chan: + mutex_unlock(&mhi_chan->mutex); + + return ret; + +error_pre_alloc: + mutex_unlock(&mhi_chan->mutex); + mhi_unprepare_channel(mhi_cntrl, mhi_chan); + + return ret; +} + +static void mhi_mark_stale_events(struct mhi_controller *mhi_cntrl, + struct mhi_event *mhi_event, + struct mhi_event_ctxt *er_ctxt, + int chan) + +{ + struct mhi_tre *dev_rp, *local_rp; + struct mhi_ring *ev_ring; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + unsigned long flags; + dma_addr_t ptr; + + MHI_VERB("Marking all events for chan: %d as stale\n", chan); + + ev_ring = &mhi_event->ring; + + /* mark all stale events related to channel as STALE event */ + spin_lock_irqsave(&mhi_event->lock, flags); + + ptr = er_ctxt->rp; + if (!is_valid_ring_ptr(ev_ring, ptr)) { + MHI_ERR("Event ring rp points outside of the event ring\n"); + dev_rp = ev_ring->rp; + } else { + dev_rp = mhi_to_virtual(ev_ring, ptr); + } + + local_rp = ev_ring->rp; + while (dev_rp != local_rp) { + if (MHI_TRE_GET_EV_TYPE(local_rp) == MHI_PKT_TYPE_TX_EVENT && + chan == MHI_TRE_GET_EV_CHID(local_rp)) + local_rp->dword[1] = MHI_TRE_EV_DWORD1(chan, + MHI_PKT_TYPE_STALE_EVENT); + local_rp++; + if (local_rp == (ev_ring->base + ev_ring->len)) + local_rp = ev_ring->base; + } + + MHI_VERB("Finished marking events as stale events\n"); + spin_unlock_irqrestore(&mhi_event->lock, flags); +} + +static void mhi_reset_data_chan(struct mhi_controller *mhi_cntrl, + struct mhi_chan *mhi_chan) +{ + struct mhi_ring *buf_ring, *tre_ring; + struct mhi_result result; + + /* Reset any pending buffers */ + buf_ring = &mhi_chan->buf_ring; + tre_ring = &mhi_chan->tre_ring; + result.transaction_status = -ENOTCONN; + result.bytes_xferd = 0; + while (tre_ring->rp != tre_ring->wp) { + struct mhi_buf_info *buf_info = buf_ring->rp; + + if (mhi_chan->dir == DMA_TO_DEVICE) + atomic_dec(&mhi_cntrl->pending_pkts); + + if (!buf_info->pre_mapped) + mhi_cntrl->unmap_single(mhi_cntrl, buf_info); + + mhi_del_ring_element(mhi_cntrl, buf_ring); + mhi_del_ring_element(mhi_cntrl, tre_ring); + + if (mhi_chan->pre_alloc) { + kfree(buf_info->cb_buf); + } else { + result.buf_addr = buf_info->cb_buf; + mhi_chan->xfer_cb(mhi_chan->mhi_dev, &result); + } + } +} + +void mhi_reset_chan(struct mhi_controller *mhi_cntrl, struct mhi_chan *mhi_chan) +{ + struct mhi_event *mhi_event; + struct mhi_event_ctxt *er_ctxt; + int chan = mhi_chan->chan; + + /* Nothing to reset, client doesn't queue buffers */ + if (mhi_chan->offload_ch) + return; + + read_lock_bh(&mhi_cntrl->pm_lock); + mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index]; + er_ctxt = &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_chan->er_index]; + + mhi_mark_stale_events(mhi_cntrl, mhi_event, er_ctxt, chan); + + mhi_reset_data_chan(mhi_cntrl, mhi_chan); + + read_unlock_bh(&mhi_cntrl->pm_lock); +} + +/* Move channel to start state */ +int mhi_prepare_for_transfer(struct mhi_device *mhi_dev) +{ + int ret, dir; + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct mhi_chan *mhi_chan; + + for (dir = 0; dir < 2; dir++) { + mhi_chan = dir ? mhi_dev->dl_chan : mhi_dev->ul_chan; + if (!mhi_chan) + continue; + + ret = mhi_prepare_channel(mhi_cntrl, mhi_chan); + if (ret) + goto error_open_chan; + } + + return 0; + +error_open_chan: + for (--dir; dir >= 0; dir--) { + mhi_chan = dir ? mhi_dev->dl_chan : mhi_dev->ul_chan; + if (!mhi_chan) + continue; + + mhi_unprepare_channel(mhi_cntrl, mhi_chan); + } + + return ret; +} +EXPORT_SYMBOL_GPL(mhi_prepare_for_transfer); + +void mhi_unprepare_from_transfer(struct mhi_device *mhi_dev) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct mhi_chan *mhi_chan; + int dir; + + for (dir = 0; dir < 2; dir++) { + mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan; + if (!mhi_chan) + continue; + + mhi_unprepare_channel(mhi_cntrl, mhi_chan); + } +} +EXPORT_SYMBOL_GPL(mhi_unprepare_from_transfer); + +static int mhi_update_transfer_state(struct mhi_device *mhi_dev, + enum mhi_ch_state_type to_state) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_chan *mhi_chan; + struct mhi_chan_ctxt *chan_ctxt; + int dir, ret; + + for (dir = 0; dir < 2; dir++) { + mhi_chan = dir ? mhi_dev->ul_chan : mhi_dev->dl_chan; + + if (!mhi_chan) + continue; + + /* + * Bail out if one of the channels fails as client will reset + * both upon failure + */ + mutex_lock(&mhi_chan->mutex); + chan_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[mhi_chan->chan]; + if (!(chan_ctxt->chcfg & CHAN_CTX_CHSTATE_MASK)) { + mutex_unlock(&mhi_chan->mutex); + MHI_ERR("Channel %s(%u) context not initialized\n", + mhi_chan->name, mhi_chan->chan); + return -EINVAL; + } + ret = mhi_update_channel_state(mhi_cntrl, mhi_chan, to_state); + if (ret) { + mutex_unlock(&mhi_chan->mutex); + return ret; + } + mutex_unlock(&mhi_chan->mutex); + } + + return 0; +} + +int mhi_stop_transfer(struct mhi_device *mhi_dev) +{ + return mhi_update_transfer_state(mhi_dev, MHI_CH_STATE_TYPE_STOP); +} +EXPORT_SYMBOL(mhi_stop_transfer); + +int mhi_start_transfer(struct mhi_device *mhi_dev) +{ + return mhi_update_transfer_state(mhi_dev, MHI_CH_STATE_TYPE_START); +} +EXPORT_SYMBOL(mhi_start_transfer); + +int mhi_poll(struct mhi_device *mhi_dev, u32 budget) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct mhi_chan *mhi_chan = mhi_dev->dl_chan; + struct mhi_event *mhi_event = &mhi_cntrl->mhi_event[mhi_chan->er_index]; + int ret; + + spin_lock_bh(&mhi_event->lock); + ret = mhi_event->process_event(mhi_cntrl, mhi_event, budget); + spin_unlock_bh(&mhi_event->lock); + + return ret; +} +EXPORT_SYMBOL_GPL(mhi_poll); diff --git a/mhi/core/misc.c b/mhi/core/misc.c new file mode 100644 index 0000000..03792de --- /dev/null +++ b/mhi/core/misc.c @@ -0,0 +1,1674 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * + */ + +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/mod_devicetable.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/wait.h> +#include "internal.h" + +#ifdef CONFIG_MHI_BUS_DEBUG +#define MHI_MISC_DEBUG_LEVEL MHI_MSG_LVL_VERBOSE +#else +#define MHI_MISC_DEBUG_LEVEL MHI_MSG_LVL_ERROR +#endif + +const char * const mhi_log_level_str[MHI_MSG_LVL_MAX] = { + [MHI_MSG_LVL_VERBOSE] = "Verbose", + [MHI_MSG_LVL_INFO] = "Info", + [MHI_MSG_LVL_ERROR] = "Error", + [MHI_MSG_LVL_CRITICAL] = "Critical", + [MHI_MSG_LVL_MASK_ALL] = "Mask all", +}; +#define TO_MHI_LOG_LEVEL_STR(level) ((level >= MHI_MSG_LVL_MAX || \ + !mhi_log_level_str[level]) ? \ + "Mask all" : mhi_log_level_str[level]) + +struct mhi_bus mhi_bus; + +void mhi_misc_init(void) +{ + mutex_init(&mhi_bus.lock); + INIT_LIST_HEAD(&mhi_bus.controller_list); +} + +void mhi_misc_exit(void) +{ + mutex_destroy(&mhi_bus.lock); +} + +static ssize_t time_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mhi_device *mhi_dev = to_mhi_device(dev); + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + u64 t_host, t_device; + int ret; + + ret = mhi_get_remote_time_sync(mhi_dev, &t_host, &t_device); + if (ret) { + MHI_ERR("Failed to obtain time, ret:%d\n", ret); + return scnprintf(buf, PAGE_SIZE, + "Request failed or feature unsupported\n"); + } + + return scnprintf(buf, PAGE_SIZE, "local: %llu remote: %llu (ticks)\n", + t_host, t_device); +} +static DEVICE_ATTR_RO(time); + +static void mhi_time_async_cb(struct mhi_device *mhi_dev, u32 sequence, + u64 local_time, u64 remote_time) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct device *dev = &mhi_dev->dev; + + MHI_LOG("Time response: seq:%llx local: %llu remote: %llu (ticks)\n", + sequence, local_time, remote_time); +} + +static ssize_t time_async_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mhi_device *mhi_dev = to_mhi_device(dev); + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + u32 seq = prandom_u32(); + int ret; + + if (!seq) + seq = 1; + + ret = mhi_get_remote_time(mhi_dev, seq, &mhi_time_async_cb); + if (ret) { + MHI_ERR("Failed to request time, seq:%llx, ret:%d\n", seq, ret); + return scnprintf(buf, PAGE_SIZE, + "Request failed or feature unsupported\n"); + } + + return scnprintf(buf, PAGE_SIZE, + "Requested time asynchronously with seq:%llx\n", seq); +} +static DEVICE_ATTR_RO(time_async); + +static struct attribute *mhi_tsync_attrs[] = { + &dev_attr_time.attr, + &dev_attr_time_async.attr, + NULL, +}; + +static const struct attribute_group mhi_tsync_group = { + .attrs = mhi_tsync_attrs, +}; + +static ssize_t log_level_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct mhi_device *mhi_dev = to_mhi_device(dev); + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct mhi_private *mhi_priv = dev_get_drvdata(&mhi_cntrl->mhi_dev->dev); + + if (!mhi_priv) + return -EIO; + + return scnprintf(buf, PAGE_SIZE, "IPC log level begins from: %s\n", + TO_MHI_LOG_LEVEL_STR(mhi_priv->log_lvl)); +} + +static ssize_t log_level_store(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct mhi_device *mhi_dev = to_mhi_device(dev); + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct mhi_private *mhi_priv = dev_get_drvdata(&mhi_cntrl->mhi_dev->dev); + enum MHI_DEBUG_LEVEL log_level; + + if (kstrtou32(buf, 0, &log_level) < 0) + return -EINVAL; + + if (!mhi_priv) + return -EIO; + + mhi_priv->log_lvl = log_level; + + MHI_LOG("IPC log level changed to: %s\n", + TO_MHI_LOG_LEVEL_STR(log_level)); + + return count; +} +static DEVICE_ATTR_RW(log_level); + +static struct attribute *mhi_misc_attrs[] = { + &dev_attr_log_level.attr, + NULL, +}; + +static const struct attribute_group mhi_misc_group = { + .attrs = mhi_misc_attrs, +}; + +int mhi_misc_register_controller(struct mhi_controller *mhi_cntrl) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = kzalloc(sizeof(*mhi_priv), GFP_KERNEL); + struct mhi_device *mhi_dev = mhi_cntrl->mhi_dev; + struct pci_dev *parent = to_pci_dev(mhi_cntrl->cntrl_dev); + int ret; + + if (!mhi_priv) + return -ENOMEM; + + if (parent) { + dev_set_name(&mhi_dev->dev, "mhi_%04x_%02u.%02u.%02u", + parent->device, pci_domain_nr(parent->bus), + parent->bus->number, PCI_SLOT(parent->devfn)); + mhi_dev->name = dev_name(&mhi_dev->dev); + } + + mhi_priv->log_buf = ipc_log_context_create(MHI_IPC_LOG_PAGES, + mhi_dev->name, 0); + mhi_priv->log_lvl = MHI_MISC_DEBUG_LEVEL; + mhi_priv->mhi_cntrl = mhi_cntrl; + + /* adding it to this list only for debug purpose */ + mutex_lock(&mhi_bus.lock); + list_add_tail(&mhi_priv->node, &mhi_bus.controller_list); + mutex_unlock(&mhi_bus.lock); + + dev_set_drvdata(dev, mhi_priv); + + ret = sysfs_create_group(&dev->kobj, &mhi_misc_group); + if (ret) + MHI_ERR("Failed to create misc sysfs group\n"); + + ret = sysfs_create_group(&dev->kobj, &mhi_tsync_group); + if (ret) + MHI_ERR("Failed to create time synchronization sysfs group\n"); + + return 0; +} + +void mhi_misc_unregister_controller(struct mhi_controller *mhi_cntrl) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(&mhi_cntrl->mhi_dev->dev); + + if (!mhi_priv) + return; + + mutex_lock(&mhi_bus.lock); + list_del(&mhi_priv->node); + mutex_unlock(&mhi_bus.lock); + + sysfs_remove_group(&dev->kobj, &mhi_tsync_group); + sysfs_remove_group(&dev->kobj, &mhi_misc_group); + + if (mhi_priv->sfr_info) + kfree(mhi_priv->sfr_info->str); + kfree(mhi_priv->sfr_info); + kfree(mhi_priv->timesync); + kfree(mhi_priv); +} + +void *mhi_controller_get_privdata(struct mhi_controller *mhi_cntrl) +{ + struct mhi_device *mhi_dev; + struct mhi_private *mhi_priv; + + if (!mhi_cntrl) + return NULL; + + mhi_dev = mhi_cntrl->mhi_dev; + if (!mhi_dev) + return NULL; + + mhi_priv = dev_get_drvdata(&mhi_dev->dev); + if (!mhi_priv) + return NULL; + + return mhi_priv->priv_data; +} +EXPORT_SYMBOL(mhi_controller_get_privdata); + +void mhi_controller_set_privdata(struct mhi_controller *mhi_cntrl, void *priv) +{ + struct mhi_device *mhi_dev; + struct mhi_private *mhi_priv; + + if (!mhi_cntrl) + return; + + mhi_dev = mhi_cntrl->mhi_dev; + if (!mhi_dev) + return; + + mhi_priv = dev_get_drvdata(&mhi_dev->dev); + if (!mhi_priv) + return; + + mhi_priv->priv_data = priv; +} +EXPORT_SYMBOL(mhi_controller_set_privdata); + +static struct mhi_controller *find_mhi_controller_by_name(const char *name) +{ + struct mhi_private *mhi_priv, *tmp_priv; + struct mhi_controller *mhi_cntrl; + + list_for_each_entry_safe(mhi_priv, tmp_priv, &mhi_bus.controller_list, + node) { + mhi_cntrl = mhi_priv->mhi_cntrl; + if (mhi_cntrl->mhi_dev->name && (!strcmp(name, mhi_cntrl->mhi_dev->name))) + return mhi_cntrl; + } + + return NULL; +} + +struct mhi_controller *mhi_bdf_to_controller(u32 domain, + u32 bus, + u32 slot, + u32 dev_id) +{ + char name[32]; + + snprintf(name, sizeof(name), "mhi_%04x_%02u.%02u.%02u", dev_id, domain, + bus, slot); + + return find_mhi_controller_by_name(name); +} +EXPORT_SYMBOL(mhi_bdf_to_controller); + +static int mhi_notify_fatal_cb(struct device *dev, void *data) +{ + mhi_notify(to_mhi_device(dev), MHI_CB_FATAL_ERROR); + + return 0; +} + +int mhi_report_error(struct mhi_controller *mhi_cntrl) +{ + struct device *dev; + struct mhi_private *mhi_priv; + struct mhi_sfr_info *sfr_info; + enum mhi_pm_state cur_state; + + if (!mhi_cntrl) + return -EINVAL; + + dev = &mhi_cntrl->mhi_dev->dev; + mhi_priv = dev_get_drvdata(dev); + sfr_info = mhi_priv->sfr_info; + + write_lock_irq(&mhi_cntrl->pm_lock); + + cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_SYS_ERR_DETECT); + if (cur_state != MHI_PM_SYS_ERR_DETECT) { + dev_err(mhi_cntrl->cntrl_dev, + "Failed to move to state: %s from: %s\n", + to_mhi_pm_state_str(MHI_PM_SYS_ERR_DETECT), + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + return -EPERM; + } + + /* force inactive/error state */ + mhi_cntrl->dev_state = MHI_STATE_SYS_ERR; + wake_up_all(&mhi_cntrl->state_event); + write_unlock_irq(&mhi_cntrl->pm_lock); + + /* copy subsystem failure reason string if supported */ + if (sfr_info && sfr_info->buf_addr) { + memcpy(sfr_info->str, sfr_info->buf_addr, sfr_info->len); + MHI_ERR("mhi: %s sfr: %s\n", dev_name(dev), sfr_info->buf_addr); + } + + /* Notify fatal error to all client drivers to halt processing */ + device_for_each_child(&mhi_cntrl->mhi_dev->dev, NULL, + mhi_notify_fatal_cb); + + return 0; +} +EXPORT_SYMBOL(mhi_report_error); + +int mhi_device_configure(struct mhi_device *mhi_dev, + enum dma_data_direction dir, + struct mhi_buf *cfg_tbl, + int elements) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_chan *mhi_chan; + struct mhi_event_ctxt *er_ctxt; + struct mhi_chan_ctxt *ch_ctxt; + int er_index, chan; + + switch (dir) { + case DMA_TO_DEVICE: + mhi_chan = mhi_dev->ul_chan; + break; + case DMA_BIDIRECTIONAL: + case DMA_FROM_DEVICE: + case DMA_NONE: + mhi_chan = mhi_dev->dl_chan; + break; + default: + return -EINVAL; + } + + er_index = mhi_chan->er_index; + chan = mhi_chan->chan; + + for (; elements > 0; elements--, cfg_tbl++) { + /* update event context array */ + if (!strcmp(cfg_tbl->name, "ECA")) { + er_ctxt = &mhi_cntrl->mhi_ctxt->er_ctxt[er_index]; + if (sizeof(*er_ctxt) != cfg_tbl->len) { + MHI_ERR( + "Invalid ECA size, expected:%zu actual%zu\n", + sizeof(*er_ctxt), cfg_tbl->len); + return -EINVAL; + } + memcpy((void *)er_ctxt, cfg_tbl->buf, sizeof(*er_ctxt)); + continue; + } + + /* update channel context array */ + if (!strcmp(cfg_tbl->name, "CCA")) { + ch_ctxt = &mhi_cntrl->mhi_ctxt->chan_ctxt[chan]; + if (cfg_tbl->len != sizeof(*ch_ctxt)) { + MHI_ERR( + "Invalid CCA size, expected:%zu actual:%zu\n", + sizeof(*ch_ctxt), cfg_tbl->len); + return -EINVAL; + } + memcpy((void *)ch_ctxt, cfg_tbl->buf, sizeof(*ch_ctxt)); + continue; + } + + return -EINVAL; + } + + return 0; +} +EXPORT_SYMBOL(mhi_device_configure); + +void mhi_set_m2_timeout_ms(struct mhi_controller *mhi_cntrl, u32 timeout) +{ + struct mhi_private *mhi_priv = dev_get_drvdata(&mhi_cntrl->mhi_dev->dev); + + mhi_priv->m2_timeout_ms = timeout; +} +EXPORT_SYMBOL(mhi_set_m2_timeout_ms); + +int mhi_pm_fast_resume(struct mhi_controller *mhi_cntrl, bool notify_clients) +{ + struct mhi_chan *itr, *tmp; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + + MHI_VERB("Entered with PM state: %s, MHI state: %s notify: %s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state), + TO_MHI_STATE_STR(mhi_cntrl->dev_state), + notify_clients ? "true" : "false"); + + if (mhi_cntrl->pm_state == MHI_PM_DISABLE) + return 0; + + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) + return -EIO; + + read_lock_bh(&mhi_cntrl->pm_lock); + WARN_ON(mhi_cntrl->pm_state != MHI_PM_M3); + read_unlock_bh(&mhi_cntrl->pm_lock); + + if (mhi_cntrl->rddm_image && mhi_get_exec_env(mhi_cntrl) == MHI_EE_RDDM + && mhi_is_active(mhi_cntrl)) { + mhi_cntrl->ee = MHI_EE_RDDM; + + MHI_ERR("RDDM event occurred!\n"); + + /* notify critical clients with early notifications */ + mhi_report_error(mhi_cntrl); + + mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_RDDM); + wake_up_all(&mhi_cntrl->state_event); + + return 0; + } + + /* Notify clients about exiting LPM */ + if (notify_clients) { + list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, + node) { + mutex_lock(&itr->mutex); + if (itr->mhi_dev) + mhi_notify(itr->mhi_dev, MHI_CB_LPM_EXIT); + mutex_unlock(&itr->mutex); + } + } + + /* disable primary event ring processing to prevent interference */ + tasklet_disable(&mhi_cntrl->mhi_event->task); + + write_lock_irq(&mhi_cntrl->pm_lock); + + /* re-check to make sure no error has occurred before proceeding */ + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) { + write_unlock_irq(&mhi_cntrl->pm_lock); + tasklet_enable(&mhi_cntrl->mhi_event->task); + return -EIO; + } + + /* restore the states */ + mhi_cntrl->pm_state = mhi_priv->saved_pm_state; + mhi_cntrl->dev_state = mhi_priv->saved_dev_state; + + write_unlock_irq(&mhi_cntrl->pm_lock); + + switch (mhi_cntrl->pm_state) { + case MHI_PM_M0: + mhi_pm_m0_transition(mhi_cntrl); + break; + case MHI_PM_M2: + read_lock_bh(&mhi_cntrl->pm_lock); + mhi_cntrl->wake_get(mhi_cntrl, true); + mhi_cntrl->wake_put(mhi_cntrl, true); + read_unlock_bh(&mhi_cntrl->pm_lock); + break; + default: + MHI_ERR("Unexpected PM state:%s after restore\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + } + + /* enable primary event ring processing and check for events */ + tasklet_enable(&mhi_cntrl->mhi_event->task); + mhi_irq_handler(0, mhi_cntrl->mhi_event); + + return 0; +} +EXPORT_SYMBOL(mhi_pm_fast_resume); + +int mhi_pm_fast_suspend(struct mhi_controller *mhi_cntrl, bool notify_clients) +{ + struct mhi_chan *itr, *tmp; + struct mhi_device *mhi_dev = mhi_cntrl->mhi_dev; + struct device *dev = &mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + enum mhi_pm_state new_state; + int ret; + + if (mhi_cntrl->pm_state == MHI_PM_DISABLE) + return -EINVAL; + + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) + return -EIO; + + /* check if host/clients have any bus votes or packets to be sent */ + if (atomic_read(&mhi_cntrl->pending_pkts)) + return -EBUSY; + + /* wait for the device to attempt a low power mode (M2 entry) */ + wait_event_timeout(mhi_cntrl->state_event, + mhi_cntrl->dev_state == MHI_STATE_M2, + msecs_to_jiffies(mhi_priv->m2_timeout_ms)); + + /* disable primary event ring processing to prevent interference */ + tasklet_disable(&mhi_cntrl->mhi_event->task); + + write_lock_irq(&mhi_cntrl->pm_lock); + + /* re-check if host/clients have any bus votes or packets to be sent */ + if (atomic_read(&mhi_cntrl->pending_pkts)) { + ret = -EBUSY; + goto error_suspend; + } + + /* re-check to make sure no error has occurred before proceeding */ + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) { + ret = -EIO; + goto error_suspend; + } + + MHI_VERB("Allowing Fast M3 transition with notify: %s\n", + notify_clients ? "true" : "false"); + + /* save the current states */ + mhi_priv->saved_pm_state = mhi_cntrl->pm_state; + mhi_priv->saved_dev_state = mhi_cntrl->dev_state; + + /* move from M2 to M0 as device can allow the transition but not host */ + if (mhi_cntrl->pm_state == MHI_PM_M2) { + new_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M0); + if (new_state != MHI_PM_M0) { + MHI_ERR("Error setting to PM state: %s from: %s\n", + to_mhi_pm_state_str(MHI_PM_M0), + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + ret = -EIO; + goto error_suspend; + } + } + + new_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3_ENTER); + if (new_state != MHI_PM_M3_ENTER) { + MHI_ERR("Error setting to PM state: %s from: %s\n", + to_mhi_pm_state_str(MHI_PM_M3_ENTER), + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + ret = -EIO; + goto error_suspend; + } + + /* set dev_state to M3_FAST and host pm_state to M3 */ + new_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3); + if (new_state != MHI_PM_M3) { + MHI_ERR("Error setting to PM state: %s from: %s\n", + to_mhi_pm_state_str(MHI_PM_M3), + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + ret = -EIO; + goto error_suspend; + } + + mhi_cntrl->dev_state = MHI_STATE_M3_FAST; + mhi_cntrl->M3_fast++; + + write_unlock_irq(&mhi_cntrl->pm_lock); + + /* enable primary event ring processing and check for events */ + tasklet_enable(&mhi_cntrl->mhi_event->task); + mhi_irq_handler(0, mhi_cntrl->mhi_event); + + /* Notify clients about entering LPM */ + if (notify_clients) { + list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, + node) { + mutex_lock(&itr->mutex); + if (itr->mhi_dev) + mhi_notify(itr->mhi_dev, MHI_CB_LPM_ENTER); + mutex_unlock(&itr->mutex); + } + } + + return 0; + +error_suspend: + write_unlock_irq(&mhi_cntrl->pm_lock); + + /* enable primary event ring processing and check for events */ + tasklet_enable(&mhi_cntrl->mhi_event->task); + mhi_irq_handler(0, mhi_cntrl->mhi_event); + + return ret; +} +EXPORT_SYMBOL(mhi_pm_fast_suspend); + +static void mhi_process_sfr(struct mhi_controller *mhi_cntrl, + struct file_info *info) +{ + struct mhi_buf *mhi_buf = mhi_cntrl->rddm_image->mhi_buf; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + u8 *sfr_buf, *file_offset = info->file_offset; + u32 file_size = info->file_size; + u32 rem_seg_len = info->rem_seg_len; + u32 seg_idx = info->seg_idx; + + sfr_buf = kzalloc(file_size + 1, GFP_KERNEL); + if (!sfr_buf) + return; + + while (file_size) { + /* file offset starting from seg base */ + if (!rem_seg_len) { + file_offset = mhi_buf[seg_idx].buf; + if (file_size > mhi_buf[seg_idx].len) + rem_seg_len = mhi_buf[seg_idx].len; + else + rem_seg_len = file_size; + } + + if (file_size <= rem_seg_len) { + memcpy(sfr_buf, file_offset, file_size); + break; + } + + memcpy(sfr_buf, file_offset, rem_seg_len); + sfr_buf += rem_seg_len; + file_size -= rem_seg_len; + rem_seg_len = 0; + seg_idx++; + if (seg_idx == mhi_cntrl->rddm_image->entries) { + MHI_ERR("invalid size for SFR file\n"); + goto err; + } + } + sfr_buf[info->file_size] = '\0'; + + /* force sfr string to log in kernel msg */ + MHI_ERR("%s\n", sfr_buf); +err: + kfree(sfr_buf); +} + +static int mhi_find_next_file_offset(struct mhi_controller *mhi_cntrl, + struct file_info *info, + struct rddm_table_info *table_info) +{ + struct mhi_buf *mhi_buf = mhi_cntrl->rddm_image->mhi_buf; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + + if (info->rem_seg_len >= table_info->size) { + info->file_offset += table_info->size; + info->rem_seg_len -= table_info->size; + return 0; + } + + info->file_size = table_info->size - info->rem_seg_len; + info->rem_seg_len = 0; + /* iterate over segments until eof is reached */ + while (info->file_size) { + info->seg_idx++; + if (info->seg_idx == mhi_cntrl->rddm_image->entries) { + MHI_ERR("invalid size for file %s\n", + table_info->file_name); + return -EINVAL; + } + if (info->file_size > mhi_buf[info->seg_idx].len) { + info->file_size -= mhi_buf[info->seg_idx].len; + } else { + info->file_offset = mhi_buf[info->seg_idx].buf + + info->file_size; + info->rem_seg_len = mhi_buf[info->seg_idx].len - + info->file_size; + info->file_size = 0; + } + } + + return 0; +} + +void mhi_dump_sfr(struct mhi_controller *mhi_cntrl) +{ + struct mhi_buf *mhi_buf = mhi_cntrl->rddm_image->mhi_buf; + struct rddm_header *rddm_header = + (struct rddm_header *)mhi_buf->buf; + struct rddm_table_info *table_info; + struct file_info info; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + u32 table_size, n; + + memset(&info, 0, sizeof(info)); + + if (rddm_header->header_size > sizeof(*rddm_header) || + rddm_header->header_size < 8) { + MHI_ERR("invalid reported header size %u\n", + rddm_header->header_size); + return; + } + + table_size = (rddm_header->header_size - 8) / sizeof(*table_info); + if (!table_size) { + MHI_ERR("invalid rddm table size %u\n", table_size); + return; + } + + info.file_offset = (u8 *)rddm_header + rddm_header->header_size; + info.rem_seg_len = mhi_buf[0].len - rddm_header->header_size; + for (n = 0; n < table_size; n++) { + table_info = &rddm_header->table_info[n]; + + if (!strcmp(table_info->file_name, "Q6-SFR.bin")) { + info.file_size = table_info->size; + mhi_process_sfr(mhi_cntrl, &info); + return; + } + + if (mhi_find_next_file_offset(mhi_cntrl, &info, table_info)) + return; + } +} +EXPORT_SYMBOL(mhi_dump_sfr); + +bool mhi_scan_rddm_cookie(struct mhi_controller *mhi_cntrl, u32 cookie) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + int ret; + u32 val; + + if (!mhi_cntrl->rddm_image || !cookie) + return false; + + MHI_VERB("Checking BHI debug register for 0x%x\n", cookie); + + if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) + return false; + + ret = mhi_read_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_ERRDBG2, &val); + if (ret) + return false; + + MHI_VERB("BHI_ERRDBG2 value:0x%x\n", val); + if (val == cookie) + return true; + + return false; +} +EXPORT_SYMBOL(mhi_scan_rddm_cookie); + +void mhi_debug_reg_dump(struct mhi_controller *mhi_cntrl) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + enum mhi_state state; + enum mhi_ee_type ee; + int i, ret; + u32 val; + void __iomem *mhi_base = mhi_cntrl->regs; + void __iomem *bhi_base = mhi_cntrl->bhi; + void __iomem *bhie_base = mhi_cntrl->bhie; + void __iomem *wake_db = mhi_cntrl->wake_db; + struct { + const char *name; + int offset; + void __iomem *base; + } debug_reg[] = { + { "BHI_ERRDBG2", BHI_ERRDBG2, bhi_base}, + { "BHI_ERRDBG3", BHI_ERRDBG3, bhi_base}, + { "BHI_ERRDBG1", BHI_ERRDBG1, bhi_base}, + { "BHI_ERRCODE", BHI_ERRCODE, bhi_base}, + { "BHI_EXECENV", BHI_EXECENV, bhi_base}, + { "BHI_STATUS", BHI_STATUS, bhi_base}, + { "MHI_CNTRL", MHICTRL, mhi_base}, + { "MHI_STATUS", MHISTATUS, mhi_base}, + { "MHI_WAKE_DB", 0, wake_db}, + { "BHIE_TXVEC_DB", BHIE_TXVECDB_OFFS, bhie_base}, + { "BHIE_TXVEC_STATUS", BHIE_TXVECSTATUS_OFFS, bhie_base}, + { "BHIE_RXVEC_DB", BHIE_RXVECDB_OFFS, bhie_base}, + { "BHIE_RXVEC_STATUS", BHIE_RXVECSTATUS_OFFS, bhie_base}, + { NULL }, + }; + + MHI_ERR("host pm_state:%s dev_state:%s ee:%s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state), + TO_MHI_STATE_STR(mhi_cntrl->dev_state), + TO_MHI_EXEC_STR(mhi_cntrl->ee)); + + state = mhi_get_mhi_state(mhi_cntrl); + ee = mhi_get_exec_env(mhi_cntrl); + + MHI_ERR("device ee: %s dev_state: %s\n", TO_MHI_EXEC_STR(ee), + TO_MHI_STATE_STR(state)); + + for (i = 0; debug_reg[i].name; i++) { + if (!debug_reg[i].base) + continue; + ret = mhi_read_reg(mhi_cntrl, debug_reg[i].base, + debug_reg[i].offset, &val); + MHI_ERR("reg: %s val: 0x%x, ret: %d\n", debug_reg[i].name, + val, ret); + } +} +EXPORT_SYMBOL(mhi_debug_reg_dump); + +int mhi_device_get_sync_atomic(struct mhi_device *mhi_dev, int timeout_us, + bool in_panic) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct device *dev = &mhi_dev->dev; + + read_lock_bh(&mhi_cntrl->pm_lock); + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) { + read_unlock_bh(&mhi_cntrl->pm_lock); + return -EIO; + } + + mhi_cntrl->wake_get(mhi_cntrl, true); + read_unlock_bh(&mhi_cntrl->pm_lock); + + mhi_dev->dev_wake++; + pm_wakeup_event(&mhi_cntrl->mhi_dev->dev, 0); + mhi_cntrl->runtime_get(mhi_cntrl); + + /* Return if client doesn't want us to wait */ + if (!timeout_us) { + if (mhi_cntrl->pm_state != MHI_PM_M0) + MHI_ERR("Return without waiting for M0\n"); + + mhi_cntrl->runtime_put(mhi_cntrl); + return 0; + } + + if (in_panic) { + while (mhi_get_mhi_state(mhi_cntrl) != MHI_STATE_M0 && + !MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) && + timeout_us > 0) { + udelay(MHI_FORCE_WAKE_DELAY_US); + timeout_us -= MHI_FORCE_WAKE_DELAY_US; + } + } else { + while (mhi_cntrl->pm_state != MHI_PM_M0 && + !MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) && + timeout_us > 0) { + udelay(MHI_FORCE_WAKE_DELAY_US); + timeout_us -= MHI_FORCE_WAKE_DELAY_US; + } + } + + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) || timeout_us <= 0) { + MHI_ERR("Did not enter M0, cur_state: %s pm_state: %s\n", + TO_MHI_STATE_STR(mhi_cntrl->dev_state), + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + read_lock_bh(&mhi_cntrl->pm_lock); + mhi_cntrl->wake_put(mhi_cntrl, false); + read_unlock_bh(&mhi_cntrl->pm_lock); + mhi_dev->dev_wake--; + mhi_cntrl->runtime_put(mhi_cntrl); + return -ETIMEDOUT; + } + + mhi_cntrl->runtime_put(mhi_cntrl); + + return 0; +} +EXPORT_SYMBOL(mhi_device_get_sync_atomic); + +static int mhi_get_capability_offset(struct mhi_controller *mhi_cntrl, + u32 capability, u32 *offset) +{ + u32 cur_cap, next_offset; + int ret; + + /* get the 1st supported capability offset */ + ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, MISC_OFFSET, + MISC_CAP_MASK, MISC_CAP_SHIFT, offset); + if (ret) + return ret; + do { + ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, *offset, + CAP_CAPID_MASK, CAP_CAPID_SHIFT, + &cur_cap); + if (ret) + return ret; + + if (cur_cap == capability) + return 0; + + ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, *offset, + CAP_NEXT_CAP_MASK, CAP_NEXT_CAP_SHIFT, + &next_offset); + if (ret) + return ret; + + *offset = next_offset; + if (*offset >= MHI_REG_SIZE) + return -ENXIO; + } while (next_offset); + + return -ENXIO; +} + +/* to be used only if a single event ring with the type is present */ +static int mhi_get_er_index(struct mhi_controller *mhi_cntrl, + enum mhi_er_data_type type) +{ + int i; + struct mhi_event *mhi_event = mhi_cntrl->mhi_event; + + /* find event ring for requested type */ + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) { + if (mhi_event->data_type == type) + return mhi_event->er_index; + } + + return -ENOENT; +} + +static int mhi_init_bw_scale(struct mhi_controller *mhi_cntrl, + void __iomem *bw_scale_db) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + int ret, er_index; + u32 bw_cfg_offset; + + /* controller doesn't support dynamic bw switch */ + if (!mhi_priv->bw_scale) + return -ENODEV; + + ret = mhi_get_capability_offset(mhi_cntrl, BW_SCALE_CAP_ID, + &bw_cfg_offset); + if (ret) + return ret; + + /* No ER configured to support BW scale */ + er_index = mhi_get_er_index(mhi_cntrl, MHI_ER_BW_SCALE); + if (er_index < 0) + return er_index; + + bw_cfg_offset += BW_SCALE_CFG_OFFSET; + + mhi_priv->bw_scale_db = bw_scale_db; + + /* advertise host support */ + mhi_write_reg(mhi_cntrl, mhi_cntrl->regs, bw_cfg_offset, + MHI_BW_SCALE_SETUP(er_index)); + + MHI_VERB("Bandwidth scaling setup complete. Event ring:%d\n", + er_index); + + return 0; +} + +int mhi_controller_setup_timesync(struct mhi_controller *mhi_cntrl, + u64 (*time_get)(struct mhi_controller *c), + int (*lpm_disable)(struct mhi_controller *c), + int (*lpm_enable)(struct mhi_controller *c)) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + struct mhi_timesync *mhi_tsync = kzalloc(sizeof(*mhi_tsync), + GFP_KERNEL); + + if (!mhi_tsync) + return -ENOMEM; + + mhi_tsync->time_get = time_get; + mhi_tsync->lpm_disable = lpm_disable; + mhi_tsync->lpm_enable = lpm_enable; + + mhi_priv->timesync = mhi_tsync; + + return 0; +} +EXPORT_SYMBOL(mhi_controller_setup_timesync); + +static int mhi_init_timesync(struct mhi_controller *mhi_cntrl, + void __iomem *time_db) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + struct mhi_timesync *mhi_tsync = mhi_priv->timesync; + u32 time_offset; + int ret, er_index; + + if (!mhi_tsync) + return -EINVAL; + + ret = mhi_get_capability_offset(mhi_cntrl, TIMESYNC_CAP_ID, + &time_offset); + if (ret) + return ret; + + /* save time_offset for obtaining time via MMIO register reads */ + mhi_tsync->time_reg = mhi_cntrl->regs + time_offset; + + mutex_init(&mhi_tsync->mutex); + + /* get timesync event ring configuration */ + er_index = mhi_get_er_index(mhi_cntrl, MHI_ER_TIMESYNC); + if (er_index < 0) + return 0; + + spin_lock_init(&mhi_tsync->lock); + INIT_LIST_HEAD(&mhi_tsync->head); + + mhi_tsync->time_db = time_db; + + /* advertise host support */ + mhi_write_reg(mhi_cntrl, mhi_tsync->time_reg, TIMESYNC_CFG_OFFSET, + MHI_TIMESYNC_DB_SETUP(er_index)); + + MHI_VERB("Time synchronization DB mode setup complete. Event ring:%d\n", + er_index); + + return 0; +} + +int mhi_misc_init_mmio(struct mhi_controller *mhi_cntrl) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + u32 chdb_off; + int ret; + + /* Read channel db offset */ + ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, CHDBOFF, + CHDBOFF_CHDBOFF_MASK, + CHDBOFF_CHDBOFF_SHIFT, &chdb_off); + if (ret) { + MHI_ERR("Unable to read CHDBOFF register\n"); + return -EIO; + } + + ret = mhi_init_bw_scale(mhi_cntrl, (mhi_cntrl->regs + chdb_off + + (8 * MHI_BW_SCALE_CHAN_DB))); + if (ret) + MHI_LOG("BW scale setup failure\n"); + + ret = mhi_init_timesync(mhi_cntrl, (mhi_cntrl->regs + chdb_off + + (8 * MHI_TIMESYNC_CHAN_DB))); + if (ret) + MHI_LOG("Time synchronization setup failure\n"); + + return 0; +} + +/* Recycle by fast forwarding WP to the last posted event */ +static void mhi_recycle_fwd_ev_ring_element + (struct mhi_controller *mhi_cntrl, struct mhi_ring *ring) +{ + dma_addr_t ctxt_wp; + + /* update the WP */ + ring->wp += ring->el_size; + if (ring->wp >= (ring->base + ring->len)) + ring->wp = ring->base; + + /* update the context WP based on the RP to support fast forwarding */ + ctxt_wp = ring->iommu_base + (ring->wp - ring->base); + *ring->ctxt_wp = ctxt_wp; + + /* update the RP */ + ring->rp += ring->el_size; + if (ring->rp >= (ring->base + ring->len)) + ring->rp = ring->base; + + /* visible to other cores */ + smp_wmb(); +} + +/* dedicated bw scale event ring processing */ +int mhi_process_misc_tsync_ev_ring(struct mhi_controller *mhi_cntrl, + struct mhi_event *mhi_event, + u32 event_quota) +{ + struct mhi_tre *dev_rp; + struct mhi_ring *ev_ring = &mhi_event->ring; + struct mhi_event_ctxt *er_ctxt = + &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index]; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + struct mhi_timesync *mhi_tsync = mhi_priv->timesync; + u32 sequence; + u64 remote_time; + int ret = 0; + + spin_lock_bh(&mhi_event->lock); + dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp); + if (ev_ring->rp == dev_rp) { + spin_unlock_bh(&mhi_event->lock); + goto exit_tsync_process; + } + + /* if rp points to base, we need to wrap it around */ + if (dev_rp == ev_ring->base) + dev_rp = ev_ring->base + ev_ring->len; + dev_rp--; + + /* fast forward to currently processed element and recycle er */ + ev_ring->rp = dev_rp; + ev_ring->wp = dev_rp - 1; + if (ev_ring->wp < ev_ring->base) + ev_ring->wp = ev_ring->base + ev_ring->len - ev_ring->el_size; + mhi_recycle_fwd_ev_ring_element(mhi_cntrl, ev_ring); + + if (WARN_ON(MHI_TRE_GET_EV_TYPE(dev_rp) != MHI_PKT_TYPE_TSYNC_EVENT)) { + MHI_ERR("!TIMESYNC event\n"); + ret = -EINVAL; + spin_unlock_bh(&mhi_event->lock); + goto exit_tsync_process; + } + + sequence = MHI_TRE_GET_EV_SEQ(dev_rp); + remote_time = MHI_TRE_GET_EV_TIME(dev_rp); + + MHI_VERB("Received TSYNC event with seq: 0x%llx time: 0x%llx\n", + sequence, remote_time); + + read_lock_bh(&mhi_cntrl->pm_lock); + if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) + mhi_ring_er_db(mhi_event); + read_unlock_bh(&mhi_cntrl->pm_lock); + spin_unlock_bh(&mhi_event->lock); + + mutex_lock(&mhi_tsync->mutex); + + if (WARN_ON(mhi_tsync->int_sequence != sequence)) { + MHI_ERR("Unexpected response: 0x%llx Expected: 0x%llx\n", + sequence, mhi_tsync->int_sequence); + + mhi_cntrl->runtime_put(mhi_cntrl); + mhi_device_put(mhi_cntrl->mhi_dev); + + mutex_unlock(&mhi_tsync->mutex); + ret = -EINVAL; + goto exit_tsync_process; + } + + do { + struct tsync_node *tsync_node; + + spin_lock(&mhi_tsync->lock); + tsync_node = list_first_entry_or_null(&mhi_tsync->head, + struct tsync_node, node); + if (!tsync_node) { + spin_unlock(&mhi_tsync->lock); + break; + } + + list_del(&tsync_node->node); + spin_unlock(&mhi_tsync->lock); + + tsync_node->cb_func(tsync_node->mhi_dev, + tsync_node->sequence, + mhi_tsync->local_time, remote_time); + kfree(tsync_node); + } while (true); + + mhi_tsync->db_pending = false; + mhi_tsync->remote_time = remote_time; + complete(&mhi_tsync->completion); + + mhi_cntrl->runtime_put(mhi_cntrl); + mhi_device_put(mhi_cntrl->mhi_dev); + + mutex_unlock(&mhi_tsync->mutex); + +exit_tsync_process: + MHI_VERB("exit er_index: %u, ret: %d\n", mhi_event->er_index, ret); + + return ret; +} + +/* dedicated bw scale event ring processing */ +int mhi_process_misc_bw_ev_ring(struct mhi_controller *mhi_cntrl, + struct mhi_event *mhi_event, + u32 event_quota) +{ + struct mhi_tre *dev_rp; + struct mhi_ring *ev_ring = &mhi_event->ring; + struct mhi_event_ctxt *er_ctxt = + &mhi_cntrl->mhi_ctxt->er_ctxt[mhi_event->er_index]; + struct mhi_link_info link_info, *cur_info = &mhi_cntrl->mhi_link_info; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + u32 result = MHI_BW_SCALE_NACK; + int ret = -EINVAL; + + if (!MHI_IN_MISSION_MODE(mhi_cntrl->ee)) + goto exit_bw_scale_process; + + spin_lock_bh(&mhi_event->lock); + dev_rp = mhi_to_virtual(ev_ring, er_ctxt->rp); + + /* if rp points to base, we need to wrap it around */ + if (dev_rp == ev_ring->base) + dev_rp = ev_ring->base + ev_ring->len; + dev_rp--; + + /* fast forward to currently processed element and recycle er */ + ev_ring->rp = dev_rp; + ev_ring->wp = dev_rp - 1; + if (ev_ring->wp < ev_ring->base) + ev_ring->wp = ev_ring->base + ev_ring->len - ev_ring->el_size; + mhi_recycle_fwd_ev_ring_element(mhi_cntrl, ev_ring); + + if (WARN_ON(MHI_TRE_GET_EV_TYPE(dev_rp) != MHI_PKT_TYPE_BW_REQ_EVENT)) { + MHI_ERR("!BW SCALE REQ event\n"); + spin_unlock_bh(&mhi_event->lock); + goto exit_bw_scale_process; + } + + link_info.target_link_speed = MHI_TRE_GET_EV_LINKSPEED(dev_rp); + link_info.target_link_width = MHI_TRE_GET_EV_LINKWIDTH(dev_rp); + link_info.sequence_num = MHI_TRE_GET_EV_BW_REQ_SEQ(dev_rp); + + MHI_VERB("Received BW_REQ with seq:%d link speed:0x%x width:0x%x\n", + link_info.sequence_num, + link_info.target_link_speed, + link_info.target_link_width); + + read_lock_bh(&mhi_cntrl->pm_lock); + if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) + mhi_ring_er_db(mhi_event); + read_unlock_bh(&mhi_cntrl->pm_lock); + spin_unlock_bh(&mhi_event->lock); + + ret = mhi_device_get_sync(mhi_cntrl->mhi_dev); + if (ret) + goto exit_bw_scale_process; + mhi_cntrl->runtime_get(mhi_cntrl); + + mutex_lock(&mhi_cntrl->pm_mutex); + + ret = mhi_priv->bw_scale(mhi_cntrl, &link_info); + if (!ret) { + *cur_info = link_info; + result = 0; + } + + write_lock_bh(&mhi_cntrl->pm_lock); + mhi_priv->bw_response = MHI_BW_SCALE_RESULT(result, + link_info.sequence_num); + if (likely(MHI_DB_ACCESS_VALID(mhi_cntrl))) { + mhi_write_reg(mhi_cntrl, mhi_priv->bw_scale_db, 0, + mhi_priv->bw_response); + mhi_priv->bw_response = 0; + } else { + MHI_VERB("Cached BW response for seq: %u, result: %d\n", + link_info.sequence_num, mhi_priv->bw_response); + } + write_unlock_bh(&mhi_cntrl->pm_lock); + + mhi_cntrl->runtime_put(mhi_cntrl); + mhi_device_put(mhi_cntrl->mhi_dev); + + mutex_unlock(&mhi_cntrl->pm_mutex); + +exit_bw_scale_process: + MHI_VERB("exit er_index:%u ret:%d\n", mhi_event->er_index, ret); + + return ret; +} + +void mhi_misc_dbs_pending(struct mhi_controller *mhi_cntrl) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + + if (mhi_priv->bw_scale && mhi_priv->bw_response) { + mhi_write_reg(mhi_cntrl, mhi_priv->bw_scale_db, 0, + mhi_priv->bw_response); + MHI_VERB("Completed BW response: %d\n", mhi_priv->bw_response); + mhi_priv->bw_response = 0; + } +} + +void mhi_controller_set_bw_scale_cb(struct mhi_controller *mhi_cntrl, + int (*cb_func)(struct mhi_controller *mhi_cntrl, + struct mhi_link_info *link_info)) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + + mhi_priv->bw_scale = cb_func; +} +EXPORT_SYMBOL(mhi_controller_set_bw_scale_cb); + +void mhi_controller_set_base(struct mhi_controller *mhi_cntrl, phys_addr_t base) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + + mhi_priv->base_addr = base; +} +EXPORT_SYMBOL(mhi_controller_set_base); + +int mhi_get_channel_db_base(struct mhi_device *mhi_dev, phys_addr_t *value) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + u32 offset; + int ret; + + if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) + return -EIO; + + ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, CHDBOFF, + CHDBOFF_CHDBOFF_MASK, CHDBOFF_CHDBOFF_SHIFT, + &offset); + if (ret) + return -EIO; + + *value = mhi_priv->base_addr + offset; + + return ret; +} +EXPORT_SYMBOL(mhi_get_channel_db_base); + +int mhi_get_event_ring_db_base(struct mhi_device *mhi_dev, phys_addr_t *value) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + u32 offset; + int ret; + + if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) + return -EIO; + + ret = mhi_read_reg_field(mhi_cntrl, mhi_cntrl->regs, ERDBOFF, + ERDBOFF_ERDBOFF_MASK, ERDBOFF_ERDBOFF_SHIFT, + &offset); + if (ret) + return -EIO; + + *value = mhi_priv->base_addr + offset; + + return ret; +} +EXPORT_SYMBOL(mhi_get_event_ring_db_base); + +struct mhi_device *mhi_get_device_for_channel(struct mhi_controller *mhi_cntrl, + u32 channel) +{ + if (channel >= mhi_cntrl->max_chan) + return NULL; + + return mhi_cntrl->mhi_chan[channel].mhi_dev; +} +EXPORT_SYMBOL(mhi_get_device_for_channel); + +#if !IS_ENABLED(CONFIG_MHI_DTR) +long mhi_device_ioctl(struct mhi_device *mhi_dev, unsigned int cmd, + unsigned long arg) +{ + return -EIO; +} +EXPORT_SYMBOL(mhi_device_ioctl); +#endif + +int mhi_controller_set_sfr_support(struct mhi_controller *mhi_cntrl, size_t len) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + struct mhi_sfr_info *sfr_info; + + sfr_info = kzalloc(sizeof(*sfr_info), GFP_KERNEL); + if (!sfr_info) + return -ENOMEM; + + sfr_info->len = len; + sfr_info->str = kzalloc(len, GFP_KERNEL); + if (!sfr_info->str) + return -ENOMEM; + + mhi_priv->sfr_info = sfr_info; + + return 0; +} +EXPORT_SYMBOL(mhi_controller_set_sfr_support); + +void mhi_misc_mission_mode(struct mhi_controller *mhi_cntrl) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + struct mhi_sfr_info *sfr_info = mhi_priv->sfr_info; + u64 local, remote; + int ret = -EIO; + + /* Attempt to print local and remote SOC time delta for debug */ + ret = mhi_get_remote_time_sync(mhi_cntrl->mhi_dev, &local, &remote); + if (!ret) + MHI_LOG("Timesync: local: %llx, remote: %llx\n", local, remote); + + /* initialize SFR */ + if (!sfr_info) + return; + + /* do a clean-up if we reach here post SSR */ + memset(sfr_info->str, 0, sfr_info->len); + + sfr_info->buf_addr = dma_alloc_coherent(mhi_cntrl->cntrl_dev, + sfr_info->len, + &sfr_info->dma_addr, + GFP_KERNEL); + if (!sfr_info->buf_addr) { + MHI_ERR("Failed to allocate memory for sfr\n"); + return; + } + + init_completion(&sfr_info->completion); + + ret = mhi_send_cmd(mhi_cntrl, NULL, MHI_CMD_SFR_CFG); + if (ret) { + MHI_ERR("Failed to send sfr cfg cmd\n"); + return; + } + + ret = wait_for_completion_timeout(&sfr_info->completion, + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + if (!ret || sfr_info->ccs != MHI_EV_CC_SUCCESS) + MHI_ERR("Failed to get sfr cfg cmd completion\n"); +} + +void mhi_misc_disable(struct mhi_controller *mhi_cntrl) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + struct mhi_sfr_info *sfr_info = mhi_priv->sfr_info; + + if (sfr_info && sfr_info->buf_addr) { + dma_free_coherent(mhi_cntrl->cntrl_dev, sfr_info->len, + sfr_info->buf_addr, sfr_info->dma_addr); + sfr_info->buf_addr = NULL; + } +} + +void mhi_misc_cmd_configure(struct mhi_controller *mhi_cntrl, unsigned int type, + u64 *ptr, u32 *dword0, u32 *dword1) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + struct mhi_sfr_info *sfr_info = mhi_priv->sfr_info; + + if (type == MHI_CMD_SFR_CFG && sfr_info) { + *ptr = MHI_TRE_CMD_SFR_CFG_PTR(sfr_info->dma_addr); + *dword0 = MHI_TRE_CMD_SFR_CFG_DWORD0(sfr_info->len - 1); + *dword1 = MHI_TRE_CMD_SFR_CFG_DWORD1; + } +} + +void mhi_misc_cmd_completion(struct mhi_controller *mhi_cntrl, + unsigned int type, unsigned int ccs) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + struct mhi_sfr_info *sfr_info = mhi_priv->sfr_info; + + if (type == MHI_CMD_SFR_CFG && sfr_info) { + sfr_info->ccs = ccs; + complete(&sfr_info->completion); + } +} + +int mhi_get_remote_time_sync(struct mhi_device *mhi_dev, + u64 *t_host, + u64 *t_dev) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + struct mhi_timesync *mhi_tsync = mhi_priv->timesync; + u64 local_time; + u32 tdev_lo = U32_MAX, tdev_hi = U32_MAX; + int ret; + + /* not all devices support time features */ + if (!mhi_tsync) + return -EINVAL; + + if (unlikely(MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state))) { + MHI_ERR("MHI is not in active state, pm_state:%s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + return -EIO; + } + + mutex_lock(&mhi_tsync->mutex); + + /* return times from last async request completion */ + if (mhi_tsync->db_pending) { + local_time = mhi_tsync->local_time; + mutex_unlock(&mhi_tsync->mutex); + + ret = wait_for_completion_timeout(&mhi_tsync->completion, + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state) || !ret) { + MHI_ERR("Pending DB request did not complete, abort\n"); + return -EAGAIN; + } + + *t_host = local_time; + *t_dev = mhi_tsync->remote_time; + + return 0; + } + + /* bring to M0 state */ + ret = mhi_device_get_sync(mhi_cntrl->mhi_dev); + if (ret) + goto error_unlock; + mhi_cntrl->runtime_get(mhi_cntrl); + + /* disable link level low power modes */ + ret = mhi_tsync->lpm_disable(mhi_cntrl); + if (ret) + goto error_invalid_state; + + /* + * time critical code to fetch device times, + * delay between these two steps should be + * deterministic as possible. + */ + preempt_disable(); + local_irq_disable(); + + *t_host = mhi_tsync->time_get(mhi_cntrl); + + ret = mhi_read_reg(mhi_cntrl, mhi_tsync->time_reg, + TIMESYNC_TIME_LOW_OFFSET, &tdev_lo); + if (ret) + MHI_ERR("Time LOW register read error\n"); + + ret = mhi_read_reg(mhi_cntrl, mhi_tsync->time_reg, + TIMESYNC_TIME_HIGH_OFFSET, &tdev_hi); + if (ret) + MHI_ERR("Time HIGH register read error\n"); + + *t_dev = (u64) tdev_hi << 32 | tdev_lo; + + local_irq_enable(); + preempt_enable(); + + mhi_tsync->lpm_enable(mhi_cntrl); + +error_invalid_state: + mhi_cntrl->runtime_put(mhi_cntrl); + mhi_device_put(mhi_cntrl->mhi_dev); +error_unlock: + mutex_unlock(&mhi_tsync->mutex); + return ret; +} +EXPORT_SYMBOL(mhi_get_remote_time_sync); + +int mhi_get_remote_time(struct mhi_device *mhi_dev, + u32 sequence, + void (*cb_func)(struct mhi_device *mhi_dev, + u32 sequence, + u64 local_time, + u64 remote_time)) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + struct mhi_private *mhi_priv = dev_get_drvdata(dev); + struct mhi_timesync *mhi_tsync = mhi_priv->timesync; + struct tsync_node *tsync_node; + int ret = 0; + + /* not all devices support all time features */ + if (!mhi_tsync || !mhi_tsync->time_db) + return -EINVAL; + + mutex_lock(&mhi_tsync->mutex); + + ret = mhi_device_get_sync(mhi_cntrl->mhi_dev); + if (ret) + goto error_unlock; + mhi_cntrl->runtime_get(mhi_cntrl); + + MHI_LOG("Enter with pm_state:%s MHI_STATE:%s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state), + TO_MHI_STATE_STR(mhi_cntrl->dev_state)); + + /* + * technically we can use GFP_KERNEL, but wants to avoid + * # of times scheduling out + */ + tsync_node = kzalloc(sizeof(*tsync_node), GFP_ATOMIC); + if (!tsync_node) { + ret = -ENOMEM; + goto error_no_mem; + } + + tsync_node->sequence = sequence; + tsync_node->cb_func = cb_func; + tsync_node->mhi_dev = mhi_dev; + + if (mhi_tsync->db_pending) { + mhi_cntrl->runtime_put(mhi_cntrl); + mhi_device_put(mhi_cntrl->mhi_dev); + goto skip_tsync_db; + } + + mhi_tsync->int_sequence++; + if (mhi_tsync->int_sequence == 0xFFFFFFFF) + mhi_tsync->int_sequence = 0; + + /* disable link level low power modes */ + ret = mhi_tsync->lpm_disable(mhi_cntrl); + if (ret) { + MHI_ERR("LPM disable request failed for %s!\n", mhi_dev->name); + goto error_invalid_state; + } + + /* + * time critical code, delay between these two steps should be + * deterministic as possible. + */ + preempt_disable(); + local_irq_disable(); + + mhi_tsync->local_time = mhi_tsync->time_get(mhi_cntrl); + mhi_write_reg(mhi_cntrl, mhi_tsync->time_db, 0, mhi_tsync->int_sequence); + + /* write must go through immediately */ + wmb(); + + local_irq_enable(); + preempt_enable(); + + mhi_tsync->lpm_enable(mhi_cntrl); + + MHI_VERB("time DB request with seq:0x%llx\n", mhi_tsync->int_sequence); + + mhi_tsync->db_pending = true; + init_completion(&mhi_tsync->completion); + +skip_tsync_db: + spin_lock(&mhi_tsync->lock); + list_add_tail(&tsync_node->node, &mhi_tsync->head); + spin_unlock(&mhi_tsync->lock); + + mutex_unlock(&mhi_tsync->mutex); + + return 0; + +error_invalid_state: + kfree(tsync_node); +error_no_mem: + mhi_cntrl->runtime_put(mhi_cntrl); + mhi_device_put(mhi_cntrl->mhi_dev); +error_unlock: + mutex_unlock(&mhi_tsync->mutex); + return ret; +} +EXPORT_SYMBOL(mhi_get_remote_time); diff --git a/mhi/core/misc.h b/mhi/core/misc.h new file mode 100644 index 0000000..fb14894 --- /dev/null +++ b/mhi/core/misc.h @@ -0,0 +1,303 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2020-2021, The Linux Foundation. All rights reserved. + * + */ + +#ifndef _MHI_CORE_MISC_H_ +#define _MHI_CORE_MISC_H_ + +#include <linux/mhi_misc.h> + +#define MHI_FORCE_WAKE_DELAY_US (100) +#define MHI_IPC_LOG_PAGES (100) +#define MAX_RDDM_TABLE_SIZE (7) +#define MHI_REG_SIZE (SZ_4K) + +/* MHI misc capability registers */ +#define MISC_OFFSET (0x24) +#define MISC_CAP_MASK (0xFFFFFFFF) +#define MISC_CAP_SHIFT (0) + +#define CAP_CAPID_MASK (0xFF000000) +#define CAP_CAPID_SHIFT (24) +#define CAP_NEXT_CAP_MASK (0x00FFF000) +#define CAP_NEXT_CAP_SHIFT (12) + +/* MHI Bandwidth scaling offsets */ +#define BW_SCALE_CFG_OFFSET (0x04) +#define BW_SCALE_CFG_CHAN_DB_ID_MASK (0xFE000000) +#define BW_SCALE_CFG_CHAN_DB_ID_SHIFT (25) +#define BW_SCALE_CFG_ENABLED_MASK (0x01000000) +#define BW_SCALE_CFG_ENABLED_SHIFT (24) +#define BW_SCALE_CFG_ER_ID_MASK (0x00F80000) +#define BW_SCALE_CFG_ER_ID_SHIFT (19) + +#define BW_SCALE_CAP_ID (3) +#define MHI_TRE_GET_EV_BW_REQ_SEQ(tre) (((tre)->dword[0] >> 8) & 0xFF) +#define MHI_BW_SCALE_CHAN_DB 126 + +#define MHI_BW_SCALE_SETUP(er_index) (((MHI_BW_SCALE_CHAN_DB << \ + BW_SCALE_CFG_CHAN_DB_ID_SHIFT) & BW_SCALE_CFG_CHAN_DB_ID_MASK) | \ + ((1 << BW_SCALE_CFG_ENABLED_SHIFT) & BW_SCALE_CFG_ENABLED_MASK) | \ + (((er_index) << BW_SCALE_CFG_ER_ID_SHIFT) & BW_SCALE_CFG_ER_ID_MASK)) + +#define MHI_BW_SCALE_RESULT(status, seq) (((status) & 0xF) << 8 | \ + ((seq) & 0xFF)) +#define MHI_BW_SCALE_NACK 0xF + +/* subsystem failure reason cfg command */ +#define MHI_TRE_CMD_SFR_CFG_PTR(ptr) (ptr) +#define MHI_TRE_CMD_SFR_CFG_DWORD0(len) (len) +#define MHI_TRE_CMD_SFR_CFG_DWORD1 (MHI_CMD_SFR_CFG << 16) + +/* MHI Timesync offsets */ +#define TIMESYNC_CFG_OFFSET (0x04) +#define TIMESYNC_CFG_ENABLED_MASK (0x80000000) +#define TIMESYNC_CFG_ENABLED_SHIFT (31) +#define TIMESYNC_CFG_CHAN_DB_ID_MASK (0x0000FF00) +#define TIMESYNC_CFG_CHAN_DB_ID_SHIFT (8) +#define TIMESYNC_CFG_ER_ID_MASK (0x000000FF) +#define TIMESYNC_CFG_ER_ID_SHIFT (0) + +#define TIMESYNC_TIME_LOW_OFFSET (0x8) +#define TIMESYNC_TIME_HIGH_OFFSET (0xC) + +#define MHI_TIMESYNC_CHAN_DB (125) +#define TIMESYNC_CAP_ID (2) + +#define MHI_TIMESYNC_DB_SETUP(er_index) ((MHI_TIMESYNC_CHAN_DB << \ + TIMESYNC_CFG_CHAN_DB_ID_SHIFT) & TIMESYNC_CFG_CHAN_DB_ID_MASK | \ + (1 << TIMESYNC_CFG_ENABLED_SHIFT) & TIMESYNC_CFG_ENABLED_MASK | \ + ((er_index) << TIMESYNC_CFG_ER_ID_SHIFT) & TIMESYNC_CFG_ER_ID_MASK) + +#define MHI_VERB(fmt, ...) do { \ + struct mhi_private *mhi_priv = \ + dev_get_drvdata(&mhi_cntrl->mhi_dev->dev); \ + dev_dbg(dev, "[D][%s] " fmt, __func__, ##__VA_ARGS__); \ + if (mhi_priv->log_lvl <= MHI_MSG_LVL_VERBOSE) \ + ipc_log_string(mhi_priv->log_buf, "[D][%s] " fmt, __func__, \ + ##__VA_ARGS__); \ +} while (0) + +#define MHI_LOG(fmt, ...) do { \ + struct mhi_private *mhi_priv = \ + dev_get_drvdata(&mhi_cntrl->mhi_dev->dev); \ + dev_dbg(dev, "[I][%s] " fmt, __func__, ##__VA_ARGS__); \ + if (mhi_priv->log_lvl <= MHI_MSG_LVL_INFO) \ + ipc_log_string(mhi_priv->log_buf, "[I][%s] " fmt, __func__, \ + ##__VA_ARGS__); \ +} while (0) + +#define MHI_ERR(fmt, ...) do { \ + struct mhi_private *mhi_priv = \ + dev_get_drvdata(&mhi_cntrl->mhi_dev->dev); \ + dev_err(dev, "[E][%s] " fmt, __func__, ##__VA_ARGS__); \ + if (mhi_priv->log_lvl <= MHI_MSG_LVL_ERROR) \ + ipc_log_string(mhi_priv->log_buf, "[E][%s] " fmt, __func__, \ + ##__VA_ARGS__); \ +} while (0) + +#define MHI_CRITICAL(fmt, ...) do { \ + struct mhi_private *mhi_priv = \ + dev_get_drvdata(&mhi_cntrl->mhi_dev->dev); \ + dev_crit(dev, "[C][%s] " fmt, __func__, ##__VA_ARGS__); \ + if (mhi_priv->log_lvl <= MHI_MSG_LVL_CRITICAL) \ + ipc_log_string(mhi_priv->log_buf, "[C][%s] " fmt, __func__, \ + ##__VA_ARGS__); \ +} while (0) + +/** + * struct rddm_table_info - rddm table info + * @base_address - Start offset of the file + * @actual_phys_address - phys addr offset of file + * @size - size of file + * @description - file description + * @file_name - name of file + */ +struct rddm_table_info { + u64 base_address; + u64 actual_phys_address; + u64 size; + char description[20]; + char file_name[20]; +}; + +/** + * struct rddm_header - rddm header + * @version - header ver + * @header_size - size of header + * @rddm_table_info - array of rddm table info + */ +struct rddm_header { + u32 version; + u32 header_size; + struct rddm_table_info table_info[MAX_RDDM_TABLE_SIZE]; +}; + +/** + * struct file_info - keeping track of file info while traversing the rddm + * table header + * @file_offset - current file offset + * @seg_idx - mhi buf seg array index + * @rem_seg_len - remaining length of the segment containing current file + */ +struct file_info { + u8 *file_offset; + u32 file_size; + u32 seg_idx; + u32 rem_seg_len; +}; + +/** + * struct mhi_private - For private variables of an MHI controller + */ +struct mhi_private { + struct list_head node; + struct mhi_controller *mhi_cntrl; + enum MHI_DEBUG_LEVEL log_lvl; + void *log_buf; + u32 saved_pm_state; + enum mhi_state saved_dev_state; + u32 m2_timeout_ms; + void *priv_data; + void __iomem *bw_scale_db; + int (*bw_scale)(struct mhi_controller *mhi_cntrl, + struct mhi_link_info *link_info); + phys_addr_t base_addr; + u32 bw_response; + struct mhi_sfr_info *sfr_info; + struct mhi_timesync *timesync; +}; + +/** + * struct mhi_bus - For MHI controller debug + */ +struct mhi_bus { + struct list_head controller_list; + struct mutex lock; +}; + +/** + * struct mhi_sfr_info - For receiving MHI subsystem failure reason + */ +struct mhi_sfr_info { + void *buf_addr; + dma_addr_t dma_addr; + size_t len; + char *str; + unsigned int ccs; + struct completion completion; +}; + +/** + * struct mhi_timesync - For enabling use of MHI time synchronization feature + */ +struct mhi_timesync { + u64 (*time_get)(struct mhi_controller *mhi_cntrl); + int (*lpm_disable)(struct mhi_controller *mhi_cntrl); + int (*lpm_enable)(struct mhi_controller *mhi_cntrl); + void __iomem *time_reg; + void __iomem *time_db; + u32 int_sequence; + u64 local_time; + u64 remote_time; + bool db_pending; + struct completion completion; + spinlock_t lock; /* list protection */ + struct list_head head; + struct mutex mutex; +}; + +/** + * struct tsync_node - Stores requests when using the timesync doorbell method + */ +struct tsync_node { + struct list_head node; + u32 sequence; + u64 remote_time; + struct mhi_device *mhi_dev; + void (*cb_func)(struct mhi_device *mhi_dev, u32 sequence, + u64 local_time, u64 remote_time); +}; + +#ifdef CONFIG_MHI_BUS_MISC +void mhi_misc_init(void); +void mhi_misc_exit(void); +int mhi_misc_init_mmio(struct mhi_controller *mhi_cntrl); +int mhi_misc_register_controller(struct mhi_controller *mhi_cntrl); +void mhi_misc_unregister_controller(struct mhi_controller *mhi_cntrl); +int mhi_process_misc_bw_ev_ring(struct mhi_controller *mhi_cntrl, + struct mhi_event *mhi_event, u32 event_quota); +int mhi_process_misc_tsync_ev_ring(struct mhi_controller *mhi_cntrl, + struct mhi_event *mhi_event, u32 event_quota); +void mhi_misc_mission_mode(struct mhi_controller *mhi_cntrl); +void mhi_misc_dbs_pending(struct mhi_controller *mhi_cntrl); +void mhi_misc_disable(struct mhi_controller *mhi_cntrl); +void mhi_misc_cmd_configure(struct mhi_controller *mhi_cntrl, + unsigned int type, u64 *ptr, u32 *dword0, + u32 *dword1); +void mhi_misc_cmd_completion(struct mhi_controller *mhi_cntrl, + unsigned int type, unsigned int ccs); +#else +static inline void mhi_misc_init(void) +{ +} + +static inline void mhi_misc_exit(void) +{ +} + +static inline int mhi_misc_init_mmio(struct mhi_controller *mhi_cntrl) +{ + return 0; +} + +static inline int mhi_misc_register_controller(struct mhi_controller *mhi_cntrl) +{ + return 0; +} + +static inline void mhi_misc_unregister_controller(struct mhi_controller + *mhi_cntrl) +{ +} + +static inline int mhi_process_misc_bw_ev_ring(struct mhi_controller *mhi_cntrl, + struct mhi_event *mhi_event, u32 event_quota) +{ + return 0; +} + +static inline int mhi_process_misc_tsync_ev_ring + (struct mhi_controller *mhi_cntrl, + struct mhi_event *mhi_event, u32 event_quota) +{ + return 0; +} + +static inline void mhi_misc_mission_mode(struct mhi_controller *mhi_cntrl) +{ +} + +static inline void mhi_special_dbs_pending(struct mhi_controller *mhi_cntrl) +{ +} + +static inline void mhi_misc_disable(struct mhi_controller *mhi_cntrl) +{ +} + +static inline void mhi_misc_cmd_configure(struct mhi_controller *mhi_cntrl, + unsigned int type, u64 *ptr, + u32 *dword0, u32 *dword1) +{ +} + +static inline void mhi_misc_cmd_completion(struct mhi_controller *mhi_cntrl, + unsigned int type, unsigned int ccs) +{ +} +#endif + +#endif /* _MHI_CORE_MISC_H_ */ diff --git a/mhi/core/pm.c b/mhi/core/pm.c new file mode 100644 index 0000000..5676f73 --- /dev/null +++ b/mhi/core/pm.c @@ -0,0 +1,1255 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved. + * + */ + +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/dma-direction.h> +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/list.h> +#include <linux/mhi.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/wait.h> +#include "internal.h" + +/* + * Not all MHI state transitions are synchronous. Transitions like Linkdown, + * SYS_ERR, and shutdown can happen anytime asynchronously. This function will + * transition to a new state only if we're allowed to. + * + * Priority increases as we go down. For instance, from any state in L0, the + * transition can be made to states in L1, L2 and L3. A notable exception to + * this rule is state DISABLE. From DISABLE state we can only transition to + * POR state. Also, while in L2 state, user cannot jump back to previous + * L1 or L0 states. + * + * Valid transitions: + * L0: DISABLE <--> POR + * POR <--> POR + * POR -> M0 -> M2 --> M0 + * POR -> FW_DL_ERR + * FW_DL_ERR <--> FW_DL_ERR + * M0 <--> M0 + * M0 -> FW_DL_ERR + * M0 -> M3_ENTER -> M3 -> M3_EXIT --> M0 + * L1: SYS_ERR_DETECT -> SYS_ERR_PROCESS --> POR + * L2: SHUTDOWN_PROCESS -> LD_ERR_FATAL_DETECT + * SHUTDOWN_PROCESS -> DISABLE + * L3: LD_ERR_FATAL_DETECT <--> LD_ERR_FATAL_DETECT + * LD_ERR_FATAL_DETECT -> DISABLE + */ +static struct mhi_pm_transitions const dev_state_transitions[] = { + /* L0 States */ + { + MHI_PM_DISABLE, + MHI_PM_POR + }, + { + MHI_PM_POR, + MHI_PM_POR | MHI_PM_DISABLE | MHI_PM_M0 | + MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS | + MHI_PM_LD_ERR_FATAL_DETECT | MHI_PM_FW_DL_ERR + }, + { + MHI_PM_M0, + MHI_PM_M0 | MHI_PM_M2 | MHI_PM_M3_ENTER | + MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS | + MHI_PM_LD_ERR_FATAL_DETECT | MHI_PM_FW_DL_ERR + }, + { + MHI_PM_M2, + MHI_PM_M0 | MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS | + MHI_PM_LD_ERR_FATAL_DETECT + }, + { + MHI_PM_M3_ENTER, + MHI_PM_M3 | MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS | + MHI_PM_LD_ERR_FATAL_DETECT + }, + { + MHI_PM_M3, + MHI_PM_M3_EXIT | MHI_PM_SYS_ERR_DETECT | + MHI_PM_LD_ERR_FATAL_DETECT + }, + { + MHI_PM_M3_EXIT, + MHI_PM_M0 | MHI_PM_SYS_ERR_DETECT | MHI_PM_SHUTDOWN_PROCESS | + MHI_PM_LD_ERR_FATAL_DETECT + }, + { + MHI_PM_FW_DL_ERR, + MHI_PM_FW_DL_ERR | MHI_PM_SYS_ERR_DETECT | + MHI_PM_SHUTDOWN_PROCESS | MHI_PM_LD_ERR_FATAL_DETECT + }, + /* L1 States */ + { + MHI_PM_SYS_ERR_DETECT, + MHI_PM_SYS_ERR_PROCESS | MHI_PM_SHUTDOWN_PROCESS | + MHI_PM_LD_ERR_FATAL_DETECT + }, + { + MHI_PM_SYS_ERR_PROCESS, + MHI_PM_POR | MHI_PM_SHUTDOWN_PROCESS | + MHI_PM_LD_ERR_FATAL_DETECT + }, + /* L2 States */ + { + MHI_PM_SHUTDOWN_PROCESS, + MHI_PM_DISABLE | MHI_PM_LD_ERR_FATAL_DETECT + }, + /* L3 States */ + { + MHI_PM_LD_ERR_FATAL_DETECT, + MHI_PM_LD_ERR_FATAL_DETECT | MHI_PM_DISABLE + }, +}; + +enum mhi_pm_state __must_check mhi_tryset_pm_state(struct mhi_controller *mhi_cntrl, + enum mhi_pm_state state) +{ + unsigned long cur_state = mhi_cntrl->pm_state; + int index = find_last_bit(&cur_state, 32); + + if (unlikely(index >= ARRAY_SIZE(dev_state_transitions))) + return cur_state; + + if (unlikely(dev_state_transitions[index].from_state != cur_state)) + return cur_state; + + if (unlikely(!(dev_state_transitions[index].to_states & state))) + return cur_state; + + mhi_cntrl->pm_state = state; + return mhi_cntrl->pm_state; +} + +void mhi_set_mhi_state(struct mhi_controller *mhi_cntrl, enum mhi_state state) +{ + if (state == MHI_STATE_RESET) { + mhi_write_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL, + MHICTRL_RESET_MASK, MHICTRL_RESET_SHIFT, 1); + } else { + mhi_write_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL, + MHICTRL_MHISTATE_MASK, + MHICTRL_MHISTATE_SHIFT, state); + } +} + +/* NOP for backward compatibility, host allowed to ring DB in M2 state */ +static void mhi_toggle_dev_wake_nop(struct mhi_controller *mhi_cntrl) +{ +} + +static void mhi_toggle_dev_wake(struct mhi_controller *mhi_cntrl) +{ + mhi_cntrl->wake_get(mhi_cntrl, false); + mhi_cntrl->wake_put(mhi_cntrl, true); +} + +/* Add event ring elements and ring er db */ +static void mhi_setup_event_rings(struct mhi_controller *mhi_cntrl, bool add_el) +{ + struct mhi_event *mhi_event; + int i; + bool skip_er_setup; + + mhi_event = mhi_cntrl->mhi_event; + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) { + struct mhi_ring *ring = &mhi_event->ring; + + if (mhi_event->offload_ev) + continue; + + /* skip HW event ring setup in ready state */ + if (mhi_cntrl->dev_state == MHI_STATE_READY) + skip_er_setup = mhi_event->hw_ring; + else + skip_er_setup = !mhi_event->hw_ring; + + /* if no er element to add, ring all er dbs */ + if (add_el && skip_er_setup) + continue; + + if (add_el) { + ring->wp = ring->base + ring->len - ring->el_size; + *ring->ctxt_wp = + ring->iommu_base + ring->len - ring->el_size; + /* Update all cores */ + smp_wmb(); + } + + /* Ring the event ring db */ + spin_lock_irq(&mhi_event->lock); + mhi_ring_er_db(mhi_event); + spin_unlock_irq(&mhi_event->lock); + } +} + +/* Handle device ready state transition */ +int mhi_ready_state_transition(struct mhi_controller *mhi_cntrl) +{ + enum mhi_pm_state cur_state; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + u32 interval_us = 25000; /* poll register field every 25 milliseconds */ + int ret; + + /* Check if device entered error state */ + if (MHI_PM_IN_FATAL_STATE(mhi_cntrl->pm_state)) { + MHI_ERR("Device link is not accessible\n"); + return -EIO; + } + + /* Wait for RESET to be cleared and READY bit to be set by the device */ + ret = mhi_poll_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL, + MHICTRL_RESET_MASK, MHICTRL_RESET_SHIFT, 0, + interval_us); + if (ret) { + MHI_ERR("Device failed to clear MHI Reset\n"); + return ret; + } + + ret = mhi_poll_reg_field(mhi_cntrl, mhi_cntrl->regs, MHISTATUS, + MHISTATUS_READY_MASK, MHISTATUS_READY_SHIFT, 1, + interval_us); + if (ret) { + MHI_ERR("Device failed to enter MHI Ready\n"); + return ret; + } + + MHI_VERB("Device in READY State\n"); + write_lock_irq(&mhi_cntrl->pm_lock); + cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_POR); + mhi_cntrl->dev_state = MHI_STATE_READY; + write_unlock_irq(&mhi_cntrl->pm_lock); + + if (cur_state != MHI_PM_POR) { + MHI_ERR("Error moving to state %s from %s\n", + to_mhi_pm_state_str(MHI_PM_POR), + to_mhi_pm_state_str(cur_state)); + return -EIO; + } + + read_lock_bh(&mhi_cntrl->pm_lock); + if (!MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) { + MHI_ERR("Device registers not accessible\n"); + goto error_mmio; + } + + /* Configure MMIO registers */ + ret = mhi_init_mmio(mhi_cntrl); + if (ret) { + MHI_ERR("Error configuring MMIO registers\n"); + goto error_mmio; + } + + /* add SW event ring elements and ring SW event ring dbs */ + mhi_setup_event_rings(mhi_cntrl, true); + + /* Set MHI to M0 state */ + mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M0); + read_unlock_bh(&mhi_cntrl->pm_lock); + + return 0; + +error_mmio: + read_unlock_bh(&mhi_cntrl->pm_lock); + + return -EIO; +} + +int mhi_pm_m0_transition(struct mhi_controller *mhi_cntrl) +{ + enum mhi_pm_state cur_state; + struct mhi_chan *mhi_chan; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + int i; + + write_lock_irq(&mhi_cntrl->pm_lock); + mhi_cntrl->dev_state = MHI_STATE_M0; + cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M0); + write_unlock_irq(&mhi_cntrl->pm_lock); + if (unlikely(cur_state != MHI_PM_M0)) { + MHI_ERR("Unable to transition to M0 state\n"); + return -EIO; + } + mhi_cntrl->M0++; + + /* Wake up the device */ + read_lock_bh(&mhi_cntrl->pm_lock); + mhi_cntrl->wake_get(mhi_cntrl, true); + + /* Ring all event rings and CMD ring only if we're in mission mode */ + if (MHI_IN_MISSION_MODE(mhi_cntrl->ee)) { + struct mhi_cmd *mhi_cmd = + &mhi_cntrl->mhi_cmd[PRIMARY_CMD_RING]; + + mhi_setup_event_rings(mhi_cntrl, false); + + /* Only ring primary cmd ring if ring is not empty */ + spin_lock_irq(&mhi_cmd->lock); + if (mhi_cmd->ring.rp != mhi_cmd->ring.wp) + mhi_ring_cmd_db(mhi_cntrl, mhi_cmd); + spin_unlock_irq(&mhi_cmd->lock); + + /* ring misc doorbells for certain controllers */ + mhi_misc_dbs_pending(mhi_cntrl); + } + + /* Ring channel DB registers */ + mhi_chan = mhi_cntrl->mhi_chan; + for (i = 0; i < mhi_cntrl->max_chan; i++, mhi_chan++) { + struct mhi_ring *tre_ring = &mhi_chan->tre_ring; + + if (mhi_chan->db_cfg.reset_req) { + write_lock_irq(&mhi_chan->lock); + mhi_chan->db_cfg.db_mode = true; + write_unlock_irq(&mhi_chan->lock); + } + + read_lock_irq(&mhi_chan->lock); + + /* Only ring DB if ring is not empty */ + if (tre_ring->base && tre_ring->wp != tre_ring->rp) + mhi_ring_chan_db(mhi_cntrl, mhi_chan); + read_unlock_irq(&mhi_chan->lock); + } + + mhi_cntrl->wake_put(mhi_cntrl, false); + read_unlock_bh(&mhi_cntrl->pm_lock); + wake_up_all(&mhi_cntrl->state_event); + + return 0; +} + +/* + * After receiving the MHI state change event from the device indicating the + * transition to M1 state, the host can transition the device to M2 state + * for keeping it in low power state. + */ +void mhi_pm_m1_transition(struct mhi_controller *mhi_cntrl) +{ + enum mhi_pm_state state; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + + write_lock_irq(&mhi_cntrl->pm_lock); + state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M2); + if (state == MHI_PM_M2) { + mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M2); + mhi_cntrl->dev_state = MHI_STATE_M2; + + write_unlock_irq(&mhi_cntrl->pm_lock); + + mhi_cntrl->M2++; + wake_up_all(&mhi_cntrl->state_event); + + /* If there are any pending resources, exit M2 immediately */ + if (unlikely(atomic_read(&mhi_cntrl->pending_pkts) || + atomic_read(&mhi_cntrl->dev_wake))) { + MHI_VERB( + "Exiting M2, pending_pkts: %d dev_wake: %d\n", + atomic_read(&mhi_cntrl->pending_pkts), + atomic_read(&mhi_cntrl->dev_wake)); + read_lock_bh(&mhi_cntrl->pm_lock); + mhi_cntrl->wake_get(mhi_cntrl, true); + mhi_cntrl->wake_put(mhi_cntrl, true); + read_unlock_bh(&mhi_cntrl->pm_lock); + } else { + mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_IDLE); + } + } else { + write_unlock_irq(&mhi_cntrl->pm_lock); + } +} + +/* MHI M3 completion handler */ +int mhi_pm_m3_transition(struct mhi_controller *mhi_cntrl) +{ + enum mhi_pm_state state; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + + write_lock_irq(&mhi_cntrl->pm_lock); + mhi_cntrl->dev_state = MHI_STATE_M3; + state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3); + write_unlock_irq(&mhi_cntrl->pm_lock); + if (state != MHI_PM_M3) { + MHI_ERR("Unable to transition to M3 state\n"); + return -EIO; + } + + mhi_cntrl->M3++; + wake_up_all(&mhi_cntrl->state_event); + + return 0; +} + +/* Handle device Mission Mode transition */ +static int mhi_pm_mission_mode_transition(struct mhi_controller *mhi_cntrl) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + enum mhi_ee_type ee = MHI_EE_MAX, current_ee = mhi_cntrl->ee; + int ret; + + MHI_VERB("Processing Mission Mode transition\n"); + + write_lock_irq(&mhi_cntrl->pm_lock); + if (MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) + ee = mhi_get_exec_env(mhi_cntrl); + + if (!MHI_IN_MISSION_MODE(ee)) { + mhi_cntrl->pm_state = MHI_PM_LD_ERR_FATAL_DETECT; + write_unlock_irq(&mhi_cntrl->pm_lock); + wake_up_all(&mhi_cntrl->state_event); + return -EIO; + } + mhi_cntrl->ee = ee; + write_unlock_irq(&mhi_cntrl->pm_lock); + + wake_up_all(&mhi_cntrl->state_event); + + device_for_each_child(&mhi_cntrl->mhi_dev->dev, ¤t_ee, + mhi_destroy_device); + mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_EE_MISSION_MODE); + + /* Force MHI to be in M0 state before continuing */ + ret = __mhi_device_get_sync(mhi_cntrl); + if (ret) + return ret; + + read_lock_bh(&mhi_cntrl->pm_lock); + + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) { + ret = -EIO; + goto error_mission_mode; + } + + /* Add elements to all HW event rings and ring HW event ring dbs */ + mhi_setup_event_rings(mhi_cntrl, true); + + read_unlock_bh(&mhi_cntrl->pm_lock); + + mhi_misc_mission_mode(mhi_cntrl); + mhi_process_sleeping_events(mhi_cntrl); + + /* + * The MHI devices are only created when the client device switches its + * Execution Environment (EE) to either SBL or AMSS states + */ + mhi_create_devices(mhi_cntrl); + + read_lock_bh(&mhi_cntrl->pm_lock); + +error_mission_mode: + mhi_cntrl->wake_put(mhi_cntrl, false); + read_unlock_bh(&mhi_cntrl->pm_lock); + + return ret; +} + +/* Handle shutdown transitions */ +static void mhi_pm_disable_transition(struct mhi_controller *mhi_cntrl) +{ + enum mhi_pm_state cur_state; + struct mhi_event *mhi_event; + struct mhi_cmd_ctxt *cmd_ctxt; + struct mhi_cmd *mhi_cmd; + struct mhi_event_ctxt *er_ctxt; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + int ret, i; + + MHI_VERB("Processing disable transition with PM state: %s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + + mutex_lock(&mhi_cntrl->pm_mutex); + + /* Trigger MHI RESET so that the device will not access host memory */ + if (!MHI_PM_IN_FATAL_STATE(mhi_cntrl->pm_state)) { + MHI_VERB("Triggering MHI Reset in device\n"); + mhi_set_mhi_state(mhi_cntrl, MHI_STATE_RESET); + + /* Wait for the reset bit to be cleared by the device */ + ret = mhi_poll_reg_field(mhi_cntrl, mhi_cntrl->regs, MHICTRL, + MHICTRL_RESET_MASK, MHICTRL_RESET_SHIFT, 0, + 25000); + if (ret) + MHI_ERR("Device failed to clear MHI Reset\n"); + + /* + * Device will clear BHI_INTVEC as a part of RESET processing, + * hence re-program it + */ + mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0); + } + + MHI_VERB( + "Waiting for all pending event ring processing to complete\n"); + mhi_event = mhi_cntrl->mhi_event; + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) { + if (mhi_event->offload_ev) + continue; + free_irq(mhi_cntrl->irq[mhi_event->irq], mhi_event); + if (mhi_event->priority == MHI_ER_PRIORITY_HI_SLEEP) + cancel_work_sync(&mhi_event->work); + else + tasklet_kill(&mhi_event->task); + } + + /* Release lock and wait for all pending threads to complete */ + mutex_unlock(&mhi_cntrl->pm_mutex); + + mhi_misc_disable(mhi_cntrl); + + MHI_VERB("Waiting for all pending threads to complete\n"); + wake_up_all(&mhi_cntrl->state_event); + + MHI_VERB("Reset all active channels and remove MHI devices\n"); + device_for_each_child(&mhi_cntrl->mhi_dev->dev, NULL, mhi_destroy_device); + + mutex_lock(&mhi_cntrl->pm_mutex); + + WARN_ON(atomic_read(&mhi_cntrl->dev_wake)); + WARN_ON(atomic_read(&mhi_cntrl->pending_pkts)); + + /* Reset the ev rings and cmd rings */ + MHI_VERB("Resetting EV CTXT and CMD CTXT\n"); + mhi_cmd = mhi_cntrl->mhi_cmd; + cmd_ctxt = mhi_cntrl->mhi_ctxt->cmd_ctxt; + for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++, cmd_ctxt++) { + struct mhi_ring *ring = &mhi_cmd->ring; + + ring->rp = ring->base; + ring->wp = ring->base; + cmd_ctxt->rp = cmd_ctxt->rbase; + cmd_ctxt->wp = cmd_ctxt->rbase; + } + + mhi_event = mhi_cntrl->mhi_event; + er_ctxt = mhi_cntrl->mhi_ctxt->er_ctxt; + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, er_ctxt++, + mhi_event++) { + struct mhi_ring *ring = &mhi_event->ring; + + /* Skip offload events */ + if (mhi_event->offload_ev) + continue; + + ring->rp = ring->base; + ring->wp = ring->base; + er_ctxt->rp = er_ctxt->rbase; + er_ctxt->wp = er_ctxt->rbase; + } + + /* Move to disable state */ + write_lock_irq(&mhi_cntrl->pm_lock); + cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_DISABLE); + write_unlock_irq(&mhi_cntrl->pm_lock); + if (unlikely(cur_state != MHI_PM_DISABLE)) + MHI_ERR("Error moving from PM state: %s to: %s\n", + to_mhi_pm_state_str(cur_state), + to_mhi_pm_state_str(MHI_PM_DISABLE)); + + MHI_VERB("Exiting with PM state: %s, MHI state: %s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state), + TO_MHI_STATE_STR(mhi_cntrl->dev_state)); + + mutex_unlock(&mhi_cntrl->pm_mutex); +} + +/* Handle system error transitions */ +static void mhi_pm_sys_error_transition(struct mhi_controller *mhi_cntrl) +{ + enum mhi_pm_state cur_state, prev_state; + struct mhi_event *mhi_event; + struct mhi_cmd_ctxt *cmd_ctxt; + struct mhi_cmd *mhi_cmd; + struct mhi_event_ctxt *er_ctxt; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + int ret, i; + + MHI_VERB("Transitioning from PM state: %s to: %s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state), + to_mhi_pm_state_str(MHI_PM_SYS_ERR_PROCESS)); + + /* We must notify MHI control driver so it can clean up first */ + mhi_cntrl->status_cb(mhi_cntrl, MHI_CB_SYS_ERROR); + + mutex_lock(&mhi_cntrl->pm_mutex); + write_lock_irq(&mhi_cntrl->pm_lock); + prev_state = mhi_cntrl->pm_state; + cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_SYS_ERR_PROCESS); + write_unlock_irq(&mhi_cntrl->pm_lock); + + if (cur_state != MHI_PM_SYS_ERR_PROCESS) { + MHI_ERR("Failed to transition from PM state: %s to: %s\n", + to_mhi_pm_state_str(cur_state), + to_mhi_pm_state_str(MHI_PM_SYS_ERR_PROCESS)); + goto exit_sys_error_transition; + } + + mhi_cntrl->ee = MHI_EE_DISABLE_TRANSITION; + mhi_cntrl->dev_state = MHI_STATE_RESET; + + /* Wake up threads waiting for state transition */ + wake_up_all(&mhi_cntrl->state_event); + + /* Trigger MHI RESET so that the device will not access host memory */ + if (MHI_REG_ACCESS_VALID(prev_state)) { + u32 in_reset = -1; + unsigned long timeout = msecs_to_jiffies(mhi_cntrl->timeout_ms); + + MHI_VERB("Triggering MHI Reset in device\n"); + mhi_set_mhi_state(mhi_cntrl, MHI_STATE_RESET); + + /* Wait for the reset bit to be cleared by the device */ + ret = wait_event_timeout(mhi_cntrl->state_event, + mhi_read_reg_field(mhi_cntrl, + mhi_cntrl->regs, + MHICTRL, + MHICTRL_RESET_MASK, + MHICTRL_RESET_SHIFT, + &in_reset) || + !in_reset, timeout); + if (!ret || in_reset) { + MHI_ERR("Device failed to exit MHI Reset state\n"); + goto exit_sys_error_transition; + } + + /* + * Device will clear BHI_INTVEC as a part of RESET processing, + * hence re-program it + */ + mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0); + } + + MHI_VERB( + "Waiting for all pending event ring processing to complete\n"); + mhi_event = mhi_cntrl->mhi_event; + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, mhi_event++) { + if (mhi_event->offload_ev) + continue; + if (mhi_event->priority == MHI_ER_PRIORITY_HI_SLEEP) + cancel_work_sync(&mhi_event->work); + else + tasklet_kill(&mhi_event->task); + } + + /* Release lock and wait for all pending threads to complete */ + mutex_unlock(&mhi_cntrl->pm_mutex); + + mhi_misc_disable(mhi_cntrl); + + MHI_VERB("Waiting for all pending threads to complete\n"); + wake_up_all(&mhi_cntrl->state_event); + + MHI_VERB("Reset all active channels and remove MHI devices\n"); + device_for_each_child(&mhi_cntrl->mhi_dev->dev, NULL, mhi_destroy_device); + + mutex_lock(&mhi_cntrl->pm_mutex); + + WARN_ON(atomic_read(&mhi_cntrl->dev_wake)); + WARN_ON(atomic_read(&mhi_cntrl->pending_pkts)); + + /* Reset the ev rings and cmd rings */ + MHI_VERB("Resetting EV CTXT and CMD CTXT\n"); + mhi_cmd = mhi_cntrl->mhi_cmd; + cmd_ctxt = mhi_cntrl->mhi_ctxt->cmd_ctxt; + for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++, cmd_ctxt++) { + struct mhi_ring *ring = &mhi_cmd->ring; + + ring->rp = ring->base; + ring->wp = ring->base; + cmd_ctxt->rp = cmd_ctxt->rbase; + cmd_ctxt->wp = cmd_ctxt->rbase; + } + + mhi_event = mhi_cntrl->mhi_event; + er_ctxt = mhi_cntrl->mhi_ctxt->er_ctxt; + for (i = 0; i < mhi_cntrl->total_ev_rings; i++, er_ctxt++, + mhi_event++) { + struct mhi_ring *ring = &mhi_event->ring; + + /* Skip offload events */ + if (mhi_event->offload_ev) + continue; + + ring->rp = ring->base; + ring->wp = ring->base; + er_ctxt->rp = er_ctxt->rbase; + er_ctxt->wp = er_ctxt->rbase; + } + + mhi_ready_state_transition(mhi_cntrl); + +exit_sys_error_transition: + MHI_VERB("Exiting with PM state: %s, MHI state: %s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state), + TO_MHI_STATE_STR(mhi_cntrl->dev_state)); + + mutex_unlock(&mhi_cntrl->pm_mutex); +} + +/* Queue a new work item and schedule work */ +int mhi_queue_state_transition(struct mhi_controller *mhi_cntrl, + enum dev_st_transition state) +{ + struct state_transition *item = kmalloc(sizeof(*item), GFP_ATOMIC); + unsigned long flags; + + if (!item) + return -ENOMEM; + + item->state = state; + spin_lock_irqsave(&mhi_cntrl->transition_lock, flags); + list_add_tail(&item->node, &mhi_cntrl->transition_list); + spin_unlock_irqrestore(&mhi_cntrl->transition_lock, flags); + + queue_work(mhi_cntrl->hiprio_wq, &mhi_cntrl->st_worker); + + return 0; +} + +/* SYS_ERR worker */ +void mhi_pm_sys_err_handler(struct mhi_controller *mhi_cntrl) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + + /* skip if controller supports RDDM */ + if (mhi_cntrl->rddm_image) { + MHI_VERB("Controller supports RDDM, skip SYS_ERROR\n"); + return; + } + + mhi_queue_state_transition(mhi_cntrl, DEV_ST_TRANSITION_SYS_ERR); +} + +/* Device State Transition worker */ +void mhi_pm_st_worker(struct work_struct *work) +{ + struct state_transition *itr, *tmp; + LIST_HEAD(head); + struct mhi_controller *mhi_cntrl = container_of(work, + struct mhi_controller, + st_worker); + struct device *dev = &mhi_cntrl->mhi_dev->dev; + int ret; + + spin_lock_irq(&mhi_cntrl->transition_lock); + list_splice_tail_init(&mhi_cntrl->transition_list, &head); + spin_unlock_irq(&mhi_cntrl->transition_lock); + + list_for_each_entry_safe(itr, tmp, &head, node) { + list_del(&itr->node); + MHI_VERB("Handling state transition: %s\n", + TO_DEV_STATE_TRANS_STR(itr->state)); + + switch (itr->state) { + case DEV_ST_TRANSITION_PBL: + write_lock_irq(&mhi_cntrl->pm_lock); + if (MHI_REG_ACCESS_VALID(mhi_cntrl->pm_state)) + mhi_cntrl->ee = mhi_get_exec_env(mhi_cntrl); + write_unlock_irq(&mhi_cntrl->pm_lock); + if (MHI_IN_PBL(mhi_cntrl->ee)) + mhi_fw_load_handler(mhi_cntrl); + break; + case DEV_ST_TRANSITION_SBL: + write_lock_irq(&mhi_cntrl->pm_lock); + mhi_cntrl->ee = MHI_EE_SBL; + write_unlock_irq(&mhi_cntrl->pm_lock); + + mhi_process_sleeping_events(mhi_cntrl); + + /* + * The MHI devices are only created when the client + * device switches its Execution Environment (EE) to + * either SBL or AMSS states + */ + mhi_create_devices(mhi_cntrl); + if (mhi_cntrl->fbc_download) + mhi_download_amss_image(mhi_cntrl); + break; + case DEV_ST_TRANSITION_MISSION_MODE: + mhi_pm_mission_mode_transition(mhi_cntrl); + break; + case DEV_ST_TRANSITION_READY: + ret = mhi_ready_state_transition(mhi_cntrl); + if (ret) { + mhi_cntrl->pm_state = MHI_PM_FW_DL_ERR; + wake_up_all(&mhi_cntrl->state_event); + } + break; + case DEV_ST_TRANSITION_SYS_ERR: + mhi_pm_sys_error_transition(mhi_cntrl); + break; + case DEV_ST_TRANSITION_DISABLE: + mhi_pm_disable_transition(mhi_cntrl); + break; + default: + break; + } + kfree(itr); + } +} + +int mhi_pm_suspend(struct mhi_controller *mhi_cntrl) +{ + struct mhi_chan *itr, *tmp; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + enum mhi_pm_state new_state; + int ret; + + if (mhi_cntrl->pm_state == MHI_PM_DISABLE) + return -EINVAL; + + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) + return -EIO; + + /* Return busy if there are any pending resources */ + if (atomic_read(&mhi_cntrl->dev_wake) || + atomic_read(&mhi_cntrl->pending_pkts)) + return -EBUSY; + + /* Take MHI out of M2 state */ + read_lock_bh(&mhi_cntrl->pm_lock); + mhi_cntrl->wake_get(mhi_cntrl, false); + read_unlock_bh(&mhi_cntrl->pm_lock); + + ret = wait_event_timeout(mhi_cntrl->state_event, + mhi_cntrl->dev_state == MHI_STATE_M0 || + mhi_cntrl->dev_state == MHI_STATE_M1 || + MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state), + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + + read_lock_bh(&mhi_cntrl->pm_lock); + mhi_cntrl->wake_put(mhi_cntrl, false); + read_unlock_bh(&mhi_cntrl->pm_lock); + + if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) { + MHI_ERR( + "Could not enter M0/M1 state"); + return -EIO; + } + + write_lock_irq(&mhi_cntrl->pm_lock); + + if (atomic_read(&mhi_cntrl->dev_wake) || + atomic_read(&mhi_cntrl->pending_pkts)) { + write_unlock_irq(&mhi_cntrl->pm_lock); + return -EBUSY; + } + + MHI_LOG("Allowing M3 transition\n"); + new_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3_ENTER); + if (new_state != MHI_PM_M3_ENTER) { + write_unlock_irq(&mhi_cntrl->pm_lock); + MHI_ERR( + "Error setting to PM state: %s from: %s\n", + to_mhi_pm_state_str(MHI_PM_M3_ENTER), + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + return -EIO; + } + + /* Set MHI to M3 and wait for completion */ + mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M3); + write_unlock_irq(&mhi_cntrl->pm_lock); + MHI_LOG("Wait for M3 completion\n"); + + ret = wait_event_timeout(mhi_cntrl->state_event, + mhi_cntrl->dev_state == MHI_STATE_M3 || + MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state), + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + + if (!ret) { + mhi_debug_reg_dump(mhi_cntrl); + panic("Timedout waiting for M3 ACK"); + return -EIO; + } else if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) { + MHI_ERR( + "Did not enter M3 state, MHI state: %s, PM state: %s\n", + TO_MHI_STATE_STR(mhi_cntrl->dev_state), + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + return -EIO; + } + + /* Notify clients about entering LPM */ + list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, node) { + mutex_lock(&itr->mutex); + if (itr->mhi_dev) + mhi_notify(itr->mhi_dev, MHI_CB_LPM_ENTER); + mutex_unlock(&itr->mutex); + } + + return 0; +} +EXPORT_SYMBOL_GPL(mhi_pm_suspend); + +int mhi_pm_resume(struct mhi_controller *mhi_cntrl) +{ + struct mhi_chan *itr, *tmp; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + enum mhi_pm_state cur_state; + int ret; + + MHI_LOG("Entered with PM state: %s, MHI state: %s\n", + to_mhi_pm_state_str(mhi_cntrl->pm_state), + TO_MHI_STATE_STR(mhi_cntrl->dev_state)); + + if (mhi_cntrl->pm_state == MHI_PM_DISABLE) + return 0; + + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) + return -EIO; + + if (mhi_cntrl->pm_state != MHI_PM_M3) + panic("mhi_pm_state != M3"); + + /* Notify clients about exiting LPM */ + list_for_each_entry_safe(itr, tmp, &mhi_cntrl->lpm_chans, node) { + mutex_lock(&itr->mutex); + if (itr->mhi_dev) + mhi_notify(itr->mhi_dev, MHI_CB_LPM_EXIT); + mutex_unlock(&itr->mutex); + } + + write_lock_irq(&mhi_cntrl->pm_lock); + cur_state = mhi_tryset_pm_state(mhi_cntrl, MHI_PM_M3_EXIT); + if (cur_state != MHI_PM_M3_EXIT) { + write_unlock_irq(&mhi_cntrl->pm_lock); + MHI_LOG( + "Error setting to PM state: %s from: %s\n", + to_mhi_pm_state_str(MHI_PM_M3_EXIT), + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + return -EIO; + } + + /* Set MHI to M0 and wait for completion */ + mhi_set_mhi_state(mhi_cntrl, MHI_STATE_M0); + write_unlock_irq(&mhi_cntrl->pm_lock); + + ret = wait_event_timeout(mhi_cntrl->state_event, + mhi_cntrl->dev_state == MHI_STATE_M0 || + mhi_cntrl->dev_state == MHI_STATE_M2 || + MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state), + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + + if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) { + MHI_ERR( + "Did not enter M0 state, MHI state: %s, PM state: %s\n", + TO_MHI_STATE_STR(mhi_cntrl->dev_state), + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + return -EIO; + } + + return 0; +} +EXPORT_SYMBOL_GPL(mhi_pm_resume); + +int __mhi_device_get_sync(struct mhi_controller *mhi_cntrl) +{ + int ret; + + /* Wake up the device */ + read_lock_bh(&mhi_cntrl->pm_lock); + if (MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) { + read_unlock_bh(&mhi_cntrl->pm_lock); + return -EIO; + } + mhi_cntrl->wake_get(mhi_cntrl, true); + if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) + mhi_trigger_resume(mhi_cntrl); + read_unlock_bh(&mhi_cntrl->pm_lock); + + ret = wait_event_timeout(mhi_cntrl->state_event, + mhi_cntrl->pm_state == MHI_PM_M0 || + MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state), + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + + if (!ret || MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state)) { + read_lock_bh(&mhi_cntrl->pm_lock); + mhi_cntrl->wake_put(mhi_cntrl, false); + read_unlock_bh(&mhi_cntrl->pm_lock); + return -EIO; + } + + return 0; +} + +/* Assert device wake db */ +static void mhi_assert_dev_wake(struct mhi_controller *mhi_cntrl, bool force) +{ + unsigned long flags; + + /* + * If force flag is set, then increment the wake count value and + * ring wake db + */ + if (unlikely(force)) { + spin_lock_irqsave(&mhi_cntrl->wlock, flags); + atomic_inc(&mhi_cntrl->dev_wake); + if (MHI_WAKE_DB_FORCE_SET_VALID(mhi_cntrl->pm_state) && + !mhi_cntrl->wake_set) { + mhi_write_db(mhi_cntrl, mhi_cntrl->wake_db, 1); + mhi_cntrl->wake_set = true; + } + spin_unlock_irqrestore(&mhi_cntrl->wlock, flags); + } else { + /* + * If resources are already requested, then just increment + * the wake count value and return + */ + if (likely(atomic_add_unless(&mhi_cntrl->dev_wake, 1, 0))) + return; + + spin_lock_irqsave(&mhi_cntrl->wlock, flags); + if ((atomic_inc_return(&mhi_cntrl->dev_wake) == 1) && + MHI_WAKE_DB_SET_VALID(mhi_cntrl->pm_state) && + !mhi_cntrl->wake_set) { + mhi_write_db(mhi_cntrl, mhi_cntrl->wake_db, 1); + mhi_cntrl->wake_set = true; + } + spin_unlock_irqrestore(&mhi_cntrl->wlock, flags); + } +} + +/* De-assert device wake db */ +static void mhi_deassert_dev_wake(struct mhi_controller *mhi_cntrl, + bool override) +{ + unsigned long flags; + + /* + * Only continue if there is a single resource, else just decrement + * and return + */ + if (likely(atomic_add_unless(&mhi_cntrl->dev_wake, -1, 1))) + return; + + spin_lock_irqsave(&mhi_cntrl->wlock, flags); + if ((atomic_dec_return(&mhi_cntrl->dev_wake) == 0) && + MHI_WAKE_DB_CLEAR_VALID(mhi_cntrl->pm_state) && !override && + mhi_cntrl->wake_set) { + mhi_write_db(mhi_cntrl, mhi_cntrl->wake_db, 0); + mhi_cntrl->wake_set = false; + } + spin_unlock_irqrestore(&mhi_cntrl->wlock, flags); +} + +int mhi_async_power_up(struct mhi_controller *mhi_cntrl) +{ + enum mhi_state state; + enum mhi_ee_type current_ee; + enum dev_st_transition next_state; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + u32 val; + int ret; + + MHI_LOG("Requested to power ON\n"); + + /* Supply default wake routines if not provided by controller driver */ + if (!mhi_cntrl->wake_get || !mhi_cntrl->wake_put || + !mhi_cntrl->wake_toggle) { + mhi_cntrl->wake_get = mhi_assert_dev_wake; + mhi_cntrl->wake_put = mhi_deassert_dev_wake; + mhi_cntrl->wake_toggle = (mhi_cntrl->db_access & MHI_PM_M2) ? + mhi_toggle_dev_wake_nop : mhi_toggle_dev_wake; + } + + mutex_lock(&mhi_cntrl->pm_mutex); + mhi_cntrl->pm_state = MHI_PM_DISABLE; + + ret = mhi_init_irq_setup(mhi_cntrl); + if (ret) + goto error_setup_irq; + + /* Setup BHI INTVEC */ + write_lock_irq(&mhi_cntrl->pm_lock); + mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0); + mhi_cntrl->pm_state = MHI_PM_POR; + mhi_cntrl->ee = MHI_EE_MAX; + current_ee = mhi_get_exec_env(mhi_cntrl); + write_unlock_irq(&mhi_cntrl->pm_lock); + + /* Confirm that the device is in valid exec env */ + if (!MHI_IN_PBL(current_ee) && current_ee != MHI_EE_AMSS) { + MHI_ERR("%s is not a valid EE for power on\n", + TO_MHI_EXEC_STR(current_ee)); + ret = -EIO; + goto error_async_power_up; + } + + state = mhi_get_mhi_state(mhi_cntrl); + MHI_VERB("Attempt power on with EE: %s and state: %s\n", + TO_MHI_EXEC_STR(current_ee), TO_MHI_STATE_STR(state)); + + if (state == MHI_STATE_SYS_ERR) { + mhi_set_mhi_state(mhi_cntrl, MHI_STATE_RESET); + ret = wait_event_timeout(mhi_cntrl->state_event, + MHI_PM_IN_FATAL_STATE(mhi_cntrl->pm_state) || + mhi_read_reg_field(mhi_cntrl, + mhi_cntrl->regs, + MHICTRL, + MHICTRL_RESET_MASK, + MHICTRL_RESET_SHIFT, + &val) || + !val, + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + if (!ret) { + ret = -EIO; + MHI_LOG("Failed to reset MHI due to syserr state\n"); + goto error_async_power_up; + } + + /* + * device cleares INTVEC as part of RESET processing, + * re-program it + */ + mhi_write_reg(mhi_cntrl, mhi_cntrl->bhi, BHI_INTVEC, 0); + } + + /* Transition to next state */ + next_state = MHI_IN_PBL(current_ee) ? + DEV_ST_TRANSITION_PBL : DEV_ST_TRANSITION_READY; + + mhi_queue_state_transition(mhi_cntrl, next_state); + + mutex_unlock(&mhi_cntrl->pm_mutex); + + MHI_LOG("Power on setup success\n"); + + return 0; + +error_async_power_up: + mhi_deinit_free_irq(mhi_cntrl); + +error_setup_irq: + mutex_unlock(&mhi_cntrl->pm_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(mhi_async_power_up); + +void mhi_power_down(struct mhi_controller *mhi_cntrl, bool graceful) +{ + enum mhi_pm_state cur_state, transition_state; + struct device *dev = &mhi_cntrl->mhi_dev->dev; + + /* If it's not a graceful shutdown, force MHI to linkdown state */ + transition_state = (graceful) ? MHI_PM_SHUTDOWN_PROCESS : + MHI_PM_LD_ERR_FATAL_DETECT; + + mutex_lock(&mhi_cntrl->pm_mutex); + write_lock_irq(&mhi_cntrl->pm_lock); + cur_state = mhi_tryset_pm_state(mhi_cntrl, transition_state); + if (cur_state != transition_state) { + MHI_ERR("Failed to move to state: %s from: %s\n", + to_mhi_pm_state_str(transition_state), + to_mhi_pm_state_str(mhi_cntrl->pm_state)); + /* Force link down or error fatal detected state */ + mhi_cntrl->pm_state = MHI_PM_LD_ERR_FATAL_DETECT; + } + + /* mark device inactive to avoid any further host processing */ + mhi_cntrl->ee = MHI_EE_DISABLE_TRANSITION; + mhi_cntrl->dev_state = MHI_STATE_RESET; + + wake_up_all(&mhi_cntrl->state_event); + + write_unlock_irq(&mhi_cntrl->pm_lock); + mutex_unlock(&mhi_cntrl->pm_mutex); + + mhi_queue_state_transition(mhi_cntrl, DEV_ST_TRANSITION_DISABLE); + + /* Wait for shutdown to complete */ + flush_work(&mhi_cntrl->st_worker); + + free_irq(mhi_cntrl->irq[0], mhi_cntrl); + + if (mhi_cntrl->fbc_image) + mhi_free_bhie_table(mhi_cntrl, &mhi_cntrl->fbc_image); +} +EXPORT_SYMBOL_GPL(mhi_power_down); + +int mhi_sync_power_up(struct mhi_controller *mhi_cntrl) +{ + int ret = mhi_async_power_up(mhi_cntrl); + + if (ret) + return ret; + + wait_event_timeout(mhi_cntrl->state_event, + MHI_IN_MISSION_MODE(mhi_cntrl->ee) || + MHI_PM_IN_ERROR_STATE(mhi_cntrl->pm_state), + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + + ret = (MHI_IN_MISSION_MODE(mhi_cntrl->ee)) ? 0 : -ETIMEDOUT; + if (ret && !mhi_cntrl->rddm_image) + mhi_power_down(mhi_cntrl, false); + + return ret; +} +EXPORT_SYMBOL(mhi_sync_power_up); + +int mhi_force_rddm_mode(struct mhi_controller *mhi_cntrl) +{ + struct device *dev = &mhi_cntrl->mhi_dev->dev; + int ret; + + /* Check if device is already in RDDM */ + if (mhi_cntrl->ee == MHI_EE_RDDM) + return 0; + + MHI_VERB("Triggering SYS_ERR to force RDDM state\n"); + mhi_set_mhi_state(mhi_cntrl, MHI_STATE_SYS_ERR); + + /* Wait for RDDM event */ + ret = wait_event_timeout(mhi_cntrl->state_event, + mhi_cntrl->ee == MHI_EE_RDDM, + msecs_to_jiffies(mhi_cntrl->timeout_ms)); + ret = ret ? 0 : -EIO; + + return ret; +} +EXPORT_SYMBOL_GPL(mhi_force_rddm_mode); + +void mhi_device_get(struct mhi_device *mhi_dev) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + + mhi_dev->dev_wake++; + read_lock_bh(&mhi_cntrl->pm_lock); + if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) + mhi_trigger_resume(mhi_cntrl); + + mhi_cntrl->wake_get(mhi_cntrl, true); + read_unlock_bh(&mhi_cntrl->pm_lock); +} +EXPORT_SYMBOL_GPL(mhi_device_get); + +int mhi_device_get_sync(struct mhi_device *mhi_dev) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + int ret; + + ret = __mhi_device_get_sync(mhi_cntrl); + if (!ret) + mhi_dev->dev_wake++; + + return ret; +} +EXPORT_SYMBOL_GPL(mhi_device_get_sync); + +void mhi_device_put(struct mhi_device *mhi_dev) +{ + struct mhi_controller *mhi_cntrl = mhi_dev->mhi_cntrl; + + mhi_dev->dev_wake--; + read_lock_bh(&mhi_cntrl->pm_lock); + if (MHI_PM_IN_SUSPEND_STATE(mhi_cntrl->pm_state)) + mhi_trigger_resume(mhi_cntrl); + + mhi_cntrl->wake_put(mhi_cntrl, false); + read_unlock_bh(&mhi_cntrl->pm_lock); +} +EXPORT_SYMBOL_GPL(mhi_device_put); |