summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorarnav_s <arnav_s@codeaurora.org>2018-11-20 16:15:52 -0800
committerarnav_s <arnav_s@codeaurora.org>2018-11-20 16:15:52 -0800
commite28553c0ebb924b506e22775ed1ce514ee8d8394 (patch)
treebd97192a25bb1934d1e9a1eeed70cb6c828ea237
parent0b734aefaf6d8e84e25cb5710467836c5043c1ff (diff)
parent659a8ea0fb392f80d278f18f24d9a98a9f6d9b58 (diff)
downloaddata-kernel-e28553c0ebb924b506e22775ed1ce514ee8d8394.tar.gz
Fastforwarding data-kernel CRT:data.lnx.4.0-181119 to data.lnx.5.0
-rw-r--r--drivers/emac-dwc-eqos/Android.mk2
-rw-r--r--drivers/emac-dwc-eqos/DWC_ETH_QOS_desc.c2
-rw-r--r--drivers/emac-dwc-eqos/DWC_ETH_QOS_eee.c42
-rw-r--r--drivers/emac-dwc-eqos/DWC_ETH_QOS_mdio.c71
-rw-r--r--drivers/emac-dwc-eqos/DWC_ETH_QOS_platform.c15
-rw-r--r--drivers/emac-dwc-eqos/DWC_ETH_QOS_rgmii_io_macro.c27
-rw-r--r--drivers/rmnet/perf/rmnet_perf_config.c1
-rw-r--r--drivers/rmnet/perf/rmnet_perf_core.c214
-rw-r--r--drivers/rmnet/perf/rmnet_perf_core.h1
-rw-r--r--drivers/rmnet/perf/rmnet_perf_tcp_opt.c38
-rw-r--r--drivers/rmnet/shs/rmnet_shs.h32
-rw-r--r--drivers/rmnet/shs/rmnet_shs_config.c3
-rw-r--r--drivers/rmnet/shs/rmnet_shs_config.h2
-rwxr-xr-x[-rw-r--r--]drivers/rmnet/shs/rmnet_shs_main.c421
-rw-r--r--drivers/rmnet/shs/rmnet_shs_wq.c98
-rw-r--r--drivers/rmnet/shs/rmnet_shs_wq.h12
16 files changed, 690 insertions, 291 deletions
diff --git a/drivers/emac-dwc-eqos/Android.mk b/drivers/emac-dwc-eqos/Android.mk
index 1c72e94..db943b9 100644
--- a/drivers/emac-dwc-eqos/Android.mk
+++ b/drivers/emac-dwc-eqos/Android.mk
@@ -2,6 +2,7 @@
ifeq ($(TARGET_BOARD_AUTO),true)
LOCAL_PATH := $(call my-dir)
+include $(CLEAR_VARS)
# This makefile is only for DLKM
ifneq ($(findstring vendor,$(LOCAL_PATH)),)
@@ -18,7 +19,6 @@ KBUILD_OPTIONS += DCONFIG_PTPSUPPORT_OBJ=1
KBUILD_OPTIONS += DCONFIG_DEBUGFS_OBJ=1
#KBUILD_OPTIONS += DDWC_ETH_QOS_TEST=1
-include $(CLEAR_VARS)
LOCAL_MODULE := emac_dwc_eqos.ko
LOCAL_MODULE_TAGS := debug
include $(DLKM_DIR)/AndroidKernelModule.mk
diff --git a/drivers/emac-dwc-eqos/DWC_ETH_QOS_desc.c b/drivers/emac-dwc-eqos/DWC_ETH_QOS_desc.c
index c092a0e..ac56b19 100644
--- a/drivers/emac-dwc-eqos/DWC_ETH_QOS_desc.c
+++ b/drivers/emac-dwc-eqos/DWC_ETH_QOS_desc.c
@@ -1692,7 +1692,7 @@ static unsigned int DWC_ETH_QOS_map_skb(struct net_device *dev,
}
}
- if (buffer->dma == 0 && buffer->dma2 == 0)
+ if (prev_buffer != NULL && buffer->dma == 0 && buffer->dma2 == 0)
prev_buffer->skb = skb;
else
buffer->skb = skb;
diff --git a/drivers/emac-dwc-eqos/DWC_ETH_QOS_eee.c b/drivers/emac-dwc-eqos/DWC_ETH_QOS_eee.c
index 7356b0b..5671e2e 100644
--- a/drivers/emac-dwc-eqos/DWC_ETH_QOS_eee.c
+++ b/drivers/emac-dwc-eqos/DWC_ETH_QOS_eee.c
@@ -285,6 +285,28 @@ static inline u32 DWC_ETH_QOS_mmd_eee_adv_to_ethtool_adv_t(u16 eee_adv)
#endif
/*!
+* \brief API to disable EEE mode.
+*
+* \details This function disable smart EEE
+* \param[in] phydev - pointer to target phy_device structure
+*/
+
+static void DWC_ETH_QOS_disable_smart_eee(struct phy_device *phydev)
+{
+ u32 smart_eee;
+
+ smart_eee = DWC_ETH_QOS_phy_read_mmd_indirect(
+ phydev->mdio.bus, AR8035_SMART_EEE_CTRL_3,
+ MDIO_MMD_PCS, phydev->mdio.addr);
+
+ smart_eee &= ~AR8035_SMART_EEE_EN;
+ DWC_ETH_QOS_phy_write_mmd_indirect(
+ phydev->mdio.bus, AR8035_SMART_EEE_CTRL_3,
+ MDIO_MMD_PCS,phydev->mdio.addr, smart_eee);
+
+}
+
+/*!
* \brief API to initialize and check EEE mode.
*
* \details This function checks if the EEE is supported by
@@ -378,15 +400,7 @@ static int DWC_ETH_QOS_phy_init_eee(struct phy_device *phydev,
/* Disable smart EEE feature in AR8035*/
if (phydev->phy_id == ATH8035_PHY_ID || phydev->phy_id == ATH8030_PHY_ID) {
- u32 smart_eee = DWC_ETH_QOS_phy_read_mmd_indirect(
- phydev->mdio.bus, AR8035_SMART_EEE_CTRL_3,
- MDIO_MMD_PCS, phydev->mdio.addr);
-
- smart_eee &= ~AR8035_SMART_EEE_EN;
-
- DWC_ETH_QOS_phy_write_mmd_indirect(
- phydev->mdio.bus, AR8035_SMART_EEE_CTRL_3,
- MDIO_MMD_PCS,phydev->mdio.addr, smart_eee);
+ DWC_ETH_QOS_disable_smart_eee(phydev);
}
ret = 0; /* EEE supported */
@@ -420,6 +434,16 @@ bool DWC_ETH_QOS_eee_init(struct DWC_ETH_QOS_prv_data *pdata)
EMACDBG("Enter\n");
hw_if = &pdata->hw_if;
+
+ /* Disable smart EEE & EEE for ATH8030*/
+ if ((pdata->emac_hw_version_type == EMAC_HW_v2_3_1)
+ && (pdata->io_macro_phy_intf == RMII_MODE) &&
+ pdata->phydev->phy_id == ATH8030_PHY_ID) {
+ //disable smart EEE
+ DWC_ETH_QOS_disable_smart_eee(pdata->phydev);
+ EMACDBG("disable smart EEE for 8030\n");
+ }
+
/* For RMII mode EEE is not supported */
if (pdata->io_macro_phy_intf == RMII_MODE)
goto phy_eee_failed;
diff --git a/drivers/emac-dwc-eqos/DWC_ETH_QOS_mdio.c b/drivers/emac-dwc-eqos/DWC_ETH_QOS_mdio.c
index 0ea301f..f295dc0 100644
--- a/drivers/emac-dwc-eqos/DWC_ETH_QOS_mdio.c
+++ b/drivers/emac-dwc-eqos/DWC_ETH_QOS_mdio.c
@@ -507,20 +507,38 @@ static void set_phy_rx_tx_delay(struct DWC_ETH_QOS_prv_data *pdata,
if ((pdata->phydev->phy_id & pdata->phydev->drv->phy_id_mask) == MICREL_PHY_ID) {
u16 phydata = 0;
u16 rx_clk = 0;
+ if (pdata->emac_hw_version_type == EMAC_HW_v2_3_1) {
+ if(!pdata->io_macro_tx_mode_non_id){
+ EMACDBG("No PHY delay settings required for ID mode for "
+ "EMAC core version 2.3.1 \n");
+ return;
+ }
+ rx_clk = 22;
+ /* RX_CLK to 0*/
+ DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
+ DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x8,&phydata);
+ phydata &= ~(0x1F<<5);
+ phydata |= (rx_clk << 5);
+ DWC_ETH_QOS_mdio_mmd_register_write_direct(pdata, pdata->phyaddr,
+ DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x8,phydata);
- rx_clk = 0x1F;
- /* RX_CLK to 0*/
- DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
+ DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
+ DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x8,&phydata);
+ EMACDBG("Read 0x%x from offset 0x8\n",phydata);
+ } else {
+ rx_clk = 0x1F;
+ /* RX_CLK to 0*/
+ DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x8,&phydata);
- phydata &= ~(0x1F);
- phydata |= rx_clk;
- DWC_ETH_QOS_mdio_mmd_register_write_direct(pdata, pdata->phyaddr,
+ phydata &= ~(0x1F);
+ phydata |= rx_clk;
+ DWC_ETH_QOS_mdio_mmd_register_write_direct(pdata, pdata->phyaddr,
DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x8,phydata);
- DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
+ DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x8,&phydata);
- EMACDBG("Read 0x%x from offset 0x8\n",phydata);
- phydata = 0;
+ EMACDBG("Read 0x%x from offset 0x8\n",phydata);
+ phydata = 0;
if (pdata->emac_hw_version_type == EMAC_HW_v2_1_2) {
u16 tx_clk = 0xE;
@@ -547,15 +565,15 @@ static void set_phy_rx_tx_delay(struct DWC_ETH_QOS_prv_data *pdata,
/* Default settings for EMAC_HW_v2_1_0 */
phydata |= ((0x0 << 12) | (0x0 << 8) | (0x0 << 4) | 0x0);
- DWC_ETH_QOS_mdio_mmd_register_write_direct(pdata, pdata->phyaddr,
+ DWC_ETH_QOS_mdio_mmd_register_write_direct(pdata, pdata->phyaddr,
DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x5,phydata);
- DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
+ DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x5,&phydata);
- EMACDBG("Read 0x%x from offset 0x5\n",phydata);
- phydata = 0;
+ EMACDBG("Read 0x%x from offset 0x5\n",phydata);
+ phydata = 0;
- /*RX_CTL to 9*/
- DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
+ /*RX_CTL to 9*/
+ DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x4,&phydata);
phydata &= ~(0xF << 4);
if (pdata->emac_hw_version_type == EMAC_HW_v2_1_2)
@@ -565,10 +583,11 @@ static void set_phy_rx_tx_delay(struct DWC_ETH_QOS_prv_data *pdata,
phydata |= (0x0 << 4);
DWC_ETH_QOS_mdio_mmd_register_write_direct(pdata, pdata->phyaddr,
DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x4,phydata);
- DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
+ DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x4,&phydata);
- EMACDBG("Read 0x%x from offset 0x4\n",phydata);
- phydata = 0;
+ EMACDBG("Read 0x%x from offset 0x4\n",phydata);
+ phydata = 0;
+ }
} else {
/* Default values are for PHY AR8035 */
int phydata = 0;
@@ -619,6 +638,11 @@ static void set_phy_rx_tx_delay(struct DWC_ETH_QOS_prv_data *pdata,
static void configure_phy_rx_tx_delay(struct DWC_ETH_QOS_prv_data *pdata)
{
EMACDBG("Enter\n");
+ if ((pdata->emac_hw_version_type == EMAC_HW_v2_3_1)
+ && (pdata->io_macro_phy_intf == RMII_MODE)) {
+ EMACDBG("phy rx tx delay setting not required for RMII mode for 2.3.1\n");
+ return;
+ }
switch (pdata->speed) {
case SPEED_1000:
@@ -764,8 +788,8 @@ static inline int DWC_ETH_QOS_configure_io_macro_dll_settings(
EMACDBG("Enter\n");
- if (pdata->emac_hw_version_type == EMAC_HW_v2_0_0)
#ifndef DWC_ETH_QOS_EMULATION_PLATFORM
+ if (pdata->emac_hw_version_type == EMAC_HW_v2_0_0 || pdata->emac_hw_version_type == EMAC_HW_v2_3_1)
DWC_ETH_QOS_rgmii_io_macro_dll_reset(pdata);
/* For RGMII ID mode with internal delay*/
@@ -1143,15 +1167,6 @@ static int DWC_ETH_QOS_init_phy(struct net_device *dev)
phydev->irq = PHY_IGNORE_INTERRUPT;
phydev->interrupts = PHY_INTERRUPT_ENABLED;
- if ((phydev->phy_id & phydev->drv->phy_id_mask) == MICREL_PHY_ID) {
- DWC_ETH_QOS_mdio_write_direct(pdata, pdata->phyaddr,
- DWC_ETH_QOS_MICREL_PHY_CTL, DWC_ETH_QOS_MICREL_INTR_LEVEL);
- DWC_ETH_QOS_mdio_read_direct(pdata, pdata->phyaddr,
- DWC_ETH_QOS_MICREL_PHY_CTL, &phydata);
- EMACDBG("Micrel PHY Control Reg (%#x) = %#x\n",
- DWC_ETH_QOS_MICREL_PHY_CTL, phydata);
- }
-
if (phydev->drv->config_intr &&
!phydev->drv->config_intr(phydev))
DWC_ETH_QOS_request_phy_wol(pdata);
diff --git a/drivers/emac-dwc-eqos/DWC_ETH_QOS_platform.c b/drivers/emac-dwc-eqos/DWC_ETH_QOS_platform.c
index 2b85fc4..4c8445b 100644
--- a/drivers/emac-dwc-eqos/DWC_ETH_QOS_platform.c
+++ b/drivers/emac-dwc-eqos/DWC_ETH_QOS_platform.c
@@ -110,6 +110,18 @@ static ssize_t read_phy_reg_dump(struct file *file,
i, phydata);
}
+ if ((pdata->phydev->phy_id & pdata->phydev->drv->phy_id_mask) == MICREL_PHY_ID) {
+ int i = 0;
+ u16 mmd_phydata = 0;
+ for(i=0;i<=8;i++){
+ DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr,
+ DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR, i, &mmd_phydata);
+ EMACDBG("Read %#x from offset %#x", mmd_phydata, i);
+ len += scnprintf(buf + len, buf_len - len,
+ "Micrel PHY MMD Register (%#x) = %#x\n", i, mmd_phydata);
+ }
+ }
+
if (len > buf_len) {
EMACERR(" %s (len > buf_len) buffer not sufficient\n",__func__);
len = buf_len;
@@ -853,7 +865,8 @@ int DWC_ETH_QOS_enable_ptp_clk(struct device *dev)
int ret;
const char* ptp_clock_name;
- if (dwc_eth_qos_res_data.emac_hw_version_type == EMAC_HW_v2_1_0)
+ if (dwc_eth_qos_res_data.emac_hw_version_type == EMAC_HW_v2_1_0
+ || dwc_eth_qos_res_data.emac_hw_version_type == EMAC_HW_v2_1_2)
ptp_clock_name = "emac_ptp_clk";
else
ptp_clock_name = "eth_ptp_clk";
diff --git a/drivers/emac-dwc-eqos/DWC_ETH_QOS_rgmii_io_macro.c b/drivers/emac-dwc-eqos/DWC_ETH_QOS_rgmii_io_macro.c
index 982a14e..6bb7da4 100644
--- a/drivers/emac-dwc-eqos/DWC_ETH_QOS_rgmii_io_macro.c
+++ b/drivers/emac-dwc-eqos/DWC_ETH_QOS_rgmii_io_macro.c
@@ -363,7 +363,8 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata)
uint loopback_mode_en = 0;
uint rgmii_data_divide_clk;
ULONG data;
- if (pdata->emac_hw_version_type == EMAC_HW_v2_3_0) {
+
+ if (pdata->emac_hw_version_type == EMAC_HW_v2_3_0 || (pdata->emac_hw_version_type == EMAC_HW_v2_3_1)) {
if(pdata->io_macro_phy_intf == RGMII_MODE)
loopback_mode_en = 0x1;
rgmii_data_divide_clk = 0x0;
@@ -374,9 +375,6 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata)
EMACDBG("Enter\n");
- if (pdata->emac_hw_version_type == EMAC_HW_v2_3_0)
- loopback_mode = 0x1;
-
/* Loopback is disabled */
DWC_ETH_QOS_set_rgmii_loopback_mode(loopback_mode);
@@ -404,7 +402,8 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata)
RGMII_CONFIG_2_RX_PROG_SWAP_UDFWR(0x1);
RGMII_LOOPBACK_EN_UDFWR(loopback_mode_en);
if (pdata->emac_hw_version_type == EMAC_HW_v2_1_0 ||
- pdata->emac_hw_version_type == EMAC_HW_v2_1_2)
+ pdata->emac_hw_version_type == EMAC_HW_v2_1_2 ||
+ (pdata->emac_hw_version_type == EMAC_HW_v2_3_1))
RGMII_CONFIG_2_TX_CLK_PHASE_SHIFT_EN_UDFWR(0x1);
} else {
/* Enable DDR mode*/
@@ -426,8 +425,11 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata)
/* Program PRG_RCLK_DLY to 52 ns for a required delay of 2 ns
on EMAC core version 2.1.0 */
- if (pdata->emac_hw_version_type == EMAC_HW_v2_1_0)
+ if (pdata->emac_hw_version_type == EMAC_HW_v2_1_0
+ || pdata->emac_hw_version_type == EMAC_HW_v2_1_2)
SDCC_HC_PRG_RCLK_DLY_UDFWR(52);
+ else if (pdata->emac_hw_version_type == EMAC_HW_v2_3_1)
+ SDCC_HC_PRG_RCLK_DLY_UDFWR(104);
else { /* Program PRG_RCLK_DLY to 57 for a required delay of 1.8 ns */
SDCC_HC_PRG_RCLK_DLY_UDFWR(57);
}
@@ -457,7 +459,8 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata)
RGMII_CONFIG_2_RX_PROG_SWAP_UDFWR(0x0);
RGMII_LOOPBACK_EN_UDFWR(loopback_mode_en);
if (pdata->emac_hw_version_type == EMAC_HW_v2_1_0 ||
- pdata->emac_hw_version_type == EMAC_HW_v2_1_2)
+ pdata->emac_hw_version_type == EMAC_HW_v2_1_2 ||
+ (pdata->emac_hw_version_type == EMAC_HW_v2_3_1))
RGMII_CONFIG_2_RX_PROG_SWAP_UDFWR(0x1);
if (pdata->emac_hw_version_type == EMAC_HW_v2_1_2)
RGMII_CONFIG_2_TX_CLK_PHASE_SHIFT_EN_UDFWR(0x1);
@@ -498,10 +501,13 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata)
RGMII_MAX_SPD_PRG_9_UDFWR(0x13);
RGMII_CONFIG_2_RERVED_CONFIG_16_EN_UDFWR(0x1);
/* Rx Path */
+ if (pdata->emac_hw_version_type == EMAC_HW_v2_3_1)
+ RGMII_LOOPBACK_EN_UDFWR(loopback_mode_en);
RGMII_CONFIG_2_RX_PROG_SWAP_UDFWR(0x0);
RGMII_LOOPBACK_EN_UDFWR(loopback_mode_en);
if (pdata->emac_hw_version_type == EMAC_HW_v2_1_0 ||
- pdata->emac_hw_version_type == EMAC_HW_v2_1_2)
+ pdata->emac_hw_version_type == EMAC_HW_v2_1_2 ||
+ (pdata->emac_hw_version_type == EMAC_HW_v2_3_1))
RGMII_CONFIG_2_RX_PROG_SWAP_UDFWR(0x1);
if (pdata->emac_hw_version_type == EMAC_HW_v2_1_2)
RGMII_CONFIG_2_TX_CLK_PHASE_SHIFT_EN_UDFWR(0x1);
@@ -548,7 +554,7 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata)
RGMII_MAX_SPD_PRG_2_UDFWR(0x1);
RGMII_MAX_SPD_PRG_9_UDFWR(0x13);
RGMII_CONFIG_2_RERVED_CONFIG_16_EN_UDFWR(0x0);
- if (pdata->emac_hw_version_type == EMAC_HW_v2_3_0)
+ if (pdata->emac_hw_version_type == EMAC_HW_v2_3_0 || (pdata->emac_hw_version_type == EMAC_HW_v2_3_1))
RGMII_LOOPBACK_EN_UDFWR(0x0);
else
RGMII_LOOPBACK_EN_UDFWR(0x1);
@@ -567,7 +573,8 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata)
RGMII_CONFIG_2_RERVED_CONFIG_16_EN_UDFWR(0x1);
if (pdata->emac_hw_version_type == EMAC_HW_v2_1_2)
RGMII_CONFIG_2_TX_CLK_PHASE_SHIFT_EN_UDFWR(0x1);
-
+ if (pdata->emac_hw_version_type == EMAC_HW_v2_3_1)
+ RGMII_LOOPBACK_EN_UDFWR(0x1);
break;
}
diff --git a/drivers/rmnet/perf/rmnet_perf_config.c b/drivers/rmnet/perf/rmnet_perf_config.c
index 550ad07..ba64176 100644
--- a/drivers/rmnet/perf/rmnet_perf_config.c
+++ b/drivers/rmnet/perf/rmnet_perf_config.c
@@ -289,6 +289,7 @@ static int rmnet_perf_config_notify_cb(struct notifier_block *nb,
dev_net_set(dev, &init_net);
perf->core_meta->dev = dev;
+ dl_ind->priority = RMNET_PERF;
dl_ind->dl_hdr_handler =
&rmnet_perf_core_handle_map_control_start;
dl_ind->dl_trl_handler =
diff --git a/drivers/rmnet/perf/rmnet_perf_core.c b/drivers/rmnet/perf/rmnet_perf_core.c
index bc61e27..c9c2ccc 100644
--- a/drivers/rmnet/perf/rmnet_perf_core.c
+++ b/drivers/rmnet/perf/rmnet_perf_core.c
@@ -355,20 +355,25 @@ void rmnet_perf_core_send_skb(struct sk_buff *skb, struct rmnet_endpoint *ep,
if (ip_version == 0x04) {
ip4hn = (struct iphdr *) data;
rmnet_set_skb_proto(skb);
- if (ip4hn->protocol == IPPROTO_TCP) {
+ /* If the checksum is unnecessary, update the header fields.
+ * Otherwise, we know that this is a single packet that
+ * failed checksum validation, so we don't want to touch
+ * the headers.
+ */
+ if (ip4hn->protocol == IPPROTO_TCP &&
+ skb->ip_summed == CHECKSUM_UNNECESSARY) {
ip4hn->tot_len = htons(skb->len);
ip4hn->check = 0;
ip4hn->check = ip_fast_csum(ip4hn, (int)ip4hn->ihl);
- skb->ip_summed = CHECKSUM_UNNECESSARY;
}
rmnet_deliver_skb(skb, perf->rmnet_port);
} else if (ip_version == 0x06) {
ip6hn = (struct ipv6hdr *)data;
rmnet_set_skb_proto(skb);
- if (ip6hn->nexthdr == IPPROTO_TCP) {
+ if (ip6hn->nexthdr == IPPROTO_TCP &&
+ skb->ip_summed == CHECKSUM_UNNECESSARY) {
ip6hn->payload_len = htons(skb->len -
sizeof(struct ipv6hdr));
- skb->ip_summed = CHECKSUM_UNNECESSARY;
}
rmnet_deliver_skb(skb, perf->rmnet_port);
} else {
@@ -423,7 +428,12 @@ void rmnet_perf_core_flush_curr_pkt(struct rmnet_perf *perf,
__skb_set_hash(skbn, 0, 0, 0);
}
+ /* If the packet passed checksum validation, tell the stack */
+ if (pkt_info->csum_valid)
+ skbn->ip_summed = CHECKSUM_UNNECESSARY;
skbn->dev = skb->dev;
+ skbn->hash = pkt_info->hash_key;
+ skbn->sw_hash = 1;
rmnet_perf_core_send_skb(skbn, ep, perf, pkt_info);
}
@@ -468,6 +478,107 @@ void rmnet_perf_core_handle_map_control_end(struct rmnet_map_dl_ind_trl *dltrl)
0xDEF, 0xDEF, NULL, NULL);
}
+int rmnet_perf_core_validate_pkt_csum(struct sk_buff *skb,
+ struct rmnet_perf_pkt_info *pkt_info)
+{
+ int result;
+ unsigned int pkt_len = pkt_info->header_len + pkt_info->payload_len;
+
+ skb_pull(skb, sizeof(struct rmnet_map_header));
+ if (pkt_info->ip_proto == 0x04) {
+ skb->protocol = htons(ETH_P_IP);
+ } else if (pkt_info->ip_proto == 0x06) {
+ skb->protocol = htons(ETH_P_IPV6);
+ } else {
+ pr_err("%s(): protocol field not set properly, protocol = %u\n",
+ __func__, pkt_info->ip_proto);
+ }
+ result = rmnet_map_checksum_downlink_packet(skb, pkt_len);
+ skb_push(skb, sizeof(struct rmnet_map_header));
+ /* Mark the current packet as OK if csum is valid */
+ if (likely(result == 0))
+ pkt_info->csum_valid = true;
+ return result;
+}
+
+void rmnet_perf_core_handle_packet_ingress(struct sk_buff *skb,
+ struct rmnet_endpoint *ep,
+ struct rmnet_perf_pkt_info *pkt_info,
+ u32 frame_len, u32 trailer_len)
+{
+ unsigned char *payload = (unsigned char *)
+ (skb->data + sizeof(struct rmnet_map_header));
+ struct rmnet_perf *perf = rmnet_perf_config_get_perf();
+ u16 pkt_len;
+
+ pkt_info->ep = ep;
+ pkt_info->ip_proto = (*payload & 0xF0) >> 4;
+ if (pkt_info->ip_proto == 4) {
+ struct iphdr *iph = (struct iphdr *)payload;
+
+ pkt_info->iphdr.v4hdr = iph;
+ pkt_info->trans_proto = iph->protocol;
+ pkt_info->header_len = iph->ihl * 4;
+ } else if (pkt_info->ip_proto == 6) {
+ struct ipv6hdr *iph = (struct ipv6hdr *)payload;
+
+ pkt_info->iphdr.v6hdr = iph;
+ pkt_info->trans_proto = iph->nexthdr;
+ pkt_info->header_len = sizeof(*iph);
+ } else {
+ pr_err("%s(): invalid packet\n", __func__);
+ return;
+ }
+
+ pkt_len = frame_len - sizeof(struct rmnet_map_header) - trailer_len;
+
+ if (pkt_info->trans_proto == IPPROTO_TCP) {
+ struct tcphdr *tp = (struct tcphdr *)
+ (payload + pkt_info->header_len);
+
+ pkt_info->trns_hdr.tp = tp;
+ pkt_info->header_len += tp->doff * 4;
+ pkt_info->payload_len = pkt_len - pkt_info->header_len;
+ pkt_info->hash_key =
+ rmnet_perf_core_compute_flow_hash(pkt_info);
+
+ if (rmnet_perf_core_validate_pkt_csum(skb, pkt_info))
+ goto flush;
+
+ rmnet_perf_tcp_opt_ingress(perf, skb, pkt_info);
+ } else if (pkt_info->trans_proto == IPPROTO_UDP) {
+ struct udphdr *up = (struct udphdr *)
+ (payload + pkt_info->header_len);
+
+ pkt_info->trns_hdr.up = up;
+ pkt_info->header_len += sizeof(*up);
+ pkt_info->payload_len = pkt_len - pkt_info->header_len;
+ pkt_info->hash_key =
+ rmnet_perf_core_compute_flow_hash(pkt_info);
+
+ /* We flush anyway, so the result of the validation
+ * does not need to be checked.
+ */
+ rmnet_perf_core_validate_pkt_csum(skb, pkt_info);
+ goto flush;
+ } else {
+ pkt_info->payload_len = pkt_len - pkt_info->header_len;
+ pkt_info->hash_key =
+ rmnet_perf_core_compute_flow_hash(pkt_info);
+
+ /* We flush anyway, so the result of the validation
+ * does not need to be checked.
+ */
+ rmnet_perf_core_validate_pkt_csum(skb, pkt_info);
+ goto flush;
+ }
+
+ return;
+
+flush:
+ rmnet_perf_core_flush_curr_pkt(perf, skb, pkt_info, pkt_len);
+}
+
/* rmnet_perf_core_deaggregate() - Deaggregated ip packets from map frame
* @port: allows access to our required global structures
* @skb: the incoming aggregated MAP frame from PND
@@ -482,12 +593,8 @@ void rmnet_perf_core_deaggregate(struct sk_buff *skb,
struct rmnet_port *port)
{
u8 mux_id;
- u16 ip_packet_len;
struct rmnet_map_header *maph;
uint32_t map_frame_len;
- unsigned char *map_payload;
- struct iphdr *ip4h;
- struct ipv6hdr *ip6h;
struct rmnet_endpoint *ep;
struct rmnet_perf_pkt_info pkt_info;
struct rmnet_perf *perf;
@@ -569,91 +676,10 @@ skip_frame:
* with processing the packet i.e. we know we are
* dealing with a packet with no funny business inside
*/
- pkt_info.ep = ep;
- map_payload = (unsigned char *)(skb->data +
- sizeof(struct rmnet_map_header));
- pkt_info.ip_proto = (*map_payload & 0xF0) >> 4;
- if (pkt_info.ip_proto == 0x04) {
- ip4h = (struct iphdr *)map_payload;
- pkt_info.iphdr.v4hdr = (struct iphdr *) ip4h;
- pkt_info.trans_proto = ip4h->protocol;
- if (pkt_info.trans_proto == IPPROTO_TCP) {
- pkt_info.trns_hdr.tp = (struct tcphdr *)
- (map_payload + ip4h->ihl*4);
- pkt_info.header_len = (ip4h->ihl * 4) +
- (pkt_info.trns_hdr.tp->doff * 4);
- pkt_info.payload_len = map_frame_len -
- (sizeof(struct rmnet_map_header) +
- trailer_len +
- pkt_info.header_len);
- pkt_info.hash_key =
- rmnet_perf_core_compute_flow_hash(&pkt_info);
- rmnet_perf_tcp_opt_ingress(perf, skb,
- &pkt_info);
- } else if (pkt_info.trans_proto ==
- IPPROTO_UDP) {
- pkt_info.trns_hdr.up = (struct udphdr *)
- (map_payload + ip4h->ihl*4);
- ip_packet_len = map_frame_len -
- (sizeof(struct rmnet_map_header) +
- trailer_len);
- pkt_info.hash_key =
- rmnet_perf_core_compute_flow_hash(&pkt_info);
- rmnet_perf_core_flush_curr_pkt(perf,
- skb, &pkt_info, ip_packet_len);
- } else {
- ip_packet_len = map_frame_len -
- (sizeof(struct rmnet_map_header) +
- trailer_len);
- pkt_info.hash_key =
- rmnet_perf_core_compute_flow_hash(&pkt_info);
- rmnet_perf_core_flush_curr_pkt(perf,
- skb, &pkt_info, ip_packet_len);
- }
- } else if (pkt_info.ip_proto == 0x06) {
- ip6h = (struct ipv6hdr *)map_payload;
- pkt_info.iphdr.v6hdr = (struct ipv6hdr *) ip6h;
- pkt_info.trans_proto = ip6h->nexthdr;
- if (pkt_info.trans_proto == IPPROTO_TCP) {
- pkt_info.trns_hdr.tp = (struct tcphdr *)
- (map_payload +
- sizeof(struct ipv6hdr));
- pkt_info.header_len =
- sizeof(struct ipv6hdr) +
- (pkt_info.trns_hdr.tp->doff *
- 4);
- pkt_info.payload_len = map_frame_len -
- (sizeof(struct rmnet_map_header) +
- trailer_len +
- pkt_info.header_len);
- pkt_info.hash_key =
- rmnet_perf_core_compute_flow_hash(&pkt_info);
- rmnet_perf_tcp_opt_ingress(perf, skb,
- &pkt_info);
- } else if (pkt_info.trans_proto ==
- IPPROTO_UDP) {
- pkt_info.trns_hdr.up = (struct udphdr *)
- (map_payload +
- sizeof(struct ipv6hdr));
- ip_packet_len = map_frame_len -
- (sizeof(struct rmnet_map_header) +
- trailer_len);
- pkt_info.hash_key =
- rmnet_perf_core_compute_flow_hash(&pkt_info);
- rmnet_perf_core_flush_curr_pkt(perf,
- skb, &pkt_info, ip_packet_len);
- } else {
- ip_packet_len = map_frame_len -
- (sizeof(struct rmnet_map_header) +
- trailer_len);
- pkt_info.hash_key =
- rmnet_perf_core_compute_flow_hash(&pkt_info);
- rmnet_perf_core_flush_curr_pkt(perf,
- skb, &pkt_info, ip_packet_len);
- }
- } else {
- pr_err("%s(): invalid packet\n", __func__);
- }
+ rmnet_perf_core_handle_packet_ingress(skb, ep,
+ &pkt_info,
+ map_frame_len,
+ trailer_len);
bad_data:
skb_pull(skb, map_frame_len);
co++;
@@ -667,7 +693,7 @@ next_chain:
* then we can flush everything
*/
if (!rmnet_perf_core_bm_flush_on ||
- perf->core_meta->bm_state->expect_packets <= 0) {
+ (int) perf->core_meta->bm_state->expect_packets <= 0) {
rmnet_perf_tcp_opt_flush_all_flow_nodes(perf);
rmnet_perf_core_free_held_skbs(perf);
rmnet_perf_core_flush_reason_cnt[
diff --git a/drivers/rmnet/perf/rmnet_perf_core.h b/drivers/rmnet/perf/rmnet_perf_core.h
index da35804..4019bc7 100644
--- a/drivers/rmnet/perf/rmnet_perf_core.h
+++ b/drivers/rmnet/perf/rmnet_perf_core.h
@@ -31,6 +31,7 @@ struct rmnet_perf {
*/
struct rmnet_perf_pkt_info {
bool first_packet;
+ bool csum_valid;
unsigned char ip_proto;
unsigned char trans_proto;
u16 header_len;
diff --git a/drivers/rmnet/perf/rmnet_perf_tcp_opt.c b/drivers/rmnet/perf/rmnet_perf_tcp_opt.c
index 2fa7453..f673884 100644
--- a/drivers/rmnet/perf/rmnet_perf_tcp_opt.c
+++ b/drivers/rmnet/perf/rmnet_perf_tcp_opt.c
@@ -136,9 +136,7 @@ rmnet_perf_tcp_opt_tcp_flag_flush(struct rmnet_perf_pkt_info *pkt_info)
* @pkt_info: characteristics of the current packet
*
* 1. check src/dest IP addr and TCP port & next seq match
- * 2. validate checksum - it also checks the ip flag
- * 3. check if timestamp changed
- * 4. check if size overflow
+ * 2. check if size overflow
*
* Return:
* - true if Pkt can be merged
@@ -151,7 +149,6 @@ rmnet_perf_tcp_opt_pkt_can_be_merged(struct sk_buff *skb,
{
struct iphdr *ip4h;
struct ipv6hdr *ip6h;
- int result = 0;
u16 payload_len = pkt_info->payload_len;
struct tcphdr *tp = pkt_info->trns_hdr.tp;
@@ -186,29 +183,7 @@ rmnet_perf_tcp_opt_pkt_can_be_merged(struct sk_buff *skb,
return RMNET_PERF_TCP_OPT_FLUSH_SOME;
}
- /* 2. validate checksum - it also checks the ip flag */
- skb_pull(skb, sizeof(struct rmnet_map_header));
- if (pkt_info->ip_proto == 0x04) {
- skb->protocol = htons(ETH_P_IP);
- } else if (pkt_info->ip_proto == 0x06) {
- skb->protocol = htons(ETH_P_IPV6);
- } else {
- pr_err("%s(): protocol field not set properly, protocol = %u\n",
- __func__, pkt_info->ip_proto);
- }
- result = rmnet_map_checksum_downlink_packet(skb, pkt_info->header_len +
- payload_len);
- skb_push(skb, sizeof(struct rmnet_map_header));
- if (!(likely((result == 0)))) {
- //TODO: later on might want to make macro out of this
- //(result == RMNET_MAP_CHECKSUM_SKIPPED)))) {
- pr_err("Wrong csum %d", result);
- rmnet_perf_tcp_opt_flush_reason_cnt[
- RMNET_PERF_TCP_OPT_CHECKSUM_ERR]++;
- return RMNET_PERF_TCP_OPT_FLUSH_ALL;
- }
-
- /* 3. check if size overflow */
+ /* 2. check if size overflow */
if ((payload_len + flow_node->len >= rmnet_perf_tcp_opt_flush_limit)) {
rmnet_perf_tcp_opt_flush_reason_cnt[
RMNET_PERF_TCP_OPT_64K_LIMIT]++;
@@ -332,6 +307,9 @@ static void rmnet_perf_tcp_opt_flush_single_flow_node(struct rmnet_perf *perf,
__func__);
} else {
skbn->hash = flow_node->hash_value;
+ skbn->sw_hash = 1;
+ /* data is already validated */
+ skbn->ip_summed = CHECKSUM_UNNECESSARY;
rmnet_perf_core_send_skb(skbn, ep,
perf, NULL);
}
@@ -593,6 +571,7 @@ void rmnet_perf_tcp_opt_ingress(struct rmnet_perf *perf, struct sk_buff *skb,
bool match;
enum rmnet_perf_tcp_opt_merge_check_rc rc = 0;
bool flow_node_exists = 0;
+ struct napi_struct *napi = NULL;
//pkt_info->hash_key = rmnet_perf_core_compute_flow_hash(pkt_info);
handle_pkt:
@@ -624,9 +603,12 @@ handle_pkt:
pkt_info);
rmnet_perf_tcp_opt_flush_single_flow_node(perf,
flow_node);
+ napi = get_current_napi_context();
+ napi_gro_flush(napi, false);
rmnet_perf_core_flush_curr_pkt(perf, skb,
pkt_info,
pkt_info->header_len + pkt_info->payload_len);
+ napi_gro_flush(napi, false);
rmnet_perf_tcp_opt_flush_reason_cnt[
RMNET_PERF_TCP_OPT_TCP_FLUSH_FORCE]++;
} else if (rc == RMNET_PERF_TCP_OPT_FLUSH_ALL) {
@@ -663,6 +645,8 @@ handle_pkt:
rmnet_perf_tcp_opt_update_flow(flow_node, pkt_info);
rmnet_perf_core_flush_curr_pkt(perf, skb, pkt_info,
pkt_info->header_len + pkt_info->payload_len);
+ napi = get_current_napi_context();
+ napi_gro_flush(napi, false);
rmnet_perf_tcp_opt_flush_reason_cnt[
RMNET_PERF_TCP_OPT_TCP_FLUSH_FORCE]++;
} else
diff --git a/drivers/rmnet/shs/rmnet_shs.h b/drivers/rmnet/shs/rmnet_shs.h
index 7f6336a..ee12024 100644
--- a/drivers/rmnet/shs/rmnet_shs.h
+++ b/drivers/rmnet/shs/rmnet_shs.h
@@ -16,6 +16,9 @@
#include <linux/skbuff.h>
#include "rmnet_shs_wq.h"
+#ifndef _RMNET_SHS_H_
+#define _RMNET_SHS_H_
+
#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h>
#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h>
#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h>
@@ -24,14 +27,11 @@
#include <../include/soc/qcom/qmi_rmnet.h>
-
-#ifndef _RMNET_SHS_H_
-#define _RMNET_SHS_H_
-
#define RMNET_SHS_HT rmnet_shs_ht
#define RMNET_SHS_HT_SIZE 9
#define RMNET_SHS_MAX_SKB_INACTIVE_TSEC 30
#define MAX_SILVER_CORES 4
+#define MAX_CPUS 8
//#define RMNET_SHS_MAX_UDP_SILVER_CORE_DATA_RATE 1073741824 //1.0Gbps
//#define RMNET_SHS_MAX_UDP_SILVER_CORE_DATA_RATE 320787200 //320 Mbps
@@ -53,18 +53,26 @@
#define RMNET_SHS_UDP_PPS_PERF_CPU_LTHRESH 40000
#define RMNET_SHS_TCP_PPS_PERF_CPU_LTHRESH (40000*RMNET_SHS_TCP_COALESCING_RATIO)
+struct core_flush_s {
+ struct hrtimer core_timer;
+ struct work_struct work;
+ struct timespec coretime;
+ int coresum;
+ u8 core;
+};
+
struct rmnet_shs_cfg_s {
struct hrtimer hrtimer_shs;
- struct rps_map *map;
struct rmnet_map_dl_ind dl_mrk_ind_cb;
struct qmi_rmnet_ps_ind rmnet_idl_ind_cb;
struct rmnet_port *port;
+ struct core_flush_s core_flush[MAX_CPUS];
+ u64 core_skbs[MAX_CPUS];
long int num_bytes_parked;
long int num_pkts_parked;
u32 is_reg_dl_mrk_ind;
u8 is_pkt_parked;
u8 is_timer_init;
- u8 high_prio;
u8 force_flush_state;
};
@@ -93,8 +101,6 @@ struct rmnet_shs_skbn_s {
/* n/w stack CPU pkt processing queue head */
u32 hash;
/*incoming hash*/
- u32 parked_skbs;
- /*num skbs parked per flow*/
u16 map_index;
/* rps map index assigned*/
u16 map_cpu;
@@ -132,6 +138,10 @@ struct rmnet_shs_cpu_node_s {
struct list_head node_list_id;
u32 qhead;
u32 qtail;
+ u32 qdiff;
+ u32 parkedlen;
+ u8 prio;
+ u8 wqprio;
};
enum rmnet_shs_trace_func {
@@ -215,6 +225,7 @@ enum rmnet_shs_trace_evt {
RMNET_SHS_DL_MRK_END,
};
+extern struct rmnet_shs_flush_work shs_delayed_work;
extern spinlock_t rmnet_shs_ht_splock;
extern struct hlist_head RMNET_SHS_HT[1 << (RMNET_SHS_HT_SIZE)];
@@ -225,8 +236,9 @@ extern int (*rmnet_shs_skb_entry)(struct sk_buff *skb,
struct rmnet_port *port);
int rmnet_shs_is_lpwr_cpu(u16 cpu);
void rmnet_shs_cancel_table(void);
-
void rmnet_shs_aggregate_init(void);
+
+int rmnet_shs_chk_and_flush_node(struct rmnet_shs_skbn_s *node, u8 force_flush);
void rmnet_shs_dl_hdr_handler(struct rmnet_map_dl_ind_hdr *dlhdr);
void rmnet_shs_dl_trl_handler(struct rmnet_map_dl_ind_trl *dltrl);
void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port);
@@ -236,5 +248,5 @@ void rmnet_shs_init(struct net_device *dev);
void rmnet_shs_exit(void);
void rmnet_shs_ps_on_hdlr(void *port);
void rmnet_shs_ps_off_hdlr(void *port);
-
+void rmnet_shs_update_cpu_proc_q_all_cpus(void);
#endif /* _RMNET_SHS_H_ */
diff --git a/drivers/rmnet/shs/rmnet_shs_config.c b/drivers/rmnet/shs/rmnet_shs_config.c
index 2d84488..3a29f04 100644
--- a/drivers/rmnet/shs/rmnet_shs_config.c
+++ b/drivers/rmnet/shs/rmnet_shs_config.c
@@ -15,6 +15,7 @@
#include <linux/kernel.h>
#include <linux/netdevice.h>
#include <linux/module.h>
+#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h>
#include "rmnet_shs_config.h"
#include "rmnet_shs.h"
#include "rmnet_shs_wq.h"
@@ -115,6 +116,8 @@ static int rmnet_shs_dev_notify_cb(struct notifier_block *nb,
rmnet_shs_wq_init(phy_dev);
rmnet_shs_aggregate_init();
rmnet_shs_cfg.is_timer_init = 1;
+ rmnet_shs_cfg.dl_mrk_ind_cb.priority =
+ RMNET_SHS;
rmnet_shs_cfg.dl_mrk_ind_cb.dl_hdr_handler =
&rmnet_shs_dl_hdr_handler;
rmnet_shs_cfg.dl_mrk_ind_cb.dl_trl_handler =
diff --git a/drivers/rmnet/shs/rmnet_shs_config.h b/drivers/rmnet/shs/rmnet_shs_config.h
index 679fc7f..0c00b4f 100644
--- a/drivers/rmnet/shs/rmnet_shs_config.h
+++ b/drivers/rmnet/shs/rmnet_shs_config.h
@@ -36,6 +36,8 @@ enum rmnet_shs_crit_err_e {
RMNET_SHS_WQ_ALLOC_EP_TBL_ERR,
RMNET_SHS_WQ_GET_RMNET_PORT_ERR,
RMNET_SHS_WQ_EP_ACCESS_ERR,
+ RMNET_SHS_CPU_PKTLEN_ERR,
+ RMNET_SHS_NULL_SKB_HEAD,
RMNET_SHS_CRIT_ERR_MAX
};
diff --git a/drivers/rmnet/shs/rmnet_shs_main.c b/drivers/rmnet/shs/rmnet_shs_main.c
index 680cd6b..469dd37 100644..100755
--- a/drivers/rmnet/shs/rmnet_shs_main.c
+++ b/drivers/rmnet/shs/rmnet_shs_main.c
@@ -24,21 +24,20 @@
#include "rmnet_shs_wq.h"
/* Local Macros */
-#define RMNET_SHS_MAX_BYTES_TO_PARK 271800
-#define RMNET_SHS_MAX_PKTS_TO_PARK 180
#define RMNET_SHS_FORCE_FLUSH_TIME_NSEC 2000000
+#define NS_IN_MS 1000000
#define LPWR_CLUSTER 0
#define PERF_CLUSTER 4
#define INVALID_CPU -1
#define GET_QTAIL(SD, CPU) (per_cpu(SD, CPU).input_queue_tail)
#define GET_QHEAD(SD, CPU) (per_cpu(SD, CPU).input_queue_head)
+#define GET_CTIMER(CPU) rmnet_shs_cfg.core_flush[CPU].core_timer
/* Local Definitions and Declarations */
-spinlock_t rmnet_shs_ht_splock;
+DEFINE_SPINLOCK(rmnet_shs_ht_splock);
DEFINE_HASHTABLE(RMNET_SHS_HT, RMNET_SHS_HT_SIZE);
-
-static struct rmnet_shs_cpu_node_s rmnet_shs_cpu_node_tbl[MAX_CPUS];
+struct rmnet_shs_cpu_node_s rmnet_shs_cpu_node_tbl[MAX_CPUS];
/* Maintains a list of flows associated with a core
* Also keeps track of number of packets processed on that core
*/
@@ -54,10 +53,41 @@ unsigned long int rmnet_shs_flush_reason[RMNET_SHS_FLUSH_MAX_REASON];
module_param_array(rmnet_shs_flush_reason, ulong, 0, 0444);
MODULE_PARM_DESC(rmnet_shs_flush_reason, "rmnet shs skb flush trigger type");
+unsigned int rmnet_shs_byte_store_limit __read_mostly = 271800 * 8;
+module_param(rmnet_shs_byte_store_limit, uint, 0644);
+MODULE_PARM_DESC(rmnet_shs_byte_store_limit, "Maximum byte module will park");
+
+unsigned int rmnet_shs_pkts_store_limit __read_mostly = 2100;
+module_param(rmnet_shs_pkts_store_limit, uint, 0644);
+MODULE_PARM_DESC(rmnet_shs_pkts_store_limit, "Maximum pkts module will park");
+
+unsigned int rmnet_shs_max_core_wait __read_mostly = 10;
+module_param(rmnet_shs_max_core_wait, uint, 0644);
+MODULE_PARM_DESC(rmnet_shs_max_core_wait,
+ "Max wait module will wait during move to perf core in ms");
+
+unsigned int rmnet_shs_inst_rate_interval __read_mostly = 15;
+module_param(rmnet_shs_inst_rate_interval, uint, 0644);
+MODULE_PARM_DESC(rmnet_shs_inst_rate_interval,
+ "Max interval we sample for instant burst prioritizing");
+
+unsigned int rmnet_shs_inst_rate_max_pkts __read_mostly = 1800;
+module_param(rmnet_shs_inst_rate_max_pkts, uint, 0644);
+MODULE_PARM_DESC(rmnet_shs_inst_rate_max_pkts,
+ "Max pkts in a instant burst interval before prioritizing");
+
unsigned int rmnet_shs_switch_cores __read_mostly = 1;
module_param(rmnet_shs_switch_cores, uint, 0644);
MODULE_PARM_DESC(rmnet_shs_switch_cores, "Switch core upon hitting threshold");
+unsigned int rmnet_shs_cpu_max_qdiff[MAX_CPUS];
+module_param_array(rmnet_shs_cpu_max_qdiff, uint, 0, 0644);
+MODULE_PARM_DESC(rmnet_shs_cpu_max_qdiff, "Max queue length seen of each core");
+
+unsigned int rmnet_shs_cpu_max_coresum[MAX_CPUS];
+module_param_array(rmnet_shs_cpu_max_coresum, uint, 0, 0644);
+MODULE_PARM_DESC(rmnet_shs_cpu_max_coresum, "Max coresum seen of each core");
+
void rmnet_shs_cpu_node_remove(struct rmnet_shs_skbn_s *node)
{
trace_rmnet_shs_low(RMNET_SHS_CPU_NODE, RMNET_SHS_CPU_NODE_FUNC_REMOVE,
@@ -116,6 +146,44 @@ int rmnet_shs_is_skb_stamping_reqd(struct sk_buff *skb)
return ret_val;
}
+static void rmnet_shs_update_core_load(int cpu, int burst)
+{
+
+ struct timespec time1;
+ struct timespec *time2;
+ long int curinterval;
+ int maxinterval = (rmnet_shs_inst_rate_interval < 5) ? 5 :
+ rmnet_shs_inst_rate_interval;
+
+ getnstimeofday(&time1);
+ time2 = &rmnet_shs_cfg.core_flush[cpu].coretime;
+
+ curinterval = RMNET_SHS_SEC_TO_NSEC(time1.tv_sec - time2->tv_sec) +
+ time1.tv_nsec - time2->tv_nsec;
+
+ if (curinterval >= maxinterval * NS_IN_MS) {
+ if (rmnet_shs_cfg.core_flush[cpu].coresum >
+ rmnet_shs_cpu_max_coresum[cpu])
+ rmnet_shs_cpu_max_coresum[cpu] = rmnet_shs_cfg.core_flush[cpu].coresum;
+
+ rmnet_shs_cfg.core_flush[cpu].coretime.tv_sec = time1.tv_sec;
+ rmnet_shs_cfg.core_flush[cpu].coretime.tv_nsec = time1.tv_nsec;
+ rmnet_shs_cfg.core_flush[cpu].coresum = burst;
+
+ } else {
+ rmnet_shs_cfg.core_flush[cpu].coresum += burst;
+ }
+
+}
+
+static int rmnet_shs_is_core_loaded(int cpu)
+{
+
+ return rmnet_shs_cfg.core_flush[cpu].coresum >=
+ rmnet_shs_inst_rate_max_pkts;
+
+}
+
/* We deliver packets to GRO module only for TCP traffic*/
static int rmnet_shs_check_skb_can_gro(struct sk_buff *skb)
{
@@ -146,14 +214,18 @@ static int rmnet_shs_check_skb_can_gro(struct sk_buff *skb)
static void rmnet_shs_deliver_skb(struct sk_buff *skb)
{
struct rmnet_priv *priv;
+ struct napi_struct *napi;
trace_rmnet_shs_low(RMNET_SHS_DELIVER_SKB, RMNET_SHS_DELIVER_SKB_START,
0xDEF, 0xDEF, 0xDEF, 0xDEF, skb, NULL);
if (!rmnet_shs_check_skb_can_gro(skb)) {
- priv = netdev_priv(skb->dev);
- gro_cells_receive(&priv->gro_cells, skb);
-
+ if ((napi = get_current_napi_context())) {
+ napi_gro_receive(napi, skb);
+ } else {
+ priv = netdev_priv(skb->dev);
+ gro_cells_receive(&priv->gro_cells, skb);
+ }
} else {
netif_receive_skb(skb);
}
@@ -196,10 +268,24 @@ int rmnet_shs_num_perf_cores_configured(struct rps_map *map)
return ret;
}
+int rmnet_shs_flow_num_perf_cores(struct rmnet_shs_skbn_s *node_p)
+{
+ int ret = 0;
+ int core = 1;
+ u16 idx = 0;
+
+ for (idx = 0; idx < MAX_CPUS; idx++) {
+ if (node_p->hstats->pri_core_msk & core)
+ ret++;
+ core = core << 1;
+ }
+ return ret;
+}
+
int rmnet_shs_is_lpwr_cpu(u16 cpu)
{
int ret = 1;
- u32 big_cluster_mask = (1 << PERF_CLUSTER);
+ u32 big_cluster_mask = (1 << PERF_CLUSTER) - 1;
if ((1 << cpu) >= big_cluster_mask)
ret = 0;
@@ -220,14 +306,17 @@ u32 rmnet_shs_form_hash(u32 index, u32 maplen, u32 hash)
int offsetmap[MAX_CPUS / 2] = {8, 4, 3, 2};
u32 ret = 0;
+ if (!maplen) {
+ rmnet_shs_crit_err[RMNET_SHS_MAIN_MAP_LEN_INVALID]++;
+ return ret;
+ }
+
if (maplen < MAX_CPUS)
ret = ((((index + ((maplen % 2) ? 1 : 0))) << 28)
* offsetmap[(maplen - 1) >> 1]) | (hash & 0xFFFFFF);
trace_rmnet_shs_low(RMNET_SHS_HASH_MAP, RMNET_SHS_HASH_MAP_FORM_HASH,
ret, hash, index, maplen, NULL, NULL);
- if (maplen == 0)
- rmnet_shs_crit_err[RMNET_SHS_MAIN_MAP_LEN_INVALID]++;
return ret;
}
@@ -273,24 +362,34 @@ int rmnet_shs_new_flow_cpu(u64 burst_size, struct net_device *dev)
return flow_cpu;
}
-int rmnet_shs_get_suggested_cpu(struct rmnet_shs_skbn_s *node_p)
+int rmnet_shs_get_suggested_cpu(struct rmnet_shs_skbn_s *node)
{
int cpu = INVALID_CPU;
- if (node_p->hstats != NULL)
- cpu = node_p->hstats->suggested_cpu;
+ /* Return same perf core unless moving to gold from silver*/
+ if (rmnet_shs_cpu_node_tbl[node->map_cpu].prio &&
+ rmnet_shs_is_lpwr_cpu(node->map_cpu)) {
+ cpu = rmnet_shs_wq_get_least_utilized_core(0xF0);
+ if (cpu < 0)
+ cpu = node->hstats->suggested_cpu;
+ } else if (node->hstats != NULL)
+ cpu = node->hstats->suggested_cpu;
return cpu;
}
int rmnet_shs_get_hash_map_idx_to_stamp(struct rmnet_shs_skbn_s *node_p)
{
- int cpu, idx;
+ int cpu, idx = INVALID_CPU;
struct rps_map *map;
cpu = rmnet_shs_get_suggested_cpu(node_p);
+
map = rcu_dereference(node_p->dev->_rx->rps_map);
+ if (!node_p->dev || !node_p->dev->_rx || !map)
+ return idx;
+
idx = rmnet_shs_map_idx_from_cpu(cpu, map);
trace_rmnet_shs_low(RMNET_SHS_HASH_MAP,
@@ -324,6 +423,18 @@ u32 rmnet_shs_get_cpu_qtail(u8 cpu_num)
return ret;
}
+u32 rmnet_shs_get_cpu_qdiff(u8 cpu_num)
+{
+ u32 ret = 0;
+
+ if (cpu_num < MAX_CPUS)
+ ret = rmnet_shs_cpu_node_tbl[cpu_num].qdiff;
+
+ trace_rmnet_shs_low(RMNET_SHS_CORE_CFG, RMNET_SHS_CORE_CFG_GET_QTAIL,
+ cpu_num, ret, 0xDEF, 0xDEF, NULL, NULL);
+
+ return ret;
+}
/* Takes a snapshot of absolute value of the CPU Qhead and Qtail counts for
* a given core.
*
@@ -341,6 +452,9 @@ void rmnet_shs_update_cpu_proc_q(u8 cpu_num)
GET_QHEAD(softnet_data, cpu_num);
rmnet_shs_cpu_node_tbl[cpu_num].qtail =
GET_QTAIL(softnet_data, cpu_num);
+ rmnet_shs_cpu_node_tbl[cpu_num].qdiff =
+ rmnet_shs_cpu_node_tbl[cpu_num].qtail -
+ rmnet_shs_cpu_node_tbl[cpu_num].qhead;
rcu_read_unlock();
trace_rmnet_shs_low(RMNET_SHS_CORE_CFG,
@@ -363,10 +477,7 @@ void rmnet_shs_update_cpu_proc_q_all_cpus(void)
rcu_read_lock();
for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) {
- rmnet_shs_cpu_node_tbl[cpu_num].qhead = GET_QHEAD(softnet_data,
- cpu_num);
- rmnet_shs_cpu_node_tbl[cpu_num].qtail = GET_QTAIL(softnet_data,
- cpu_num);
+ rmnet_shs_update_cpu_proc_q(cpu_num);
trace_rmnet_shs_low(RMNET_SHS_CORE_CFG,
RMNET_SHS_CORE_CFG_GET_CPU_PROC_PARAMS,
@@ -376,8 +487,8 @@ void rmnet_shs_update_cpu_proc_q_all_cpus(void)
0xDEF, NULL, NULL);
}
rcu_read_unlock();
-}
+}
int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush)
{
struct rps_map *map;
@@ -386,6 +497,8 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush)
u32 node_qhead;
int ret = 0;
int prev_cpu = -1;
+ int ccpu;
+ int cpu_num;
struct rmnet_shs_cpu_node_s *cpun;
cpu_map_index = rmnet_shs_get_hash_map_idx_to_stamp(node);
@@ -398,6 +511,12 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush)
}
node->is_shs_enabled = 1;
map = rcu_dereference(node->dev->_rx->rps_map);
+ if (!node->dev->_rx || !map){
+ node->is_shs_enabled = 0;
+ ret = 1;
+ break;
+ }
+
/* If the flow is going to the same core itself
*/
@@ -408,16 +527,33 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush)
cur_cpu_qhead = rmnet_shs_get_cpu_qhead(node->map_cpu);
node_qhead = node->queue_head;
-
+ cpu_num = node->map_cpu;
if ((cur_cpu_qhead >= node_qhead) ||
(node->skb_tport_proto == IPPROTO_TCP) ||
- (force_flush)
- ) {
+ (force_flush)) {
if (rmnet_shs_switch_cores) {
+
+ /* Move the amount parked to other core's count
+ * Update old core's parked to not include diverted
+ * packets and update new core's packets
+ */
+ rmnet_shs_cpu_node_tbl[map->cpus[cpu_map_index]].parkedlen +=
+ node->skb_list.num_parked_skbs;
+ rmnet_shs_cpu_node_tbl[node->map_cpu].parkedlen -=
+ node->skb_list.num_parked_skbs;
node->map_index = cpu_map_index;
node->map_cpu = map->cpus[cpu_map_index];
+ ccpu = node->map_cpu;
+
+ /* Mark gold core as prio to prevent
+ * flows from moving in wq
+ */
+ if (rmnet_shs_cpu_node_tbl[cpu_num].prio) {
+ node->hstats->suggested_cpu = ccpu;
+ rmnet_shs_cpu_node_tbl[ccpu].wqprio = 1;
+ }
cpun = &rmnet_shs_cpu_node_tbl[node->map_cpu];
- rmnet_shs_update_cpu_proc_q(node->map_cpu);
+ rmnet_shs_update_cpu_proc_q_all_cpus();
node->queue_head = cpun->qhead;
rmnet_shs_cpu_node_move(node,
&cpun->node_list_id);
@@ -437,6 +573,69 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush)
return ret;
}
+void rmnet_shs_flush_core(u8 cpu_num)
+{
+ struct rmnet_shs_skbn_s *n;
+ struct list_head *ptr, *next;
+ unsigned long ht_flags;
+ u32 cpu_tail;
+ u32 num_pkts_flush = 0;
+ u32 num_bytes_flush = 0;
+ u32 total_pkts_flush = 0;
+ u32 total_bytes_flush = 0;
+
+ /* Record a qtail + pkts flushed or move if reqd
+ * currently only use qtail for non TCP flows
+ */
+ rmnet_shs_update_cpu_proc_q_all_cpus();
+ trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_START,
+ rmnet_shs_cfg.num_pkts_parked,
+ rmnet_shs_cfg.num_bytes_parked,
+ 0xDEF, 0xDEF, NULL, NULL);
+
+ spin_lock_irqsave(&rmnet_shs_ht_splock, ht_flags);
+ cpu_tail = rmnet_shs_get_cpu_qtail(cpu_num);
+ list_for_each_safe(ptr, next,
+ &rmnet_shs_cpu_node_tbl[cpu_num].node_list_id) {
+ n = list_entry(ptr, struct rmnet_shs_skbn_s, node_id);
+ if (n != NULL && n->skb_list.num_parked_skbs) {
+ num_pkts_flush = n->skb_list.num_parked_skbs;
+ num_bytes_flush = n->skb_list.num_parked_bytes;
+
+ rmnet_shs_chk_and_flush_node(n, 1);
+
+ total_pkts_flush += num_pkts_flush;
+ total_bytes_flush += num_bytes_flush;
+ if (n->map_cpu == cpu_num) {
+ cpu_tail += num_pkts_flush;
+ n->queue_head = cpu_tail;
+
+ }
+
+ }
+ }
+
+ rmnet_shs_cfg.num_bytes_parked -= total_bytes_flush;
+ rmnet_shs_cfg.num_pkts_parked -= total_pkts_flush;
+ rmnet_shs_cpu_node_tbl[cpu_num].prio = 0;
+ rmnet_shs_cpu_node_tbl[cpu_num].parkedlen = 0;
+ spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags);
+
+ trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_END,
+ rmnet_shs_cfg.num_pkts_parked,
+ rmnet_shs_cfg.num_bytes_parked,
+ total_pkts_flush, total_bytes_flush, NULL, NULL);
+
+}
+
+static void rmnet_shs_flush_core_work(struct work_struct *work)
+{
+ struct core_flush_s *core_work = container_of(work,
+ struct core_flush_s, work);
+
+ rmnet_shs_flush_core(core_work->core);
+}
+
/* Flushes all the packets parked in order for this flow */
void rmnet_shs_flush_node(struct rmnet_shs_skbn_s *node)
{
@@ -447,13 +646,17 @@ void rmnet_shs_flush_node(struct rmnet_shs_skbn_s *node)
u32 skb_bytes_delivered = 0;
u32 hash2stamp;
- map = rcu_dereference(node->dev->_rx->rps_map);
-
if (!node->skb_list.head)
return;
- hash2stamp = rmnet_shs_form_hash(node->map_index,
+ map = rcu_dereference(node->dev->_rx->rps_map);
+
+ if (!map) {
+ hash2stamp = rmnet_shs_form_hash(node->map_index,
map->len, node->skb_list.head->hash);
+ } else {
+ node->is_shs_enabled = 0;
+ }
trace_rmnet_shs_high(RMNET_SHS_FLUSH,
RMNET_SHS_FLUSH_NODE_START,
node->hash, hash2stamp,
@@ -470,8 +673,11 @@ void rmnet_shs_flush_node(struct rmnet_shs_skbn_s *node)
skb->next = NULL;
skbs_delivered += 1;
skb_bytes_delivered += skb->len;
+
rmnet_shs_deliver_skb(skb);
+
}
+
node->skb_list.num_parked_skbs = 0;
node->skb_list.num_parked_bytes = 0;
node->skb_list.head = NULL;
@@ -503,7 +709,6 @@ int rmnet_shs_chk_and_flush_node(struct rmnet_shs_skbn_s *node, u8 force_flush)
node, NULL);
return ret_val;
}
-
/* Flushes all the packets that have been parked so far across all the flows
* The order of flushing depends on the CPU<=>flow association
* The flows associated with low power cores are flushed before flushing
@@ -527,6 +732,7 @@ void rmnet_shs_flush_table(u8 flsh)
u32 total_pkts_flush = 0;
u32 total_bytes_flush = 0;
u8 is_flushed = 0;
+ u32 wait = (!rmnet_shs_max_core_wait) ? 1 : rmnet_shs_max_core_wait;
/* Record a qtail + pkts flushed or move if reqd
* currently only use qtail for non TCP flows
@@ -539,12 +745,36 @@ void rmnet_shs_flush_table(u8 flsh)
spin_lock_irqsave(&rmnet_shs_ht_splock, ht_flags);
for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) {
+
cpu_tail = rmnet_shs_get_cpu_qtail(cpu_num);
+
+ /* If core is loaded set core flows as priority and
+ * start a 10ms hard flush timer
+ */
+ if (rmnet_shs_is_lpwr_cpu(cpu_num) &&
+ !rmnet_shs_cpu_node_tbl[cpu_num].prio)
+ rmnet_shs_update_core_load(cpu_num,
+ rmnet_shs_cpu_node_tbl[cpu_num].parkedlen);
+
+ if (rmnet_shs_is_core_loaded(cpu_num) &&
+ rmnet_shs_is_lpwr_cpu(cpu_num) &&
+ !rmnet_shs_cpu_node_tbl[cpu_num].prio) {
+
+ rmnet_shs_cpu_node_tbl[cpu_num].prio = 1;
+ if (hrtimer_active(&GET_CTIMER(cpu_num)))
+ hrtimer_cancel(&GET_CTIMER(cpu_num));
+
+ hrtimer_start(&GET_CTIMER(cpu_num),
+ ns_to_ktime(wait * NS_IN_MS),
+ HRTIMER_MODE_REL);
+
+ }
+
list_for_each_safe(ptr, next,
&rmnet_shs_cpu_node_tbl[cpu_num].node_list_id) {
n = list_entry(ptr, struct rmnet_shs_skbn_s, node_id);
- if (n != NULL) {
+ if (n != NULL && n->skb_list.num_parked_skbs) {
num_pkts_flush = n->skb_list.num_parked_skbs;
num_bytes_flush = n->skb_list.num_parked_bytes;
is_flushed = rmnet_shs_chk_and_flush_node(n,
@@ -553,21 +783,30 @@ void rmnet_shs_flush_table(u8 flsh)
if (is_flushed) {
total_pkts_flush += num_pkts_flush;
total_bytes_flush += num_bytes_flush;
+ rmnet_shs_cpu_node_tbl[n->map_cpu].parkedlen -= num_pkts_flush;
+
if (n->map_cpu == cpu_num) {
cpu_tail += num_pkts_flush;
n->queue_head = cpu_tail;
+
}
}
}
}
- }
+ if (rmnet_shs_cpu_node_tbl[cpu_num].parkedlen < 0)
+ rmnet_shs_crit_err[RMNET_SHS_CPU_PKTLEN_ERR]++;
+
+ if (rmnet_shs_get_cpu_qdiff(cpu_num) >=
+ rmnet_shs_cpu_max_qdiff[cpu_num])
+ rmnet_shs_cpu_max_qdiff[cpu_num] =
+ rmnet_shs_get_cpu_qdiff(cpu_num);
+ }
spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags);
rmnet_shs_cfg.num_bytes_parked -= total_bytes_flush;
rmnet_shs_cfg.num_pkts_parked -= total_pkts_flush;
-
trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_END,
rmnet_shs_cfg.num_pkts_parked,
rmnet_shs_cfg.num_bytes_parked,
@@ -600,10 +839,10 @@ void rmnet_shs_chain_to_skb_list(struct sk_buff *skb,
node->skb_list.num_parked_bytes += skb->len;
rmnet_shs_cfg.num_bytes_parked += skb->len;
+ rmnet_shs_cpu_node_tbl[node->map_cpu].parkedlen++;
node->skb_list.num_parked_skbs += 1;
rmnet_shs_cfg.num_pkts_parked += 1;
-
trace_rmnet_shs_high(RMNET_SHS_ASSIGN,
RMNET_SHS_ASSIGN_PARK_PKT_COMPLETE,
node->skb_list.num_parked_skbs,
@@ -612,13 +851,12 @@ void rmnet_shs_chain_to_skb_list(struct sk_buff *skb,
rmnet_shs_cfg.num_bytes_parked,
skb, node);
}
-
/* Invoked when all the packets that are parked to be flushed through
* the workqueue.
*/
static void rmnet_flush_buffered(struct work_struct *work)
{
- u8 is_force_flush = 1;
+ u8 is_force_flush = 0;
trace_rmnet_shs_high(RMNET_SHS_FLUSH,
RMNET_SHS_FLUSH_DELAY_WQ_START, is_force_flush,
@@ -626,16 +864,14 @@ static void rmnet_flush_buffered(struct work_struct *work)
0xDEF, NULL, NULL);
if (rmnet_shs_cfg.is_pkt_parked &&
- rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_ON) {
+ rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_ON) {
+
rmnet_shs_flush_table(is_force_flush);
rmnet_shs_flush_reason[RMNET_SHS_FLUSH_TIMER_EXPIRY]++;
}
trace_rmnet_shs_high(RMNET_SHS_FLUSH,
RMNET_SHS_FLUSH_DELAY_WQ_END,
is_force_flush, 0xDEF, 0xDEF, 0xDEF, NULL, NULL);
-
-
-
}
/* Invoked when the flushing timer has expired.
* Upon first expiry, we set the flag that will trigger force flushing of all
@@ -652,9 +888,8 @@ enum hrtimer_restart rmnet_shs_map_flush_queue(struct hrtimer *t)
RMNET_SHS_FLUSH_PARK_TMR_EXPIRY,
rmnet_shs_cfg.force_flush_state, 0xDEF,
0xDEF, 0xDEF, NULL, NULL);
-
if (rmnet_shs_cfg.num_pkts_parked > 0) {
- if (rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_OFF) {
+ if (rmnet_shs_cfg.force_flush_state != RMNET_SHS_FLUSH_ON) {
rmnet_shs_cfg.force_flush_state = RMNET_SHS_FLUSH_ON;
hrtimer_forward(t, hrtimer_cb_get_time(t),
ns_to_ktime(2000000));
@@ -664,6 +899,10 @@ enum hrtimer_restart rmnet_shs_map_flush_queue(struct hrtimer *t)
RMNET_SHS_FLUSH_PARK_TMR_RESTART,
rmnet_shs_cfg.num_pkts_parked,
0xDEF, 0xDEF, 0xDEF, NULL, NULL);
+ } else if (rmnet_shs_cfg.force_flush_state ==
+ RMNET_SHS_FLUSH_DONE) {
+ rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_OFF;
+
} else {
trace_rmnet_shs_high(RMNET_SHS_FLUSH,
RMNET_SHS_FLUSH_DELAY_WQ_TRIGGER,
@@ -675,27 +914,44 @@ enum hrtimer_restart rmnet_shs_map_flush_queue(struct hrtimer *t)
return ret;
}
-/* Initializes the flushing timer. The timer is initialized upon receiving
- * first NET UP event
- */
+enum hrtimer_restart rmnet_shs_queue_core(struct hrtimer *t)
+{
+ const enum hrtimer_restart ret = HRTIMER_NORESTART;
+ struct core_flush_s *core_work = container_of(t,
+ struct core_flush_s, core_timer);
+
+ schedule_work(&core_work->work);
+ return ret;
+}
+
void rmnet_shs_aggregate_init(void)
{
+ int i;
+
+ for (i = 0; i < MAX_CPUS; i++) {
+ rmnet_shs_cfg.core_flush[i].core = i;
+ INIT_WORK(&rmnet_shs_cfg.core_flush[i].work,
+ rmnet_shs_flush_core_work);
- hrtimer_init(&rmnet_shs_cfg.hrtimer_shs,
- CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- rmnet_shs_cfg.hrtimer_shs.function = rmnet_shs_map_flush_queue;
+ hrtimer_init(&rmnet_shs_cfg.core_flush[i].core_timer,
+ CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ rmnet_shs_cfg.core_flush[i].core_timer.function =
+ rmnet_shs_queue_core;
+ }
+ hrtimer_init(&rmnet_shs_cfg.hrtimer_shs,
+ CLOCK_MONOTONIC, HRTIMER_MODE_REL);
+ rmnet_shs_cfg.hrtimer_shs.function = rmnet_shs_map_flush_queue;
+ INIT_WORK(&shs_delayed_work.work, rmnet_flush_buffered);
}
void rmnet_shs_ps_on_hdlr(void *port)
{
- rmnet_shs_flush_table(1);
rmnet_shs_wq_pause();
}
void rmnet_shs_ps_off_hdlr(void *port)
{
rmnet_shs_wq_restart();
-
}
void rmnet_shs_dl_hdr_handler(struct rmnet_map_dl_ind_hdr *dlhdr)
@@ -732,7 +988,6 @@ void rmnet_shs_init(struct net_device *dev)
return;
rmnet_shs_cfg.port = rmnet_get_port(dev);
- spin_lock_init(&rmnet_shs_ht_splock);
for (num_cpu = 0; num_cpu < MAX_CPUS; num_cpu++)
INIT_LIST_HEAD(&rmnet_shs_cpu_node_tbl[num_cpu].node_list_id);
@@ -814,12 +1069,17 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port)
struct rmnet_shs_cpu_node_s *cpu_node_tbl_p;
/*deliver non TCP/UDP packets right away*/
+ if (!rmnet_shs_is_skb_stamping_reqd(skb)) {
+ rmnet_shs_deliver_skb(skb);
+ return;
+ }
- if (unlikely(!map)) {
+ if ((unlikely(!map))|| !rmnet_shs_init_complete) {
+ rmnet_shs_deliver_skb(skb);
trace_rmnet_shs_err(RMNET_SHS_ASSIGN,
- RMNET_SHS_ASSIGN_CRIT_ERROR_NO_MSK_SET,
+ RMNET_SHS_ASSIGN_CRIT_ERROR_NO_SHS_REQD,
0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL);
- rmnet_shs_crit_err[RMNET_SHS_MAIN_MAP_LEN_INVALID]++;
+ rmnet_shs_crit_err[RMNET_SHS_MAIN_SHS_NOT_REQD]++;
return;
}
@@ -845,7 +1105,7 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port)
map_cpu = rmnet_shs_new_flow_cpu(brate, dev);
node_p->map_cpu = map_cpu;
node_p->map_index =
- rmnet_shs_map_idx_from_cpu(map_cpu, map);
+ rmnet_shs_map_idx_from_cpu(map_cpu, map);
trace_rmnet_shs_err(RMNET_SHS_ASSIGN,
RMNET_SHS_ASSIGN_MASK_CHNG,
@@ -876,8 +1136,10 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port)
node_p = kzalloc(sizeof(*node_p), 0);
- if (!node_p)
+ if (!node_p) {
+ rmnet_shs_crit_err[RMNET_SHS_MAIN_MALLOC_ERR]++;
break;
+ }
node_p->dev = skb->dev;
node_p->hash = skb->hash;
@@ -899,12 +1161,12 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port)
cpu_node_tbl_p = &rmnet_shs_cpu_node_tbl[map_cpu];
rmnet_shs_cpu_node_add(node_p, &cpu_node_tbl_p->node_list_id);
-
hash_add_rcu(RMNET_SHS_HT, &node_p->list, skb->hash);
/* Chain this pkt to skb list (most likely to skb_list.head)
* because this is the first packet for this flow
*/
rmnet_shs_chain_to_skb_list(skb, node_p);
+
is_shs_reqd = 1;
break;
@@ -926,14 +1188,15 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port)
&rmnet_shs_cfg.rmnet_idl_ind_cb);
rmnet_shs_cfg.is_reg_dl_mrk_ind = 1;
- INIT_WORK(&shs_delayed_work.work, rmnet_flush_buffered);
shs_delayed_work.port = port;
+
}
/* We got the first packet after a previous successdul flush. Arm the
* flushing timer.
*/
if (!rmnet_shs_cfg.is_pkt_parked) {
rmnet_shs_cfg.is_pkt_parked = 1;
+ rmnet_shs_cfg.force_flush_state = RMNET_SHS_FLUSH_OFF;
if (hrtimer_active(&rmnet_shs_cfg.hrtimer_shs)) {
trace_rmnet_shs_low(RMNET_SHS_ASSIGN,
RMNET_SHS_ASSIGN_PARK_TMR_CANCEL,
@@ -949,26 +1212,8 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port)
0xDEF, 0xDEF, 0xDEF, skb, NULL);
}
- /* Flushing timer that was armed previously has successfully fired.
- * Now we trigger force flushing of all packets. If a flow is waiting
- * to switch to another core, it will be forcefully moved during this
- * trigger.
- *
- * In case the previously delivered packets haven't been processed by
- * the next layers, the parked packets may be delivered out of order
- * until all the previously delivered packets have been processed
- * successully
- */
- if (rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_ON) {
- rmnet_shs_flush_reason[RMNET_SHS_FLUSH_TIMER_EXPIRY]++;
- trace_rmnet_shs_high(RMNET_SHS_FLUSH,
- RMNET_SHS_FLUSH_FORCE_TRIGGER, 1,
- rmnet_shs_cfg.num_pkts_parked,
- 0xDEF, 0xDEF, NULL, NULL);
- rmnet_shs_flush_table(1);
-
- } else if (rmnet_shs_cfg.num_pkts_parked >
- RMNET_SHS_MAX_PKTS_TO_PARK) {
+ if (rmnet_shs_cfg.num_pkts_parked >
+ rmnet_shs_pkts_store_limit) {
if (rmnet_shs_stats_enabled)
rmnet_shs_flush_reason[RMNET_SHS_FLUSH_PKT_LIMIT]++;
@@ -976,10 +1221,10 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port)
trace_rmnet_shs_high(RMNET_SHS_FLUSH,
RMNET_SHS_FLUSH_PKT_LIMIT_TRIGGER, 0,
0xDEF, 0xDEF, 0xDEF, NULL, NULL);
- rmnet_shs_flush_table(0);
+ rmnet_shs_flush_table(1);
} else if (rmnet_shs_cfg.num_bytes_parked >
- RMNET_SHS_MAX_BYTES_TO_PARK) {
+ rmnet_shs_byte_store_limit) {
if (rmnet_shs_stats_enabled)
rmnet_shs_flush_reason[RMNET_SHS_FLUSH_BYTE_LIMIT]++;
@@ -987,7 +1232,27 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port)
trace_rmnet_shs_high(RMNET_SHS_FLUSH,
RMNET_SHS_FLUSH_BYTE_LIMIT_TRIGGER, 0,
0xDEF, 0xDEF, 0xDEF, NULL, NULL);
+ rmnet_shs_flush_table(1);
+
+ }
+ /* Flushing timer that was armed previously has successfully fired.
+ * Now we trigger force flushing of all packets. If a flow is waiting
+ * to switch to another core, it will be forcefully moved during this
+ * trigger.
+ *
+ * In case the previously delivered packets haven't been processed by
+ * the next layers, the parked packets may be delivered out of order
+ * until all the previously delivered packets have been processed
+ * successully
+ */
+ else if (rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_ON) {
+ rmnet_shs_flush_reason[RMNET_SHS_FLUSH_TIMER_EXPIRY]++;
+ trace_rmnet_shs_high(RMNET_SHS_FLUSH,
+ RMNET_SHS_FLUSH_FORCE_TRIGGER, 1,
+ rmnet_shs_cfg.num_pkts_parked,
+ 0xDEF, 0xDEF, NULL, NULL);
rmnet_shs_flush_table(0);
+
}
}
diff --git a/drivers/rmnet/shs/rmnet_shs_wq.c b/drivers/rmnet/shs/rmnet_shs_wq.c
index d4d9d2f..42f316c 100644
--- a/drivers/rmnet/shs/rmnet_shs_wq.c
+++ b/drivers/rmnet/shs/rmnet_shs_wq.c
@@ -25,12 +25,21 @@ MODULE_LICENSE("GPL v2");
#define RMNET_SHS_BYTE_TO_BIT(x) ((x)*8)
#define RMNET_SHS_MIN_HSTAT_NODES_REQD 16
#define RMNET_SHS_WQ_DELAY_TICKS 10
+
+#define PERIODIC_CLEAN 0
+/* FORCE_CLEAN should only used during module de-ini.*/
+#define FORCE_CLEAN 1
/* Time to wait (in time ticks) before re-triggering the workqueue
* 1 tick = 10 ms (Maximum possible resolution)
* 100 ticks = 1 second
*/
/* Local Definitions and Declarations */
+unsigned int rmnet_shs_cpu_prio_dur __read_mostly = RMNET_SHS_WQ_DELAY_TICKS;
+module_param(rmnet_shs_cpu_prio_dur, uint, 0644);
+MODULE_PARM_DESC(rmnet_shs_cpu_prio_dur, "Priority ignore duration(ticks)");
+
+#define PRIO_BACKOFF ((!rmnet_shs_cpu_prio_dur) ? 2 : rmnet_shs_cpu_prio_dur)
unsigned int rmnet_shs_wq_frequency __read_mostly = RMNET_SHS_WQ_DELAY_TICKS;
module_param(rmnet_shs_wq_frequency, uint, 0644);
@@ -141,7 +150,6 @@ static struct workqueue_struct *rmnet_shs_wq;
static struct rmnet_shs_delay_wq_s *rmnet_shs_delayed_wq;
static struct rmnet_shs_wq_rx_flow_s rmnet_shs_rx_flow_tbl;
-
static struct list_head rmnet_shs_wq_hstat_tbl =
LIST_HEAD_INIT(rmnet_shs_wq_hstat_tbl);
static int rmnet_shs_flow_dbg_stats_idx_cnt;
@@ -153,21 +161,23 @@ static struct list_head rmnet_shs_wq_ep_tbl =
*/
void rmnet_shs_wq_ep_tbl_add(struct rmnet_shs_wq_ep_s *ep)
{
+ unsigned long flags;
trace_rmnet_shs_wq_low(RMNET_SHS_WQ_EP_TBL, RMNET_SHS_WQ_EP_TBL_ADD,
0xDEF, 0xDEF, 0xDEF, 0xDEF, ep, NULL);
- spin_lock(&rmnet_shs_hstat_tbl_lock);
+ spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
list_add(&ep->ep_list_id, &rmnet_shs_wq_ep_tbl);
- spin_unlock(&rmnet_shs_hstat_tbl_lock);
+ spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);
}
void rmnet_shs_wq_ep_tbl_remove(struct rmnet_shs_wq_ep_s *ep)
{
+ unsigned long flags;
trace_rmnet_shs_wq_low(RMNET_SHS_WQ_EP_TBL, RMNET_SHS_WQ_EP_TBL_DEL,
0xDEF, 0xDEF, 0xDEF, 0xDEF, ep, NULL);
- spin_lock(&rmnet_shs_hstat_tbl_lock);
+ spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
list_del_init(&ep->ep_list_id);
- spin_unlock(&rmnet_shs_hstat_tbl_lock);
+ spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);
}
@@ -176,23 +186,27 @@ void rmnet_shs_wq_ep_tbl_remove(struct rmnet_shs_wq_ep_s *ep)
*/
void rmnet_shs_wq_hstat_tbl_add(struct rmnet_shs_wq_hstat_s *hnode)
{
+ unsigned long flags;
+
trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL,
RMNET_SHS_WQ_HSTAT_TBL_ADD,
0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL);
- spin_lock(&rmnet_shs_hstat_tbl_lock);
+ spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
list_add(&hnode->hstat_node_id, &rmnet_shs_wq_hstat_tbl);
- spin_unlock(&rmnet_shs_hstat_tbl_lock);
+ spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);
}
void rmnet_shs_wq_hstat_tbl_remove(struct rmnet_shs_wq_hstat_s *hnode)
{
+ unsigned long flags;
+
trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL,
RMNET_SHS_WQ_HSTAT_TBL_DEL,
0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL);
- spin_lock(&rmnet_shs_hstat_tbl_lock);
+ spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
list_del_init(&hnode->hstat_node_id);
- spin_unlock(&rmnet_shs_hstat_tbl_lock);
+ spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);
}
@@ -202,37 +216,43 @@ void rmnet_shs_wq_hstat_tbl_remove(struct rmnet_shs_wq_hstat_s *hnode)
*/
void rmnet_shs_wq_cpu_list_remove(struct rmnet_shs_wq_hstat_s *hnode)
{
+ unsigned long flags;
+
trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_HSTAT_TBL,
RMNET_SHS_WQ_CPU_HSTAT_TBL_DEL,
0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL);
- spin_lock(&rmnet_shs_hstat_tbl_lock);
+ spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
list_del_init(&hnode->cpu_node_id);
- spin_unlock(&rmnet_shs_hstat_tbl_lock);
+ spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);
}
void rmnet_shs_wq_cpu_list_add(struct rmnet_shs_wq_hstat_s *hnode,
struct list_head *head)
{
+ unsigned long flags;
+
trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_HSTAT_TBL,
RMNET_SHS_WQ_CPU_HSTAT_TBL_ADD,
0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL);
- spin_lock(&rmnet_shs_hstat_tbl_lock);
+ spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
list_add(&hnode->cpu_node_id, head);
- spin_unlock(&rmnet_shs_hstat_tbl_lock);
+ spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);
}
void rmnet_shs_wq_cpu_list_move(struct rmnet_shs_wq_hstat_s *hnode,
struct list_head *head)
{
+ unsigned long flags;
+
trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_HSTAT_TBL,
RMNET_SHS_WQ_CPU_HSTAT_TBL_MOVE,
hnode->current_cpu,
0xDEF, 0xDEF, 0xDEF, hnode, NULL);
- spin_lock(&rmnet_shs_hstat_tbl_lock);
+ spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
list_move(&hnode->cpu_node_id, head);
- spin_unlock(&rmnet_shs_hstat_tbl_lock);
+ spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);
}
@@ -300,8 +320,9 @@ struct rmnet_shs_wq_hstat_s *rmnet_shs_wq_get_new_hstat_node(void)
{
struct rmnet_shs_wq_hstat_s *hnode;
struct rmnet_shs_wq_hstat_s *ret_node = NULL;
+ unsigned long flags;
- spin_lock(&rmnet_shs_hstat_tbl_lock);
+ spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags);
list_for_each_entry(hnode, &rmnet_shs_wq_hstat_tbl, hstat_node_id) {
if (hnode == NULL)
continue;
@@ -313,7 +334,7 @@ struct rmnet_shs_wq_hstat_s *rmnet_shs_wq_get_new_hstat_node(void)
break;
}
}
- spin_unlock(&rmnet_shs_hstat_tbl_lock);
+ spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags);
if (ret_node) {
trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL,
@@ -334,7 +355,6 @@ struct rmnet_shs_wq_hstat_s *rmnet_shs_wq_get_new_hstat_node(void)
return NULL;
}
-
rmnet_shs_wq_hstat_reset_node(ret_node);
ret_node->is_perm = 0;
ret_node->in_use = 1;
@@ -477,7 +497,6 @@ void rmnet_shs_wq_update_hash_stats(struct rmnet_shs_wq_hstat_s *hstats_p)
hstats_p->hash, 0xDEF, hstats_p->rx_pps,
hstats_p->rx_bps, hstats_p, NULL);
-
rmnet_shs_wq_update_hstat_rps_msk(hstats_p);
hstats_p->inactive_duration = 0;
hstats_p->l_epoch = node_p->hstats->c_epoch;
@@ -581,6 +600,9 @@ static void rmnet_shs_wq_refresh_cpu_stats(u16 cpu)
cpu_p = &rmnet_shs_rx_flow_tbl.cpu_list[cpu];
new_skbs = cpu_p->rx_skbs - cpu_p->last_rx_skbs;
+ if (rmnet_shs_cpu_node_tbl[cpu].wqprio)
+ rmnet_shs_cpu_node_tbl[cpu].wqprio = (rmnet_shs_cpu_node_tbl[cpu].wqprio + 1)
+ % (PRIO_BACKOFF);
if (new_skbs == 0) {
cpu_p->l_epoch = rmnet_shs_wq_tnsec;
cpu_p->rx_bps = 0;
@@ -640,10 +662,13 @@ void rmnet_shs_wq_update_cpu_rx_tbl(struct rmnet_shs_wq_hstat_s *hstat_p)
return;
map = rcu_dereference(node_p->dev->_rx->rps_map);
+
+ if (!map)
+ return;
+
map_idx = node_p->map_index;
cpu_num = map->cpus[map_idx];
-
skb_diff = hstat_p->rx_skb - hstat_p->last_rx_skb;
byte_diff = hstat_p->rx_bytes - hstat_p->last_rx_bytes;
@@ -818,6 +843,13 @@ u16 rmnet_shs_wq_find_cpu_to_move_flows(u16 current_cpu,
(cur_cpu_rx_pps > pps_uthresh)) {
return cpu_to_move;
}
+ /* If a core (should only be lpwr was marked prio we don't touch it
+ * for a few ticks and reset it afterwards
+ */
+
+ if (rmnet_shs_cpu_node_tbl[current_cpu].wqprio) {
+ return current_cpu;
+ }
for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) {
@@ -826,7 +858,8 @@ u16 rmnet_shs_wq_find_cpu_to_move_flows(u16 current_cpu,
/* We are looking for a core that is configured and that
* can handle traffic better than the current core
*/
- if ((cpu_num == current_cpu) || (!is_core_in_msk))
+ if ((cpu_num == current_cpu) || (!is_core_in_msk) ||
+ !cpu_online(current_cpu))
continue;
pps_uthresh = rmnet_shs_cpu_rx_max_pps_thresh[cpu_num];
@@ -837,9 +870,9 @@ u16 rmnet_shs_wq_find_cpu_to_move_flows(u16 current_cpu,
reqd_pps = cpu_rx_pps + cur_cpu_rx_pps;
trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS,
- RMNET_SHS_WQ_CPU_STATS_CORE2SWITCH_FIND,
- current_cpu, cpu_num, reqd_pps,
- cpu_rx_pps, NULL, NULL);
+ RMNET_SHS_WQ_CPU_STATS_CORE2SWITCH_FIND,
+ current_cpu, cpu_num, reqd_pps,
+ cpu_rx_pps, NULL, NULL);
/* Return the first available CPU */
if ((reqd_pps > pps_lthresh) && (reqd_pps < pps_uthresh)) {
@@ -1119,7 +1152,7 @@ int rmnet_shs_wq_get_perf_cpu_new_flow(struct net_device *dev)
return cpu_assigned;
}
-void rmnet_shs_wq_cleanup_hash_tbl(void)
+void rmnet_shs_wq_cleanup_hash_tbl(u8 force_clean)
{
struct rmnet_shs_skbn_s *node_p;
time_t tns2s;
@@ -1133,9 +1166,12 @@ void rmnet_shs_wq_cleanup_hash_tbl(void)
if (hnode == NULL)
continue;
+ if (hnode->node == NULL)
+ continue;
+
node_p = hnode->node;
tns2s = RMNET_SHS_NSEC_TO_SEC(hnode->inactive_duration);
- if (tns2s > rmnet_shs_max_flow_inactivity_sec) {
+ if (tns2s > rmnet_shs_max_flow_inactivity_sec || force_clean) {
trace_rmnet_shs_wq_low(RMNET_SHS_WQ_FLOW_STATS,
RMNET_SHS_WQ_FLOW_STATS_FLOW_INACTIVE_TIMEOUT,
@@ -1149,11 +1185,12 @@ void rmnet_shs_wq_cleanup_hash_tbl(void)
kfree(node_p);
}
rmnet_shs_wq_cpu_list_remove(hnode);
- if (hnode->is_perm == 0) {
+ if (hnode->is_perm == 0 || force_clean) {
rmnet_shs_wq_hstat_tbl_remove(hnode);
kfree(hnode);
- } else
+ } else {
rmnet_shs_wq_hstat_reset_node(hnode);
+ }
spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags);
}
}
@@ -1250,7 +1287,7 @@ static void rmnet_shs_wq_update_stats(void)
rmnet_shs_wq_eval_suggested_cpu();
rmnet_shs_wq_refresh_new_flow_list();
/*Invoke after both the locks are released*/
- rmnet_shs_wq_cleanup_hash_tbl();
+ rmnet_shs_wq_cleanup_hash_tbl(PERIODIC_CLEAN);
}
void rmnet_shs_wq_process_wq(struct work_struct *work)
@@ -1289,7 +1326,6 @@ void rmnet_shs_wq_clean_ep_tbl(void)
void rmnet_shs_wq_exit(void)
{
-
/*If Wq is not initialized, nothing to cleanup */
if (!rmnet_shs_wq || !rmnet_shs_delayed_wq)
return;
@@ -1304,8 +1340,8 @@ void rmnet_shs_wq_exit(void)
rmnet_shs_delayed_wq = NULL;
rmnet_shs_wq = NULL;
+ rmnet_shs_wq_cleanup_hash_tbl(FORCE_CLEAN);
rmnet_shs_wq_clean_ep_tbl();
-
trace_rmnet_shs_wq_high(RMNET_SHS_WQ_EXIT, RMNET_SHS_WQ_EXIT_END,
0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL);
}
diff --git a/drivers/rmnet/shs/rmnet_shs_wq.h b/drivers/rmnet/shs/rmnet_shs_wq.h
index e16b0a9..96843f5 100644
--- a/drivers/rmnet/shs/rmnet_shs_wq.h
+++ b/drivers/rmnet/shs/rmnet_shs_wq.h
@@ -17,10 +17,17 @@
#define _RMNET_SHS_WQ_H_
#include "rmnet_shs_config.h"
+#include "rmnet_shs.h"
-#define MAX_CPUS 8
#define MAX_SUPPORTED_FLOWS_DEBUG 16
+#define RMNET_SHS_RX_BPNSEC_TO_BPSEC(x) ((x)*1000000000)
+#define RMNET_SHS_SEC_TO_NSEC(x) ((x)*1000000000)
+#define RMNET_SHS_NSEC_TO_SEC(x) ((x)/1000000000)
+#define RMNET_SHS_BYTE_TO_BIT(x) ((x)*8)
+#define RMNET_SHS_MIN_HSTAT_NODES_REQD 16
+#define RMNET_SHS_WQ_DELAY_TICKS 10
+
/* stores wq and end point details */
struct rmnet_shs_wq_ep_s {
@@ -193,6 +200,9 @@ enum rmnet_shs_wq_trace_evt {
};
+
+extern struct rmnet_shs_cpu_node_s rmnet_shs_cpu_node_tbl[MAX_CPUS];
+
void rmnet_shs_wq_init(struct net_device *dev);
void rmnet_shs_wq_exit(void);
void rmnet_shs_wq_restart(void);