diff options
author | arnav_s <arnav_s@codeaurora.org> | 2018-11-20 16:15:52 -0800 |
---|---|---|
committer | arnav_s <arnav_s@codeaurora.org> | 2018-11-20 16:15:52 -0800 |
commit | e28553c0ebb924b506e22775ed1ce514ee8d8394 (patch) | |
tree | bd97192a25bb1934d1e9a1eeed70cb6c828ea237 | |
parent | 0b734aefaf6d8e84e25cb5710467836c5043c1ff (diff) | |
parent | 659a8ea0fb392f80d278f18f24d9a98a9f6d9b58 (diff) | |
download | data-kernel-e28553c0ebb924b506e22775ed1ce514ee8d8394.tar.gz |
Fastforwarding data-kernel CRT:data.lnx.4.0-181119 to data.lnx.5.0
-rw-r--r-- | drivers/emac-dwc-eqos/Android.mk | 2 | ||||
-rw-r--r-- | drivers/emac-dwc-eqos/DWC_ETH_QOS_desc.c | 2 | ||||
-rw-r--r-- | drivers/emac-dwc-eqos/DWC_ETH_QOS_eee.c | 42 | ||||
-rw-r--r-- | drivers/emac-dwc-eqos/DWC_ETH_QOS_mdio.c | 71 | ||||
-rw-r--r-- | drivers/emac-dwc-eqos/DWC_ETH_QOS_platform.c | 15 | ||||
-rw-r--r-- | drivers/emac-dwc-eqos/DWC_ETH_QOS_rgmii_io_macro.c | 27 | ||||
-rw-r--r-- | drivers/rmnet/perf/rmnet_perf_config.c | 1 | ||||
-rw-r--r-- | drivers/rmnet/perf/rmnet_perf_core.c | 214 | ||||
-rw-r--r-- | drivers/rmnet/perf/rmnet_perf_core.h | 1 | ||||
-rw-r--r-- | drivers/rmnet/perf/rmnet_perf_tcp_opt.c | 38 | ||||
-rw-r--r-- | drivers/rmnet/shs/rmnet_shs.h | 32 | ||||
-rw-r--r-- | drivers/rmnet/shs/rmnet_shs_config.c | 3 | ||||
-rw-r--r-- | drivers/rmnet/shs/rmnet_shs_config.h | 2 | ||||
-rwxr-xr-x[-rw-r--r--] | drivers/rmnet/shs/rmnet_shs_main.c | 421 | ||||
-rw-r--r-- | drivers/rmnet/shs/rmnet_shs_wq.c | 98 | ||||
-rw-r--r-- | drivers/rmnet/shs/rmnet_shs_wq.h | 12 |
16 files changed, 690 insertions, 291 deletions
diff --git a/drivers/emac-dwc-eqos/Android.mk b/drivers/emac-dwc-eqos/Android.mk index 1c72e94..db943b9 100644 --- a/drivers/emac-dwc-eqos/Android.mk +++ b/drivers/emac-dwc-eqos/Android.mk @@ -2,6 +2,7 @@ ifeq ($(TARGET_BOARD_AUTO),true) LOCAL_PATH := $(call my-dir) +include $(CLEAR_VARS) # This makefile is only for DLKM ifneq ($(findstring vendor,$(LOCAL_PATH)),) @@ -18,7 +19,6 @@ KBUILD_OPTIONS += DCONFIG_PTPSUPPORT_OBJ=1 KBUILD_OPTIONS += DCONFIG_DEBUGFS_OBJ=1 #KBUILD_OPTIONS += DDWC_ETH_QOS_TEST=1 -include $(CLEAR_VARS) LOCAL_MODULE := emac_dwc_eqos.ko LOCAL_MODULE_TAGS := debug include $(DLKM_DIR)/AndroidKernelModule.mk diff --git a/drivers/emac-dwc-eqos/DWC_ETH_QOS_desc.c b/drivers/emac-dwc-eqos/DWC_ETH_QOS_desc.c index c092a0e..ac56b19 100644 --- a/drivers/emac-dwc-eqos/DWC_ETH_QOS_desc.c +++ b/drivers/emac-dwc-eqos/DWC_ETH_QOS_desc.c @@ -1692,7 +1692,7 @@ static unsigned int DWC_ETH_QOS_map_skb(struct net_device *dev, } } - if (buffer->dma == 0 && buffer->dma2 == 0) + if (prev_buffer != NULL && buffer->dma == 0 && buffer->dma2 == 0) prev_buffer->skb = skb; else buffer->skb = skb; diff --git a/drivers/emac-dwc-eqos/DWC_ETH_QOS_eee.c b/drivers/emac-dwc-eqos/DWC_ETH_QOS_eee.c index 7356b0b..5671e2e 100644 --- a/drivers/emac-dwc-eqos/DWC_ETH_QOS_eee.c +++ b/drivers/emac-dwc-eqos/DWC_ETH_QOS_eee.c @@ -285,6 +285,28 @@ static inline u32 DWC_ETH_QOS_mmd_eee_adv_to_ethtool_adv_t(u16 eee_adv) #endif /*! +* \brief API to disable EEE mode. +* +* \details This function disable smart EEE +* \param[in] phydev - pointer to target phy_device structure +*/ + +static void DWC_ETH_QOS_disable_smart_eee(struct phy_device *phydev) +{ + u32 smart_eee; + + smart_eee = DWC_ETH_QOS_phy_read_mmd_indirect( + phydev->mdio.bus, AR8035_SMART_EEE_CTRL_3, + MDIO_MMD_PCS, phydev->mdio.addr); + + smart_eee &= ~AR8035_SMART_EEE_EN; + DWC_ETH_QOS_phy_write_mmd_indirect( + phydev->mdio.bus, AR8035_SMART_EEE_CTRL_3, + MDIO_MMD_PCS,phydev->mdio.addr, smart_eee); + +} + +/*! * \brief API to initialize and check EEE mode. * * \details This function checks if the EEE is supported by @@ -378,15 +400,7 @@ static int DWC_ETH_QOS_phy_init_eee(struct phy_device *phydev, /* Disable smart EEE feature in AR8035*/ if (phydev->phy_id == ATH8035_PHY_ID || phydev->phy_id == ATH8030_PHY_ID) { - u32 smart_eee = DWC_ETH_QOS_phy_read_mmd_indirect( - phydev->mdio.bus, AR8035_SMART_EEE_CTRL_3, - MDIO_MMD_PCS, phydev->mdio.addr); - - smart_eee &= ~AR8035_SMART_EEE_EN; - - DWC_ETH_QOS_phy_write_mmd_indirect( - phydev->mdio.bus, AR8035_SMART_EEE_CTRL_3, - MDIO_MMD_PCS,phydev->mdio.addr, smart_eee); + DWC_ETH_QOS_disable_smart_eee(phydev); } ret = 0; /* EEE supported */ @@ -420,6 +434,16 @@ bool DWC_ETH_QOS_eee_init(struct DWC_ETH_QOS_prv_data *pdata) EMACDBG("Enter\n"); hw_if = &pdata->hw_if; + + /* Disable smart EEE & EEE for ATH8030*/ + if ((pdata->emac_hw_version_type == EMAC_HW_v2_3_1) + && (pdata->io_macro_phy_intf == RMII_MODE) && + pdata->phydev->phy_id == ATH8030_PHY_ID) { + //disable smart EEE + DWC_ETH_QOS_disable_smart_eee(pdata->phydev); + EMACDBG("disable smart EEE for 8030\n"); + } + /* For RMII mode EEE is not supported */ if (pdata->io_macro_phy_intf == RMII_MODE) goto phy_eee_failed; diff --git a/drivers/emac-dwc-eqos/DWC_ETH_QOS_mdio.c b/drivers/emac-dwc-eqos/DWC_ETH_QOS_mdio.c index 0ea301f..f295dc0 100644 --- a/drivers/emac-dwc-eqos/DWC_ETH_QOS_mdio.c +++ b/drivers/emac-dwc-eqos/DWC_ETH_QOS_mdio.c @@ -507,20 +507,38 @@ static void set_phy_rx_tx_delay(struct DWC_ETH_QOS_prv_data *pdata, if ((pdata->phydev->phy_id & pdata->phydev->drv->phy_id_mask) == MICREL_PHY_ID) { u16 phydata = 0; u16 rx_clk = 0; + if (pdata->emac_hw_version_type == EMAC_HW_v2_3_1) { + if(!pdata->io_macro_tx_mode_non_id){ + EMACDBG("No PHY delay settings required for ID mode for " + "EMAC core version 2.3.1 \n"); + return; + } + rx_clk = 22; + /* RX_CLK to 0*/ + DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, + DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x8,&phydata); + phydata &= ~(0x1F<<5); + phydata |= (rx_clk << 5); + DWC_ETH_QOS_mdio_mmd_register_write_direct(pdata, pdata->phyaddr, + DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x8,phydata); - rx_clk = 0x1F; - /* RX_CLK to 0*/ - DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, + DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, + DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x8,&phydata); + EMACDBG("Read 0x%x from offset 0x8\n",phydata); + } else { + rx_clk = 0x1F; + /* RX_CLK to 0*/ + DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x8,&phydata); - phydata &= ~(0x1F); - phydata |= rx_clk; - DWC_ETH_QOS_mdio_mmd_register_write_direct(pdata, pdata->phyaddr, + phydata &= ~(0x1F); + phydata |= rx_clk; + DWC_ETH_QOS_mdio_mmd_register_write_direct(pdata, pdata->phyaddr, DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x8,phydata); - DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, + DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x8,&phydata); - EMACDBG("Read 0x%x from offset 0x8\n",phydata); - phydata = 0; + EMACDBG("Read 0x%x from offset 0x8\n",phydata); + phydata = 0; if (pdata->emac_hw_version_type == EMAC_HW_v2_1_2) { u16 tx_clk = 0xE; @@ -547,15 +565,15 @@ static void set_phy_rx_tx_delay(struct DWC_ETH_QOS_prv_data *pdata, /* Default settings for EMAC_HW_v2_1_0 */ phydata |= ((0x0 << 12) | (0x0 << 8) | (0x0 << 4) | 0x0); - DWC_ETH_QOS_mdio_mmd_register_write_direct(pdata, pdata->phyaddr, + DWC_ETH_QOS_mdio_mmd_register_write_direct(pdata, pdata->phyaddr, DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x5,phydata); - DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, + DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x5,&phydata); - EMACDBG("Read 0x%x from offset 0x5\n",phydata); - phydata = 0; + EMACDBG("Read 0x%x from offset 0x5\n",phydata); + phydata = 0; - /*RX_CTL to 9*/ - DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, + /*RX_CTL to 9*/ + DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x4,&phydata); phydata &= ~(0xF << 4); if (pdata->emac_hw_version_type == EMAC_HW_v2_1_2) @@ -565,10 +583,11 @@ static void set_phy_rx_tx_delay(struct DWC_ETH_QOS_prv_data *pdata, phydata |= (0x0 << 4); DWC_ETH_QOS_mdio_mmd_register_write_direct(pdata, pdata->phyaddr, DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x4,phydata); - DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, + DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR,0x4,&phydata); - EMACDBG("Read 0x%x from offset 0x4\n",phydata); - phydata = 0; + EMACDBG("Read 0x%x from offset 0x4\n",phydata); + phydata = 0; + } } else { /* Default values are for PHY AR8035 */ int phydata = 0; @@ -619,6 +638,11 @@ static void set_phy_rx_tx_delay(struct DWC_ETH_QOS_prv_data *pdata, static void configure_phy_rx_tx_delay(struct DWC_ETH_QOS_prv_data *pdata) { EMACDBG("Enter\n"); + if ((pdata->emac_hw_version_type == EMAC_HW_v2_3_1) + && (pdata->io_macro_phy_intf == RMII_MODE)) { + EMACDBG("phy rx tx delay setting not required for RMII mode for 2.3.1\n"); + return; + } switch (pdata->speed) { case SPEED_1000: @@ -764,8 +788,8 @@ static inline int DWC_ETH_QOS_configure_io_macro_dll_settings( EMACDBG("Enter\n"); - if (pdata->emac_hw_version_type == EMAC_HW_v2_0_0) #ifndef DWC_ETH_QOS_EMULATION_PLATFORM + if (pdata->emac_hw_version_type == EMAC_HW_v2_0_0 || pdata->emac_hw_version_type == EMAC_HW_v2_3_1) DWC_ETH_QOS_rgmii_io_macro_dll_reset(pdata); /* For RGMII ID mode with internal delay*/ @@ -1143,15 +1167,6 @@ static int DWC_ETH_QOS_init_phy(struct net_device *dev) phydev->irq = PHY_IGNORE_INTERRUPT; phydev->interrupts = PHY_INTERRUPT_ENABLED; - if ((phydev->phy_id & phydev->drv->phy_id_mask) == MICREL_PHY_ID) { - DWC_ETH_QOS_mdio_write_direct(pdata, pdata->phyaddr, - DWC_ETH_QOS_MICREL_PHY_CTL, DWC_ETH_QOS_MICREL_INTR_LEVEL); - DWC_ETH_QOS_mdio_read_direct(pdata, pdata->phyaddr, - DWC_ETH_QOS_MICREL_PHY_CTL, &phydata); - EMACDBG("Micrel PHY Control Reg (%#x) = %#x\n", - DWC_ETH_QOS_MICREL_PHY_CTL, phydata); - } - if (phydev->drv->config_intr && !phydev->drv->config_intr(phydev)) DWC_ETH_QOS_request_phy_wol(pdata); diff --git a/drivers/emac-dwc-eqos/DWC_ETH_QOS_platform.c b/drivers/emac-dwc-eqos/DWC_ETH_QOS_platform.c index 2b85fc4..4c8445b 100644 --- a/drivers/emac-dwc-eqos/DWC_ETH_QOS_platform.c +++ b/drivers/emac-dwc-eqos/DWC_ETH_QOS_platform.c @@ -110,6 +110,18 @@ static ssize_t read_phy_reg_dump(struct file *file, i, phydata); } + if ((pdata->phydev->phy_id & pdata->phydev->drv->phy_id_mask) == MICREL_PHY_ID) { + int i = 0; + u16 mmd_phydata = 0; + for(i=0;i<=8;i++){ + DWC_ETH_QOS_mdio_mmd_register_read_direct(pdata, pdata->phyaddr, + DWC_ETH_QOS_MICREL_PHY_DEBUG_MMD_DEV_ADDR, i, &mmd_phydata); + EMACDBG("Read %#x from offset %#x", mmd_phydata, i); + len += scnprintf(buf + len, buf_len - len, + "Micrel PHY MMD Register (%#x) = %#x\n", i, mmd_phydata); + } + } + if (len > buf_len) { EMACERR(" %s (len > buf_len) buffer not sufficient\n",__func__); len = buf_len; @@ -853,7 +865,8 @@ int DWC_ETH_QOS_enable_ptp_clk(struct device *dev) int ret; const char* ptp_clock_name; - if (dwc_eth_qos_res_data.emac_hw_version_type == EMAC_HW_v2_1_0) + if (dwc_eth_qos_res_data.emac_hw_version_type == EMAC_HW_v2_1_0 + || dwc_eth_qos_res_data.emac_hw_version_type == EMAC_HW_v2_1_2) ptp_clock_name = "emac_ptp_clk"; else ptp_clock_name = "eth_ptp_clk"; diff --git a/drivers/emac-dwc-eqos/DWC_ETH_QOS_rgmii_io_macro.c b/drivers/emac-dwc-eqos/DWC_ETH_QOS_rgmii_io_macro.c index 982a14e..6bb7da4 100644 --- a/drivers/emac-dwc-eqos/DWC_ETH_QOS_rgmii_io_macro.c +++ b/drivers/emac-dwc-eqos/DWC_ETH_QOS_rgmii_io_macro.c @@ -363,7 +363,8 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata) uint loopback_mode_en = 0; uint rgmii_data_divide_clk; ULONG data; - if (pdata->emac_hw_version_type == EMAC_HW_v2_3_0) { + + if (pdata->emac_hw_version_type == EMAC_HW_v2_3_0 || (pdata->emac_hw_version_type == EMAC_HW_v2_3_1)) { if(pdata->io_macro_phy_intf == RGMII_MODE) loopback_mode_en = 0x1; rgmii_data_divide_clk = 0x0; @@ -374,9 +375,6 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata) EMACDBG("Enter\n"); - if (pdata->emac_hw_version_type == EMAC_HW_v2_3_0) - loopback_mode = 0x1; - /* Loopback is disabled */ DWC_ETH_QOS_set_rgmii_loopback_mode(loopback_mode); @@ -404,7 +402,8 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata) RGMII_CONFIG_2_RX_PROG_SWAP_UDFWR(0x1); RGMII_LOOPBACK_EN_UDFWR(loopback_mode_en); if (pdata->emac_hw_version_type == EMAC_HW_v2_1_0 || - pdata->emac_hw_version_type == EMAC_HW_v2_1_2) + pdata->emac_hw_version_type == EMAC_HW_v2_1_2 || + (pdata->emac_hw_version_type == EMAC_HW_v2_3_1)) RGMII_CONFIG_2_TX_CLK_PHASE_SHIFT_EN_UDFWR(0x1); } else { /* Enable DDR mode*/ @@ -426,8 +425,11 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata) /* Program PRG_RCLK_DLY to 52 ns for a required delay of 2 ns on EMAC core version 2.1.0 */ - if (pdata->emac_hw_version_type == EMAC_HW_v2_1_0) + if (pdata->emac_hw_version_type == EMAC_HW_v2_1_0 + || pdata->emac_hw_version_type == EMAC_HW_v2_1_2) SDCC_HC_PRG_RCLK_DLY_UDFWR(52); + else if (pdata->emac_hw_version_type == EMAC_HW_v2_3_1) + SDCC_HC_PRG_RCLK_DLY_UDFWR(104); else { /* Program PRG_RCLK_DLY to 57 for a required delay of 1.8 ns */ SDCC_HC_PRG_RCLK_DLY_UDFWR(57); } @@ -457,7 +459,8 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata) RGMII_CONFIG_2_RX_PROG_SWAP_UDFWR(0x0); RGMII_LOOPBACK_EN_UDFWR(loopback_mode_en); if (pdata->emac_hw_version_type == EMAC_HW_v2_1_0 || - pdata->emac_hw_version_type == EMAC_HW_v2_1_2) + pdata->emac_hw_version_type == EMAC_HW_v2_1_2 || + (pdata->emac_hw_version_type == EMAC_HW_v2_3_1)) RGMII_CONFIG_2_RX_PROG_SWAP_UDFWR(0x1); if (pdata->emac_hw_version_type == EMAC_HW_v2_1_2) RGMII_CONFIG_2_TX_CLK_PHASE_SHIFT_EN_UDFWR(0x1); @@ -498,10 +501,13 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata) RGMII_MAX_SPD_PRG_9_UDFWR(0x13); RGMII_CONFIG_2_RERVED_CONFIG_16_EN_UDFWR(0x1); /* Rx Path */ + if (pdata->emac_hw_version_type == EMAC_HW_v2_3_1) + RGMII_LOOPBACK_EN_UDFWR(loopback_mode_en); RGMII_CONFIG_2_RX_PROG_SWAP_UDFWR(0x0); RGMII_LOOPBACK_EN_UDFWR(loopback_mode_en); if (pdata->emac_hw_version_type == EMAC_HW_v2_1_0 || - pdata->emac_hw_version_type == EMAC_HW_v2_1_2) + pdata->emac_hw_version_type == EMAC_HW_v2_1_2 || + (pdata->emac_hw_version_type == EMAC_HW_v2_3_1)) RGMII_CONFIG_2_RX_PROG_SWAP_UDFWR(0x1); if (pdata->emac_hw_version_type == EMAC_HW_v2_1_2) RGMII_CONFIG_2_TX_CLK_PHASE_SHIFT_EN_UDFWR(0x1); @@ -548,7 +554,7 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata) RGMII_MAX_SPD_PRG_2_UDFWR(0x1); RGMII_MAX_SPD_PRG_9_UDFWR(0x13); RGMII_CONFIG_2_RERVED_CONFIG_16_EN_UDFWR(0x0); - if (pdata->emac_hw_version_type == EMAC_HW_v2_3_0) + if (pdata->emac_hw_version_type == EMAC_HW_v2_3_0 || (pdata->emac_hw_version_type == EMAC_HW_v2_3_1)) RGMII_LOOPBACK_EN_UDFWR(0x0); else RGMII_LOOPBACK_EN_UDFWR(0x1); @@ -567,7 +573,8 @@ int DWC_ETH_QOS_rgmii_io_macro_init(struct DWC_ETH_QOS_prv_data *pdata) RGMII_CONFIG_2_RERVED_CONFIG_16_EN_UDFWR(0x1); if (pdata->emac_hw_version_type == EMAC_HW_v2_1_2) RGMII_CONFIG_2_TX_CLK_PHASE_SHIFT_EN_UDFWR(0x1); - + if (pdata->emac_hw_version_type == EMAC_HW_v2_3_1) + RGMII_LOOPBACK_EN_UDFWR(0x1); break; } diff --git a/drivers/rmnet/perf/rmnet_perf_config.c b/drivers/rmnet/perf/rmnet_perf_config.c index 550ad07..ba64176 100644 --- a/drivers/rmnet/perf/rmnet_perf_config.c +++ b/drivers/rmnet/perf/rmnet_perf_config.c @@ -289,6 +289,7 @@ static int rmnet_perf_config_notify_cb(struct notifier_block *nb, dev_net_set(dev, &init_net); perf->core_meta->dev = dev; + dl_ind->priority = RMNET_PERF; dl_ind->dl_hdr_handler = &rmnet_perf_core_handle_map_control_start; dl_ind->dl_trl_handler = diff --git a/drivers/rmnet/perf/rmnet_perf_core.c b/drivers/rmnet/perf/rmnet_perf_core.c index bc61e27..c9c2ccc 100644 --- a/drivers/rmnet/perf/rmnet_perf_core.c +++ b/drivers/rmnet/perf/rmnet_perf_core.c @@ -355,20 +355,25 @@ void rmnet_perf_core_send_skb(struct sk_buff *skb, struct rmnet_endpoint *ep, if (ip_version == 0x04) { ip4hn = (struct iphdr *) data; rmnet_set_skb_proto(skb); - if (ip4hn->protocol == IPPROTO_TCP) { + /* If the checksum is unnecessary, update the header fields. + * Otherwise, we know that this is a single packet that + * failed checksum validation, so we don't want to touch + * the headers. + */ + if (ip4hn->protocol == IPPROTO_TCP && + skb->ip_summed == CHECKSUM_UNNECESSARY) { ip4hn->tot_len = htons(skb->len); ip4hn->check = 0; ip4hn->check = ip_fast_csum(ip4hn, (int)ip4hn->ihl); - skb->ip_summed = CHECKSUM_UNNECESSARY; } rmnet_deliver_skb(skb, perf->rmnet_port); } else if (ip_version == 0x06) { ip6hn = (struct ipv6hdr *)data; rmnet_set_skb_proto(skb); - if (ip6hn->nexthdr == IPPROTO_TCP) { + if (ip6hn->nexthdr == IPPROTO_TCP && + skb->ip_summed == CHECKSUM_UNNECESSARY) { ip6hn->payload_len = htons(skb->len - sizeof(struct ipv6hdr)); - skb->ip_summed = CHECKSUM_UNNECESSARY; } rmnet_deliver_skb(skb, perf->rmnet_port); } else { @@ -423,7 +428,12 @@ void rmnet_perf_core_flush_curr_pkt(struct rmnet_perf *perf, __skb_set_hash(skbn, 0, 0, 0); } + /* If the packet passed checksum validation, tell the stack */ + if (pkt_info->csum_valid) + skbn->ip_summed = CHECKSUM_UNNECESSARY; skbn->dev = skb->dev; + skbn->hash = pkt_info->hash_key; + skbn->sw_hash = 1; rmnet_perf_core_send_skb(skbn, ep, perf, pkt_info); } @@ -468,6 +478,107 @@ void rmnet_perf_core_handle_map_control_end(struct rmnet_map_dl_ind_trl *dltrl) 0xDEF, 0xDEF, NULL, NULL); } +int rmnet_perf_core_validate_pkt_csum(struct sk_buff *skb, + struct rmnet_perf_pkt_info *pkt_info) +{ + int result; + unsigned int pkt_len = pkt_info->header_len + pkt_info->payload_len; + + skb_pull(skb, sizeof(struct rmnet_map_header)); + if (pkt_info->ip_proto == 0x04) { + skb->protocol = htons(ETH_P_IP); + } else if (pkt_info->ip_proto == 0x06) { + skb->protocol = htons(ETH_P_IPV6); + } else { + pr_err("%s(): protocol field not set properly, protocol = %u\n", + __func__, pkt_info->ip_proto); + } + result = rmnet_map_checksum_downlink_packet(skb, pkt_len); + skb_push(skb, sizeof(struct rmnet_map_header)); + /* Mark the current packet as OK if csum is valid */ + if (likely(result == 0)) + pkt_info->csum_valid = true; + return result; +} + +void rmnet_perf_core_handle_packet_ingress(struct sk_buff *skb, + struct rmnet_endpoint *ep, + struct rmnet_perf_pkt_info *pkt_info, + u32 frame_len, u32 trailer_len) +{ + unsigned char *payload = (unsigned char *) + (skb->data + sizeof(struct rmnet_map_header)); + struct rmnet_perf *perf = rmnet_perf_config_get_perf(); + u16 pkt_len; + + pkt_info->ep = ep; + pkt_info->ip_proto = (*payload & 0xF0) >> 4; + if (pkt_info->ip_proto == 4) { + struct iphdr *iph = (struct iphdr *)payload; + + pkt_info->iphdr.v4hdr = iph; + pkt_info->trans_proto = iph->protocol; + pkt_info->header_len = iph->ihl * 4; + } else if (pkt_info->ip_proto == 6) { + struct ipv6hdr *iph = (struct ipv6hdr *)payload; + + pkt_info->iphdr.v6hdr = iph; + pkt_info->trans_proto = iph->nexthdr; + pkt_info->header_len = sizeof(*iph); + } else { + pr_err("%s(): invalid packet\n", __func__); + return; + } + + pkt_len = frame_len - sizeof(struct rmnet_map_header) - trailer_len; + + if (pkt_info->trans_proto == IPPROTO_TCP) { + struct tcphdr *tp = (struct tcphdr *) + (payload + pkt_info->header_len); + + pkt_info->trns_hdr.tp = tp; + pkt_info->header_len += tp->doff * 4; + pkt_info->payload_len = pkt_len - pkt_info->header_len; + pkt_info->hash_key = + rmnet_perf_core_compute_flow_hash(pkt_info); + + if (rmnet_perf_core_validate_pkt_csum(skb, pkt_info)) + goto flush; + + rmnet_perf_tcp_opt_ingress(perf, skb, pkt_info); + } else if (pkt_info->trans_proto == IPPROTO_UDP) { + struct udphdr *up = (struct udphdr *) + (payload + pkt_info->header_len); + + pkt_info->trns_hdr.up = up; + pkt_info->header_len += sizeof(*up); + pkt_info->payload_len = pkt_len - pkt_info->header_len; + pkt_info->hash_key = + rmnet_perf_core_compute_flow_hash(pkt_info); + + /* We flush anyway, so the result of the validation + * does not need to be checked. + */ + rmnet_perf_core_validate_pkt_csum(skb, pkt_info); + goto flush; + } else { + pkt_info->payload_len = pkt_len - pkt_info->header_len; + pkt_info->hash_key = + rmnet_perf_core_compute_flow_hash(pkt_info); + + /* We flush anyway, so the result of the validation + * does not need to be checked. + */ + rmnet_perf_core_validate_pkt_csum(skb, pkt_info); + goto flush; + } + + return; + +flush: + rmnet_perf_core_flush_curr_pkt(perf, skb, pkt_info, pkt_len); +} + /* rmnet_perf_core_deaggregate() - Deaggregated ip packets from map frame * @port: allows access to our required global structures * @skb: the incoming aggregated MAP frame from PND @@ -482,12 +593,8 @@ void rmnet_perf_core_deaggregate(struct sk_buff *skb, struct rmnet_port *port) { u8 mux_id; - u16 ip_packet_len; struct rmnet_map_header *maph; uint32_t map_frame_len; - unsigned char *map_payload; - struct iphdr *ip4h; - struct ipv6hdr *ip6h; struct rmnet_endpoint *ep; struct rmnet_perf_pkt_info pkt_info; struct rmnet_perf *perf; @@ -569,91 +676,10 @@ skip_frame: * with processing the packet i.e. we know we are * dealing with a packet with no funny business inside */ - pkt_info.ep = ep; - map_payload = (unsigned char *)(skb->data + - sizeof(struct rmnet_map_header)); - pkt_info.ip_proto = (*map_payload & 0xF0) >> 4; - if (pkt_info.ip_proto == 0x04) { - ip4h = (struct iphdr *)map_payload; - pkt_info.iphdr.v4hdr = (struct iphdr *) ip4h; - pkt_info.trans_proto = ip4h->protocol; - if (pkt_info.trans_proto == IPPROTO_TCP) { - pkt_info.trns_hdr.tp = (struct tcphdr *) - (map_payload + ip4h->ihl*4); - pkt_info.header_len = (ip4h->ihl * 4) + - (pkt_info.trns_hdr.tp->doff * 4); - pkt_info.payload_len = map_frame_len - - (sizeof(struct rmnet_map_header) + - trailer_len + - pkt_info.header_len); - pkt_info.hash_key = - rmnet_perf_core_compute_flow_hash(&pkt_info); - rmnet_perf_tcp_opt_ingress(perf, skb, - &pkt_info); - } else if (pkt_info.trans_proto == - IPPROTO_UDP) { - pkt_info.trns_hdr.up = (struct udphdr *) - (map_payload + ip4h->ihl*4); - ip_packet_len = map_frame_len - - (sizeof(struct rmnet_map_header) + - trailer_len); - pkt_info.hash_key = - rmnet_perf_core_compute_flow_hash(&pkt_info); - rmnet_perf_core_flush_curr_pkt(perf, - skb, &pkt_info, ip_packet_len); - } else { - ip_packet_len = map_frame_len - - (sizeof(struct rmnet_map_header) + - trailer_len); - pkt_info.hash_key = - rmnet_perf_core_compute_flow_hash(&pkt_info); - rmnet_perf_core_flush_curr_pkt(perf, - skb, &pkt_info, ip_packet_len); - } - } else if (pkt_info.ip_proto == 0x06) { - ip6h = (struct ipv6hdr *)map_payload; - pkt_info.iphdr.v6hdr = (struct ipv6hdr *) ip6h; - pkt_info.trans_proto = ip6h->nexthdr; - if (pkt_info.trans_proto == IPPROTO_TCP) { - pkt_info.trns_hdr.tp = (struct tcphdr *) - (map_payload + - sizeof(struct ipv6hdr)); - pkt_info.header_len = - sizeof(struct ipv6hdr) + - (pkt_info.trns_hdr.tp->doff * - 4); - pkt_info.payload_len = map_frame_len - - (sizeof(struct rmnet_map_header) + - trailer_len + - pkt_info.header_len); - pkt_info.hash_key = - rmnet_perf_core_compute_flow_hash(&pkt_info); - rmnet_perf_tcp_opt_ingress(perf, skb, - &pkt_info); - } else if (pkt_info.trans_proto == - IPPROTO_UDP) { - pkt_info.trns_hdr.up = (struct udphdr *) - (map_payload + - sizeof(struct ipv6hdr)); - ip_packet_len = map_frame_len - - (sizeof(struct rmnet_map_header) + - trailer_len); - pkt_info.hash_key = - rmnet_perf_core_compute_flow_hash(&pkt_info); - rmnet_perf_core_flush_curr_pkt(perf, - skb, &pkt_info, ip_packet_len); - } else { - ip_packet_len = map_frame_len - - (sizeof(struct rmnet_map_header) + - trailer_len); - pkt_info.hash_key = - rmnet_perf_core_compute_flow_hash(&pkt_info); - rmnet_perf_core_flush_curr_pkt(perf, - skb, &pkt_info, ip_packet_len); - } - } else { - pr_err("%s(): invalid packet\n", __func__); - } + rmnet_perf_core_handle_packet_ingress(skb, ep, + &pkt_info, + map_frame_len, + trailer_len); bad_data: skb_pull(skb, map_frame_len); co++; @@ -667,7 +693,7 @@ next_chain: * then we can flush everything */ if (!rmnet_perf_core_bm_flush_on || - perf->core_meta->bm_state->expect_packets <= 0) { + (int) perf->core_meta->bm_state->expect_packets <= 0) { rmnet_perf_tcp_opt_flush_all_flow_nodes(perf); rmnet_perf_core_free_held_skbs(perf); rmnet_perf_core_flush_reason_cnt[ diff --git a/drivers/rmnet/perf/rmnet_perf_core.h b/drivers/rmnet/perf/rmnet_perf_core.h index da35804..4019bc7 100644 --- a/drivers/rmnet/perf/rmnet_perf_core.h +++ b/drivers/rmnet/perf/rmnet_perf_core.h @@ -31,6 +31,7 @@ struct rmnet_perf { */ struct rmnet_perf_pkt_info { bool first_packet; + bool csum_valid; unsigned char ip_proto; unsigned char trans_proto; u16 header_len; diff --git a/drivers/rmnet/perf/rmnet_perf_tcp_opt.c b/drivers/rmnet/perf/rmnet_perf_tcp_opt.c index 2fa7453..f673884 100644 --- a/drivers/rmnet/perf/rmnet_perf_tcp_opt.c +++ b/drivers/rmnet/perf/rmnet_perf_tcp_opt.c @@ -136,9 +136,7 @@ rmnet_perf_tcp_opt_tcp_flag_flush(struct rmnet_perf_pkt_info *pkt_info) * @pkt_info: characteristics of the current packet * * 1. check src/dest IP addr and TCP port & next seq match - * 2. validate checksum - it also checks the ip flag - * 3. check if timestamp changed - * 4. check if size overflow + * 2. check if size overflow * * Return: * - true if Pkt can be merged @@ -151,7 +149,6 @@ rmnet_perf_tcp_opt_pkt_can_be_merged(struct sk_buff *skb, { struct iphdr *ip4h; struct ipv6hdr *ip6h; - int result = 0; u16 payload_len = pkt_info->payload_len; struct tcphdr *tp = pkt_info->trns_hdr.tp; @@ -186,29 +183,7 @@ rmnet_perf_tcp_opt_pkt_can_be_merged(struct sk_buff *skb, return RMNET_PERF_TCP_OPT_FLUSH_SOME; } - /* 2. validate checksum - it also checks the ip flag */ - skb_pull(skb, sizeof(struct rmnet_map_header)); - if (pkt_info->ip_proto == 0x04) { - skb->protocol = htons(ETH_P_IP); - } else if (pkt_info->ip_proto == 0x06) { - skb->protocol = htons(ETH_P_IPV6); - } else { - pr_err("%s(): protocol field not set properly, protocol = %u\n", - __func__, pkt_info->ip_proto); - } - result = rmnet_map_checksum_downlink_packet(skb, pkt_info->header_len + - payload_len); - skb_push(skb, sizeof(struct rmnet_map_header)); - if (!(likely((result == 0)))) { - //TODO: later on might want to make macro out of this - //(result == RMNET_MAP_CHECKSUM_SKIPPED)))) { - pr_err("Wrong csum %d", result); - rmnet_perf_tcp_opt_flush_reason_cnt[ - RMNET_PERF_TCP_OPT_CHECKSUM_ERR]++; - return RMNET_PERF_TCP_OPT_FLUSH_ALL; - } - - /* 3. check if size overflow */ + /* 2. check if size overflow */ if ((payload_len + flow_node->len >= rmnet_perf_tcp_opt_flush_limit)) { rmnet_perf_tcp_opt_flush_reason_cnt[ RMNET_PERF_TCP_OPT_64K_LIMIT]++; @@ -332,6 +307,9 @@ static void rmnet_perf_tcp_opt_flush_single_flow_node(struct rmnet_perf *perf, __func__); } else { skbn->hash = flow_node->hash_value; + skbn->sw_hash = 1; + /* data is already validated */ + skbn->ip_summed = CHECKSUM_UNNECESSARY; rmnet_perf_core_send_skb(skbn, ep, perf, NULL); } @@ -593,6 +571,7 @@ void rmnet_perf_tcp_opt_ingress(struct rmnet_perf *perf, struct sk_buff *skb, bool match; enum rmnet_perf_tcp_opt_merge_check_rc rc = 0; bool flow_node_exists = 0; + struct napi_struct *napi = NULL; //pkt_info->hash_key = rmnet_perf_core_compute_flow_hash(pkt_info); handle_pkt: @@ -624,9 +603,12 @@ handle_pkt: pkt_info); rmnet_perf_tcp_opt_flush_single_flow_node(perf, flow_node); + napi = get_current_napi_context(); + napi_gro_flush(napi, false); rmnet_perf_core_flush_curr_pkt(perf, skb, pkt_info, pkt_info->header_len + pkt_info->payload_len); + napi_gro_flush(napi, false); rmnet_perf_tcp_opt_flush_reason_cnt[ RMNET_PERF_TCP_OPT_TCP_FLUSH_FORCE]++; } else if (rc == RMNET_PERF_TCP_OPT_FLUSH_ALL) { @@ -663,6 +645,8 @@ handle_pkt: rmnet_perf_tcp_opt_update_flow(flow_node, pkt_info); rmnet_perf_core_flush_curr_pkt(perf, skb, pkt_info, pkt_info->header_len + pkt_info->payload_len); + napi = get_current_napi_context(); + napi_gro_flush(napi, false); rmnet_perf_tcp_opt_flush_reason_cnt[ RMNET_PERF_TCP_OPT_TCP_FLUSH_FORCE]++; } else diff --git a/drivers/rmnet/shs/rmnet_shs.h b/drivers/rmnet/shs/rmnet_shs.h index 7f6336a..ee12024 100644 --- a/drivers/rmnet/shs/rmnet_shs.h +++ b/drivers/rmnet/shs/rmnet_shs.h @@ -16,6 +16,9 @@ #include <linux/skbuff.h> #include "rmnet_shs_wq.h" +#ifndef _RMNET_SHS_H_ +#define _RMNET_SHS_H_ + #include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h> #include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h> #include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h> @@ -24,14 +27,11 @@ #include <../include/soc/qcom/qmi_rmnet.h> - -#ifndef _RMNET_SHS_H_ -#define _RMNET_SHS_H_ - #define RMNET_SHS_HT rmnet_shs_ht #define RMNET_SHS_HT_SIZE 9 #define RMNET_SHS_MAX_SKB_INACTIVE_TSEC 30 #define MAX_SILVER_CORES 4 +#define MAX_CPUS 8 //#define RMNET_SHS_MAX_UDP_SILVER_CORE_DATA_RATE 1073741824 //1.0Gbps //#define RMNET_SHS_MAX_UDP_SILVER_CORE_DATA_RATE 320787200 //320 Mbps @@ -53,18 +53,26 @@ #define RMNET_SHS_UDP_PPS_PERF_CPU_LTHRESH 40000 #define RMNET_SHS_TCP_PPS_PERF_CPU_LTHRESH (40000*RMNET_SHS_TCP_COALESCING_RATIO) +struct core_flush_s { + struct hrtimer core_timer; + struct work_struct work; + struct timespec coretime; + int coresum; + u8 core; +}; + struct rmnet_shs_cfg_s { struct hrtimer hrtimer_shs; - struct rps_map *map; struct rmnet_map_dl_ind dl_mrk_ind_cb; struct qmi_rmnet_ps_ind rmnet_idl_ind_cb; struct rmnet_port *port; + struct core_flush_s core_flush[MAX_CPUS]; + u64 core_skbs[MAX_CPUS]; long int num_bytes_parked; long int num_pkts_parked; u32 is_reg_dl_mrk_ind; u8 is_pkt_parked; u8 is_timer_init; - u8 high_prio; u8 force_flush_state; }; @@ -93,8 +101,6 @@ struct rmnet_shs_skbn_s { /* n/w stack CPU pkt processing queue head */ u32 hash; /*incoming hash*/ - u32 parked_skbs; - /*num skbs parked per flow*/ u16 map_index; /* rps map index assigned*/ u16 map_cpu; @@ -132,6 +138,10 @@ struct rmnet_shs_cpu_node_s { struct list_head node_list_id; u32 qhead; u32 qtail; + u32 qdiff; + u32 parkedlen; + u8 prio; + u8 wqprio; }; enum rmnet_shs_trace_func { @@ -215,6 +225,7 @@ enum rmnet_shs_trace_evt { RMNET_SHS_DL_MRK_END, }; +extern struct rmnet_shs_flush_work shs_delayed_work; extern spinlock_t rmnet_shs_ht_splock; extern struct hlist_head RMNET_SHS_HT[1 << (RMNET_SHS_HT_SIZE)]; @@ -225,8 +236,9 @@ extern int (*rmnet_shs_skb_entry)(struct sk_buff *skb, struct rmnet_port *port); int rmnet_shs_is_lpwr_cpu(u16 cpu); void rmnet_shs_cancel_table(void); - void rmnet_shs_aggregate_init(void); + +int rmnet_shs_chk_and_flush_node(struct rmnet_shs_skbn_s *node, u8 force_flush); void rmnet_shs_dl_hdr_handler(struct rmnet_map_dl_ind_hdr *dlhdr); void rmnet_shs_dl_trl_handler(struct rmnet_map_dl_ind_trl *dltrl); void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port); @@ -236,5 +248,5 @@ void rmnet_shs_init(struct net_device *dev); void rmnet_shs_exit(void); void rmnet_shs_ps_on_hdlr(void *port); void rmnet_shs_ps_off_hdlr(void *port); - +void rmnet_shs_update_cpu_proc_q_all_cpus(void); #endif /* _RMNET_SHS_H_ */ diff --git a/drivers/rmnet/shs/rmnet_shs_config.c b/drivers/rmnet/shs/rmnet_shs_config.c index 2d84488..3a29f04 100644 --- a/drivers/rmnet/shs/rmnet_shs_config.c +++ b/drivers/rmnet/shs/rmnet_shs_config.c @@ -15,6 +15,7 @@ #include <linux/kernel.h> #include <linux/netdevice.h> #include <linux/module.h> +#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h> #include "rmnet_shs_config.h" #include "rmnet_shs.h" #include "rmnet_shs_wq.h" @@ -115,6 +116,8 @@ static int rmnet_shs_dev_notify_cb(struct notifier_block *nb, rmnet_shs_wq_init(phy_dev); rmnet_shs_aggregate_init(); rmnet_shs_cfg.is_timer_init = 1; + rmnet_shs_cfg.dl_mrk_ind_cb.priority = + RMNET_SHS; rmnet_shs_cfg.dl_mrk_ind_cb.dl_hdr_handler = &rmnet_shs_dl_hdr_handler; rmnet_shs_cfg.dl_mrk_ind_cb.dl_trl_handler = diff --git a/drivers/rmnet/shs/rmnet_shs_config.h b/drivers/rmnet/shs/rmnet_shs_config.h index 679fc7f..0c00b4f 100644 --- a/drivers/rmnet/shs/rmnet_shs_config.h +++ b/drivers/rmnet/shs/rmnet_shs_config.h @@ -36,6 +36,8 @@ enum rmnet_shs_crit_err_e { RMNET_SHS_WQ_ALLOC_EP_TBL_ERR, RMNET_SHS_WQ_GET_RMNET_PORT_ERR, RMNET_SHS_WQ_EP_ACCESS_ERR, + RMNET_SHS_CPU_PKTLEN_ERR, + RMNET_SHS_NULL_SKB_HEAD, RMNET_SHS_CRIT_ERR_MAX }; diff --git a/drivers/rmnet/shs/rmnet_shs_main.c b/drivers/rmnet/shs/rmnet_shs_main.c index 680cd6b..469dd37 100644..100755 --- a/drivers/rmnet/shs/rmnet_shs_main.c +++ b/drivers/rmnet/shs/rmnet_shs_main.c @@ -24,21 +24,20 @@ #include "rmnet_shs_wq.h" /* Local Macros */ -#define RMNET_SHS_MAX_BYTES_TO_PARK 271800 -#define RMNET_SHS_MAX_PKTS_TO_PARK 180 #define RMNET_SHS_FORCE_FLUSH_TIME_NSEC 2000000 +#define NS_IN_MS 1000000 #define LPWR_CLUSTER 0 #define PERF_CLUSTER 4 #define INVALID_CPU -1 #define GET_QTAIL(SD, CPU) (per_cpu(SD, CPU).input_queue_tail) #define GET_QHEAD(SD, CPU) (per_cpu(SD, CPU).input_queue_head) +#define GET_CTIMER(CPU) rmnet_shs_cfg.core_flush[CPU].core_timer /* Local Definitions and Declarations */ -spinlock_t rmnet_shs_ht_splock; +DEFINE_SPINLOCK(rmnet_shs_ht_splock); DEFINE_HASHTABLE(RMNET_SHS_HT, RMNET_SHS_HT_SIZE); - -static struct rmnet_shs_cpu_node_s rmnet_shs_cpu_node_tbl[MAX_CPUS]; +struct rmnet_shs_cpu_node_s rmnet_shs_cpu_node_tbl[MAX_CPUS]; /* Maintains a list of flows associated with a core * Also keeps track of number of packets processed on that core */ @@ -54,10 +53,41 @@ unsigned long int rmnet_shs_flush_reason[RMNET_SHS_FLUSH_MAX_REASON]; module_param_array(rmnet_shs_flush_reason, ulong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flush_reason, "rmnet shs skb flush trigger type"); +unsigned int rmnet_shs_byte_store_limit __read_mostly = 271800 * 8; +module_param(rmnet_shs_byte_store_limit, uint, 0644); +MODULE_PARM_DESC(rmnet_shs_byte_store_limit, "Maximum byte module will park"); + +unsigned int rmnet_shs_pkts_store_limit __read_mostly = 2100; +module_param(rmnet_shs_pkts_store_limit, uint, 0644); +MODULE_PARM_DESC(rmnet_shs_pkts_store_limit, "Maximum pkts module will park"); + +unsigned int rmnet_shs_max_core_wait __read_mostly = 10; +module_param(rmnet_shs_max_core_wait, uint, 0644); +MODULE_PARM_DESC(rmnet_shs_max_core_wait, + "Max wait module will wait during move to perf core in ms"); + +unsigned int rmnet_shs_inst_rate_interval __read_mostly = 15; +module_param(rmnet_shs_inst_rate_interval, uint, 0644); +MODULE_PARM_DESC(rmnet_shs_inst_rate_interval, + "Max interval we sample for instant burst prioritizing"); + +unsigned int rmnet_shs_inst_rate_max_pkts __read_mostly = 1800; +module_param(rmnet_shs_inst_rate_max_pkts, uint, 0644); +MODULE_PARM_DESC(rmnet_shs_inst_rate_max_pkts, + "Max pkts in a instant burst interval before prioritizing"); + unsigned int rmnet_shs_switch_cores __read_mostly = 1; module_param(rmnet_shs_switch_cores, uint, 0644); MODULE_PARM_DESC(rmnet_shs_switch_cores, "Switch core upon hitting threshold"); +unsigned int rmnet_shs_cpu_max_qdiff[MAX_CPUS]; +module_param_array(rmnet_shs_cpu_max_qdiff, uint, 0, 0644); +MODULE_PARM_DESC(rmnet_shs_cpu_max_qdiff, "Max queue length seen of each core"); + +unsigned int rmnet_shs_cpu_max_coresum[MAX_CPUS]; +module_param_array(rmnet_shs_cpu_max_coresum, uint, 0, 0644); +MODULE_PARM_DESC(rmnet_shs_cpu_max_coresum, "Max coresum seen of each core"); + void rmnet_shs_cpu_node_remove(struct rmnet_shs_skbn_s *node) { trace_rmnet_shs_low(RMNET_SHS_CPU_NODE, RMNET_SHS_CPU_NODE_FUNC_REMOVE, @@ -116,6 +146,44 @@ int rmnet_shs_is_skb_stamping_reqd(struct sk_buff *skb) return ret_val; } +static void rmnet_shs_update_core_load(int cpu, int burst) +{ + + struct timespec time1; + struct timespec *time2; + long int curinterval; + int maxinterval = (rmnet_shs_inst_rate_interval < 5) ? 5 : + rmnet_shs_inst_rate_interval; + + getnstimeofday(&time1); + time2 = &rmnet_shs_cfg.core_flush[cpu].coretime; + + curinterval = RMNET_SHS_SEC_TO_NSEC(time1.tv_sec - time2->tv_sec) + + time1.tv_nsec - time2->tv_nsec; + + if (curinterval >= maxinterval * NS_IN_MS) { + if (rmnet_shs_cfg.core_flush[cpu].coresum > + rmnet_shs_cpu_max_coresum[cpu]) + rmnet_shs_cpu_max_coresum[cpu] = rmnet_shs_cfg.core_flush[cpu].coresum; + + rmnet_shs_cfg.core_flush[cpu].coretime.tv_sec = time1.tv_sec; + rmnet_shs_cfg.core_flush[cpu].coretime.tv_nsec = time1.tv_nsec; + rmnet_shs_cfg.core_flush[cpu].coresum = burst; + + } else { + rmnet_shs_cfg.core_flush[cpu].coresum += burst; + } + +} + +static int rmnet_shs_is_core_loaded(int cpu) +{ + + return rmnet_shs_cfg.core_flush[cpu].coresum >= + rmnet_shs_inst_rate_max_pkts; + +} + /* We deliver packets to GRO module only for TCP traffic*/ static int rmnet_shs_check_skb_can_gro(struct sk_buff *skb) { @@ -146,14 +214,18 @@ static int rmnet_shs_check_skb_can_gro(struct sk_buff *skb) static void rmnet_shs_deliver_skb(struct sk_buff *skb) { struct rmnet_priv *priv; + struct napi_struct *napi; trace_rmnet_shs_low(RMNET_SHS_DELIVER_SKB, RMNET_SHS_DELIVER_SKB_START, 0xDEF, 0xDEF, 0xDEF, 0xDEF, skb, NULL); if (!rmnet_shs_check_skb_can_gro(skb)) { - priv = netdev_priv(skb->dev); - gro_cells_receive(&priv->gro_cells, skb); - + if ((napi = get_current_napi_context())) { + napi_gro_receive(napi, skb); + } else { + priv = netdev_priv(skb->dev); + gro_cells_receive(&priv->gro_cells, skb); + } } else { netif_receive_skb(skb); } @@ -196,10 +268,24 @@ int rmnet_shs_num_perf_cores_configured(struct rps_map *map) return ret; } +int rmnet_shs_flow_num_perf_cores(struct rmnet_shs_skbn_s *node_p) +{ + int ret = 0; + int core = 1; + u16 idx = 0; + + for (idx = 0; idx < MAX_CPUS; idx++) { + if (node_p->hstats->pri_core_msk & core) + ret++; + core = core << 1; + } + return ret; +} + int rmnet_shs_is_lpwr_cpu(u16 cpu) { int ret = 1; - u32 big_cluster_mask = (1 << PERF_CLUSTER); + u32 big_cluster_mask = (1 << PERF_CLUSTER) - 1; if ((1 << cpu) >= big_cluster_mask) ret = 0; @@ -220,14 +306,17 @@ u32 rmnet_shs_form_hash(u32 index, u32 maplen, u32 hash) int offsetmap[MAX_CPUS / 2] = {8, 4, 3, 2}; u32 ret = 0; + if (!maplen) { + rmnet_shs_crit_err[RMNET_SHS_MAIN_MAP_LEN_INVALID]++; + return ret; + } + if (maplen < MAX_CPUS) ret = ((((index + ((maplen % 2) ? 1 : 0))) << 28) * offsetmap[(maplen - 1) >> 1]) | (hash & 0xFFFFFF); trace_rmnet_shs_low(RMNET_SHS_HASH_MAP, RMNET_SHS_HASH_MAP_FORM_HASH, ret, hash, index, maplen, NULL, NULL); - if (maplen == 0) - rmnet_shs_crit_err[RMNET_SHS_MAIN_MAP_LEN_INVALID]++; return ret; } @@ -273,24 +362,34 @@ int rmnet_shs_new_flow_cpu(u64 burst_size, struct net_device *dev) return flow_cpu; } -int rmnet_shs_get_suggested_cpu(struct rmnet_shs_skbn_s *node_p) +int rmnet_shs_get_suggested_cpu(struct rmnet_shs_skbn_s *node) { int cpu = INVALID_CPU; - if (node_p->hstats != NULL) - cpu = node_p->hstats->suggested_cpu; + /* Return same perf core unless moving to gold from silver*/ + if (rmnet_shs_cpu_node_tbl[node->map_cpu].prio && + rmnet_shs_is_lpwr_cpu(node->map_cpu)) { + cpu = rmnet_shs_wq_get_least_utilized_core(0xF0); + if (cpu < 0) + cpu = node->hstats->suggested_cpu; + } else if (node->hstats != NULL) + cpu = node->hstats->suggested_cpu; return cpu; } int rmnet_shs_get_hash_map_idx_to_stamp(struct rmnet_shs_skbn_s *node_p) { - int cpu, idx; + int cpu, idx = INVALID_CPU; struct rps_map *map; cpu = rmnet_shs_get_suggested_cpu(node_p); + map = rcu_dereference(node_p->dev->_rx->rps_map); + if (!node_p->dev || !node_p->dev->_rx || !map) + return idx; + idx = rmnet_shs_map_idx_from_cpu(cpu, map); trace_rmnet_shs_low(RMNET_SHS_HASH_MAP, @@ -324,6 +423,18 @@ u32 rmnet_shs_get_cpu_qtail(u8 cpu_num) return ret; } +u32 rmnet_shs_get_cpu_qdiff(u8 cpu_num) +{ + u32 ret = 0; + + if (cpu_num < MAX_CPUS) + ret = rmnet_shs_cpu_node_tbl[cpu_num].qdiff; + + trace_rmnet_shs_low(RMNET_SHS_CORE_CFG, RMNET_SHS_CORE_CFG_GET_QTAIL, + cpu_num, ret, 0xDEF, 0xDEF, NULL, NULL); + + return ret; +} /* Takes a snapshot of absolute value of the CPU Qhead and Qtail counts for * a given core. * @@ -341,6 +452,9 @@ void rmnet_shs_update_cpu_proc_q(u8 cpu_num) GET_QHEAD(softnet_data, cpu_num); rmnet_shs_cpu_node_tbl[cpu_num].qtail = GET_QTAIL(softnet_data, cpu_num); + rmnet_shs_cpu_node_tbl[cpu_num].qdiff = + rmnet_shs_cpu_node_tbl[cpu_num].qtail - + rmnet_shs_cpu_node_tbl[cpu_num].qhead; rcu_read_unlock(); trace_rmnet_shs_low(RMNET_SHS_CORE_CFG, @@ -363,10 +477,7 @@ void rmnet_shs_update_cpu_proc_q_all_cpus(void) rcu_read_lock(); for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) { - rmnet_shs_cpu_node_tbl[cpu_num].qhead = GET_QHEAD(softnet_data, - cpu_num); - rmnet_shs_cpu_node_tbl[cpu_num].qtail = GET_QTAIL(softnet_data, - cpu_num); + rmnet_shs_update_cpu_proc_q(cpu_num); trace_rmnet_shs_low(RMNET_SHS_CORE_CFG, RMNET_SHS_CORE_CFG_GET_CPU_PROC_PARAMS, @@ -376,8 +487,8 @@ void rmnet_shs_update_cpu_proc_q_all_cpus(void) 0xDEF, NULL, NULL); } rcu_read_unlock(); -} +} int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush) { struct rps_map *map; @@ -386,6 +497,8 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush) u32 node_qhead; int ret = 0; int prev_cpu = -1; + int ccpu; + int cpu_num; struct rmnet_shs_cpu_node_s *cpun; cpu_map_index = rmnet_shs_get_hash_map_idx_to_stamp(node); @@ -398,6 +511,12 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush) } node->is_shs_enabled = 1; map = rcu_dereference(node->dev->_rx->rps_map); + if (!node->dev->_rx || !map){ + node->is_shs_enabled = 0; + ret = 1; + break; + } + /* If the flow is going to the same core itself */ @@ -408,16 +527,33 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush) cur_cpu_qhead = rmnet_shs_get_cpu_qhead(node->map_cpu); node_qhead = node->queue_head; - + cpu_num = node->map_cpu; if ((cur_cpu_qhead >= node_qhead) || (node->skb_tport_proto == IPPROTO_TCP) || - (force_flush) - ) { + (force_flush)) { if (rmnet_shs_switch_cores) { + + /* Move the amount parked to other core's count + * Update old core's parked to not include diverted + * packets and update new core's packets + */ + rmnet_shs_cpu_node_tbl[map->cpus[cpu_map_index]].parkedlen += + node->skb_list.num_parked_skbs; + rmnet_shs_cpu_node_tbl[node->map_cpu].parkedlen -= + node->skb_list.num_parked_skbs; node->map_index = cpu_map_index; node->map_cpu = map->cpus[cpu_map_index]; + ccpu = node->map_cpu; + + /* Mark gold core as prio to prevent + * flows from moving in wq + */ + if (rmnet_shs_cpu_node_tbl[cpu_num].prio) { + node->hstats->suggested_cpu = ccpu; + rmnet_shs_cpu_node_tbl[ccpu].wqprio = 1; + } cpun = &rmnet_shs_cpu_node_tbl[node->map_cpu]; - rmnet_shs_update_cpu_proc_q(node->map_cpu); + rmnet_shs_update_cpu_proc_q_all_cpus(); node->queue_head = cpun->qhead; rmnet_shs_cpu_node_move(node, &cpun->node_list_id); @@ -437,6 +573,69 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush) return ret; } +void rmnet_shs_flush_core(u8 cpu_num) +{ + struct rmnet_shs_skbn_s *n; + struct list_head *ptr, *next; + unsigned long ht_flags; + u32 cpu_tail; + u32 num_pkts_flush = 0; + u32 num_bytes_flush = 0; + u32 total_pkts_flush = 0; + u32 total_bytes_flush = 0; + + /* Record a qtail + pkts flushed or move if reqd + * currently only use qtail for non TCP flows + */ + rmnet_shs_update_cpu_proc_q_all_cpus(); + trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_START, + rmnet_shs_cfg.num_pkts_parked, + rmnet_shs_cfg.num_bytes_parked, + 0xDEF, 0xDEF, NULL, NULL); + + spin_lock_irqsave(&rmnet_shs_ht_splock, ht_flags); + cpu_tail = rmnet_shs_get_cpu_qtail(cpu_num); + list_for_each_safe(ptr, next, + &rmnet_shs_cpu_node_tbl[cpu_num].node_list_id) { + n = list_entry(ptr, struct rmnet_shs_skbn_s, node_id); + if (n != NULL && n->skb_list.num_parked_skbs) { + num_pkts_flush = n->skb_list.num_parked_skbs; + num_bytes_flush = n->skb_list.num_parked_bytes; + + rmnet_shs_chk_and_flush_node(n, 1); + + total_pkts_flush += num_pkts_flush; + total_bytes_flush += num_bytes_flush; + if (n->map_cpu == cpu_num) { + cpu_tail += num_pkts_flush; + n->queue_head = cpu_tail; + + } + + } + } + + rmnet_shs_cfg.num_bytes_parked -= total_bytes_flush; + rmnet_shs_cfg.num_pkts_parked -= total_pkts_flush; + rmnet_shs_cpu_node_tbl[cpu_num].prio = 0; + rmnet_shs_cpu_node_tbl[cpu_num].parkedlen = 0; + spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); + + trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_END, + rmnet_shs_cfg.num_pkts_parked, + rmnet_shs_cfg.num_bytes_parked, + total_pkts_flush, total_bytes_flush, NULL, NULL); + +} + +static void rmnet_shs_flush_core_work(struct work_struct *work) +{ + struct core_flush_s *core_work = container_of(work, + struct core_flush_s, work); + + rmnet_shs_flush_core(core_work->core); +} + /* Flushes all the packets parked in order for this flow */ void rmnet_shs_flush_node(struct rmnet_shs_skbn_s *node) { @@ -447,13 +646,17 @@ void rmnet_shs_flush_node(struct rmnet_shs_skbn_s *node) u32 skb_bytes_delivered = 0; u32 hash2stamp; - map = rcu_dereference(node->dev->_rx->rps_map); - if (!node->skb_list.head) return; - hash2stamp = rmnet_shs_form_hash(node->map_index, + map = rcu_dereference(node->dev->_rx->rps_map); + + if (!map) { + hash2stamp = rmnet_shs_form_hash(node->map_index, map->len, node->skb_list.head->hash); + } else { + node->is_shs_enabled = 0; + } trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_NODE_START, node->hash, hash2stamp, @@ -470,8 +673,11 @@ void rmnet_shs_flush_node(struct rmnet_shs_skbn_s *node) skb->next = NULL; skbs_delivered += 1; skb_bytes_delivered += skb->len; + rmnet_shs_deliver_skb(skb); + } + node->skb_list.num_parked_skbs = 0; node->skb_list.num_parked_bytes = 0; node->skb_list.head = NULL; @@ -503,7 +709,6 @@ int rmnet_shs_chk_and_flush_node(struct rmnet_shs_skbn_s *node, u8 force_flush) node, NULL); return ret_val; } - /* Flushes all the packets that have been parked so far across all the flows * The order of flushing depends on the CPU<=>flow association * The flows associated with low power cores are flushed before flushing @@ -527,6 +732,7 @@ void rmnet_shs_flush_table(u8 flsh) u32 total_pkts_flush = 0; u32 total_bytes_flush = 0; u8 is_flushed = 0; + u32 wait = (!rmnet_shs_max_core_wait) ? 1 : rmnet_shs_max_core_wait; /* Record a qtail + pkts flushed or move if reqd * currently only use qtail for non TCP flows @@ -539,12 +745,36 @@ void rmnet_shs_flush_table(u8 flsh) spin_lock_irqsave(&rmnet_shs_ht_splock, ht_flags); for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) { + cpu_tail = rmnet_shs_get_cpu_qtail(cpu_num); + + /* If core is loaded set core flows as priority and + * start a 10ms hard flush timer + */ + if (rmnet_shs_is_lpwr_cpu(cpu_num) && + !rmnet_shs_cpu_node_tbl[cpu_num].prio) + rmnet_shs_update_core_load(cpu_num, + rmnet_shs_cpu_node_tbl[cpu_num].parkedlen); + + if (rmnet_shs_is_core_loaded(cpu_num) && + rmnet_shs_is_lpwr_cpu(cpu_num) && + !rmnet_shs_cpu_node_tbl[cpu_num].prio) { + + rmnet_shs_cpu_node_tbl[cpu_num].prio = 1; + if (hrtimer_active(&GET_CTIMER(cpu_num))) + hrtimer_cancel(&GET_CTIMER(cpu_num)); + + hrtimer_start(&GET_CTIMER(cpu_num), + ns_to_ktime(wait * NS_IN_MS), + HRTIMER_MODE_REL); + + } + list_for_each_safe(ptr, next, &rmnet_shs_cpu_node_tbl[cpu_num].node_list_id) { n = list_entry(ptr, struct rmnet_shs_skbn_s, node_id); - if (n != NULL) { + if (n != NULL && n->skb_list.num_parked_skbs) { num_pkts_flush = n->skb_list.num_parked_skbs; num_bytes_flush = n->skb_list.num_parked_bytes; is_flushed = rmnet_shs_chk_and_flush_node(n, @@ -553,21 +783,30 @@ void rmnet_shs_flush_table(u8 flsh) if (is_flushed) { total_pkts_flush += num_pkts_flush; total_bytes_flush += num_bytes_flush; + rmnet_shs_cpu_node_tbl[n->map_cpu].parkedlen -= num_pkts_flush; + if (n->map_cpu == cpu_num) { cpu_tail += num_pkts_flush; n->queue_head = cpu_tail; + } } } } - } + if (rmnet_shs_cpu_node_tbl[cpu_num].parkedlen < 0) + rmnet_shs_crit_err[RMNET_SHS_CPU_PKTLEN_ERR]++; + + if (rmnet_shs_get_cpu_qdiff(cpu_num) >= + rmnet_shs_cpu_max_qdiff[cpu_num]) + rmnet_shs_cpu_max_qdiff[cpu_num] = + rmnet_shs_get_cpu_qdiff(cpu_num); + } spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); rmnet_shs_cfg.num_bytes_parked -= total_bytes_flush; rmnet_shs_cfg.num_pkts_parked -= total_pkts_flush; - trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_END, rmnet_shs_cfg.num_pkts_parked, rmnet_shs_cfg.num_bytes_parked, @@ -600,10 +839,10 @@ void rmnet_shs_chain_to_skb_list(struct sk_buff *skb, node->skb_list.num_parked_bytes += skb->len; rmnet_shs_cfg.num_bytes_parked += skb->len; + rmnet_shs_cpu_node_tbl[node->map_cpu].parkedlen++; node->skb_list.num_parked_skbs += 1; rmnet_shs_cfg.num_pkts_parked += 1; - trace_rmnet_shs_high(RMNET_SHS_ASSIGN, RMNET_SHS_ASSIGN_PARK_PKT_COMPLETE, node->skb_list.num_parked_skbs, @@ -612,13 +851,12 @@ void rmnet_shs_chain_to_skb_list(struct sk_buff *skb, rmnet_shs_cfg.num_bytes_parked, skb, node); } - /* Invoked when all the packets that are parked to be flushed through * the workqueue. */ static void rmnet_flush_buffered(struct work_struct *work) { - u8 is_force_flush = 1; + u8 is_force_flush = 0; trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_DELAY_WQ_START, is_force_flush, @@ -626,16 +864,14 @@ static void rmnet_flush_buffered(struct work_struct *work) 0xDEF, NULL, NULL); if (rmnet_shs_cfg.is_pkt_parked && - rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_ON) { + rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_ON) { + rmnet_shs_flush_table(is_force_flush); rmnet_shs_flush_reason[RMNET_SHS_FLUSH_TIMER_EXPIRY]++; } trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_DELAY_WQ_END, is_force_flush, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); - - - } /* Invoked when the flushing timer has expired. * Upon first expiry, we set the flag that will trigger force flushing of all @@ -652,9 +888,8 @@ enum hrtimer_restart rmnet_shs_map_flush_queue(struct hrtimer *t) RMNET_SHS_FLUSH_PARK_TMR_EXPIRY, rmnet_shs_cfg.force_flush_state, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); - if (rmnet_shs_cfg.num_pkts_parked > 0) { - if (rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_OFF) { + if (rmnet_shs_cfg.force_flush_state != RMNET_SHS_FLUSH_ON) { rmnet_shs_cfg.force_flush_state = RMNET_SHS_FLUSH_ON; hrtimer_forward(t, hrtimer_cb_get_time(t), ns_to_ktime(2000000)); @@ -664,6 +899,10 @@ enum hrtimer_restart rmnet_shs_map_flush_queue(struct hrtimer *t) RMNET_SHS_FLUSH_PARK_TMR_RESTART, rmnet_shs_cfg.num_pkts_parked, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); + } else if (rmnet_shs_cfg.force_flush_state == + RMNET_SHS_FLUSH_DONE) { + rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_OFF; + } else { trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_DELAY_WQ_TRIGGER, @@ -675,27 +914,44 @@ enum hrtimer_restart rmnet_shs_map_flush_queue(struct hrtimer *t) return ret; } -/* Initializes the flushing timer. The timer is initialized upon receiving - * first NET UP event - */ +enum hrtimer_restart rmnet_shs_queue_core(struct hrtimer *t) +{ + const enum hrtimer_restart ret = HRTIMER_NORESTART; + struct core_flush_s *core_work = container_of(t, + struct core_flush_s, core_timer); + + schedule_work(&core_work->work); + return ret; +} + void rmnet_shs_aggregate_init(void) { + int i; + + for (i = 0; i < MAX_CPUS; i++) { + rmnet_shs_cfg.core_flush[i].core = i; + INIT_WORK(&rmnet_shs_cfg.core_flush[i].work, + rmnet_shs_flush_core_work); - hrtimer_init(&rmnet_shs_cfg.hrtimer_shs, - CLOCK_MONOTONIC, HRTIMER_MODE_REL); - rmnet_shs_cfg.hrtimer_shs.function = rmnet_shs_map_flush_queue; + hrtimer_init(&rmnet_shs_cfg.core_flush[i].core_timer, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + rmnet_shs_cfg.core_flush[i].core_timer.function = + rmnet_shs_queue_core; + } + hrtimer_init(&rmnet_shs_cfg.hrtimer_shs, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + rmnet_shs_cfg.hrtimer_shs.function = rmnet_shs_map_flush_queue; + INIT_WORK(&shs_delayed_work.work, rmnet_flush_buffered); } void rmnet_shs_ps_on_hdlr(void *port) { - rmnet_shs_flush_table(1); rmnet_shs_wq_pause(); } void rmnet_shs_ps_off_hdlr(void *port) { rmnet_shs_wq_restart(); - } void rmnet_shs_dl_hdr_handler(struct rmnet_map_dl_ind_hdr *dlhdr) @@ -732,7 +988,6 @@ void rmnet_shs_init(struct net_device *dev) return; rmnet_shs_cfg.port = rmnet_get_port(dev); - spin_lock_init(&rmnet_shs_ht_splock); for (num_cpu = 0; num_cpu < MAX_CPUS; num_cpu++) INIT_LIST_HEAD(&rmnet_shs_cpu_node_tbl[num_cpu].node_list_id); @@ -814,12 +1069,17 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) struct rmnet_shs_cpu_node_s *cpu_node_tbl_p; /*deliver non TCP/UDP packets right away*/ + if (!rmnet_shs_is_skb_stamping_reqd(skb)) { + rmnet_shs_deliver_skb(skb); + return; + } - if (unlikely(!map)) { + if ((unlikely(!map))|| !rmnet_shs_init_complete) { + rmnet_shs_deliver_skb(skb); trace_rmnet_shs_err(RMNET_SHS_ASSIGN, - RMNET_SHS_ASSIGN_CRIT_ERROR_NO_MSK_SET, + RMNET_SHS_ASSIGN_CRIT_ERROR_NO_SHS_REQD, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); - rmnet_shs_crit_err[RMNET_SHS_MAIN_MAP_LEN_INVALID]++; + rmnet_shs_crit_err[RMNET_SHS_MAIN_SHS_NOT_REQD]++; return; } @@ -845,7 +1105,7 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) map_cpu = rmnet_shs_new_flow_cpu(brate, dev); node_p->map_cpu = map_cpu; node_p->map_index = - rmnet_shs_map_idx_from_cpu(map_cpu, map); + rmnet_shs_map_idx_from_cpu(map_cpu, map); trace_rmnet_shs_err(RMNET_SHS_ASSIGN, RMNET_SHS_ASSIGN_MASK_CHNG, @@ -876,8 +1136,10 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) node_p = kzalloc(sizeof(*node_p), 0); - if (!node_p) + if (!node_p) { + rmnet_shs_crit_err[RMNET_SHS_MAIN_MALLOC_ERR]++; break; + } node_p->dev = skb->dev; node_p->hash = skb->hash; @@ -899,12 +1161,12 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) cpu_node_tbl_p = &rmnet_shs_cpu_node_tbl[map_cpu]; rmnet_shs_cpu_node_add(node_p, &cpu_node_tbl_p->node_list_id); - hash_add_rcu(RMNET_SHS_HT, &node_p->list, skb->hash); /* Chain this pkt to skb list (most likely to skb_list.head) * because this is the first packet for this flow */ rmnet_shs_chain_to_skb_list(skb, node_p); + is_shs_reqd = 1; break; @@ -926,14 +1188,15 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) &rmnet_shs_cfg.rmnet_idl_ind_cb); rmnet_shs_cfg.is_reg_dl_mrk_ind = 1; - INIT_WORK(&shs_delayed_work.work, rmnet_flush_buffered); shs_delayed_work.port = port; + } /* We got the first packet after a previous successdul flush. Arm the * flushing timer. */ if (!rmnet_shs_cfg.is_pkt_parked) { rmnet_shs_cfg.is_pkt_parked = 1; + rmnet_shs_cfg.force_flush_state = RMNET_SHS_FLUSH_OFF; if (hrtimer_active(&rmnet_shs_cfg.hrtimer_shs)) { trace_rmnet_shs_low(RMNET_SHS_ASSIGN, RMNET_SHS_ASSIGN_PARK_TMR_CANCEL, @@ -949,26 +1212,8 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) 0xDEF, 0xDEF, 0xDEF, skb, NULL); } - /* Flushing timer that was armed previously has successfully fired. - * Now we trigger force flushing of all packets. If a flow is waiting - * to switch to another core, it will be forcefully moved during this - * trigger. - * - * In case the previously delivered packets haven't been processed by - * the next layers, the parked packets may be delivered out of order - * until all the previously delivered packets have been processed - * successully - */ - if (rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_ON) { - rmnet_shs_flush_reason[RMNET_SHS_FLUSH_TIMER_EXPIRY]++; - trace_rmnet_shs_high(RMNET_SHS_FLUSH, - RMNET_SHS_FLUSH_FORCE_TRIGGER, 1, - rmnet_shs_cfg.num_pkts_parked, - 0xDEF, 0xDEF, NULL, NULL); - rmnet_shs_flush_table(1); - - } else if (rmnet_shs_cfg.num_pkts_parked > - RMNET_SHS_MAX_PKTS_TO_PARK) { + if (rmnet_shs_cfg.num_pkts_parked > + rmnet_shs_pkts_store_limit) { if (rmnet_shs_stats_enabled) rmnet_shs_flush_reason[RMNET_SHS_FLUSH_PKT_LIMIT]++; @@ -976,10 +1221,10 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_PKT_LIMIT_TRIGGER, 0, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); - rmnet_shs_flush_table(0); + rmnet_shs_flush_table(1); } else if (rmnet_shs_cfg.num_bytes_parked > - RMNET_SHS_MAX_BYTES_TO_PARK) { + rmnet_shs_byte_store_limit) { if (rmnet_shs_stats_enabled) rmnet_shs_flush_reason[RMNET_SHS_FLUSH_BYTE_LIMIT]++; @@ -987,7 +1232,27 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_BYTE_LIMIT_TRIGGER, 0, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); + rmnet_shs_flush_table(1); + + } + /* Flushing timer that was armed previously has successfully fired. + * Now we trigger force flushing of all packets. If a flow is waiting + * to switch to another core, it will be forcefully moved during this + * trigger. + * + * In case the previously delivered packets haven't been processed by + * the next layers, the parked packets may be delivered out of order + * until all the previously delivered packets have been processed + * successully + */ + else if (rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_ON) { + rmnet_shs_flush_reason[RMNET_SHS_FLUSH_TIMER_EXPIRY]++; + trace_rmnet_shs_high(RMNET_SHS_FLUSH, + RMNET_SHS_FLUSH_FORCE_TRIGGER, 1, + rmnet_shs_cfg.num_pkts_parked, + 0xDEF, 0xDEF, NULL, NULL); rmnet_shs_flush_table(0); + } } diff --git a/drivers/rmnet/shs/rmnet_shs_wq.c b/drivers/rmnet/shs/rmnet_shs_wq.c index d4d9d2f..42f316c 100644 --- a/drivers/rmnet/shs/rmnet_shs_wq.c +++ b/drivers/rmnet/shs/rmnet_shs_wq.c @@ -25,12 +25,21 @@ MODULE_LICENSE("GPL v2"); #define RMNET_SHS_BYTE_TO_BIT(x) ((x)*8) #define RMNET_SHS_MIN_HSTAT_NODES_REQD 16 #define RMNET_SHS_WQ_DELAY_TICKS 10 + +#define PERIODIC_CLEAN 0 +/* FORCE_CLEAN should only used during module de-ini.*/ +#define FORCE_CLEAN 1 /* Time to wait (in time ticks) before re-triggering the workqueue * 1 tick = 10 ms (Maximum possible resolution) * 100 ticks = 1 second */ /* Local Definitions and Declarations */ +unsigned int rmnet_shs_cpu_prio_dur __read_mostly = RMNET_SHS_WQ_DELAY_TICKS; +module_param(rmnet_shs_cpu_prio_dur, uint, 0644); +MODULE_PARM_DESC(rmnet_shs_cpu_prio_dur, "Priority ignore duration(ticks)"); + +#define PRIO_BACKOFF ((!rmnet_shs_cpu_prio_dur) ? 2 : rmnet_shs_cpu_prio_dur) unsigned int rmnet_shs_wq_frequency __read_mostly = RMNET_SHS_WQ_DELAY_TICKS; module_param(rmnet_shs_wq_frequency, uint, 0644); @@ -141,7 +150,6 @@ static struct workqueue_struct *rmnet_shs_wq; static struct rmnet_shs_delay_wq_s *rmnet_shs_delayed_wq; static struct rmnet_shs_wq_rx_flow_s rmnet_shs_rx_flow_tbl; - static struct list_head rmnet_shs_wq_hstat_tbl = LIST_HEAD_INIT(rmnet_shs_wq_hstat_tbl); static int rmnet_shs_flow_dbg_stats_idx_cnt; @@ -153,21 +161,23 @@ static struct list_head rmnet_shs_wq_ep_tbl = */ void rmnet_shs_wq_ep_tbl_add(struct rmnet_shs_wq_ep_s *ep) { + unsigned long flags; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_EP_TBL, RMNET_SHS_WQ_EP_TBL_ADD, 0xDEF, 0xDEF, 0xDEF, 0xDEF, ep, NULL); - spin_lock(&rmnet_shs_hstat_tbl_lock); + spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_add(&ep->ep_list_id, &rmnet_shs_wq_ep_tbl); - spin_unlock(&rmnet_shs_hstat_tbl_lock); + spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); } void rmnet_shs_wq_ep_tbl_remove(struct rmnet_shs_wq_ep_s *ep) { + unsigned long flags; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_EP_TBL, RMNET_SHS_WQ_EP_TBL_DEL, 0xDEF, 0xDEF, 0xDEF, 0xDEF, ep, NULL); - spin_lock(&rmnet_shs_hstat_tbl_lock); + spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_del_init(&ep->ep_list_id); - spin_unlock(&rmnet_shs_hstat_tbl_lock); + spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); } @@ -176,23 +186,27 @@ void rmnet_shs_wq_ep_tbl_remove(struct rmnet_shs_wq_ep_s *ep) */ void rmnet_shs_wq_hstat_tbl_add(struct rmnet_shs_wq_hstat_s *hnode) { + unsigned long flags; + trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL, RMNET_SHS_WQ_HSTAT_TBL_ADD, 0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL); - spin_lock(&rmnet_shs_hstat_tbl_lock); + spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_add(&hnode->hstat_node_id, &rmnet_shs_wq_hstat_tbl); - spin_unlock(&rmnet_shs_hstat_tbl_lock); + spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); } void rmnet_shs_wq_hstat_tbl_remove(struct rmnet_shs_wq_hstat_s *hnode) { + unsigned long flags; + trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL, RMNET_SHS_WQ_HSTAT_TBL_DEL, 0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL); - spin_lock(&rmnet_shs_hstat_tbl_lock); + spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_del_init(&hnode->hstat_node_id); - spin_unlock(&rmnet_shs_hstat_tbl_lock); + spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); } @@ -202,37 +216,43 @@ void rmnet_shs_wq_hstat_tbl_remove(struct rmnet_shs_wq_hstat_s *hnode) */ void rmnet_shs_wq_cpu_list_remove(struct rmnet_shs_wq_hstat_s *hnode) { + unsigned long flags; + trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_HSTAT_TBL, RMNET_SHS_WQ_CPU_HSTAT_TBL_DEL, 0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL); - spin_lock(&rmnet_shs_hstat_tbl_lock); + spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_del_init(&hnode->cpu_node_id); - spin_unlock(&rmnet_shs_hstat_tbl_lock); + spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); } void rmnet_shs_wq_cpu_list_add(struct rmnet_shs_wq_hstat_s *hnode, struct list_head *head) { + unsigned long flags; + trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_HSTAT_TBL, RMNET_SHS_WQ_CPU_HSTAT_TBL_ADD, 0xDEF, 0xDEF, 0xDEF, 0xDEF, hnode, NULL); - spin_lock(&rmnet_shs_hstat_tbl_lock); + spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_add(&hnode->cpu_node_id, head); - spin_unlock(&rmnet_shs_hstat_tbl_lock); + spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); } void rmnet_shs_wq_cpu_list_move(struct rmnet_shs_wq_hstat_s *hnode, struct list_head *head) { + unsigned long flags; + trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_HSTAT_TBL, RMNET_SHS_WQ_CPU_HSTAT_TBL_MOVE, hnode->current_cpu, 0xDEF, 0xDEF, 0xDEF, hnode, NULL); - spin_lock(&rmnet_shs_hstat_tbl_lock); + spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_move(&hnode->cpu_node_id, head); - spin_unlock(&rmnet_shs_hstat_tbl_lock); + spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); } @@ -300,8 +320,9 @@ struct rmnet_shs_wq_hstat_s *rmnet_shs_wq_get_new_hstat_node(void) { struct rmnet_shs_wq_hstat_s *hnode; struct rmnet_shs_wq_hstat_s *ret_node = NULL; + unsigned long flags; - spin_lock(&rmnet_shs_hstat_tbl_lock); + spin_lock_irqsave(&rmnet_shs_hstat_tbl_lock, flags); list_for_each_entry(hnode, &rmnet_shs_wq_hstat_tbl, hstat_node_id) { if (hnode == NULL) continue; @@ -313,7 +334,7 @@ struct rmnet_shs_wq_hstat_s *rmnet_shs_wq_get_new_hstat_node(void) break; } } - spin_unlock(&rmnet_shs_hstat_tbl_lock); + spin_unlock_irqrestore(&rmnet_shs_hstat_tbl_lock, flags); if (ret_node) { trace_rmnet_shs_wq_low(RMNET_SHS_WQ_HSTAT_TBL, @@ -334,7 +355,6 @@ struct rmnet_shs_wq_hstat_s *rmnet_shs_wq_get_new_hstat_node(void) return NULL; } - rmnet_shs_wq_hstat_reset_node(ret_node); ret_node->is_perm = 0; ret_node->in_use = 1; @@ -477,7 +497,6 @@ void rmnet_shs_wq_update_hash_stats(struct rmnet_shs_wq_hstat_s *hstats_p) hstats_p->hash, 0xDEF, hstats_p->rx_pps, hstats_p->rx_bps, hstats_p, NULL); - rmnet_shs_wq_update_hstat_rps_msk(hstats_p); hstats_p->inactive_duration = 0; hstats_p->l_epoch = node_p->hstats->c_epoch; @@ -581,6 +600,9 @@ static void rmnet_shs_wq_refresh_cpu_stats(u16 cpu) cpu_p = &rmnet_shs_rx_flow_tbl.cpu_list[cpu]; new_skbs = cpu_p->rx_skbs - cpu_p->last_rx_skbs; + if (rmnet_shs_cpu_node_tbl[cpu].wqprio) + rmnet_shs_cpu_node_tbl[cpu].wqprio = (rmnet_shs_cpu_node_tbl[cpu].wqprio + 1) + % (PRIO_BACKOFF); if (new_skbs == 0) { cpu_p->l_epoch = rmnet_shs_wq_tnsec; cpu_p->rx_bps = 0; @@ -640,10 +662,13 @@ void rmnet_shs_wq_update_cpu_rx_tbl(struct rmnet_shs_wq_hstat_s *hstat_p) return; map = rcu_dereference(node_p->dev->_rx->rps_map); + + if (!map) + return; + map_idx = node_p->map_index; cpu_num = map->cpus[map_idx]; - skb_diff = hstat_p->rx_skb - hstat_p->last_rx_skb; byte_diff = hstat_p->rx_bytes - hstat_p->last_rx_bytes; @@ -818,6 +843,13 @@ u16 rmnet_shs_wq_find_cpu_to_move_flows(u16 current_cpu, (cur_cpu_rx_pps > pps_uthresh)) { return cpu_to_move; } + /* If a core (should only be lpwr was marked prio we don't touch it + * for a few ticks and reset it afterwards + */ + + if (rmnet_shs_cpu_node_tbl[current_cpu].wqprio) { + return current_cpu; + } for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) { @@ -826,7 +858,8 @@ u16 rmnet_shs_wq_find_cpu_to_move_flows(u16 current_cpu, /* We are looking for a core that is configured and that * can handle traffic better than the current core */ - if ((cpu_num == current_cpu) || (!is_core_in_msk)) + if ((cpu_num == current_cpu) || (!is_core_in_msk) || + !cpu_online(current_cpu)) continue; pps_uthresh = rmnet_shs_cpu_rx_max_pps_thresh[cpu_num]; @@ -837,9 +870,9 @@ u16 rmnet_shs_wq_find_cpu_to_move_flows(u16 current_cpu, reqd_pps = cpu_rx_pps + cur_cpu_rx_pps; trace_rmnet_shs_wq_low(RMNET_SHS_WQ_CPU_STATS, - RMNET_SHS_WQ_CPU_STATS_CORE2SWITCH_FIND, - current_cpu, cpu_num, reqd_pps, - cpu_rx_pps, NULL, NULL); + RMNET_SHS_WQ_CPU_STATS_CORE2SWITCH_FIND, + current_cpu, cpu_num, reqd_pps, + cpu_rx_pps, NULL, NULL); /* Return the first available CPU */ if ((reqd_pps > pps_lthresh) && (reqd_pps < pps_uthresh)) { @@ -1119,7 +1152,7 @@ int rmnet_shs_wq_get_perf_cpu_new_flow(struct net_device *dev) return cpu_assigned; } -void rmnet_shs_wq_cleanup_hash_tbl(void) +void rmnet_shs_wq_cleanup_hash_tbl(u8 force_clean) { struct rmnet_shs_skbn_s *node_p; time_t tns2s; @@ -1133,9 +1166,12 @@ void rmnet_shs_wq_cleanup_hash_tbl(void) if (hnode == NULL) continue; + if (hnode->node == NULL) + continue; + node_p = hnode->node; tns2s = RMNET_SHS_NSEC_TO_SEC(hnode->inactive_duration); - if (tns2s > rmnet_shs_max_flow_inactivity_sec) { + if (tns2s > rmnet_shs_max_flow_inactivity_sec || force_clean) { trace_rmnet_shs_wq_low(RMNET_SHS_WQ_FLOW_STATS, RMNET_SHS_WQ_FLOW_STATS_FLOW_INACTIVE_TIMEOUT, @@ -1149,11 +1185,12 @@ void rmnet_shs_wq_cleanup_hash_tbl(void) kfree(node_p); } rmnet_shs_wq_cpu_list_remove(hnode); - if (hnode->is_perm == 0) { + if (hnode->is_perm == 0 || force_clean) { rmnet_shs_wq_hstat_tbl_remove(hnode); kfree(hnode); - } else + } else { rmnet_shs_wq_hstat_reset_node(hnode); + } spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); } } @@ -1250,7 +1287,7 @@ static void rmnet_shs_wq_update_stats(void) rmnet_shs_wq_eval_suggested_cpu(); rmnet_shs_wq_refresh_new_flow_list(); /*Invoke after both the locks are released*/ - rmnet_shs_wq_cleanup_hash_tbl(); + rmnet_shs_wq_cleanup_hash_tbl(PERIODIC_CLEAN); } void rmnet_shs_wq_process_wq(struct work_struct *work) @@ -1289,7 +1326,6 @@ void rmnet_shs_wq_clean_ep_tbl(void) void rmnet_shs_wq_exit(void) { - /*If Wq is not initialized, nothing to cleanup */ if (!rmnet_shs_wq || !rmnet_shs_delayed_wq) return; @@ -1304,8 +1340,8 @@ void rmnet_shs_wq_exit(void) rmnet_shs_delayed_wq = NULL; rmnet_shs_wq = NULL; + rmnet_shs_wq_cleanup_hash_tbl(FORCE_CLEAN); rmnet_shs_wq_clean_ep_tbl(); - trace_rmnet_shs_wq_high(RMNET_SHS_WQ_EXIT, RMNET_SHS_WQ_EXIT_END, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); } diff --git a/drivers/rmnet/shs/rmnet_shs_wq.h b/drivers/rmnet/shs/rmnet_shs_wq.h index e16b0a9..96843f5 100644 --- a/drivers/rmnet/shs/rmnet_shs_wq.h +++ b/drivers/rmnet/shs/rmnet_shs_wq.h @@ -17,10 +17,17 @@ #define _RMNET_SHS_WQ_H_ #include "rmnet_shs_config.h" +#include "rmnet_shs.h" -#define MAX_CPUS 8 #define MAX_SUPPORTED_FLOWS_DEBUG 16 +#define RMNET_SHS_RX_BPNSEC_TO_BPSEC(x) ((x)*1000000000) +#define RMNET_SHS_SEC_TO_NSEC(x) ((x)*1000000000) +#define RMNET_SHS_NSEC_TO_SEC(x) ((x)/1000000000) +#define RMNET_SHS_BYTE_TO_BIT(x) ((x)*8) +#define RMNET_SHS_MIN_HSTAT_NODES_REQD 16 +#define RMNET_SHS_WQ_DELAY_TICKS 10 + /* stores wq and end point details */ struct rmnet_shs_wq_ep_s { @@ -193,6 +200,9 @@ enum rmnet_shs_wq_trace_evt { }; + +extern struct rmnet_shs_cpu_node_s rmnet_shs_cpu_node_tbl[MAX_CPUS]; + void rmnet_shs_wq_init(struct net_device *dev); void rmnet_shs_wq_exit(void); void rmnet_shs_wq_restart(void); |