diff options
-rw-r--r-- | drivers/rmnet/shs/rmnet_shs.h | 5 | ||||
-rwxr-xr-x | drivers/rmnet/shs/rmnet_shs_main.c | 162 | ||||
-rw-r--r-- | drivers/rmnet/shs/rmnet_shs_wq.c | 18 |
3 files changed, 126 insertions, 59 deletions
diff --git a/drivers/rmnet/shs/rmnet_shs.h b/drivers/rmnet/shs/rmnet_shs.h index b7bf773..99ca7e4 100644 --- a/drivers/rmnet/shs/rmnet_shs.h +++ b/drivers/rmnet/shs/rmnet_shs.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2020 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -156,6 +156,8 @@ enum rmnet_shs_switch_reason_e { RMNET_SHS_SWITCH_WQ_RATE, RMNET_SHS_OOO_PACKET_SWITCH, RMNET_SHS_OOO_PACKET_TOTAL, + RMNET_SHS_SWITCH_PACKET_BURST, + RMNET_SHS_SWITCH_CORE_BACKLOG, RMNET_SHS_SWITCH_MAX_REASON }; @@ -195,6 +197,7 @@ struct rmnet_shs_cpu_node_s { u32 qtail; u32 qdiff; u32 parkedlen; + u32 seg; u8 prio; u8 wqprio; }; diff --git a/drivers/rmnet/shs/rmnet_shs_main.c b/drivers/rmnet/shs/rmnet_shs_main.c index 2accd29..60ba7ed 100755 --- a/drivers/rmnet/shs/rmnet_shs_main.c +++ b/drivers/rmnet/shs/rmnet_shs_main.c @@ -30,6 +30,8 @@ #define NS_IN_MS 1000000 #define LPWR_CLUSTER 0 #define PERF_CLUSTER 4 +#define DEF_CORE_WAIT 10 + #define PERF_CORES 4 #define INVALID_CPU -1 @@ -65,15 +67,15 @@ unsigned long rmnet_shs_flush_reason[RMNET_SHS_FLUSH_MAX_REASON]; module_param_array(rmnet_shs_flush_reason, ulong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flush_reason, "rmnet shs skb flush trigger type"); -unsigned int rmnet_shs_byte_store_limit __read_mostly = 271800 * 8; +unsigned int rmnet_shs_byte_store_limit __read_mostly = 271800 * 80; module_param(rmnet_shs_byte_store_limit, uint, 0644); MODULE_PARM_DESC(rmnet_shs_byte_store_limit, "Maximum byte module will park"); -unsigned int rmnet_shs_pkts_store_limit __read_mostly = 2100; +unsigned int rmnet_shs_pkts_store_limit __read_mostly = 2100 * 8; module_param(rmnet_shs_pkts_store_limit, uint, 0644); MODULE_PARM_DESC(rmnet_shs_pkts_store_limit, "Maximum pkts module will park"); -unsigned int rmnet_shs_max_core_wait __read_mostly = 10; +unsigned int rmnet_shs_max_core_wait __read_mostly = 45; module_param(rmnet_shs_max_core_wait, uint, 0644); MODULE_PARM_DESC(rmnet_shs_max_core_wait, "Max wait module will wait during move to perf core in ms"); @@ -93,6 +95,11 @@ module_param(rmnet_shs_fall_back_timer, uint, 0644); MODULE_PARM_DESC(rmnet_shs_fall_back_timer, "Option to enable fall back limit for parking"); +unsigned int rmnet_shs_backlog_max_pkts __read_mostly = 1200; +module_param(rmnet_shs_backlog_max_pkts, uint, 0644); +MODULE_PARM_DESC(rmnet_shs_backlog_max_pkts, + "Max pkts in backlog prioritizing"); + unsigned int rmnet_shs_inst_rate_max_pkts __read_mostly = 2500; module_param(rmnet_shs_inst_rate_max_pkts, uint, 0644); MODULE_PARM_DESC(rmnet_shs_inst_rate_max_pkts, @@ -110,6 +117,10 @@ unsigned int rmnet_shs_cpu_max_qdiff[MAX_CPUS]; module_param_array(rmnet_shs_cpu_max_qdiff, uint, 0, 0644); MODULE_PARM_DESC(rmnet_shs_cpu_max_qdiff, "Max queue length seen of each core"); +unsigned int rmnet_shs_cpu_ooo_count[MAX_CPUS]; +module_param_array(rmnet_shs_cpu_ooo_count, uint, 0, 0644); +MODULE_PARM_DESC(rmnet_shs_cpu_ooo_count, "OOO count for each cpu"); + unsigned int rmnet_shs_cpu_max_coresum[MAX_CPUS]; module_param_array(rmnet_shs_cpu_max_coresum, uint, 0, 0644); MODULE_PARM_DESC(rmnet_shs_cpu_max_coresum, "Max coresum seen of each core"); @@ -145,6 +156,13 @@ void rmnet_shs_cpu_node_move(struct rmnet_shs_skbn_s *node, cpu_num_flows[oldcpu]--; } +static void rmnet_shs_cpu_ooo(u8 cpu, int count) +{ + if (cpu < MAX_CPUS) + { + rmnet_shs_cpu_ooo_count[cpu]+=count; + } +} /* Evaluates the incoming transport protocol of the incoming skb. Determines * if the skb transport protocol will be supported by SHS module */ @@ -282,14 +300,6 @@ static void rmnet_shs_update_core_load(int cpu, int burst) } -static int rmnet_shs_is_core_loaded(int cpu) -{ - - return rmnet_shs_cfg.core_flush[cpu].coresum >= - rmnet_shs_inst_rate_max_pkts; - -} - /* We deliver packets to GRO module only for TCP traffic*/ static int rmnet_shs_check_skb_can_gro(struct sk_buff *skb) { @@ -423,18 +433,9 @@ int rmnet_shs_flow_num_perf_cores(struct rmnet_shs_skbn_s *node_p) return ret; } -int rmnet_shs_is_lpwr_cpu(u16 cpu) +inline int rmnet_shs_is_lpwr_cpu(u16 cpu) { - int ret = 1; - u32 big_cluster_mask = (1 << PERF_CLUSTER) - 1; - - if ((1 << cpu) >= big_cluster_mask) - ret = 0; - - SHS_TRACE_LOW(RMNET_SHS_CORE_CFG, - RMNET_SHS_CORE_CFG_CHK_LO_CPU, - ret, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); - return ret; + return !((1 << cpu) & PERF_MASK); } /* Forms a new hash from the incoming hash based on the number of cores @@ -665,6 +666,22 @@ u32 rmnet_shs_get_cpu_qdiff(u8 cpu_num) return ret; } + +static int rmnet_shs_is_core_loaded(int cpu, int backlog_check, int parked_pkts) +{ + int ret = 0; + + if (rmnet_shs_cfg.core_flush[cpu].coresum >= + rmnet_shs_inst_rate_max_pkts) { + ret = RMNET_SHS_SWITCH_PACKET_BURST; + } + if (backlog_check && ((rmnet_shs_get_cpu_qdiff(cpu) + parked_pkts) >= + rmnet_shs_backlog_max_pkts)) + ret = RMNET_SHS_SWITCH_CORE_BACKLOG; + + return ret; +} + /* Takes a snapshot of absolute value of the CPU Qhead and Qtail counts for * a given core. * @@ -784,6 +801,7 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush) rmnet_shs_switch_reason[RMNET_SHS_OOO_PACKET_TOTAL] += (node_qhead - cur_cpu_qhead); + rmnet_shs_cpu_ooo(cpu_num, node_qhead - cur_cpu_qhead); } /* Mark gold core as prio to prevent * flows from moving in wq @@ -866,6 +884,8 @@ void rmnet_shs_flush_core(u8 cpu_num) rmnet_shs_cfg.num_bytes_parked -= total_bytes_flush; rmnet_shs_cfg.num_pkts_parked -= total_pkts_flush; rmnet_shs_cpu_node_tbl[cpu_num].prio = 0; + /* Reset coresum in case of instant rate switch */ + rmnet_shs_cfg.core_flush[cpu_num].coresum = 0; rmnet_shs_cpu_node_tbl[cpu_num].parkedlen = 0; spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); local_bh_enable(); @@ -1043,6 +1063,35 @@ int rmnet_shs_chk_and_flush_node(struct rmnet_shs_skbn_s *node, node, NULL); return ret_val; } + +/* Check if cpu_num should be marked as a priority core and take care of + * marking it as priority and configuring all the changes need for a core + * switch. + */ +static void rmnet_shs_core_prio_check(u8 cpu_num, u8 segmented, u32 parked_pkts) +{ + u32 wait = (!rmnet_shs_max_core_wait) ? 1 : rmnet_shs_max_core_wait; + int load_reason; + + if ((load_reason = rmnet_shs_is_core_loaded(cpu_num, segmented, parked_pkts)) && + rmnet_shs_is_lpwr_cpu(cpu_num) && + !rmnet_shs_cpu_node_tbl[cpu_num].prio) { + + + wait = (!segmented)? DEF_CORE_WAIT: wait; + rmnet_shs_cpu_node_tbl[cpu_num].prio = 1; + rmnet_shs_boost_cpus(); + if (hrtimer_active(&GET_CTIMER(cpu_num))) + hrtimer_cancel(&GET_CTIMER(cpu_num)); + + hrtimer_start(&GET_CTIMER(cpu_num), + ns_to_ktime(wait * NS_IN_MS), + HRTIMER_MODE_REL); + + rmnet_shs_switch_reason[load_reason]++; + } +} + /* Flushes all the packets that have been parked so far across all the flows * The order of flushing depends on the CPU<=>flow association * The flows associated with low power cores are flushed before flushing @@ -1063,13 +1112,13 @@ void rmnet_shs_flush_lock_table(u8 flsh, u8 ctxt) u32 cpu_tail; u32 num_pkts_flush = 0; u32 num_bytes_flush = 0; + u32 skb_seg_pending = 0; u32 total_pkts_flush = 0; u32 total_bytes_flush = 0; u32 total_cpu_gro_flushed = 0; u32 total_node_gro_flushed = 0; - u8 is_flushed = 0; - u32 wait = (!rmnet_shs_max_core_wait) ? 1 : rmnet_shs_max_core_wait; + u8 cpu_segment = 0; /* Record a qtail + pkts flushed or move if reqd * currently only use qtail for non TCP flows @@ -1083,10 +1132,22 @@ void rmnet_shs_flush_lock_table(u8 flsh, u8 ctxt) for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) { cpu_tail = rmnet_shs_get_cpu_qtail(cpu_num); - + cpu_segment = 0; total_cpu_gro_flushed = 0; + skb_seg_pending = 0; list_for_each_safe(ptr, next, - &rmnet_shs_cpu_node_tbl[cpu_num].node_list_id) { + &rmnet_shs_cpu_node_tbl[cpu_num].node_list_id) { + n = list_entry(ptr, struct rmnet_shs_skbn_s, node_id); + skb_seg_pending += n->skb_list.skb_load; + } + if (rmnet_shs_inst_rate_switch) { + cpu_segment = rmnet_shs_cpu_node_tbl[cpu_num].seg; + rmnet_shs_core_prio_check(cpu_num, cpu_segment, + skb_seg_pending); + } + + list_for_each_safe(ptr, next, + &rmnet_shs_cpu_node_tbl[cpu_num].node_list_id) { n = list_entry(ptr, struct rmnet_shs_skbn_s, node_id); if (n != NULL && n->skb_list.num_parked_skbs) { @@ -1111,31 +1172,21 @@ void rmnet_shs_flush_lock_table(u8 flsh, u8 ctxt) } } } + } /* If core is loaded set core flows as priority and * start a 10ms hard flush timer */ if (rmnet_shs_inst_rate_switch) { + /* Update cpu load with prev flush for check */ if (rmnet_shs_is_lpwr_cpu(cpu_num) && !rmnet_shs_cpu_node_tbl[cpu_num].prio) rmnet_shs_update_core_load(cpu_num, total_cpu_gro_flushed); - if (rmnet_shs_is_core_loaded(cpu_num) && - rmnet_shs_is_lpwr_cpu(cpu_num) && - !rmnet_shs_cpu_node_tbl[cpu_num].prio) { - - rmnet_shs_cpu_node_tbl[cpu_num].prio = 1; - rmnet_shs_boost_cpus(); - if (hrtimer_active(&GET_CTIMER(cpu_num))) - hrtimer_cancel(&GET_CTIMER(cpu_num)); - - hrtimer_start(&GET_CTIMER(cpu_num), - ns_to_ktime(wait * NS_IN_MS), - HRTIMER_MODE_REL); + rmnet_shs_core_prio_check(cpu_num, cpu_segment, 0); - } } if (rmnet_shs_cpu_node_tbl[cpu_num].parkedlen < 0) @@ -1178,6 +1229,21 @@ void rmnet_shs_flush_table(u8 flsh, u8 ctxt) spin_lock_irqsave(&rmnet_shs_ht_splock, ht_flags); rmnet_shs_flush_lock_table(flsh, ctxt); + if (ctxt == RMNET_WQ_CTXT) { + /* If packets remain restart the timer in case there are no + * more NET_RX flushes coming so pkts are no lost + */ + if (rmnet_shs_fall_back_timer && + rmnet_shs_cfg.num_bytes_parked && + rmnet_shs_cfg.num_pkts_parked){ + if (hrtimer_active(&rmnet_shs_cfg.hrtimer_shs)) + hrtimer_cancel(&rmnet_shs_cfg.hrtimer_shs); + hrtimer_start(&rmnet_shs_cfg.hrtimer_shs, + ns_to_ktime(rmnet_shs_timeout * NS_IN_MS), + HRTIMER_MODE_REL); + } + rmnet_shs_flush_reason[RMNET_SHS_FLUSH_WQ_FB_FLUSH]++; + } spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); @@ -1262,21 +1328,6 @@ static void rmnet_flush_buffered(struct work_struct *work) local_bh_disable(); rmnet_shs_flush_table(is_force_flush, RMNET_WQ_CTXT); - - /* If packets remain restart the timer in case there are no - * more NET_RX flushes coming so pkts are no lost - */ - if (rmnet_shs_fall_back_timer && - rmnet_shs_cfg.num_bytes_parked && - rmnet_shs_cfg.num_pkts_parked){ - if (hrtimer_active(&rmnet_shs_cfg.hrtimer_shs)) - hrtimer_cancel(&rmnet_shs_cfg.hrtimer_shs); - - hrtimer_start(&rmnet_shs_cfg.hrtimer_shs, - ns_to_ktime(rmnet_shs_timeout * NS_IN_MS), - HRTIMER_MODE_REL); - } - rmnet_shs_flush_reason[RMNET_SHS_FLUSH_WQ_FB_FLUSH]++; local_bh_enable(); } SHS_TRACE_HIGH(RMNET_SHS_FLUSH, @@ -1649,9 +1700,9 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) break; } while (0); - spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); if (!is_shs_reqd) { + spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); rmnet_shs_crit_err[RMNET_SHS_MAIN_SHS_NOT_REQD]++; rmnet_shs_deliver_skb(skb); SHS_TRACE_ERR(RMNET_SHS_ASSIGN, @@ -1683,6 +1734,7 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) RMNET_SHS_FORCE_FLUSH_TIME_NSEC, 0xDEF, 0xDEF, 0xDEF, skb, NULL); } + spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); if (rmnet_shs_cfg.num_pkts_parked > rmnet_shs_pkts_store_limit) { diff --git a/drivers/rmnet/shs/rmnet_shs_wq.c b/drivers/rmnet/shs/rmnet_shs_wq.c index 5944f16..70589ce 100644 --- a/drivers/rmnet/shs/rmnet_shs_wq.c +++ b/drivers/rmnet/shs/rmnet_shs_wq.c @@ -1909,14 +1909,18 @@ void rmnet_shs_update_cfg_mask(void) void rmnet_shs_wq_filter(void) { - int cpu; + int cpu, cur_cpu; int temp; struct rmnet_shs_wq_hstat_s *hnode = NULL; - for (cpu = 0; cpu < MAX_CPUS; cpu++) + for (cpu = 0; cpu < MAX_CPUS; cpu++) { rmnet_shs_cpu_rx_filter_flows[cpu] = 0; + rmnet_shs_cpu_node_tbl[cpu].seg = 0; + } - /* Filter out flows with low pkt count */ + /* Filter out flows with low pkt count and + * mark CPUS with slowstart flows + */ list_for_each_entry(hnode, &rmnet_shs_wq_hstat_tbl, hstat_node_id) { if (hnode->in_use == 0) @@ -1927,6 +1931,14 @@ void rmnet_shs_wq_filter(void) temp = hnode->current_cpu; rmnet_shs_cpu_rx_filter_flows[temp]++; } + cur_cpu = hnode->current_cpu; + if (cur_cpu >= MAX_CPUS) { + continue; + } + + if (hnode->node->hstats->segment_enable) { + rmnet_shs_cpu_node_tbl[cur_cpu].seg++; + } } } |