diff options
| -rw-r--r-- | drivers/rmnet/perf/Kbuild | 3 | ||||
| -rw-r--r-- | drivers/rmnet/perf/rmnet_perf_config.c | 191 | ||||
| -rw-r--r-- | drivers/rmnet/perf/rmnet_perf_config.h | 14 | ||||
| -rw-r--r-- | drivers/rmnet/perf/rmnet_perf_core.c | 137 | ||||
| -rw-r--r-- | drivers/rmnet/perf/rmnet_perf_core.h | 11 | ||||
| -rw-r--r-- | drivers/rmnet/perf/rmnet_perf_opt.c | 681 | ||||
| -rw-r--r-- | drivers/rmnet/perf/rmnet_perf_opt.h | 91 | ||||
| -rw-r--r-- | drivers/rmnet/perf/rmnet_perf_tcp_opt.c | 516 | ||||
| -rw-r--r-- | drivers/rmnet/perf/rmnet_perf_tcp_opt.h | 81 | ||||
| -rw-r--r-- | drivers/rmnet/perf/rmnet_perf_udp_opt.c | 138 | ||||
| -rw-r--r-- | drivers/rmnet/perf/rmnet_perf_udp_opt.h | 40 | ||||
| -rw-r--r-- | drivers/rmnet/shs/rmnet_shs.h | 66 | ||||
| -rw-r--r-- | drivers/rmnet/shs/rmnet_shs_config.c | 37 | ||||
| -rw-r--r-- | drivers/rmnet/shs/rmnet_shs_config.h | 4 | ||||
| -rwxr-xr-x | drivers/rmnet/shs/rmnet_shs_main.c | 646 | ||||
| -rw-r--r-- | drivers/rmnet/shs/rmnet_shs_wq.c | 77 | ||||
| -rw-r--r-- | drivers/rmnet/shs/rmnet_shs_wq.h | 5 |
17 files changed, 1837 insertions, 901 deletions
diff --git a/drivers/rmnet/perf/Kbuild b/drivers/rmnet/perf/Kbuild index de0acb9..e3537c7 100644 --- a/drivers/rmnet/perf/Kbuild +++ b/drivers/rmnet/perf/Kbuild @@ -1,2 +1,3 @@ obj-m += rmnet_perf.o -rmnet_perf-y := rmnet_perf_config.o rmnet_perf_core.o rmnet_perf_tcp_opt.o
\ No newline at end of file +rmnet_perf-y := rmnet_perf_config.o rmnet_perf_core.o rmnet_perf_opt.o \ + rmnet_perf_tcp_opt.o rmnet_perf_udp_opt.o diff --git a/drivers/rmnet/perf/rmnet_perf_config.c b/drivers/rmnet/perf/rmnet_perf_config.c index ba64176..acf2c0f 100644 --- a/drivers/rmnet/perf/rmnet_perf_config.c +++ b/drivers/rmnet/perf/rmnet_perf_config.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -17,10 +17,9 @@ #include <linux/module.h> #include <linux/netdevice.h> #include "rmnet_perf_core.h" -#include "rmnet_perf_tcp_opt.h" +#include "rmnet_perf_opt.h" #include "rmnet_perf_config.h" #include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h> -#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h> #include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.h> #include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h> @@ -113,8 +112,9 @@ rmnet_perf_config_free_resources(struct rmnet_perf *perf, if (!perf) return RMNET_PERF_RESOURCE_MGMT_FAIL; - /* Free everything tcp_opt currently holds */ - rmnet_perf_tcp_opt_flush_all_flow_nodes(perf); + /* Free everything flow nodes currently hold */ + rmnet_perf_opt_flush_all_flow_nodes(perf); + /* Get rid of 64k sk_buff cache */ rmnet_perf_config_free_64k_buffs(perf); /* Before we free tcp_opt's structures, make sure we arent holding @@ -124,7 +124,7 @@ rmnet_perf_config_free_resources(struct rmnet_perf *perf, //rmnet_perf_core_timer_exit(perf->core_meta); /* Since we allocated in one chunk, we will also free in one chunk */ - kfree(perf->tcp_opt_meta); + kfree(perf); return RMNET_PERF_RESOURCE_MGMT_SUCCESS; } @@ -144,23 +144,24 @@ static int rmnet_perf_config_allocate_resources(struct rmnet_perf **perf) { int i; void *buffer_head; - struct rmnet_perf_tcp_opt_meta *tcp_opt_meta; + struct rmnet_perf_opt_meta *opt_meta; struct rmnet_perf_core_meta *core_meta; struct rmnet_perf *local_perf; int perf_size = sizeof(**perf); - int tcp_opt_meta_size = sizeof(struct rmnet_perf_tcp_opt_meta); + int opt_meta_size = sizeof(struct rmnet_perf_opt_meta); int flow_node_pool_size = - sizeof(struct rmnet_perf_tcp_opt_flow_node_pool); + sizeof(struct rmnet_perf_opt_flow_node_pool); int bm_state_size = sizeof(struct rmnet_perf_core_burst_marker_state); - int flow_node_size = sizeof(struct rmnet_perf_tcp_opt_flow_node); + int flow_node_size = sizeof(struct rmnet_perf_opt_flow_node); int core_meta_size = sizeof(struct rmnet_perf_core_meta); int skb_list_size = sizeof(struct rmnet_perf_core_skb_list); int skb_buff_pool_size = sizeof(struct rmnet_perf_core_64k_buff_pool); - int total_size = perf_size + tcp_opt_meta_size + flow_node_pool_size + + int total_size = perf_size + opt_meta_size + flow_node_pool_size + (flow_node_size * RMNET_PERF_NUM_FLOW_NODES) + - core_meta_size + skb_list_size + skb_buff_pool_size; + core_meta_size + skb_list_size + skb_buff_pool_size + + bm_state_size; /* allocate all the memory in one chunk for cache coherency sake */ buffer_head = kmalloc(total_size, GFP_KERNEL); @@ -171,21 +172,21 @@ static int rmnet_perf_config_allocate_resources(struct rmnet_perf **perf) local_perf = *perf; buffer_head += perf_size; - local_perf->tcp_opt_meta = buffer_head; - tcp_opt_meta = local_perf->tcp_opt_meta; - buffer_head += tcp_opt_meta_size; + local_perf->opt_meta = buffer_head; + opt_meta = local_perf->opt_meta; + buffer_head += opt_meta_size; /* assign the node pool */ - tcp_opt_meta->node_pool = buffer_head; - tcp_opt_meta->node_pool->num_flows_in_use = 0; - tcp_opt_meta->node_pool->flow_recycle_counter = 0; + opt_meta->node_pool = buffer_head; + opt_meta->node_pool->num_flows_in_use = 0; + opt_meta->node_pool->flow_recycle_counter = 0; buffer_head += flow_node_pool_size; /* assign the individual flow nodes themselves */ for (i = 0; i < RMNET_PERF_NUM_FLOW_NODES; i++) { - struct rmnet_perf_tcp_opt_flow_node **flow_node; + struct rmnet_perf_opt_flow_node **flow_node; - flow_node = &tcp_opt_meta->node_pool->node_list[i]; + flow_node = &opt_meta->node_pool->node_list[i]; *flow_node = buffer_head; buffer_head += flow_node_size; (*flow_node)->num_pkts_held = 0; @@ -196,12 +197,12 @@ static int rmnet_perf_config_allocate_resources(struct rmnet_perf **perf) //rmnet_perf_core_timer_init(core_meta); buffer_head += core_meta_size; - /* Assign common (not specific to something like tcp_opt) structures */ + /* Assign common (not specific to something like opt) structures */ core_meta->skb_needs_free_list = buffer_head; core_meta->skb_needs_free_list->num_skbs_held = 0; buffer_head += skb_list_size; - /* allocate buffer pool struct (also not specific to tcp_opt) */ + /* allocate buffer pool struct (also not specific to opt) */ core_meta->buff_pool = buffer_head; buffer_head += skb_buff_pool_size; @@ -215,6 +216,54 @@ static int rmnet_perf_config_allocate_resources(struct rmnet_perf **perf) return RMNET_PERF_RESOURCE_MGMT_SUCCESS; } +enum rmnet_perf_resource_management_e +rmnet_perf_config_register_callbacks(struct net_device *dev, + struct rmnet_port *port) +{ + struct rmnet_map_dl_ind *dl_ind; + struct qmi_rmnet_ps_ind *ps_ind; + enum rmnet_perf_resource_management_e rc = + RMNET_PERF_RESOURCE_MGMT_SUCCESS; + + perf->core_meta->dev = dev; + /* register for DL marker */ + dl_ind = kzalloc(sizeof(struct rmnet_map_dl_ind), GFP_ATOMIC); + if (dl_ind) { + dl_ind->priority = RMNET_PERF; + dl_ind->dl_hdr_handler = + &rmnet_perf_core_handle_map_control_start; + dl_ind->dl_trl_handler = + &rmnet_perf_core_handle_map_control_end; + perf->core_meta->dl_ind = dl_ind; + if (rmnet_map_dl_ind_register(port, dl_ind)) { + kfree(dl_ind); + pr_err("%s(): Failed to register dl_ind\n", __func__); + rc = RMNET_PERF_RESOURCE_MGMT_FAIL; + } + } else { + pr_err("%s(): Failed to allocate dl_ind\n", __func__); + rc = RMNET_PERF_RESOURCE_MGMT_FAIL; + } + + /* register for PS mode indications */ + ps_ind = kzalloc(sizeof(struct qmi_rmnet_ps_ind), GFP_ATOMIC); + if (ps_ind) { + ps_ind->ps_on_handler = &rmnet_perf_core_ps_on; + ps_ind->ps_off_handler = &rmnet_perf_core_ps_off; + perf->core_meta->ps_ind = ps_ind; + if (qmi_rmnet_ps_ind_register(port, ps_ind)) { + kfree(ps_ind); + rc = RMNET_PERF_RESOURCE_MGMT_FAIL; + pr_err("%s(): Failed to register ps_ind\n", __func__); + } + } else { + rc = RMNET_PERF_RESOURCE_MGMT_FAIL; + pr_err("%s(): Failed to allocate ps_ind\n", __func__); + } + + return rc; +} + static void rmnet_perf_netdev_down(struct net_device *dev) { enum rmnet_perf_resource_management_e config_status; @@ -222,35 +271,45 @@ static void rmnet_perf_netdev_down(struct net_device *dev) config_status = rmnet_perf_config_free_resources(perf, dev); } -static int rmnet_perf_netdev_up(void) +static int rmnet_perf_netdev_up(struct net_device *real_dev, + struct rmnet_port *port) { - enum rmnet_perf_resource_management_e alloc_rc; + enum rmnet_perf_resource_management_e rc; - alloc_rc = rmnet_perf_config_allocate_resources(&perf); - if (alloc_rc == RMNET_PERF_RESOURCE_MGMT_FAIL) - pr_err("Failed to allocate tcp_opt and core resources"); + rc = rmnet_perf_config_allocate_resources(&perf); + if (rc == RMNET_PERF_RESOURCE_MGMT_FAIL) { + pr_err("Failed to allocate tcp_opt and core resources\n"); + return RMNET_PERF_RESOURCE_MGMT_FAIL; + } /* structs to contain these have already been allocated. Here we are * simply allocating the buffers themselves */ - alloc_rc |= rmnet_perf_config_alloc_64k_buffs(perf); - if (alloc_rc == RMNET_PERF_RESOURCE_MGMT_FAIL) + rc = rmnet_perf_config_alloc_64k_buffs(perf); + if (rc == RMNET_PERF_RESOURCE_MGMT_FAIL) { pr_err("%s(): Failed to allocate 64k buffers for recycling\n", __func__); - else - pr_err("%s(): Allocated 64k buffers for recycling\n", - __func__); + return RMNET_PERF_RESOURCE_MGMT_SEMI_FAIL; + } + + rc = rmnet_perf_config_register_callbacks(real_dev, port); + if (rc == RMNET_PERF_RESOURCE_MGMT_FAIL) { + pr_err("%s(): Failed to register for required " + "callbacks\n", __func__); + return RMNET_PERF_RESOURCE_MGMT_SEMI_FAIL; + } - return alloc_rc; + return RMNET_PERF_RESOURCE_MGMT_SUCCESS; } /* TODO Needs modifying*/ static int rmnet_perf_config_notify_cb(struct notifier_block *nb, unsigned long event, void *data) { - /*Not sure if we need this*/ + enum rmnet_perf_resource_management_e return_val = + RMNET_PERF_RESOURCE_MGMT_SUCCESS; struct net_device *dev = netdev_notifier_info_to_dev(data); - unsigned int return_val; + struct rmnet_port *port = rmnet_get_port(dev); if (!dev) return NOTIFY_DONE; @@ -258,53 +317,54 @@ static int rmnet_perf_config_notify_cb(struct notifier_block *nb, switch (event) { case NETDEV_UNREGISTER: if (rmnet_is_real_dev_registered(dev) && + rmnet_perf_deag_entry && !strncmp(dev->name, "rmnet_ipa0", 10)) { - pr_err("%s(): rmnet_perf netdevice unregister,", + struct rmnet_perf_core_meta *core_meta = + perf->core_meta; + pr_err("%s(): rmnet_perf netdevice unregister\n", __func__); /* Unregister for DL marker */ - rmnet_map_dl_ind_deregister(rmnet_get_port(dev), - perf->core_meta->dl_ind); + rmnet_map_dl_ind_deregister(port, + core_meta->dl_ind); + kfree(core_meta->dl_ind); + qmi_rmnet_ps_ind_deregister(port, + core_meta->ps_ind); + kfree(core_meta->ps_ind); rmnet_perf_netdev_down(dev); + RCU_INIT_POINTER(rmnet_perf_deag_entry, NULL); } break; case NETDEV_REGISTER: - pr_err("%s(): rmnet_perf netdevice register, name = %s,", + pr_err("%s(): rmnet_perf netdevice register, name = %s\n", __func__, dev->name); /* Check prevents us from allocating resources for every * interface */ - if (!rmnet_perf_deag_entry) { - rmnet_perf_netdev_up(); + + + if (!rmnet_perf_deag_entry && + strncmp(dev->name, "rmnet_data", 10) == 0) { + struct rmnet_priv *priv = netdev_priv(dev); + port = rmnet_get_port(priv->real_dev); + return_val |= rmnet_perf_netdev_up(priv->real_dev, + port); + if (return_val == RMNET_PERF_RESOURCE_MGMT_FAIL) { + pr_err("%s(): rmnet_perf allocation or " + "registry failed. Potentially falling " + "back on legacy path\n", + __func__); + goto exit; + } RCU_INIT_POINTER(rmnet_perf_deag_entry, rmnet_perf_core_deaggregate); - } - if (strncmp(dev->name, "rmnet_ipa0", 10) == 0 && - rmnet_perf_deag_entry) { - struct rmnet_map_dl_ind *dl_ind; - - /* register for DL marker */ - dl_ind = kzalloc(sizeof(struct rmnet_map_dl_ind), - GFP_ATOMIC); - if (dl_ind) { - dev_net_set(dev, &init_net); - perf->core_meta->dev = dev; - - dl_ind->priority = RMNET_PERF; - dl_ind->dl_hdr_handler = - &rmnet_perf_core_handle_map_control_start; - dl_ind->dl_trl_handler = - &rmnet_perf_core_handle_map_control_end; - perf->core_meta->dl_ind = dl_ind; - return_val = - rmnet_map_dl_ind_register(rmnet_get_port(dev), - dl_ind); - } + pr_err("%s(): rmnet_perf registered on " + "name = %s\n", __func__, dev->name); } break; default: break; } - +exit: return NOTIFY_DONE; } @@ -314,14 +374,13 @@ static struct notifier_block rmnet_perf_dev_notifier __read_mostly = { int __init rmnet_perf_init(void) { - pr_err("%s(): initializing rmnet_perf, 5\n", __func__); + pr_err("%s(): initializing rmnet_perf\n", __func__); return register_netdevice_notifier(&rmnet_perf_dev_notifier); } void __exit rmnet_perf_exit(void) { pr_err("%s(): exiting rmnet_perf\n", __func__); - RCU_INIT_POINTER(rmnet_perf_deag_entry, NULL); unregister_netdevice_notifier(&rmnet_perf_dev_notifier); } diff --git a/drivers/rmnet/perf/rmnet_perf_config.h b/drivers/rmnet/perf/rmnet_perf_config.h index 52c092e..a8bf12f 100644 --- a/drivers/rmnet/perf/rmnet_perf_config.h +++ b/drivers/rmnet/perf/rmnet_perf_config.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2013-2014, 2016-2017 The Linux Foundation. All rights reserved. +/* Copyright (c) 2013-2014, 2016-2017, 2019 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -15,15 +15,20 @@ #include <linux/skbuff.h> #include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h> +#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.h> +#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h> #include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_trace.h> +#include <../include/soc/qcom/qmi_rmnet.h> #include "rmnet_perf_core.h" + #ifndef _RMNET_PERF_CONFIG_H_ #define _RMNET_PERF_CONFIG_H_ enum rmnet_perf_resource_management_e { RMNET_PERF_RESOURCE_MGMT_SUCCESS, - RMNET_PERF_RESOURCE_MGMT_FAIL + RMNET_PERF_RESOURCE_MGMT_SEMI_FAIL, + RMNET_PERF_RESOURCE_MGMT_FAIL, }; /* rmnet based variables that we rely on*/ @@ -47,6 +52,11 @@ extern struct rmnet_port *rmnet_get_port(struct net_device *real_dev); extern void rmnet_map_cmd_init(struct rmnet_port *port); extern void rmnet_map_cmd_exit(struct rmnet_port *port); +/* Function declarations */ struct rmnet_perf *rmnet_perf_config_get_perf(void); +enum rmnet_perf_resource_management_e + rmnet_perf_config_register_callbacks(struct net_device *dev, + struct rmnet_port *port); + #endif /* _RMNET_PERF_CONFIG_H_ */ diff --git a/drivers/rmnet/perf/rmnet_perf_core.c b/drivers/rmnet/perf/rmnet_perf_core.c index c9c2ccc..fe7fa04 100644 --- a/drivers/rmnet/perf/rmnet_perf_core.c +++ b/drivers/rmnet/perf/rmnet_perf_core.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -25,10 +25,14 @@ #include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h> #include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.h> #include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h> -#include "rmnet_perf_tcp_opt.h" +#include "rmnet_perf_opt.h" #include "rmnet_perf_core.h" #include "rmnet_perf_config.h" +#ifdef CONFIG_QCOM_QMI_POWER_COLLAPSE +#include <soc/qcom/qmi_rmnet.h> +#endif + /* Each index tells us the number of iterations it took us to find a recycled * skb */ @@ -90,6 +94,17 @@ unsigned long int packet_dropper_time = 1; module_param(packet_dropper_time, ulong, 0644); MODULE_PARM_DESC(packet_dropper_time, "packet_dropper_time"); +unsigned long int rmnet_perf_flush_shs = 0; +module_param(rmnet_perf_flush_shs, ulong, 0644); +MODULE_PARM_DESC(rmnet_perf_flush_shs, "rmnet_perf_flush_shs"); + +unsigned long int rmnet_perf_frag_flush = 0; +module_param(rmnet_perf_frag_flush, ulong, 0444); +MODULE_PARM_DESC(rmnet_perf_frag_flush, + "Number of packet fragments flushed to stack"); + +#define SHS_FLUSH 0 + /* rmnet_perf_core_free_held_skbs() - Free held SKBs given to us by physical * device * @perf: allows access to our required global structures @@ -357,21 +372,19 @@ void rmnet_perf_core_send_skb(struct sk_buff *skb, struct rmnet_endpoint *ep, rmnet_set_skb_proto(skb); /* If the checksum is unnecessary, update the header fields. * Otherwise, we know that this is a single packet that - * failed checksum validation, so we don't want to touch - * the headers. + * either failed checksum validation, or is not coalescable + * (fragment, ICMP, etc), so don't touch the headers. */ - if (ip4hn->protocol == IPPROTO_TCP && - skb->ip_summed == CHECKSUM_UNNECESSARY) { + if (skb_csum_unnecessary(skb)) { ip4hn->tot_len = htons(skb->len); ip4hn->check = 0; ip4hn->check = ip_fast_csum(ip4hn, (int)ip4hn->ihl); } rmnet_deliver_skb(skb, perf->rmnet_port); - } else if (ip_version == 0x06) { + } else if (ip_version == 0x06) { ip6hn = (struct ipv6hdr *)data; rmnet_set_skb_proto(skb); - if (ip6hn->nexthdr == IPPROTO_TCP && - skb->ip_summed == CHECKSUM_UNNECESSARY) { + if (skb_csum_unnecessary(skb)) { ip6hn->payload_len = htons(skb->len - sizeof(struct ipv6hdr)); } @@ -397,7 +410,8 @@ void rmnet_perf_core_send_skb(struct sk_buff *skb, struct rmnet_endpoint *ep, void rmnet_perf_core_flush_curr_pkt(struct rmnet_perf *perf, struct sk_buff *skb, struct rmnet_perf_pkt_info *pkt_info, - u16 packet_len) + u16 packet_len, bool flush_shs, + bool skip_hash) { struct sk_buff *skbn; struct rmnet_endpoint *ep = pkt_info->ep; @@ -407,36 +421,49 @@ void rmnet_perf_core_flush_curr_pkt(struct rmnet_perf *perf, return; } - if (pkt_info->trans_proto != IPPROTO_UDP || packet_len < 64) { - /* allocate the sk_buff of proper size for this packet */ - skbn = alloc_skb(packet_len + RMNET_MAP_DEAGGR_SPACING, - GFP_ATOMIC); - if (!skbn) - return; + /* allocate the sk_buff of proper size for this packet */ + skbn = alloc_skb(packet_len + RMNET_MAP_DEAGGR_SPACING, + GFP_ATOMIC); + if (!skbn) + return; - skb_reserve(skbn, RMNET_MAP_DEAGGR_HEADROOM); - skb_put(skbn, packet_len); - memcpy(skbn->data, pkt_info->iphdr.v4hdr, packet_len); - } else { - skbn = skb_clone(skb, GFP_ATOMIC); - if (!skbn) - return; - - skb_pull(skbn, sizeof(struct rmnet_map_header)); - skb_trim(skbn, packet_len); - skbn->truesize = SKB_TRUESIZE(packet_len); - __skb_set_hash(skbn, 0, 0, 0); - } + skb_reserve(skbn, RMNET_MAP_DEAGGR_HEADROOM); + skb_put(skbn, packet_len); + memcpy(skbn->data, pkt_info->iphdr.v4hdr, packet_len); /* If the packet passed checksum validation, tell the stack */ if (pkt_info->csum_valid) skbn->ip_summed = CHECKSUM_UNNECESSARY; skbn->dev = skb->dev; - skbn->hash = pkt_info->hash_key; - skbn->sw_hash = 1; + + /* Only set hash info if we actually calculated it */ + if (!skip_hash) { + skbn->hash = pkt_info->hash_key; + skbn->sw_hash = 1; + } + + skbn->cb[SHS_FLUSH] = (char) flush_shs; rmnet_perf_core_send_skb(skbn, ep, perf, pkt_info); } +/* DL marker is off, we need to flush more aggresively at end of chains */ +void rmnet_perf_core_ps_on(void *port) +{ + struct rmnet_perf *perf = rmnet_perf_config_get_perf(); + + rmnet_perf_core_bm_flush_on = 0; + rmnet_perf_opt_flush_all_flow_nodes(perf); + rmnet_perf_core_flush_reason_cnt[RMNET_PERF_CORE_PS_MODE_ON]++; + /* Essentially resets expected packet count to safe state */ + perf->core_meta->bm_state->expect_packets = -1; +} + +/* DL marker on, we can try to coalesce more packets */ +void rmnet_perf_core_ps_off(void *port) +{ + rmnet_perf_core_bm_flush_on = 1; +} + void rmnet_perf_core_handle_map_control_start(struct rmnet_map_dl_ind_hdr *dlhdr) { @@ -449,7 +476,7 @@ rmnet_perf_core_handle_map_control_start(struct rmnet_map_dl_ind_hdr *dlhdr) */ if (!bm_state->wait_for_start) { /* flush everything, we got a 2nd start */ - rmnet_perf_tcp_opt_flush_all_flow_nodes(perf); + rmnet_perf_opt_flush_all_flow_nodes(perf); rmnet_perf_core_flush_reason_cnt[ RMNET_PERF_CORE_DL_MARKER_FLUSHES]++; } else { @@ -469,7 +496,7 @@ void rmnet_perf_core_handle_map_control_end(struct rmnet_map_dl_ind_trl *dltrl) struct rmnet_perf_core_burst_marker_state *bm_state; bm_state = perf->core_meta->bm_state; - rmnet_perf_tcp_opt_flush_all_flow_nodes(perf); + rmnet_perf_opt_flush_all_flow_nodes(perf); rmnet_perf_core_flush_reason_cnt[RMNET_PERF_CORE_DL_MARKER_FLUSHES]++; bm_state->wait_for_start = true; bm_state->curr_seq = 0; @@ -510,7 +537,9 @@ void rmnet_perf_core_handle_packet_ingress(struct sk_buff *skb, (skb->data + sizeof(struct rmnet_map_header)); struct rmnet_perf *perf = rmnet_perf_config_get_perf(); u16 pkt_len; + bool skip_hash = false; + pkt_len = frame_len - sizeof(struct rmnet_map_header) - trailer_len; pkt_info->ep = ep; pkt_info->ip_proto = (*payload & 0xF0) >> 4; if (pkt_info->ip_proto == 4) { @@ -519,18 +548,23 @@ void rmnet_perf_core_handle_packet_ingress(struct sk_buff *skb, pkt_info->iphdr.v4hdr = iph; pkt_info->trans_proto = iph->protocol; pkt_info->header_len = iph->ihl * 4; + skip_hash = !!(ntohs(iph->frag_off) & (IP_MF | IP_OFFSET)); } else if (pkt_info->ip_proto == 6) { struct ipv6hdr *iph = (struct ipv6hdr *)payload; pkt_info->iphdr.v6hdr = iph; pkt_info->trans_proto = iph->nexthdr; pkt_info->header_len = sizeof(*iph); + skip_hash = iph->nexthdr == NEXTHDR_FRAGMENT; } else { - pr_err("%s(): invalid packet\n", __func__); return; } - pkt_len = frame_len - sizeof(struct rmnet_map_header) - trailer_len; + /* Push out fragments immediately */ + if (skip_hash) { + rmnet_perf_frag_flush++; + goto flush; + } if (pkt_info->trans_proto == IPPROTO_TCP) { struct tcphdr *tp = (struct tcphdr *) @@ -545,7 +579,8 @@ void rmnet_perf_core_handle_packet_ingress(struct sk_buff *skb, if (rmnet_perf_core_validate_pkt_csum(skb, pkt_info)) goto flush; - rmnet_perf_tcp_opt_ingress(perf, skb, pkt_info); + if (!rmnet_perf_opt_ingress(perf, skb, pkt_info)) + goto flush; } else if (pkt_info->trans_proto == IPPROTO_UDP) { struct udphdr *up = (struct udphdr *) (payload + pkt_info->header_len); @@ -556,16 +591,14 @@ void rmnet_perf_core_handle_packet_ingress(struct sk_buff *skb, pkt_info->hash_key = rmnet_perf_core_compute_flow_hash(pkt_info); - /* We flush anyway, so the result of the validation - * does not need to be checked. - */ - rmnet_perf_core_validate_pkt_csum(skb, pkt_info); - goto flush; + if (rmnet_perf_core_validate_pkt_csum(skb, pkt_info)) + goto flush; + + if (!rmnet_perf_opt_ingress(perf, skb, pkt_info)) + goto flush; } else { pkt_info->payload_len = pkt_len - pkt_info->header_len; - pkt_info->hash_key = - rmnet_perf_core_compute_flow_hash(pkt_info); - + skip_hash = true; /* We flush anyway, so the result of the validation * does not need to be checked. */ @@ -576,7 +609,8 @@ void rmnet_perf_core_handle_packet_ingress(struct sk_buff *skb, return; flush: - rmnet_perf_core_flush_curr_pkt(perf, skb, pkt_info, pkt_len); + rmnet_perf_core_flush_curr_pkt(perf, skb, pkt_info, pkt_len, false, + skip_hash); } /* rmnet_perf_core_deaggregate() - Deaggregated ip packets from map frame @@ -659,6 +693,13 @@ skip_frame: goto bad_data; skb->dev = ep->egress_dev; +#ifdef CONFIG_QCOM_QMI_POWER_COLLAPSE + /* Wakeup PS work on DL packets */ + if ((port->data_format & RMNET_INGRESS_FORMAT_PS) && + !RMNET_MAP_GET_CD_BIT(skb)) + qmi_rmnet_work_maybe_restart(port); +#endif + if (enable_packet_dropper) { getnstimeofday(&curr_time); if (last_drop_time.tv_sec == 0 && @@ -694,13 +735,13 @@ next_chain: */ if (!rmnet_perf_core_bm_flush_on || (int) perf->core_meta->bm_state->expect_packets <= 0) { - rmnet_perf_tcp_opt_flush_all_flow_nodes(perf); + rmnet_perf_opt_flush_all_flow_nodes(perf); rmnet_perf_core_free_held_skbs(perf); rmnet_perf_core_flush_reason_cnt[ RMNET_PERF_CORE_IPA_ZERO_FLUSH]++; } else if (perf->core_meta->skb_needs_free_list->num_skbs_held >= rmnet_perf_core_num_skbs_max) { - rmnet_perf_tcp_opt_flush_all_flow_nodes(perf); + rmnet_perf_opt_flush_all_flow_nodes(perf); rmnet_perf_core_free_held_skbs(perf); rmnet_perf_core_flush_reason_cnt[ RMNET_PERF_CORE_SK_BUFF_HELD_LIMIT]++; @@ -708,7 +749,7 @@ next_chain: goto update_stats; drop_packets: - rmnet_perf_tcp_opt_flush_all_flow_nodes(perf); + rmnet_perf_opt_flush_all_flow_nodes(perf); rmnet_perf_core_free_held_skbs(perf); update_stats: rmnet_perf_core_pre_ip_count += co; diff --git a/drivers/rmnet/perf/rmnet_perf_core.h b/drivers/rmnet/perf/rmnet_perf_core.h index 4019bc7..91557fc 100644 --- a/drivers/rmnet/perf/rmnet_perf_core.h +++ b/drivers/rmnet/perf/rmnet_perf_core.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -20,7 +20,7 @@ #define RMNET_PERF_CORE_RECYCLE_SKB_SIZE 65600//33000//32768//65600 struct rmnet_perf { - struct rmnet_perf_tcp_opt_meta *tcp_opt_meta; + struct rmnet_perf_opt_meta *opt_meta; struct rmnet_perf_core_meta *core_meta; struct rmnet_port *rmnet_port; }; @@ -79,12 +79,14 @@ struct rmnet_perf_core_meta { //spinlock_t timer_lock; struct rmnet_perf_core_burst_marker_state *bm_state; struct rmnet_map_dl_ind *dl_ind; + struct qmi_rmnet_ps_ind *ps_ind; }; enum rmnet_perf_core_flush_reasons { RMNET_PERF_CORE_IPA_ZERO_FLUSH, RMNET_PERF_CORE_SK_BUFF_HELD_LIMIT, RMNET_PERF_CORE_DL_MARKER_FLUSHES, + RMNET_PERF_CORE_PS_MODE_ON, RMNET_PERF_CORE_NUM_CONDITIONS }; @@ -109,6 +111,8 @@ enum rmnet_perf_trace_evt { RMNET_PERF_DEAG_PKT, }; +void rmnet_perf_core_ps_on(void *port); +void rmnet_perf_core_ps_off(void *port); void rmnet_perf_core_reset_recycled_skb(struct sk_buff *skb); struct sk_buff *rmnet_perf_core_elligible_for_cache_skb(struct rmnet_perf *perf, u32 len); @@ -119,7 +123,8 @@ void rmnet_perf_core_send_skb(struct sk_buff *skb, struct rmnet_endpoint *ep, void rmnet_perf_core_flush_curr_pkt(struct rmnet_perf *perf, struct sk_buff *skb, struct rmnet_perf_pkt_info *pkt_info, - u16 packet_len); + u16 packet_len, bool flush_shs, + bool skip_hash); void rmnet_perf_core_deaggregate(struct sk_buff *skb, struct rmnet_port *port); u32 rmnet_perf_core_compute_flow_hash(struct rmnet_perf_pkt_info *pkt_info); diff --git a/drivers/rmnet/perf/rmnet_perf_opt.c b/drivers/rmnet/perf/rmnet_perf_opt.c new file mode 100644 index 0000000..91d7e72 --- /dev/null +++ b/drivers/rmnet/perf/rmnet_perf_opt.c @@ -0,0 +1,681 @@ +/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * RMNET generic protocol optimization handlers + * + */ + +#include <linux/skbuff.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/module.h> +#include <linux/spinlock.h> +#include <net/ip.h> +#include <net/checksum.h> +#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h> +#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h> +#include "rmnet_perf_opt.h" +#include "rmnet_perf_tcp_opt.h" +#include "rmnet_perf_udp_opt.h" +#include "rmnet_perf_core.h" +#include "rmnet_perf_config.h" + +/* If true then we allocate all large SKBs */ +unsigned long int rmnet_perf_opt_skb_recycle_off = 1; +module_param(rmnet_perf_opt_skb_recycle_off, ulong, 0644); +MODULE_PARM_DESC(rmnet_perf_opt_skb_recycle_off, "Num skbs max held"); + +/* Stat showing reason for flushes of flow nodes */ +unsigned long int +rmnet_perf_opt_flush_reason_cnt[RMNET_PERF_OPT_NUM_CONDITIONS]; +module_param_array(rmnet_perf_opt_flush_reason_cnt, ulong, 0, 0444); +MODULE_PARM_DESC(rmnet_perf_opt_flush_reason_cnt, + "opt performance statistics"); + +/* Stat showing packets dropped due to lack of memory */ +unsigned long int rmnet_perf_opt_oom_drops = 0; +module_param(rmnet_perf_opt_oom_drops, ulong, 0644); +MODULE_PARM_DESC(rmnet_perf_opt_oom_drops, + "Number of packets dropped because we couldn't allocate SKBs"); + +enum { + RMNET_PERF_OPT_MODE_TCP, + RMNET_PERF_OPT_MODE_UDP, + RMNET_PERF_OPT_MODE_ALL, + RMNET_PERF_OPT_MODE_NON, +}; + +/* What protocols we optimize */ +static int rmnet_perf_opt_mode = RMNET_PERF_OPT_MODE_ALL; + +/* Lock around flow nodes for syncornization with rmnet_perf_opt_mode changes */ +static DEFINE_SPINLOCK(rmnet_perf_opt_lock); + +/* flow hash table */ +DEFINE_HASHTABLE(rmnet_perf_opt_fht, RMNET_PERF_FLOW_HASH_TABLE_BITS); + +static void flush_flow_nodes_by_protocol(struct rmnet_perf *perf, u8 protocol) +{ + struct rmnet_perf_opt_flow_node *flow_node; + int bkt_cursor; + + hash_for_each(rmnet_perf_opt_fht, bkt_cursor, flow_node, list) { + if (flow_node->num_pkts_held > 0 && + flow_node->protocol == protocol) + rmnet_perf_opt_flush_single_flow_node(perf, flow_node); + } +} + +static int rmnet_perf_set_opt_mode(const char *val, + const struct kernel_param *kp) +{ + struct rmnet_perf *perf; + int old_mode = rmnet_perf_opt_mode; + int rc = -EINVAL; + char value[4]; + + strlcpy(value, val, 4); + value[3] = '\0'; + spin_lock(&rmnet_perf_opt_lock); + if (!strcmp(value, "tcp")) + rmnet_perf_opt_mode = RMNET_PERF_OPT_MODE_TCP; + else if (!strcmp(value, "udp")) + rmnet_perf_opt_mode = RMNET_PERF_OPT_MODE_UDP; + else if (!strcmp(value, "all")) + rmnet_perf_opt_mode = RMNET_PERF_OPT_MODE_ALL; + else if (!strcmp(value, "non")) + rmnet_perf_opt_mode = RMNET_PERF_OPT_MODE_NON; + else + goto out; + + rc = 0; + + /* If we didn't change anything, or if all we did was add the + * other protocol, no nodes need to be flushed out + */ + if (old_mode == rmnet_perf_opt_mode || + rmnet_perf_opt_mode == RMNET_PERF_OPT_MODE_ALL) + goto out; + + /* Flush out any nodes of the protocol we are no longer optimizing */ + perf = rmnet_perf_config_get_perf(); + switch (rmnet_perf_opt_mode) { + case RMNET_PERF_OPT_MODE_TCP: + flush_flow_nodes_by_protocol(perf, IPPROTO_UDP); + break; + case RMNET_PERF_OPT_MODE_UDP: + flush_flow_nodes_by_protocol(perf, IPPROTO_TCP); + break; + case RMNET_PERF_OPT_MODE_NON: + flush_flow_nodes_by_protocol(perf, IPPROTO_TCP); + flush_flow_nodes_by_protocol(perf, IPPROTO_UDP); + break; + } + +out: + spin_unlock(&rmnet_perf_opt_lock); + return rc; +} + +static int rmnet_perf_get_opt_mode(char *buf, + const struct kernel_param *kp) +{ + switch (rmnet_perf_opt_mode) { + case RMNET_PERF_OPT_MODE_TCP: + strlcpy(buf, "tcp\n", 5); + break; + case RMNET_PERF_OPT_MODE_UDP: + strlcpy(buf, "udp\n", 5); + break; + case RMNET_PERF_OPT_MODE_ALL: + strlcpy(buf, "all\n", 5); + break; + case RMNET_PERF_OPT_MODE_NON: + strlcpy(buf, "non\n", 5); + break; + } + + return strlen(buf); +} + +static const struct kernel_param_ops rmnet_perf_opt_mode_ops = { + .set = rmnet_perf_set_opt_mode, + .get = rmnet_perf_get_opt_mode, +}; + +module_param_cb(rmnet_perf_opt_mode, &rmnet_perf_opt_mode_ops, NULL, 0644); + +/* optimize_protocol() - Check if we should optimize the given protocol + * @protocol: The IP protocol number to check + * + * Return: + * - true if protocol should use the flow node infrastructure + * - false if packets og the given protocol should be flushed + **/ +static bool optimize_protocol(u8 protocol) +{ + if (rmnet_perf_opt_mode == RMNET_PERF_OPT_MODE_ALL) + return true; + else if (protocol == IPPROTO_TCP) + return rmnet_perf_opt_mode == RMNET_PERF_OPT_MODE_TCP; + else if (protocol == IPPROTO_UDP) + return rmnet_perf_opt_mode == RMNET_PERF_OPT_MODE_UDP; + + return false; +} + +/* ip_flag_flush() - Check IP header flags to decide if + * immediate flush required + * @pkt_info: characteristics of the current packet + * + * If the IP header has any flag set that GRO won't accept, we will flush the + * packet right away. + * + * Return: + * - true if need flush + * - false if immediate flush may not be needed + **/ +static bool ip_flag_flush(struct rmnet_perf_opt_flow_node *flow_node, + struct rmnet_perf_pkt_info *pkt_info) +{ + struct iphdr *ip4h; + struct ipv6hdr *ip6h; + __be32 first_word; + + switch (pkt_info->ip_proto) { + case 0x04: + ip4h = pkt_info->iphdr.v4hdr; + + if ((ip4h->ttl ^ flow_node->ip_flags.ip4_flags.ip_ttl) || + (ip4h->tos ^ flow_node->ip_flags.ip4_flags.ip_tos) || + (ip4h->frag_off ^ + flow_node->ip_flags.ip4_flags.ip_frag_off) || + ip4h->ihl > 5) + return true; + + break; + case 0x06: + ip6h = (struct ipv6hdr *) pkt_info->iphdr.v6hdr; + first_word = *(__be32 *)ip6h ^ flow_node->ip_flags.first_word; + + if (!!(first_word & htonl(0x0FF00000))) + return true; + + break; + default: + pr_err("Unsupported ip version %d", pkt_info->ip_proto); + rmnet_perf_opt_flush_reason_cnt[ + RMNET_PERF_OPT_PACKET_CORRUPT_ERROR]++; + } + return false; +} + +/* identify_flow() - Tell whether packet corresponds to + * given flow + * @flow_node: Node we are checking against + * @pkt_info: characteristics of the current packet + * + * Checks to see if the incoming packet is a match for a given flow node + * + * Return: + * - true: it is a match + * - false: not a match + **/ +static bool identify_flow(struct rmnet_perf_opt_flow_node *flow_node, + struct rmnet_perf_pkt_info *pkt_info) +{ + struct iphdr *ip4h; + struct ipv6hdr *ip6h; + /* Actually protocol generic. UDP and TCP headers have the source + * and dest ports in the same location. ;) + */ + struct udphdr *up = pkt_info->trns_hdr.up; + + /* if pkt count == 0 and hash is the same, then we give this one as + * pass as good enough since at this point there is no address stuff + * to check/verify. + */ + if (flow_node->num_pkts_held == 0 && + flow_node->hash_value == pkt_info->hash_key) + return true; + + /* protocol must match */ + if (flow_node->protocol != pkt_info->trans_proto) + return false; + + /* cast iph to right ip header struct for ip_version */ + switch (pkt_info->ip_proto) { + case 0x04: + ip4h = pkt_info->iphdr.v4hdr; + if (((__force u32)flow_node->saddr.saddr4 ^ + (__force u32)ip4h->saddr) | + ((__force u32)flow_node->daddr.daddr4 ^ + (__force u32)ip4h->daddr) | + ((__force u16)flow_node->src_port ^ + (__force u16)up->source) | + ((__force u16)flow_node->dest_port ^ + (__force u16)up->dest)) + return false; + break; + case 0x06: + ip6h = pkt_info->iphdr.v6hdr; + if ((ipv6_addr_cmp(&(flow_node->saddr.saddr6), &ip6h->saddr)) | + (ipv6_addr_cmp(&(flow_node->daddr.daddr6), + &ip6h->daddr)) | + ((__force u16)flow_node->src_port ^ + (__force u16)up->source) | + ((__force u16)flow_node->dest_port ^ + (__force u16)up->dest)) + return false; + break; + default: + pr_err("Unsupported ip version %d", pkt_info->ip_proto); + rmnet_perf_opt_flush_reason_cnt[ + RMNET_PERF_OPT_PACKET_CORRUPT_ERROR]++; + return false; + } + return true; +} + +/* make_flow_skb() - Allocate and populate SKB for + * flow node that is being pushed up the stack + * @perf: allows access to our required global structures + * @flow_node: opt structure containing packet we are allocating for + * + * Allocate skb of proper size for opt'd packet, and memcpy data + * into the buffer + * + * Return: + * - skbn: sk_buff to then push up the NW stack + * - NULL: if memory allocation failed + **/ +static struct sk_buff *make_flow_skb(struct rmnet_perf *perf, + struct rmnet_perf_opt_flow_node *flow_node) +{ + struct sk_buff *skbn; + struct rmnet_perf_opt_pkt_node *pkt_list; + int i; + u32 pkt_size; + u32 total_pkt_size = 0; + + if (rmnet_perf_opt_skb_recycle_off) { + skbn = alloc_skb(flow_node->len + RMNET_MAP_DEAGGR_SPACING, + GFP_ATOMIC); + if (!skbn) + return NULL; + } else { + skbn = rmnet_perf_core_elligible_for_cache_skb(perf, + flow_node->len); + if (!skbn) { + skbn = alloc_skb(flow_node->len + RMNET_MAP_DEAGGR_SPACING, + GFP_ATOMIC); + if (!skbn) + return NULL; + } + } + + skb_reserve(skbn, RMNET_MAP_DEAGGR_HEADROOM); + pkt_list = flow_node->pkt_list; + + for (i = 0; i < flow_node->num_pkts_held; i++) { + pkt_size = pkt_list[i].data_end - pkt_list[i].data_start; + memcpy(skbn->data + skbn->len, pkt_list[i].data_start, + pkt_size); + skb_put(skbn, pkt_size); + total_pkt_size += pkt_size; + } + if (flow_node->len != total_pkt_size) + pr_err("%s(): skbn = %pK, flow_node->len = %u, pkt_size = %u\n", + __func__, skbn, flow_node->len, total_pkt_size); + + return skbn; +} + +static void flow_skb_fixup(struct sk_buff *skb, + struct rmnet_perf_opt_flow_node *flow_node) +{ + struct skb_shared_info *shinfo; + struct iphdr *iph = (struct iphdr *)skb->data; + struct tcphdr *tp; + struct udphdr *up; + __wsum pseudo; + u16 datagram_len, ip_len; + u16 proto; + bool ipv4 = (iph->version == 4); + skb->hash = flow_node->hash_value; + skb->sw_hash = 1; + /* We've already validated all data */ + skb->ip_summed = CHECKSUM_UNNECESSARY; + + /* Aggregated flows can be segmented by the stack + * during forwarding/tethering scenarios, so pretend + * we ran through the GRO logic to coalesce the packets + */ + + if (flow_node->num_pkts_held <= 1) + return; + + datagram_len = flow_node->gso_len * flow_node->num_pkts_held; + + /* Update transport header fields to reflect new length. + * Checksum is set to the pseudoheader checksum value + * since we'll need to mark the SKB as CHECKSUM_PARTIAL. + */ + if (ipv4) { + ip_len = iph->ihl * 4; + pseudo = csum_partial(&iph->saddr, + sizeof(iph->saddr) * 2, 0); + proto = iph->protocol; + } else { + struct ipv6hdr *ip6h = (struct ipv6hdr *)iph; + + ip_len = sizeof(*ip6h); + pseudo = csum_partial(&ip6h->saddr, + sizeof(ip6h->saddr) * 2, 0); + proto = ip6h->nexthdr; + } + + pseudo = csum16_add(pseudo, htons(proto)); + switch (proto) { + case IPPROTO_TCP: + tp = (struct tcphdr *)((char *)iph + ip_len); + datagram_len += tp->doff * 4; + pseudo = csum16_add(pseudo, htons(datagram_len)); + tp->check = ~csum_fold(pseudo); + skb->csum_start = (unsigned char *) tp - skb->head; + skb->csum_offset = offsetof(struct tcphdr, check); + skb_shinfo(skb)->gso_type = (ipv4) ? SKB_GSO_TCPV4: + SKB_GSO_TCPV6; + break; + case IPPROTO_UDP: + up = (struct udphdr *)((char *)iph + ip_len); + datagram_len += sizeof(*up); + up->len = htons(datagram_len); + pseudo = csum16_add(pseudo, up->len); + up->check = ~csum_fold(pseudo); + skb->csum_start = (unsigned char *)up - skb->head; + skb->csum_offset = offsetof(struct udphdr, check); + skb_shinfo(skb)->gso_type = SKB_GSO_UDP_L4; + break; + default: + return; + } + + /* Update GSO metadata */ + shinfo = skb_shinfo(skb); + shinfo->gso_size = flow_node->gso_len; + shinfo->gso_segs = flow_node->num_pkts_held; + skb->ip_summed = CHECKSUM_PARTIAL; +} + +/* get_new_flow_index() - Pull flow node from node pool + * @perf: allows access to our required global structures + * + * Fetch the flow node from the node pool. If we have already given + * out all the flow nodes then we will always hit the else case and + * thereafter we will use modulo arithmetic to choose which flow node + * to evict and use. + * + * Return: + * - flow_node: node to be used by caller function + **/ +static struct rmnet_perf_opt_flow_node * +get_new_flow_index(struct rmnet_perf *perf) +{ + struct rmnet_perf_opt_flow_node_pool *node_pool; + struct rmnet_perf_opt_flow_node *flow_node_ejected; + + node_pool = perf->opt_meta->node_pool; + /* once this value gets too big it never goes back down. + * from that point forward we use flow node repurposing techniques + * instead + */ + if (node_pool->num_flows_in_use < RMNET_PERF_NUM_FLOW_NODES) + return node_pool->node_list[node_pool->num_flows_in_use++]; + + flow_node_ejected = node_pool->node_list[ + node_pool->flow_recycle_counter++ % RMNET_PERF_NUM_FLOW_NODES]; + rmnet_perf_opt_flush_single_flow_node(perf, flow_node_ejected); + hash_del(&flow_node_ejected->list); + return flow_node_ejected; +} + +/* rmnet_perf_opt_update_flow() - Update stored IP flow information + * @flow_node: opt structure containing flow information + * @pkt_info: characteristics of the current packet + * + * Update IP-specific flags stored about the flow (i.e. ttl, tos/traffic class, + * fragment information, flags). + * + * Return: + * - void + **/ +void +rmnet_perf_opt_update_flow(struct rmnet_perf_opt_flow_node *flow_node, + struct rmnet_perf_pkt_info *pkt_info) +{ + if (pkt_info->ip_proto == 0x04) { + struct iphdr *iph = pkt_info->iphdr.v4hdr; + /* Frags don't make it this far, so this is all we care about */ + __be16 flags = iph->frag_off & htons(IP_CE | IP_DF); + + flow_node->ip_flags.ip4_flags.ip_ttl = iph->ttl; + flow_node->ip_flags.ip4_flags.ip_tos = iph->tos; + flow_node->ip_flags.ip4_flags.ip_frag_off = flags; + } else if (pkt_info->ip_proto == 0x06) { + __be32 *word = (__be32 *)pkt_info->iphdr.v6hdr; + + flow_node->ip_flags.first_word = *word; + } +} + +/* rmnet_perf_opt_flush_single_flow_node() - Send a given flow node up + * NW stack. + * @perf: allows access to our required global structures + * @flow_node: opt structure containing packet we are allocating for + * + * Send a given flow up NW stack via specific VND + * + * Return: + * - skbn: sk_buff to then push up the NW stack + **/ +void rmnet_perf_opt_flush_single_flow_node(struct rmnet_perf *perf, + struct rmnet_perf_opt_flow_node *flow_node) +{ + struct sk_buff *skbn; + struct rmnet_endpoint *ep; + + /* future change: when inserting the first packet in a flow, + * save away the ep value so we dont have to look it up every flush + */ + hlist_for_each_entry_rcu(ep, + &perf->rmnet_port->muxed_ep[flow_node->mux_id], + hlnode) { + if (ep->mux_id == flow_node->mux_id && + flow_node->num_pkts_held) { + skbn = make_flow_skb(perf, flow_node); + if (skbn) { + flow_skb_fixup(skbn, flow_node); + rmnet_perf_core_send_skb(skbn, ep, perf, NULL); + } else { + rmnet_perf_opt_oom_drops += + flow_node->num_pkts_held; + } + /* equivalent to memsetting the flow node */ + flow_node->num_pkts_held = 0; + } + } +} + +/* rmnet_perf_opt_flush_all_flow_nodes() - Iterate through all flow nodes + * and flush them individually + * @perf: allows access to our required global structures + * + * Return: + * - void + **/ +void rmnet_perf_opt_flush_all_flow_nodes(struct rmnet_perf *perf) +{ + struct rmnet_perf_opt_flow_node *flow_node; + int bkt_cursor; + int num_pkts_held; + u32 hash_val; + + hash_for_each(rmnet_perf_opt_fht, bkt_cursor, flow_node, list) { + hash_val = flow_node->hash_value; + num_pkts_held = flow_node->num_pkts_held; + if (num_pkts_held > 0) { + rmnet_perf_opt_flush_single_flow_node(perf, flow_node); + //rmnet_perf_core_flush_single_gro_flow(hash_val); + } + } +} + +/* rmnet_perf_opt_insert_pkt_in_flow() - Inserts single IP packet into + * opt meta structure + * @skb: pointer to packet given to us by physical device + * @flow_node: flow node we are going to insert the ip packet into + * @pkt_info: characteristics of the current packet + * + * Return: + * - void + **/ +void rmnet_perf_opt_insert_pkt_in_flow(struct sk_buff *skb, + struct rmnet_perf_opt_flow_node *flow_node, + struct rmnet_perf_pkt_info *pkt_info) +{ + struct rmnet_perf_opt_pkt_node *pkt_node; + struct tcphdr *tp = pkt_info->trns_hdr.tp; + void *iph = (void *) pkt_info->iphdr.v4hdr; + u16 header_len = pkt_info->header_len; + u16 payload_len = pkt_info->payload_len; + unsigned char ip_version = pkt_info->ip_proto; + + pkt_node = &flow_node->pkt_list[flow_node->num_pkts_held]; + pkt_node->data_end = (unsigned char *) iph + header_len + payload_len; + if (pkt_info->trans_proto == IPPROTO_TCP) + flow_node->next_seq = ntohl(tp->seq) + + (__force u32) payload_len; + + if (pkt_info->first_packet) { + pkt_node->ip_start = (unsigned char *) iph; + pkt_node->data_start = (unsigned char *) iph; + flow_node->len = header_len + payload_len; + flow_node->mux_id = RMNET_MAP_GET_MUX_ID(skb); + flow_node->src_port = tp->source; + flow_node->dest_port = tp->dest; + flow_node->hash_value = pkt_info->hash_key; + flow_node->gso_len = payload_len; + + if (pkt_info->trans_proto == IPPROTO_TCP) + flow_node->timestamp = pkt_info->curr_timestamp; + + if (ip_version == 0x04) { + flow_node->saddr.saddr4 = + (__be32) ((struct iphdr *) iph)->saddr; + flow_node->daddr.daddr4 = + (__be32) ((struct iphdr *) iph)->daddr; + flow_node->protocol = ((struct iphdr *) iph)->protocol; + } else if (ip_version == 0x06) { + flow_node->saddr.saddr6 = + ((struct ipv6hdr *) iph)->saddr; + flow_node->daddr.daddr6 = + ((struct ipv6hdr *) iph)->daddr; + flow_node->protocol = ((struct ipv6hdr *) iph)->nexthdr; + } else { + pr_err("%s(): Encountered invalid ip version\n", + __func__); + /* TODO as Vamsi mentioned get a way to handle + * this case... still want to send packet up NW stack + */ + } + flow_node->num_pkts_held = 1; + } else { + pkt_node->ip_start = (unsigned char *) iph; + pkt_node->data_start = (unsigned char *) iph + header_len; + flow_node->len += payload_len; + flow_node->num_pkts_held++; + } +} + +/* rmnet_perf_opt_ingress() - Core business logic of optimization framework + * @perf: allows access to our required global structures + * @skb: the incoming ip packet + * @pkt_info: characteristics of the current packet + * + * Makes determination of what to do with a given incoming + * ip packet. Find matching flow if it exists and call protocol- + * specific helper to try and insert the packet and handle any + * flushing needed. + * + * Return: + * - true if packet has been handled + * - false if caller needs to flush packet + **/ +bool rmnet_perf_opt_ingress(struct rmnet_perf *perf, struct sk_buff *skb, + struct rmnet_perf_pkt_info *pkt_info) +{ + struct rmnet_perf_opt_flow_node *flow_node; + struct rmnet_perf_opt_flow_node *flow_node_recycled; + bool flush; + bool handled = false; + bool flow_node_exists = false; + + spin_lock(&rmnet_perf_opt_lock); + if (!optimize_protocol(pkt_info->trans_proto)) + goto out; + +handle_pkt: + hash_for_each_possible(rmnet_perf_opt_fht, flow_node, list, + pkt_info->hash_key) { + if (!identify_flow(flow_node, pkt_info)) + continue; + + flush = ip_flag_flush(flow_node, pkt_info); + + /* set this to true by default. Let the protocol helpers + * change this if it is needed. + */ + pkt_info->first_packet = true; + flow_node_exists = true; + + switch (pkt_info->trans_proto) { + case IPPROTO_TCP: + rmnet_perf_tcp_opt_ingress(perf, skb, flow_node, + pkt_info, flush); + handled = true; + goto out; + case IPPROTO_UDP: + rmnet_perf_udp_opt_ingress(perf, skb, flow_node, + pkt_info, flush); + handled = true; + goto out; + default: + pr_err("%s(): Unhandled protocol %u\n", + __func__, pkt_info->trans_proto); + goto out; + } + } + + /* If we didn't find the flow, we need to add it and try again */ + if (!flow_node_exists) { + flow_node_recycled = get_new_flow_index(perf); + flow_node_recycled->hash_value = pkt_info->hash_key; + rmnet_perf_opt_update_flow(flow_node_recycled, pkt_info); + hash_add(rmnet_perf_opt_fht, &flow_node_recycled->list, + pkt_info->hash_key); + goto handle_pkt; + } + +out: + spin_unlock(&rmnet_perf_opt_lock); + return handled; +} diff --git a/drivers/rmnet/perf/rmnet_perf_opt.h b/drivers/rmnet/perf/rmnet_perf_opt.h new file mode 100644 index 0000000..4a785cb --- /dev/null +++ b/drivers/rmnet/perf/rmnet_perf_opt.h @@ -0,0 +1,91 @@ +/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef _RMNET_PERF_OPT_H_ +#define _RMNET_PERF_OPT_H_ + +#include <linux/skbuff.h> +#include "rmnet_perf_core.h" + +#define RMNET_PERF_FLOW_HASH_TABLE_BITS 4 +#define RMNET_PERF_NUM_FLOW_NODES 8 + +struct rmnet_perf_opt_pkt_node { + unsigned char *ip_start; /* This is simply used for debug purposes */ + unsigned char *data_start; + unsigned char *data_end; +}; + +struct rmnet_perf_opt_ip_flags { + u8 ip_ttl; + u8 ip_tos; + u16 ip_frag_off; +}; + +struct rmnet_perf_opt_flow_node { + u8 mux_id; + u8 protocol; + u8 num_pkts_held; + union { + struct rmnet_perf_opt_ip_flags ip4_flags; + __be32 first_word; + } ip_flags; + u32 timestamp; + __be32 next_seq; + u32 gso_len; + u32 len; + u32 hash_value; + + __be16 src_port; + __be16 dest_port; + union { + __be32 saddr4; + struct in6_addr saddr6; + } saddr; + union { + __be32 daddr4; + struct in6_addr daddr6; + } daddr; + + struct hlist_node list; + struct rmnet_perf_opt_pkt_node pkt_list[50]; +}; + +struct rmnet_perf_opt_flow_node_pool { + u8 num_flows_in_use; + u16 flow_recycle_counter; + struct rmnet_perf_opt_flow_node * + node_list[RMNET_PERF_NUM_FLOW_NODES]; +}; + +struct rmnet_perf_opt_meta { + struct rmnet_perf_opt_flow_node_pool *node_pool; +}; + +enum rmnet_perf_opt_flush_reasons { + RMNET_PERF_OPT_PACKET_CORRUPT_ERROR, + RMNET_PERF_OPT_NUM_CONDITIONS +}; + +void +rmnet_perf_opt_update_flow(struct rmnet_perf_opt_flow_node *flow_node, + struct rmnet_perf_pkt_info *pkt_info); +void rmnet_perf_opt_flush_single_flow_node(struct rmnet_perf *perf, + struct rmnet_perf_opt_flow_node *flow_node); +void rmnet_perf_opt_flush_all_flow_nodes(struct rmnet_perf *perf); +void rmnet_perf_opt_insert_pkt_in_flow(struct sk_buff *skb, + struct rmnet_perf_opt_flow_node *flow_node, + struct rmnet_perf_pkt_info *pkt_info); +bool rmnet_perf_opt_ingress(struct rmnet_perf *perf, struct sk_buff *skb, + struct rmnet_perf_pkt_info *pkt_info); + +#endif /* _RMNET_PERF_OPT_H_ */ diff --git a/drivers/rmnet/perf/rmnet_perf_tcp_opt.c b/drivers/rmnet/perf/rmnet_perf_tcp_opt.c index f673884..5dc2224 100644 --- a/drivers/rmnet/perf/rmnet_perf_tcp_opt.c +++ b/drivers/rmnet/perf/rmnet_perf_tcp_opt.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -16,15 +16,12 @@ #include <linux/netdevice.h> #include <linux/ip.h> #include <linux/ipv6.h> -#include <linux/jhash.h> #include <net/ip6_checksum.h> #include <net/tcp.h> #include <linux/module.h> #include <asm/unaligned.h> -#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_config.h> #include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h> -#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_private.h> -#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_handlers.h> +#include "rmnet_perf_opt.h" #include "rmnet_perf_tcp_opt.h" #include "rmnet_perf_core.h" #include "rmnet_perf_config.h" @@ -36,8 +33,8 @@ MODULE_PARM_DESC(rmnet_perf_tcp_opt_flush_limit, "Max flush limiit for tcp_opt"); /* Stat showing reason for flushes of flow nodes */ -unsigned long int rmnet_perf_tcp_opt_flush_reason_cnt[ - RMNET_PERF_TCP_OPT_NUM_CONDITIONS]; +unsigned long int +rmnet_perf_tcp_opt_flush_reason_cnt[RMNET_PERF_TCP_OPT_NUM_CONDITIONS]; module_param_array(rmnet_perf_tcp_opt_flush_reason_cnt, ulong, 0, 0444); MODULE_PARM_DESC(rmnet_perf_tcp_opt_flush_reason_cnt, "tcp_opt performance statistics"); @@ -48,11 +45,6 @@ module_param(rmnet_perf_tcp_opt_post_ip_count, ulong, 0644); MODULE_PARM_DESC(rmnet_perf_tcp_opt_post_ip_count, "Number of packets of MTU size, post-tcp_opt"); -/* If true then we allocate all large SKBs */ -unsigned long int rmnet_perf_tcp_opt_skb_recycle_off = 1; -module_param(rmnet_perf_tcp_opt_skb_recycle_off, ulong, 0644); -MODULE_PARM_DESC(rmnet_perf_tcp_opt_skb_recycle_off, "Num skbs max held"); - unsigned long int rmnet_perf_tcp_opt_fn_seq = 0; module_param(rmnet_perf_tcp_opt_fn_seq, ulong, 0644); MODULE_PARM_DESC(rmnet_perf_tcp_opt_fn_seq, "flow node seq"); @@ -61,52 +53,6 @@ unsigned long int rmnet_perf_tcp_opt_pkt_seq = 0; module_param(rmnet_perf_tcp_opt_pkt_seq, ulong, 0644); MODULE_PARM_DESC(rmnet_perf_tcp_opt_pkt_seq, "incoming pkt seq"); -/* flow hash table */ -DEFINE_HASHTABLE(rmnet_perf_tcp_opt_fht, RMNET_PERF_FLOW_HASH_TABLE_BITS); - -/* rmnet_perf_tcp_opt_ip_flag_flush() - Check IP header flags to decide if - * immediate flush required - * @pkt_info: characteristics of the current packet - * - * If the IP header has any flag set that GRO won't accept, we will flush the - * packet right away. - * - * Return: - * - true if need flush - * - false if immediate flush may not be needed - **/ -static bool -rmnet_perf_tcp_opt_ip_flag_flush(struct rmnet_perf_tcp_opt_flow_node *flow_node, - struct rmnet_perf_pkt_info *pkt_info) -{ - struct iphdr *ip4h; - struct ipv6hdr *ip6h; - __be32 first_word; - //struct rmnet_perf_tcp_opt_ip_flags *flags; - - switch (pkt_info->ip_proto) { - case 0x04: - ip4h = pkt_info->iphdr.v4hdr; - if ((ip4h->ttl ^ flow_node->ip_flags.ip4_flags.ip_ttl) || - (ip4h->tos ^ flow_node->ip_flags.ip4_flags.ip_tos) || - (ip4h->frag_off ^ - flow_node->ip_flags.ip4_flags.ip_frag_off)) - return true; - break; - case 0x06: - ip6h = (struct ipv6hdr *) pkt_info->iphdr.v6hdr; - first_word = *(__be32 *)ip6h ^ flow_node->ip_flags.first_word; - if (!!(first_word & htonl(0x0FF00000))) - return true; - break; - default: - pr_err("Unsupported ip version %d", pkt_info->ip_proto); - rmnet_perf_tcp_opt_flush_reason_cnt[ - RMNET_PERF_TCP_OPT_PACKET_CORRUPT_ERROR]++; - } - return false; -} - /* rmnet_perf_tcp_opt_tcp_flag_flush() - Check TCP header flag to decide if * immediate flush required * @pkt_info: characteristics of the current packet @@ -144,7 +90,7 @@ rmnet_perf_tcp_opt_tcp_flag_flush(struct rmnet_perf_pkt_info *pkt_info) **/ static enum rmnet_perf_tcp_opt_merge_check_rc rmnet_perf_tcp_opt_pkt_can_be_merged(struct sk_buff *skb, - struct rmnet_perf_tcp_opt_flow_node *flow_node, + struct rmnet_perf_opt_flow_node *flow_node, struct rmnet_perf_pkt_info *pkt_info) { struct iphdr *ip4h; @@ -192,221 +138,14 @@ rmnet_perf_tcp_opt_pkt_can_be_merged(struct sk_buff *skb, rmnet_perf_tcp_opt_flush_reason_cnt[ RMNET_PERF_TCP_OPT_NO_SPACE_IN_NODE]++; return RMNET_PERF_TCP_OPT_FLUSH_SOME; + } else if (flow_node->gso_len != payload_len) { + rmnet_perf_tcp_opt_flush_reason_cnt[ + RMNET_PERF_TCP_OPT_LENGTH_MISMATCH]++; + return RMNET_PERF_TCP_OPT_FLUSH_SOME; } return RMNET_PERF_TCP_OPT_MERGE_SUCCESS; } -/* rmnet_perf_tcp_opt_make_new_skb_for_flow() - Allocate and populate SKB for - * flow node that is being pushed up the stack - * @perf: allows access to our required global structures - * @flow_node: tcp_opt structure containing packet we are allocating for - * - * Allocate skb of proper size for tcp_opt'd packet, and memcpy data - * into the buffer - * - * Return: - * - skbn: sk_buff to then push up the NW stack - * - NULL: if memory allocation failed - **/ -static struct sk_buff * -rmnet_perf_tcp_opt_make_new_skb_for_flow(struct rmnet_perf *perf, - struct rmnet_perf_tcp_opt_flow_node *flow_node) -{ - struct sk_buff *skbn; - struct rmnet_perf_tcp_opt_pkt_node *pkt_list; - int i; - u32 pkt_size; - u32 total_pkt_size = 0; - - if (rmnet_perf_tcp_opt_skb_recycle_off) { - skbn = alloc_skb(flow_node->len + RMNET_MAP_DEAGGR_SPACING, - GFP_ATOMIC); - if (!skbn) - return NULL; - } else { - skbn = rmnet_perf_core_elligible_for_cache_skb(perf, flow_node->len); - if (!skbn) { - skbn = alloc_skb(flow_node->len + RMNET_MAP_DEAGGR_SPACING, - GFP_ATOMIC); - if (!skbn) - return NULL; - } - } - pkt_list = flow_node->pkt_list; - - for (i = 0; i < flow_node->num_pkts_held; i++) { - pkt_size = pkt_list[i].data_end - pkt_list[i].data_start; - memcpy(skbn->data + skbn->len, pkt_list[i].data_start, - pkt_size); - skb_put(skbn, pkt_size); - total_pkt_size += pkt_size; - } - if (flow_node->len != total_pkt_size) - pr_err("%s(): skbn = %pK, flow_node->len = %u, pkt_size = %u\n", - __func__, skbn, flow_node->len, total_pkt_size); - - return skbn; -} - -/* rmnet_perf_tcp_opt_update_flow() - Update stored IP flow information - * @flow_node: tcp_opt structure containing flow information - * @pkt_info: characteristics of the current packet - * - * Update IP-specific flags stored about the flow (i.e. ttl, tos/traffic class, - * fragment information, flags). - * - * Return: - * - void - **/ -static void -rmnet_perf_tcp_opt_update_flow(struct rmnet_perf_tcp_opt_flow_node *flow_node, - struct rmnet_perf_pkt_info *pkt_info) -{ - if (pkt_info->ip_proto == 0x04) { - struct iphdr *iph = pkt_info->iphdr.v4hdr; - - flow_node->ip_flags.ip4_flags.ip_ttl = iph->ttl; - flow_node->ip_flags.ip4_flags.ip_tos = iph->tos; - flow_node->ip_flags.ip4_flags.ip_frag_off = iph->frag_off; - } else if (pkt_info->ip_proto == 0x06) { - __be32 *word = (__be32 *)pkt_info->iphdr.v6hdr; - - flow_node->ip_flags.first_word = *word; - } -} - -/* rmnet_perf_tcp_opt_flush_single_flow_node() - Send a given flow node up - * NW stack. - * @perf: allows access to our required global structures - * @flow_node: tcp_opt structure containing packet we are allocating for - * - * Send a given flow up NW stack via specific VND - * - * Return: - * - skbn: sk_buff to then push up the NW stack - **/ -static void rmnet_perf_tcp_opt_flush_single_flow_node(struct rmnet_perf *perf, - struct rmnet_perf_tcp_opt_flow_node *flow_node) -{ - struct sk_buff *skbn; - struct rmnet_endpoint *ep; - - /* future change: when inserting the first packet in a flow, - * save away the ep value so we dont have to look it up every flush - */ - hlist_for_each_entry_rcu(ep, - &perf->rmnet_port->muxed_ep[flow_node->mux_id], - hlnode) { - if (ep->mux_id == flow_node->mux_id) { - if (flow_node->num_pkts_held) { - skbn = - rmnet_perf_tcp_opt_make_new_skb_for_flow(perf, - flow_node); - if (!skbn) { - pr_err("%s(): skbn is NULL\n", - __func__); - } else { - skbn->hash = flow_node->hash_value; - skbn->sw_hash = 1; - /* data is already validated */ - skbn->ip_summed = CHECKSUM_UNNECESSARY; - rmnet_perf_core_send_skb(skbn, ep, - perf, NULL); - } - /* equivalent to memsetting the flow node */ - flow_node->num_pkts_held = 0; - } - } - } -} - -/* rmnet_perf_tcp_opt_flush_all_flow_nodes() - Iterate through all flow nodes - * and flush them individually - * @perf: allows access to our required global structures - * - * Return: - * - void - **/ -void rmnet_perf_tcp_opt_flush_all_flow_nodes(struct rmnet_perf *perf) -{ - struct rmnet_perf_tcp_opt_flow_node *flow_node; - int bkt_cursor; - int num_pkts_held; - u32 hash_val; - - hash_for_each(rmnet_perf_tcp_opt_fht, bkt_cursor, flow_node, list) { - hash_val = flow_node->hash_value; - num_pkts_held = flow_node->num_pkts_held; - //if (num_pkts_held > 0 && num_pkts_held < 3) - if (num_pkts_held > 0) { - rmnet_perf_tcp_opt_flush_single_flow_node(perf, - flow_node); - //rmnet_perf_core_flush_single_gro_flow(hash_val); - } - } -} - -/* rmnet_perf_tcp_opt_insert_pkt_in_flow() - Inserts single IP packet into - * tcp_opt meta structure - * @skb: pointer to packet given to us by physical device - * @flow_node: flow node we are going to insert the ip packet into - * @pkt_info: characteristics of the current packet - * - * Return: - * - void - **/ -static void rmnet_perf_tcp_opt_insert_pkt_in_flow(struct sk_buff *skb, - struct rmnet_perf_tcp_opt_flow_node *flow_node, - struct rmnet_perf_pkt_info *pkt_info) -{ - struct rmnet_perf_tcp_opt_pkt_node *pkt_node; - struct tcphdr *tp = pkt_info->trns_hdr.tp; - void *iph = (void *) pkt_info->iphdr.v4hdr; - u16 header_len = pkt_info->header_len; - u16 payload_len = pkt_info->payload_len; - unsigned char ip_version = pkt_info->ip_proto; - - pkt_node = &flow_node->pkt_list[flow_node->num_pkts_held]; - pkt_node->data_end = (unsigned char *) iph + header_len + payload_len; - flow_node->next_seq = ntohl(tp->seq) + (__force u32) payload_len; - - if (pkt_info->first_packet) { - pkt_node->ip_start = (unsigned char *) iph; - pkt_node->data_start = (unsigned char *) iph; - flow_node->len = header_len + payload_len; - flow_node->mux_id = RMNET_MAP_GET_MUX_ID(skb); - flow_node->src_port = tp->source; - flow_node->dest_port = tp->dest; - flow_node->timestamp = pkt_info->curr_timestamp; - flow_node->hash_value = pkt_info->hash_key; - if (ip_version == 0x04) { - flow_node->saddr.saddr4 = - (__be32) ((struct iphdr *) iph)->saddr; - flow_node->daddr.daddr4 = - (__be32) ((struct iphdr *) iph)->daddr; - flow_node->protocol = ((struct iphdr *) iph)->protocol; - } else if (ip_version == 0x06) { - flow_node->saddr.saddr6 = - ((struct ipv6hdr *) iph)->saddr; - flow_node->daddr.daddr6 = - ((struct ipv6hdr *) iph)->daddr; - flow_node->protocol = ((struct ipv6hdr *) iph)->nexthdr; - } else { - pr_err("%s(): Encountered invalid ip version\n", - __func__); - /* TODO as Vamsi mentioned get a way to handle - * this case... still want to send packet up NW stack - */ - } - flow_node->num_pkts_held = 1; - } else { - pkt_node->ip_start = (unsigned char *) iph; - pkt_node->data_start = (unsigned char *) iph + header_len; - flow_node->len += payload_len; - flow_node->num_pkts_held++; - } -} - /* rmnet_perf_tcp_opt_check_timestamp() -Check timestamp of incoming packet * @skb: incoming packet to check * @tp: pointer to tcp header of incoming packet @@ -455,103 +194,11 @@ static u32 rmnet_perf_tcp_opt_check_timestamp(struct sk_buff *skb, return 0; } -/* rmnet_perf_tcp_opt_identify_flow() - Tell whether packet corresponds to - * given flow - * @flow_node: Node we are checking against - * @pkt_info: characteristics of the current packet - * - * Checks to see if the incoming packet is a match for a given flow node - * - * Return: - * - true: it is a match - * - false: not a match - **/ -static bool -rmnet_perf_tcp_opt_identify_flow(struct rmnet_perf_tcp_opt_flow_node *flow_node, - struct rmnet_perf_pkt_info *pkt_info) -{ - struct iphdr *ip4h; - struct ipv6hdr *ip6h; - struct tcphdr *tp = pkt_info->trns_hdr.tp; - - //if pkt count == 0 and hash is the same, then we give this one as - //pass as good enough - //since at this point there is no address stuff to check/verify. - if (flow_node->num_pkts_held == 0 && - flow_node->hash_value == pkt_info->hash_key) - return true; - - /* cast iph to right ip header struct for ip_version */ - switch (pkt_info->ip_proto) { - case 0x04: - ip4h = pkt_info->iphdr.v4hdr; - if (((__force u32)flow_node->saddr.saddr4 ^ - (__force u32)ip4h->saddr) | - ((__force u32)flow_node->daddr.daddr4 ^ - (__force u32)ip4h->daddr) | - ((__force u16)flow_node->src_port ^ - (__force u16)tp->source) | - ((__force u16)flow_node->dest_port ^ - (__force u16)tp->dest)) - return false; - break; - case 0x06: - ip6h = pkt_info->iphdr.v6hdr; - if ((ipv6_addr_cmp(&(flow_node->saddr.saddr6), &ip6h->saddr)) | - (ipv6_addr_cmp(&(flow_node->daddr.daddr6), - &ip6h->daddr)) | - ((__force u16)flow_node->src_port ^ - (__force u16)tp->source) | - ((__force u16)flow_node->dest_port ^ - (__force u16)tp->dest)) - return false; - break; - default: - pr_err("Unsupported ip version %d", pkt_info->ip_proto); - rmnet_perf_tcp_opt_flush_reason_cnt[ - RMNET_PERF_TCP_OPT_PACKET_CORRUPT_ERROR]++; - return false; - } - return true; -} - -/* rmnet_perf_tcp_opt_get_new_flow_node_index() - Pull flow node from node pool - * @perf: allows access to our required global structures - * - * Fetch the flow node from the node pool. If we have already given - * out all the flow nodes then we will always hit the else case and - * thereafter we will use modulo arithmetic to choose which flow node - * to evict and use. - * - * Return: - * - flow_node: node to be used by caller function - **/ -static struct rmnet_perf_tcp_opt_flow_node * -rmnet_perf_tcp_opt_get_new_flow_node_index(struct rmnet_perf *perf) -{ - struct rmnet_perf_tcp_opt_flow_node_pool *node_pool; - struct rmnet_perf_tcp_opt_flow_node *flow_node_ejected; - - node_pool = perf->tcp_opt_meta->node_pool; - /* once this value gets too big it never goes back down. - * from that point forward we use flow node repurposing techniques - * instead - */ - if (node_pool->num_flows_in_use < RMNET_PERF_NUM_FLOW_NODES) - return node_pool->node_list[node_pool->num_flows_in_use++]; - - flow_node_ejected = node_pool->node_list[ - node_pool->flow_recycle_counter++ % RMNET_PERF_NUM_FLOW_NODES]; - rmnet_perf_tcp_opt_flush_single_flow_node(perf, - flow_node_ejected); - hash_del(&flow_node_ejected->list); - return flow_node_ejected; -} - /* rmnet_perf_tcp_opt_ingress() - Core business logic of tcp_opt * @perf: allows access to our required global structures * @skb: the incoming ip packet * @pkt_info: characteristics of the current packet + * @flush: IP flag mismatch detected * * Makes determination of what to do with a given incoming * ip packet. All other tcp_opt based checks originate from here. @@ -562,105 +209,60 @@ rmnet_perf_tcp_opt_get_new_flow_node_index(struct rmnet_perf *perf) * - void **/ void rmnet_perf_tcp_opt_ingress(struct rmnet_perf *perf, struct sk_buff *skb, - struct rmnet_perf_pkt_info *pkt_info) + struct rmnet_perf_opt_flow_node *flow_node, + struct rmnet_perf_pkt_info *pkt_info, + bool flush) { - struct rmnet_perf_tcp_opt_flow_node *flow_node; - struct rmnet_perf_tcp_opt_flow_node *flow_node_recycled; - bool flush_pkt_now; bool timestamp_mismatch; - bool match; - enum rmnet_perf_tcp_opt_merge_check_rc rc = 0; - bool flow_node_exists = 0; + enum rmnet_perf_tcp_opt_merge_check_rc rc; struct napi_struct *napi = NULL; - //pkt_info->hash_key = rmnet_perf_core_compute_flow_hash(pkt_info); -handle_pkt: - hash_for_each_possible(rmnet_perf_tcp_opt_fht, flow_node, list, - pkt_info->hash_key) { - match = rmnet_perf_tcp_opt_identify_flow(flow_node, pkt_info); - if (!match) - continue; - - flush_pkt_now = rmnet_perf_tcp_opt_tcp_flag_flush(pkt_info) | - rmnet_perf_tcp_opt_ip_flag_flush(flow_node, pkt_info); - pkt_info->curr_timestamp = - rmnet_perf_tcp_opt_check_timestamp(skb, - pkt_info->trns_hdr.tp, - perf->core_meta->dev); - /* set this to true by default... only change it if we - * identify that we have merged successfully - */ - pkt_info->first_packet = true; - timestamp_mismatch = (flow_node->timestamp != - pkt_info->curr_timestamp) ? 1 : 0; - flow_node_exists = 1; + if (flush || rmnet_perf_tcp_opt_tcp_flag_flush(pkt_info)) { + rmnet_perf_opt_update_flow(flow_node, pkt_info); + rmnet_perf_opt_flush_single_flow_node(perf, flow_node); + napi = get_current_napi_context(); + napi_gro_flush(napi, false); + rmnet_perf_core_flush_curr_pkt(perf, skb, pkt_info, + pkt_info->header_len + + pkt_info->payload_len, true, + false); + napi_gro_flush(napi, false); + rmnet_perf_tcp_opt_flush_reason_cnt[ + RMNET_PERF_TCP_OPT_TCP_FLUSH_FORCE]++; + return; + } - if (flow_node->num_pkts_held > 0) { - rc = rmnet_perf_tcp_opt_pkt_can_be_merged(skb, - flow_node, pkt_info); - if (flush_pkt_now) { - rmnet_perf_tcp_opt_update_flow(flow_node, - pkt_info); - rmnet_perf_tcp_opt_flush_single_flow_node(perf, - flow_node); - napi = get_current_napi_context(); - napi_gro_flush(napi, false); - rmnet_perf_core_flush_curr_pkt(perf, skb, - pkt_info, - pkt_info->header_len + pkt_info->payload_len); - napi_gro_flush(napi, false); - rmnet_perf_tcp_opt_flush_reason_cnt[ - RMNET_PERF_TCP_OPT_TCP_FLUSH_FORCE]++; - } else if (rc == RMNET_PERF_TCP_OPT_FLUSH_ALL) { - rmnet_perf_tcp_opt_flush_single_flow_node(perf, - flow_node); - rmnet_perf_core_flush_curr_pkt(perf, skb, - pkt_info, - pkt_info->header_len + pkt_info->payload_len); - } else if (rc == RMNET_PERF_TCP_OPT_FLUSH_SOME) { - rmnet_perf_tcp_opt_flush_single_flow_node(perf, - flow_node); - rmnet_perf_tcp_opt_insert_pkt_in_flow(skb, - flow_node, pkt_info); - } else if (rc == RMNET_PERF_TCP_OPT_FLUSH_SOME_GRO) { - rmnet_perf_tcp_opt_flush_single_flow_node(perf, - flow_node); - //rmnet_perf_core_flush_single_gro_flow( - // pkt_info->hash_key); - rmnet_perf_tcp_opt_insert_pkt_in_flow(skb, - flow_node, pkt_info); - } else if (timestamp_mismatch) { - rmnet_perf_tcp_opt_flush_single_flow_node(perf, - flow_node); - rmnet_perf_tcp_opt_insert_pkt_in_flow(skb, - flow_node, pkt_info); - rmnet_perf_tcp_opt_flush_reason_cnt[ - RMNET_PERF_TCP_OPT_TIMESTAMP_MISMATCH]++; - } else if (rc == RMNET_PERF_TCP_OPT_MERGE_SUCCESS) { - pkt_info->first_packet = false; - rmnet_perf_tcp_opt_insert_pkt_in_flow(skb, - flow_node, pkt_info); - } - } else if (flush_pkt_now) { - rmnet_perf_tcp_opt_update_flow(flow_node, pkt_info); - rmnet_perf_core_flush_curr_pkt(perf, skb, pkt_info, - pkt_info->header_len + pkt_info->payload_len); - napi = get_current_napi_context(); - napi_gro_flush(napi, false); - rmnet_perf_tcp_opt_flush_reason_cnt[ - RMNET_PERF_TCP_OPT_TCP_FLUSH_FORCE]++; - } else - rmnet_perf_tcp_opt_insert_pkt_in_flow(skb, flow_node, - pkt_info); - break; + /* Go ahead and insert the packet now if we're not holding anything. + * We know at this point that it's a normal packet in the flow + */ + if (!flow_node->num_pkts_held) { + rmnet_perf_opt_insert_pkt_in_flow(skb, flow_node, pkt_info); + return; } - if (!flow_node_exists) { - flow_node_recycled = - rmnet_perf_tcp_opt_get_new_flow_node_index(perf); - flow_node_recycled->hash_value = pkt_info->hash_key; - rmnet_perf_tcp_opt_update_flow(flow_node_recycled, pkt_info); - hash_add(rmnet_perf_tcp_opt_fht, &flow_node_recycled->list, - pkt_info->hash_key); - goto handle_pkt; + + pkt_info->curr_timestamp = + rmnet_perf_tcp_opt_check_timestamp(skb, + pkt_info->trns_hdr.tp, + perf->core_meta->dev); + timestamp_mismatch = flow_node->timestamp != pkt_info->curr_timestamp; + + rc = rmnet_perf_tcp_opt_pkt_can_be_merged(skb, flow_node, pkt_info); + if (rc == RMNET_PERF_TCP_OPT_FLUSH_ALL) { + rmnet_perf_opt_flush_single_flow_node(perf, flow_node); + rmnet_perf_core_flush_curr_pkt(perf, skb, pkt_info, + pkt_info->header_len + + pkt_info->payload_len, false, + false); + } else if (rc == RMNET_PERF_TCP_OPT_FLUSH_SOME) { + rmnet_perf_opt_flush_single_flow_node(perf, flow_node); + rmnet_perf_opt_insert_pkt_in_flow(skb, flow_node, pkt_info); + } else if (timestamp_mismatch) { + rmnet_perf_opt_flush_single_flow_node(perf, flow_node); + rmnet_perf_opt_insert_pkt_in_flow(skb, flow_node, pkt_info); + rmnet_perf_tcp_opt_flush_reason_cnt[ + RMNET_PERF_TCP_OPT_TIMESTAMP_MISMATCH]++; + } else if (rc == RMNET_PERF_TCP_OPT_MERGE_SUCCESS) { + pkt_info->first_packet = false; + rmnet_perf_opt_insert_pkt_in_flow(skb, flow_node, pkt_info); } } diff --git a/drivers/rmnet/perf/rmnet_perf_tcp_opt.h b/drivers/rmnet/perf/rmnet_perf_tcp_opt.h index dc2f4f8..2a6f2c6 100644 --- a/drivers/rmnet/perf/rmnet_perf_tcp_opt.h +++ b/drivers/rmnet/perf/rmnet_perf_tcp_opt.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -17,12 +17,6 @@ #ifndef _RMNET_PERF_TCP_OPT_H_ #define _RMNET_PERF_TCP_OPT_H_ -#define RMNET_PERF_FLOW_HASH_TABLE_BITS 4 -#define RMNET_PERF_FLOW_HASH_TABLE_BUCKETS 16 -#define RMNET_PERF_NUM_FLOW_NODES 8 -#define RMNET_PERF_TCP_OPT_HEADER_PKT 1 -#define RMNET_PERF_TCP_OPT_PAYLOAD_PKT 0 - enum rmnet_perf_tcp_opt_merge_check_rc { /* merge pkt into flow node */ RMNET_PERF_TCP_OPT_MERGE_SUCCESS, @@ -30,8 +24,6 @@ enum rmnet_perf_tcp_opt_merge_check_rc { RMNET_PERF_TCP_OPT_FLUSH_ALL, /* flush flow nodes, but insert pkt into newly empty flow */ RMNET_PERF_TCP_OPT_FLUSH_SOME, - /* flush flow nodes, but insert pkt into newly empty flow, flush GRO*/ - RMNET_PERF_TCP_OPT_FLUSH_SOME_GRO, }; enum rmnet_perf_tcp_opt_flush_reasons { @@ -41,76 +33,13 @@ enum rmnet_perf_tcp_opt_flush_reasons { RMNET_PERF_TCP_OPT_NO_SPACE_IN_NODE, RMNET_PERF_TCP_OPT_FLOW_NODE_SHORTAGE, RMNET_PERF_TCP_OPT_OUT_OF_ORDER_SEQ, - RMNET_PERF_TCP_OPT_CHECKSUM_ERR, RMNET_PERF_TCP_OPT_PACKET_CORRUPT_ERROR, + RMNET_PERF_TCP_OPT_LENGTH_MISMATCH, RMNET_PERF_TCP_OPT_NUM_CONDITIONS }; -struct rmnet_perf_tcp_opt_pkt_node { - unsigned char *ip_start; /* This is simply used for debug purposes */ - unsigned char *data_start; - unsigned char *data_end; -}; - -struct rmnet_perf_tcp_opt_ip_flags { - u8 ip_ttl; - u8 ip_tos; - u16 ip_frag_off; -}; - -struct rmnet_perf_tcp_opt_flow_node { - u8 mux_id; - u8 protocol; - u8 num_pkts_held; - union { - struct rmnet_perf_tcp_opt_ip_flags ip4_flags; - __be32 first_word; - } ip_flags; - u32 timestamp; - __be32 next_seq; - u32 len; - u32 hash_value; - - __be16 src_port; - __be16 dest_port; - union { - __be32 saddr4; - struct in6_addr saddr6; - } saddr; - union { - __be32 daddr4; - struct in6_addr daddr6; - } daddr; - - struct hlist_node list; - struct rmnet_perf_tcp_opt_pkt_node pkt_list[50]; -}; - -struct rmnet_perf_tcp_opt_flow_node_pool { - u8 num_flows_in_use; - u16 flow_recycle_counter; - struct rmnet_perf_tcp_opt_flow_node * - node_list[RMNET_PERF_NUM_FLOW_NODES]; -}; - -struct rmnet_perf_tcp_opt_meta { - struct rmnet_perf_tcp_opt_flow_node_pool *node_pool; -}; - -void rmnet_perf_tcp_opt_dealloc_64k_buffs(struct rmnet_perf *perf); - -enum rmnet_perf_resource_management_e -rmnet_perf_tcp_opt_alloc_64k_buffs(struct rmnet_perf *perf); - -void rmnet_perf_tcp_opt_deaggregate(struct sk_buff *skb, - struct rmnet_perf *perf, - unsigned int more); - -enum rmnet_perf_resource_management_e -rmnet_perf_tcp_opt_config_free_resources(struct rmnet_perf *perf); - -void rmnet_perf_tcp_opt_free_held_skbs(struct rmnet_perf *perf); -void rmnet_perf_tcp_opt_flush_all_flow_nodes(struct rmnet_perf *perf); void rmnet_perf_tcp_opt_ingress(struct rmnet_perf *perf, struct sk_buff *skb, - struct rmnet_perf_pkt_info *pkt_info); + struct rmnet_perf_opt_flow_node *flow_node, + struct rmnet_perf_pkt_info *pkt_info, + bool flush); #endif /* _RMNET_PERF_TCP_OPT_H_ */ diff --git a/drivers/rmnet/perf/rmnet_perf_udp_opt.c b/drivers/rmnet/perf/rmnet_perf_udp_opt.c new file mode 100644 index 0000000..d730820 --- /dev/null +++ b/drivers/rmnet/perf/rmnet_perf_udp_opt.c @@ -0,0 +1,138 @@ +/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * RMNET udp_opt + * + */ + +#include <linux/netdevice.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <net/ip6_checksum.h> +#include <net/udp.h> +#include <linux/module.h> +#include <../drivers/net/ethernet/qualcomm/rmnet/rmnet_map.h> +#include "rmnet_perf_opt.h" +#include "rmnet_perf_udp_opt.h" +#include "rmnet_perf_core.h" +#include "rmnet_perf_config.h" + +/* Max number of bytes we allow udp_opt to aggregate per flow */ +unsigned int rmnet_perf_udp_opt_flush_limit __read_mostly = 65536; +module_param(rmnet_perf_udp_opt_flush_limit, uint, 0644); +MODULE_PARM_DESC(rmnet_perf_udp_opt_flush_limit, + "Max flush limiit for udp_opt"); + +/* Stat showing reason for flushes of flow nodes */ +unsigned long int +rmnet_perf_udp_opt_flush_reason_cnt[RMNET_PERF_UDP_OPT_NUM_CONDITIONS]; +module_param_array(rmnet_perf_udp_opt_flush_reason_cnt, ulong, 0, 0444); +MODULE_PARM_DESC(rmnet_perf_udp_opt_flush_reason_cnt, + "udp_opt performance statistics"); + +/* update_udp_flush_stat() - Increment a given flush statistic + * @stat: The statistic to increment + * + * Return: + * - void + */ +static inline void +update_udp_flush_stat(enum rmnet_perf_udp_opt_flush_reasons stat) +{ + if (stat < RMNET_PERF_UDP_OPT_NUM_CONDITIONS) + rmnet_perf_udp_opt_flush_reason_cnt[stat]++; +} + +/* udp_pkt_can_be_merged() - Check if packet can be merged + * @skb: Source socket buffer containing current MAP frames + * @flow_node: flow node meta data for checking condition + * @pkt_info: characteristics of the current packet + * + * 1. validate packet length + * 2. check for size overflow + * + * Return: + * - rmnet_perf_upd_opt_merge_check_rc enum indicating + * merge status + **/ +static enum rmnet_perf_udp_opt_merge_check_rc +udp_pkt_can_be_merged(struct sk_buff *skb, + struct rmnet_perf_opt_flow_node *flow_node, + struct rmnet_perf_pkt_info *pkt_info) +{ + u16 payload_len = pkt_info->payload_len; + + /* 1. validate length */ + if (flow_node->gso_len != payload_len) { + update_udp_flush_stat(RMNET_PERF_UDP_OPT_LENGTH_MISMATCH); + return RMNET_PERF_UDP_OPT_FLUSH_SOME; + } + + /* 2. check for size/count overflow */ + if ((payload_len + flow_node->len >= rmnet_perf_udp_opt_flush_limit)) { + update_udp_flush_stat(RMNET_PERF_UDP_OPT_64K_LIMIT); + return RMNET_PERF_UDP_OPT_FLUSH_SOME; + } else if ((flow_node->num_pkts_held >= 50)) { + update_udp_flush_stat(RMNET_PERF_UDP_OPT_NO_SPACE_IN_NODE); + return RMNET_PERF_UDP_OPT_FLUSH_SOME; + } + return RMNET_PERF_UDP_OPT_MERGE_SUCCESS; +} + +/* rmnet_perf_udp_opt_ingress() - Core business logic of udp_opt + * @perf: allows access to our required global structures + * @skb: the incoming ip packet + * @pkt_info: characteristics of the current packet + * @flush: IP flag mismatch detected + * + * Makes determination of what to do with a given incoming + * ip packet. All other udp_opt based checks originate from here. + * If we are working within this context then we know that + * we are operating on UDP packets. + * + * Return: + * - void + **/ +void rmnet_perf_udp_opt_ingress(struct rmnet_perf *perf, struct sk_buff *skb, + struct rmnet_perf_opt_flow_node *flow_node, + struct rmnet_perf_pkt_info *pkt_info, + bool flush) +{ + enum rmnet_perf_udp_opt_merge_check_rc rc; + + if (flush) { + rmnet_perf_opt_update_flow(flow_node, pkt_info); + rmnet_perf_opt_flush_single_flow_node(perf, flow_node); + rmnet_perf_core_flush_curr_pkt(perf, skb, pkt_info, + pkt_info->header_len + + pkt_info->payload_len, false, + false); + update_udp_flush_stat(RMNET_PERF_UDP_OPT_FLAG_MISMATCH); + return; + } + + /* Go ahead and insert the packet now if we're not holding anything. + * We know at this point that it's a normal packet in the flow + */ + if (!flow_node->num_pkts_held) { + rmnet_perf_opt_insert_pkt_in_flow(skb, flow_node, pkt_info); + return; + } + + rc = udp_pkt_can_be_merged(skb, flow_node, pkt_info); + if (rc == RMNET_PERF_UDP_OPT_FLUSH_SOME) { + rmnet_perf_opt_flush_single_flow_node(perf, flow_node); + rmnet_perf_opt_insert_pkt_in_flow(skb, flow_node, pkt_info); + } else if (rc == RMNET_PERF_UDP_OPT_MERGE_SUCCESS) { + pkt_info->first_packet = false; + rmnet_perf_opt_insert_pkt_in_flow(skb, flow_node, pkt_info); + } +} diff --git a/drivers/rmnet/perf/rmnet_perf_udp_opt.h b/drivers/rmnet/perf/rmnet_perf_udp_opt.h new file mode 100644 index 0000000..b48d79a --- /dev/null +++ b/drivers/rmnet/perf/rmnet_perf_udp_opt.h @@ -0,0 +1,40 @@ +/* Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/skbuff.h> +#include <net/udp.h> +#include "rmnet_perf_core.h" +#include "rmnet_perf_opt.h" + +#ifndef _RMNET_PERF_UDP_OPT_H_ +#define _RMNET_PERF_UDP_OPT_H_ + +enum rmnet_perf_udp_opt_merge_check_rc { + /* merge pkt into flow node */ + RMNET_PERF_UDP_OPT_MERGE_SUCCESS, + /* flush flow nodes, but insert pkt into newly empty flow */ + RMNET_PERF_UDP_OPT_FLUSH_SOME, +}; + +enum rmnet_perf_udp_opt_flush_reasons { + RMNET_PERF_UDP_OPT_FLAG_MISMATCH, + RMNET_PERF_UDP_OPT_LENGTH_MISMATCH, + RMNET_PERF_UDP_OPT_64K_LIMIT, + RMNET_PERF_UDP_OPT_NO_SPACE_IN_NODE, + RMNET_PERF_UDP_OPT_NUM_CONDITIONS +}; + +void rmnet_perf_udp_opt_ingress(struct rmnet_perf *perf, struct sk_buff *skb, + struct rmnet_perf_opt_flow_node *flow_node, + struct rmnet_perf_pkt_info *pkt_info, + bool flush); +#endif /* _RMNET_PERF_UDP_OPT_H_ */ diff --git a/drivers/rmnet/shs/rmnet_shs.h b/drivers/rmnet/shs/rmnet_shs.h index ee12024..e4450d1 100644 --- a/drivers/rmnet/shs/rmnet_shs.h +++ b/drivers/rmnet/shs/rmnet_shs.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -33,17 +33,30 @@ #define MAX_SILVER_CORES 4 #define MAX_CPUS 8 +/* RPS mask change's Default core for orphaned CPU flows */ +#define MAIN_CORE 0 +#define UPDATE_MASK 0xFF + //#define RMNET_SHS_MAX_UDP_SILVER_CORE_DATA_RATE 1073741824 //1.0Gbps //#define RMNET_SHS_MAX_UDP_SILVER_CORE_DATA_RATE 320787200 //320 Mbps //#define RMNET_SHS_MAX_UDP_GOLD_CORE_DATA_RATE 3650722201 //3.4 Gbps //#define RMNET_SHS_UDP_PPS_SILVER_CORE_UPPER_THRESH 90000 //#define RMNET_SHS_TCP_PPS_SILVER_CORE_UPPER_THRESH 90000 +#define SHS_TRACE_ERR(...) if (rmnet_shs_debug) \ + trace_rmnet_shs_err(__VA_ARGS__) + +#define SHS_TRACE_HIGH(...) if (rmnet_shs_debug) \ + trace_rmnet_shs_high(__VA_ARGS__) + +#define SHS_TRACE_LOW(...) if (rmnet_shs_debug) \ + trace_rmnet_shs_low(__VA_ARGS__) + #define RMNET_SHS_MAX_SILVER_CORE_BURST_CAPACITY 204800 #define RMNET_SHS_TCP_COALESCING_RATIO 23 //Heuristic -#define RMNET_SHS_UDP_PPS_LPWR_CPU_UTHRESH 70000 -#define RMNET_SHS_TCP_PPS_LPWR_CPU_UTHRESH (70000*RMNET_SHS_TCP_COALESCING_RATIO) +#define RMNET_SHS_UDP_PPS_LPWR_CPU_UTHRESH 80000 +#define RMNET_SHS_TCP_PPS_LPWR_CPU_UTHRESH (80000*RMNET_SHS_TCP_COALESCING_RATIO) #define RMNET_SHS_UDP_PPS_PERF_CPU_UTHRESH 210000 #define RMNET_SHS_TCP_PPS_PERF_CPU_UTHRESH (210000*RMNET_SHS_TCP_COALESCING_RATIO) @@ -74,6 +87,11 @@ struct rmnet_shs_cfg_s { u8 is_pkt_parked; u8 is_timer_init; u8 force_flush_state; + u8 rmnet_shs_init_complete; + u8 dl_ind_state; + u8 map_mask; + u8 map_len; + }; struct rmnet_shs_skb_list { @@ -81,6 +99,7 @@ struct rmnet_shs_skb_list { struct sk_buff *tail; u64 num_parked_bytes; u32 num_parked_skbs; + u32 skb_load; }; struct rmnet_shs_skbn_s { @@ -116,11 +135,32 @@ enum rmnet_shs_tmr_force_flush_state_e { RMNET_SHS_FLUSH_ON, RMNET_SHS_FLUSH_DONE }; + +enum rmnet_shs_switch_reason_e { + RMNET_SHS_SWITCH_INSTANT_RATE, + RMNET_SHS_SWITCH_WQ_RATE, + RMNET_SHS_OOO_PACKET_SWITCH, + RMNET_SHS_OOO_PACKET_TOTAL, + RMNET_SHS_SWITCH_MAX_REASON +}; + +enum rmnet_shs_dl_ind_state { + RMNET_SHS_HDR_PENDING, + RMNET_SHS_END_PENDING, + RMNET_SHS_IND_COMPLETE, + RMNET_SHS_DL_IND_MAX_STATE +}; + + enum rmnet_shs_flush_reason_e { RMNET_SHS_FLUSH_PKT_LIMIT, RMNET_SHS_FLUSH_BYTE_LIMIT, RMNET_SHS_FLUSH_TIMER_EXPIRY, RMNET_SHS_FLUSH_RX_DL_TRAILER, + RMNET_SHS_FLUSH_INV_DL_IND, + RMNET_SHS_FLUSH_WQ_FB_FLUSH, + RMNET_SHS_FLUSH_WQ_CORE_FLUSH, + RMNET_SHS_FLUSH_PSH_PKT_FLUSH, RMNET_SHS_FLUSH_MAX_REASON }; @@ -157,6 +197,13 @@ enum rmnet_shs_trace_func { RMNET_SHS_DL_MRK, }; +enum rmnet_shs_flush_context { + RMNET_RX_CTXT, + RMNET_WQ_CTXT, + RMNET_MAX_CTXT +}; + + /* Trace events and functions */ enum rmnet_shs_trace_evt { RMNET_SHS_MODULE_INIT, @@ -236,17 +283,22 @@ extern int (*rmnet_shs_skb_entry)(struct sk_buff *skb, struct rmnet_port *port); int rmnet_shs_is_lpwr_cpu(u16 cpu); void rmnet_shs_cancel_table(void); -void rmnet_shs_aggregate_init(void); +void rmnet_shs_rx_wq_init(void); +void rmnet_shs_rx_wq_exit(void); +int rmnet_shs_get_mask_len(u8 mask); -int rmnet_shs_chk_and_flush_node(struct rmnet_shs_skbn_s *node, u8 force_flush); +int rmnet_shs_chk_and_flush_node(struct rmnet_shs_skbn_s *node, + u8 force_flush, u8 ctxt); void rmnet_shs_dl_hdr_handler(struct rmnet_map_dl_ind_hdr *dlhdr); void rmnet_shs_dl_trl_handler(struct rmnet_map_dl_ind_trl *dltrl); void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port); -void rmnet_shs_flush_table(u8 is_force_flush); +void rmnet_shs_flush_table(u8 is_force_flush, u8 ctxt); void rmnet_shs_cpu_node_remove(struct rmnet_shs_skbn_s *node); -void rmnet_shs_init(struct net_device *dev); +void rmnet_shs_init(struct net_device *dev, struct net_device *vnd); void rmnet_shs_exit(void); void rmnet_shs_ps_on_hdlr(void *port); void rmnet_shs_ps_off_hdlr(void *port); void rmnet_shs_update_cpu_proc_q_all_cpus(void); + +u32 rmnet_shs_get_cpu_qhead(u8 cpu_num); #endif /* _RMNET_SHS_H_ */ diff --git a/drivers/rmnet/shs/rmnet_shs_config.c b/drivers/rmnet/shs/rmnet_shs_config.c index 3a29f04..c0c7e7b 100644 --- a/drivers/rmnet/shs/rmnet_shs_config.c +++ b/drivers/rmnet/shs/rmnet_shs_config.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -45,6 +45,7 @@ static struct notifier_block rmnet_shs_dev_notifier __read_mostly = { static int rmnet_shs_dev_notify_cb(struct notifier_block *nb, unsigned long event, void *data); +static int rmnet_vnd_total; /* Enable smart hashing capability upon call to initialize module*/ int __init rmnet_shs_module_init(void) { @@ -64,9 +65,13 @@ int __init rmnet_shs_module_init(void) void __exit rmnet_shs_module_exit(void) { RCU_INIT_POINTER(rmnet_shs_skb_entry, NULL); - rmnet_shs_cancel_table(); - rmnet_shs_wq_exit(); - rmnet_shs_exit(); + + if (rmnet_shs_cfg.rmnet_shs_init_complete) { + rmnet_shs_cancel_table(); + rmnet_shs_rx_wq_exit(); + rmnet_shs_wq_exit(); + rmnet_shs_exit(); + } unregister_netdevice_notifier(&rmnet_shs_dev_notifier); if (unlikely(rmnet_shs_debug)) pr_info("Exiting rmnet_shs module"); @@ -89,10 +94,19 @@ static int rmnet_shs_dev_notify_cb(struct notifier_block *nb, switch (event) { case NETDEV_GOING_DOWN: rmnet_shs_wq_reset_ep_active(dev); - if (rmnet_is_real_dev_registered(dev) && - !strcmp(dev->name, "rmnet_ipa0")) { + + if (strncmp(dev->name, "rmnet_data", 10) == 0) + rmnet_vnd_total--; + + /* Deinitialize if last vnd is going down or if + * phy_dev is going down. + */ + if ((rmnet_is_real_dev_registered(dev) && + !strcmp(dev->name, "rmnet_ipa0")) && + rmnet_shs_cfg.rmnet_shs_init_complete) { RCU_INIT_POINTER(rmnet_shs_skb_entry, NULL); rmnet_shs_cancel_table(); + rmnet_shs_rx_wq_exit(); rmnet_shs_wq_exit(); rmnet_shs_exit(); trace_rmnet_shs_high(RMNET_SHS_MODULE, @@ -106,15 +120,20 @@ static int rmnet_shs_dev_notify_cb(struct notifier_block *nb, if (strncmp(dev->name, "rmnet_ipa0", 10) == 0) phy_dev = dev; + + if (strncmp(dev->name, "rmnet_data", 10) == 0){ + rmnet_vnd_total++; + } + if (strncmp(dev->name, "rmnet_data", 10) == 0) { /* Need separate if check to avoid * NULL dereferencing */ - if (phy_dev) { - rmnet_shs_init(phy_dev); + if (phy_dev && !rmnet_shs_cfg.rmnet_shs_init_complete) { + rmnet_shs_init(phy_dev, dev); rmnet_shs_wq_init(phy_dev); - rmnet_shs_aggregate_init(); + rmnet_shs_rx_wq_init(); rmnet_shs_cfg.is_timer_init = 1; rmnet_shs_cfg.dl_mrk_ind_cb.priority = RMNET_SHS; diff --git a/drivers/rmnet/shs/rmnet_shs_config.h b/drivers/rmnet/shs/rmnet_shs_config.h index 0c00b4f..ca5bec0 100644 --- a/drivers/rmnet/shs/rmnet_shs_config.h +++ b/drivers/rmnet/shs/rmnet_shs_config.h @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -28,6 +28,7 @@ enum rmnet_shs_crit_err_e { RMNET_SHS_NETDEV_ERR, RMNET_SHS_INVALID_CPU_ERR, RMNET_SHS_MAIN_SHS_NOT_REQD, + RMNET_SHS_MAIN_SHS_RPS_INIT_ERR, RMNET_SHS_MAIN_MALLOC_ERR, RMNET_SHS_MAIN_MAP_LEN_INVALID, RMNET_SHS_WQ_ALLOC_WQ_ERR, @@ -38,6 +39,7 @@ enum rmnet_shs_crit_err_e { RMNET_SHS_WQ_EP_ACCESS_ERR, RMNET_SHS_CPU_PKTLEN_ERR, RMNET_SHS_NULL_SKB_HEAD, + RMNET_SHS_RPS_MASK_CHANGE, RMNET_SHS_CRIT_ERR_MAX }; diff --git a/drivers/rmnet/shs/rmnet_shs_main.c b/drivers/rmnet/shs/rmnet_shs_main.c index 469dd37..f09e833 100755 --- a/drivers/rmnet/shs/rmnet_shs_main.c +++ b/drivers/rmnet/shs/rmnet_shs_main.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -16,6 +16,8 @@ #include <net/sock.h> #include <linux/netlink.h> #include <linux/ip.h> +#include <net/ip.h> + #include <linux/ipv6.h> #include <linux/netdevice.h> #include <linux/percpu-defs.h> @@ -30,10 +32,14 @@ #define PERF_CLUSTER 4 #define INVALID_CPU -1 +#define WQ_DELAY 2000000 +#define MIN_MS 5 + #define GET_QTAIL(SD, CPU) (per_cpu(SD, CPU).input_queue_tail) #define GET_QHEAD(SD, CPU) (per_cpu(SD, CPU).input_queue_head) #define GET_CTIMER(CPU) rmnet_shs_cfg.core_flush[CPU].core_timer +#define SKB_FLUSH 0 /* Local Definitions and Declarations */ DEFINE_SPINLOCK(rmnet_shs_ht_splock); DEFINE_HASHTABLE(RMNET_SHS_HT, RMNET_SHS_HT_SIZE); @@ -43,11 +49,13 @@ struct rmnet_shs_cpu_node_s rmnet_shs_cpu_node_tbl[MAX_CPUS]; */ struct rmnet_shs_cfg_s rmnet_shs_cfg; -static u8 rmnet_shs_init_complete; /* This flag is set to true after a successful SHS module init*/ -struct rmnet_shs_flush_work shs_delayed_work; +struct rmnet_shs_flush_work shs_rx_work; /* Delayed workqueue that will be used to flush parked packets*/ +unsigned long int rmnet_shs_switch_reason[RMNET_SHS_SWITCH_MAX_REASON]; +module_param_array(rmnet_shs_switch_reason, ulong, 0, 0444); +MODULE_PARM_DESC(rmnet_shs_switch_reason, "rmnet shs skb core swtich type"); unsigned long int rmnet_shs_flush_reason[RMNET_SHS_FLUSH_MAX_REASON]; module_param_array(rmnet_shs_flush_reason, ulong, 0, 0444); @@ -66,16 +74,30 @@ module_param(rmnet_shs_max_core_wait, uint, 0644); MODULE_PARM_DESC(rmnet_shs_max_core_wait, "Max wait module will wait during move to perf core in ms"); -unsigned int rmnet_shs_inst_rate_interval __read_mostly = 15; +unsigned int rmnet_shs_inst_rate_interval __read_mostly = 20; module_param(rmnet_shs_inst_rate_interval, uint, 0644); MODULE_PARM_DESC(rmnet_shs_inst_rate_interval, "Max interval we sample for instant burst prioritizing"); -unsigned int rmnet_shs_inst_rate_max_pkts __read_mostly = 1800; +unsigned int rmnet_shs_inst_rate_switch __read_mostly = 1; +module_param(rmnet_shs_inst_rate_switch, uint, 0644); +MODULE_PARM_DESC(rmnet_shs_inst_rate_switch, + "Configurable option to enable rx rate cpu switching"); + +unsigned int rmnet_shs_fall_back_timer __read_mostly = 1; +module_param(rmnet_shs_fall_back_timer, uint, 0644); +MODULE_PARM_DESC(rmnet_shs_fall_back_timer, + "Option to enable fall back limit for parking"); + +unsigned int rmnet_shs_inst_rate_max_pkts __read_mostly = 2500; module_param(rmnet_shs_inst_rate_max_pkts, uint, 0644); MODULE_PARM_DESC(rmnet_shs_inst_rate_max_pkts, "Max pkts in a instant burst interval before prioritizing"); +unsigned int rmnet_shs_timeout __read_mostly = 6; +module_param(rmnet_shs_timeout, uint, 0644); +MODULE_PARM_DESC(rmnet_shs_timeout, "Option to configure fall back duration"); + unsigned int rmnet_shs_switch_cores __read_mostly = 1; module_param(rmnet_shs_switch_cores, uint, 0644); MODULE_PARM_DESC(rmnet_shs_switch_cores, "Switch core upon hitting threshold"); @@ -90,7 +112,7 @@ MODULE_PARM_DESC(rmnet_shs_cpu_max_coresum, "Max coresum seen of each core"); void rmnet_shs_cpu_node_remove(struct rmnet_shs_skbn_s *node) { - trace_rmnet_shs_low(RMNET_SHS_CPU_NODE, RMNET_SHS_CPU_NODE_FUNC_REMOVE, + SHS_TRACE_LOW(RMNET_SHS_CPU_NODE, RMNET_SHS_CPU_NODE_FUNC_REMOVE, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); list_del_init(&node->node_id); @@ -99,7 +121,7 @@ void rmnet_shs_cpu_node_remove(struct rmnet_shs_skbn_s *node) void rmnet_shs_cpu_node_add(struct rmnet_shs_skbn_s *node, struct list_head *hd) { - trace_rmnet_shs_low(RMNET_SHS_CPU_NODE, RMNET_SHS_CPU_NODE_FUNC_ADD, + SHS_TRACE_LOW(RMNET_SHS_CPU_NODE, RMNET_SHS_CPU_NODE_FUNC_ADD, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); list_add(&node->node_id, hd); @@ -108,7 +130,7 @@ void rmnet_shs_cpu_node_add(struct rmnet_shs_skbn_s *node, void rmnet_shs_cpu_node_move(struct rmnet_shs_skbn_s *node, struct list_head *hd) { - trace_rmnet_shs_low(RMNET_SHS_CPU_NODE, RMNET_SHS_CPU_NODE_FUNC_MOVE, + SHS_TRACE_LOW(RMNET_SHS_CPU_NODE, RMNET_SHS_CPU_NODE_FUNC_MOVE, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); list_move(&node->node_id, hd); @@ -121,17 +143,20 @@ int rmnet_shs_is_skb_stamping_reqd(struct sk_buff *skb) { int ret_val = 0; + /* SHS will ignore ICMP and frag pkts completely */ switch (skb->protocol) { case htons(ETH_P_IP): - if ((ip_hdr(skb)->protocol == IPPROTO_TCP) || - (ip_hdr(skb)->protocol == IPPROTO_UDP)) + if (!ip_is_fragment(ip_hdr(skb)) && + ((ip_hdr(skb)->protocol == IPPROTO_TCP) || + (ip_hdr(skb)->protocol == IPPROTO_UDP))) ret_val = 1; break; case htons(ETH_P_IPV6): - if ((ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) || - (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)) + if (!(ipv6_hdr(skb)->nexthdr == NEXTHDR_FRAGMENT) && + ((ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) || + (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP))) ret_val = 1; break; @@ -140,7 +165,7 @@ int rmnet_shs_is_skb_stamping_reqd(struct sk_buff *skb) break; } - trace_rmnet_shs_low(RMNET_SHS_SKB_STAMPING, RMNET_SHS_SKB_STAMPING_END, + SHS_TRACE_LOW(RMNET_SHS_SKB_STAMPING, RMNET_SHS_SKB_STAMPING_END, ret_val, 0xDEF, 0xDEF, 0xDEF, skb, NULL); return ret_val; @@ -152,7 +177,7 @@ static void rmnet_shs_update_core_load(int cpu, int burst) struct timespec time1; struct timespec *time2; long int curinterval; - int maxinterval = (rmnet_shs_inst_rate_interval < 5) ? 5 : + int maxinterval = (rmnet_shs_inst_rate_interval < MIN_MS) ? MIN_MS : rmnet_shs_inst_rate_interval; getnstimeofday(&time1); @@ -204,7 +229,7 @@ static int rmnet_shs_check_skb_can_gro(struct sk_buff *skb) break; } - trace_rmnet_shs_low(RMNET_SHS_SKB_CAN_GRO, RMNET_SHS_SKB_CAN_GRO_END, + SHS_TRACE_LOW(RMNET_SHS_SKB_CAN_GRO, RMNET_SHS_SKB_CAN_GRO_END, ret_val, 0xDEF, 0xDEF, 0xDEF, skb, NULL); return ret_val; @@ -216,7 +241,7 @@ static void rmnet_shs_deliver_skb(struct sk_buff *skb) struct rmnet_priv *priv; struct napi_struct *napi; - trace_rmnet_shs_low(RMNET_SHS_DELIVER_SKB, RMNET_SHS_DELIVER_SKB_START, + SHS_TRACE_LOW(RMNET_SHS_DELIVER_SKB, RMNET_SHS_DELIVER_SKB_START, 0xDEF, 0xDEF, 0xDEF, 0xDEF, skb, NULL); if (!rmnet_shs_check_skb_can_gro(skb)) { @@ -231,41 +256,15 @@ static void rmnet_shs_deliver_skb(struct sk_buff *skb) } } -/* Returns the number of low power cores configured and available - * for packet processing - */ -int rmnet_shs_num_lpwr_cores_configured(struct rps_map *map) +static void rmnet_shs_deliver_skb_wq(struct sk_buff *skb) { - int ret = 0; - u16 idx = 0; + struct rmnet_priv *priv; - for (idx = 0; idx < map->len; idx++) - if (map->cpus[idx] < PERF_CLUSTER) - ret += 1; + SHS_TRACE_LOW(RMNET_SHS_DELIVER_SKB, RMNET_SHS_DELIVER_SKB_START, + 0xDEF, 0xDEF, 0xDEF, 0xDEF, skb, NULL); - trace_rmnet_shs_low(RMNET_SHS_CORE_CFG, - RMNET_SHS_CORE_CFG_NUM_LO_CORES, - ret, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); - return ret; -} - -/* Returns the number of performance cores configured and available - * for packet processing - */ -int rmnet_shs_num_perf_cores_configured(struct rps_map *map) -{ - int ret = 0; - u16 idx = 0; - - for (idx = 0; idx < map->len; idx++) - if (map->cpus[idx] >= PERF_CLUSTER) - ret += 1; - - trace_rmnet_shs_low(RMNET_SHS_CORE_CFG, - RMNET_SHS_CORE_CFG_NUM_HI_CORES, - ret, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); - - return ret; + priv = netdev_priv(skb->dev); + gro_cells_receive(&priv->gro_cells, skb); } int rmnet_shs_flow_num_perf_cores(struct rmnet_shs_skbn_s *node_p) @@ -290,7 +289,7 @@ int rmnet_shs_is_lpwr_cpu(u16 cpu) if ((1 << cpu) >= big_cluster_mask) ret = 0; - trace_rmnet_shs_low(RMNET_SHS_CORE_CFG, + SHS_TRACE_LOW(RMNET_SHS_CORE_CFG, RMNET_SHS_CORE_CFG_CHK_LO_CPU, ret, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); return ret; @@ -311,28 +310,86 @@ u32 rmnet_shs_form_hash(u32 index, u32 maplen, u32 hash) return ret; } + /* Override MSB of skb hash to steer. Save most of Hash bits + * Leave some as 0 to allow for easy debugging. + */ if (maplen < MAX_CPUS) ret = ((((index + ((maplen % 2) ? 1 : 0))) << 28) - * offsetmap[(maplen - 1) >> 1]) | (hash & 0xFFFFFF); + * offsetmap[(maplen - 1) >> 1]) | (hash & 0x0FFFFF); - trace_rmnet_shs_low(RMNET_SHS_HASH_MAP, RMNET_SHS_HASH_MAP_FORM_HASH, + SHS_TRACE_LOW(RMNET_SHS_HASH_MAP, RMNET_SHS_HASH_MAP_FORM_HASH, ret, hash, index, maplen, NULL, NULL); return ret; } -int rmnet_shs_map_idx_from_cpu(u16 cpu, struct rps_map *map) +u8 rmnet_shs_mask_from_map(struct rps_map *map) +{ + u8 mask = 0; + u8 i; + + for (i = 0; i < map->len; i++) { + mask |= 1 << map->cpus[i]; + } + return mask; +} + +int rmnet_shs_get_mask_len(u8 mask) +{ + u8 i; + u8 sum = 0; + + for (i = 0; i < MAX_CPUS; i++) { + if (mask & (1 << i)) + sum++; + } + return sum; +} + +/* Take a index and a mask and returns what active CPU is + * in that index. + */ +int rmnet_shs_cpu_from_idx(u8 index, u8 mask) +{ + int ret = INVALID_CPU; + u8 curr_idx = 0; + u8 i; + + for (i = 0; i < MAX_CPUS; i++) { + /* If core is enabled & is the index'th core + * return that CPU + */ + if (curr_idx == index && (mask & (1 << i))) + return i; + + if (mask & (1 << i)) + curr_idx++; + } + return ret; +} + +/* Takes a CPU and a CPU mask and computes what index of configured + * the CPU is in. Returns INVALID_CPU if CPU is not enabled in the mask. + */ +int rmnet_shs_idx_from_cpu(u8 cpu, u8 mask) { int ret = INVALID_CPU; - u16 idx; + u8 idx = 0; + u8 i; + + /* If not in mask return invalid*/ + if (!(mask & 1 << cpu)) + return ret; - for (idx = 0; idx < map->len; idx++) { - if (cpu == map->cpus[idx]) { + /* Find idx by counting all other configed CPUs*/ + for (i = 0; i < MAX_CPUS; i++) { + if (i == cpu && (mask & (1 << i))) { ret = idx; break; } + if(mask & (1 << i)) + idx++; } - return ret; } @@ -352,10 +409,11 @@ int rmnet_shs_new_flow_cpu(u64 burst_size, struct net_device *dev) if (burst_size < RMNET_SHS_MAX_SILVER_CORE_BURST_CAPACITY) flow_cpu = rmnet_shs_wq_get_lpwr_cpu_new_flow(dev); - else + if (flow_cpu == INVALID_CPU || + burst_size >= RMNET_SHS_MAX_SILVER_CORE_BURST_CAPACITY) flow_cpu = rmnet_shs_wq_get_perf_cpu_new_flow(dev); - trace_rmnet_shs_high(RMNET_SHS_ASSIGN, + SHS_TRACE_HIGH(RMNET_SHS_ASSIGN, RMNET_SHS_ASSIGN_GET_NEW_FLOW_CPU, flow_cpu, burst_size, 0xDEF, 0xDEF, NULL, NULL); @@ -370,7 +428,7 @@ int rmnet_shs_get_suggested_cpu(struct rmnet_shs_skbn_s *node) if (rmnet_shs_cpu_node_tbl[node->map_cpu].prio && rmnet_shs_is_lpwr_cpu(node->map_cpu)) { cpu = rmnet_shs_wq_get_least_utilized_core(0xF0); - if (cpu < 0) + if (cpu < 0 && node->hstats != NULL) cpu = node->hstats->suggested_cpu; } else if (node->hstats != NULL) cpu = node->hstats->suggested_cpu; @@ -378,23 +436,21 @@ int rmnet_shs_get_suggested_cpu(struct rmnet_shs_skbn_s *node) return cpu; } -int rmnet_shs_get_hash_map_idx_to_stamp(struct rmnet_shs_skbn_s *node_p) +int rmnet_shs_get_hash_map_idx_to_stamp(struct rmnet_shs_skbn_s *node) { int cpu, idx = INVALID_CPU; - struct rps_map *map; - - cpu = rmnet_shs_get_suggested_cpu(node_p); - + cpu = rmnet_shs_get_suggested_cpu(node); - map = rcu_dereference(node_p->dev->_rx->rps_map); - if (!node_p->dev || !node_p->dev->_rx || !map) - return idx; + idx = rmnet_shs_idx_from_cpu(cpu, rmnet_shs_cfg.map_mask); - idx = rmnet_shs_map_idx_from_cpu(cpu, map); + /* If suggested CPU is no longer in mask. Try using current.*/ + if (unlikely(idx < 0)) + idx = rmnet_shs_idx_from_cpu(node->map_cpu, + rmnet_shs_cfg.map_mask); - trace_rmnet_shs_low(RMNET_SHS_HASH_MAP, + SHS_TRACE_LOW(RMNET_SHS_HASH_MAP, RMNET_SHS_HASH_MAP_IDX_TO_STAMP, - node_p->hash, cpu, idx, 0xDEF, node_p, NULL); + node->hash, cpu, idx, 0xDEF, node, NULL); return idx; } @@ -405,7 +461,7 @@ u32 rmnet_shs_get_cpu_qhead(u8 cpu_num) if (cpu_num < MAX_CPUS) ret = rmnet_shs_cpu_node_tbl[cpu_num].qhead; - trace_rmnet_shs_low(RMNET_SHS_CORE_CFG, RMNET_SHS_CORE_CFG_GET_QHEAD, + SHS_TRACE_LOW(RMNET_SHS_CORE_CFG, RMNET_SHS_CORE_CFG_GET_QHEAD, cpu_num, ret, 0xDEF, 0xDEF, NULL, NULL); return ret; } @@ -417,7 +473,7 @@ u32 rmnet_shs_get_cpu_qtail(u8 cpu_num) if (cpu_num < MAX_CPUS) ret = rmnet_shs_cpu_node_tbl[cpu_num].qtail; - trace_rmnet_shs_low(RMNET_SHS_CORE_CFG, RMNET_SHS_CORE_CFG_GET_QTAIL, + SHS_TRACE_LOW(RMNET_SHS_CORE_CFG, RMNET_SHS_CORE_CFG_GET_QTAIL, cpu_num, ret, 0xDEF, 0xDEF, NULL, NULL); return ret; @@ -430,7 +486,7 @@ u32 rmnet_shs_get_cpu_qdiff(u8 cpu_num) if (cpu_num < MAX_CPUS) ret = rmnet_shs_cpu_node_tbl[cpu_num].qdiff; - trace_rmnet_shs_low(RMNET_SHS_CORE_CFG, RMNET_SHS_CORE_CFG_GET_QTAIL, + SHS_TRACE_LOW(RMNET_SHS_CORE_CFG, RMNET_SHS_CORE_CFG_GET_QTAIL, cpu_num, ret, 0xDEF, 0xDEF, NULL, NULL); return ret; @@ -452,12 +508,13 @@ void rmnet_shs_update_cpu_proc_q(u8 cpu_num) GET_QHEAD(softnet_data, cpu_num); rmnet_shs_cpu_node_tbl[cpu_num].qtail = GET_QTAIL(softnet_data, cpu_num); + rcu_read_unlock(); + rmnet_shs_cpu_node_tbl[cpu_num].qdiff = rmnet_shs_cpu_node_tbl[cpu_num].qtail - rmnet_shs_cpu_node_tbl[cpu_num].qhead; - rcu_read_unlock(); - trace_rmnet_shs_low(RMNET_SHS_CORE_CFG, + SHS_TRACE_LOW(RMNET_SHS_CORE_CFG, RMNET_SHS_CORE_CFG_GET_CPU_PROC_PARAMS, cpu_num, rmnet_shs_cpu_node_tbl[cpu_num].qhead, rmnet_shs_cpu_node_tbl[cpu_num].qtail, @@ -479,7 +536,7 @@ void rmnet_shs_update_cpu_proc_q_all_cpus(void) for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) { rmnet_shs_update_cpu_proc_q(cpu_num); - trace_rmnet_shs_low(RMNET_SHS_CORE_CFG, + SHS_TRACE_LOW(RMNET_SHS_CORE_CFG, RMNET_SHS_CORE_CFG_GET_CPU_PROC_PARAMS, cpu_num, rmnet_shs_cpu_node_tbl[cpu_num].qhead, @@ -491,7 +548,6 @@ void rmnet_shs_update_cpu_proc_q_all_cpus(void) } int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush) { - struct rps_map *map; int cpu_map_index; u32 cur_cpu_qhead; u32 node_qhead; @@ -499,7 +555,9 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush) int prev_cpu = -1; int ccpu; int cpu_num; + int new_cpu; struct rmnet_shs_cpu_node_s *cpun; + u8 map = rmnet_shs_cfg.map_mask; cpu_map_index = rmnet_shs_get_hash_map_idx_to_stamp(node); do { @@ -510,14 +568,12 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush) break; } node->is_shs_enabled = 1; - map = rcu_dereference(node->dev->_rx->rps_map); - if (!node->dev->_rx || !map){ + if (!map){ node->is_shs_enabled = 0; ret = 1; break; } - /* If the flow is going to the same core itself */ if (cpu_map_index == node->map_index) { @@ -528,8 +584,8 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush) cur_cpu_qhead = rmnet_shs_get_cpu_qhead(node->map_cpu); node_qhead = node->queue_head; cpu_num = node->map_cpu; + if ((cur_cpu_qhead >= node_qhead) || - (node->skb_tport_proto == IPPROTO_TCP) || (force_flush)) { if (rmnet_shs_switch_cores) { @@ -537,27 +593,43 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush) * Update old core's parked to not include diverted * packets and update new core's packets */ - rmnet_shs_cpu_node_tbl[map->cpus[cpu_map_index]].parkedlen += - node->skb_list.num_parked_skbs; - rmnet_shs_cpu_node_tbl[node->map_cpu].parkedlen -= - node->skb_list.num_parked_skbs; + new_cpu = rmnet_shs_cpu_from_idx(cpu_map_index, + rmnet_shs_cfg.map_mask); + if (new_cpu < 0) { + ret = 1; + break; + } + rmnet_shs_cpu_node_tbl[new_cpu].parkedlen += node->skb_list.num_parked_skbs; + rmnet_shs_cpu_node_tbl[node->map_cpu].parkedlen -= node->skb_list.num_parked_skbs; node->map_index = cpu_map_index; - node->map_cpu = map->cpus[cpu_map_index]; + node->map_cpu = new_cpu; ccpu = node->map_cpu; + if (cur_cpu_qhead < node_qhead) { + rmnet_shs_switch_reason[RMNET_SHS_OOO_PACKET_SWITCH]++; + rmnet_shs_switch_reason[RMNET_SHS_OOO_PACKET_TOTAL]+= + (node_qhead - + cur_cpu_qhead); + } /* Mark gold core as prio to prevent * flows from moving in wq */ if (rmnet_shs_cpu_node_tbl[cpu_num].prio) { node->hstats->suggested_cpu = ccpu; rmnet_shs_cpu_node_tbl[ccpu].wqprio = 1; + rmnet_shs_switch_reason[RMNET_SHS_SWITCH_INSTANT_RATE]++; + + } else { + + rmnet_shs_switch_reason[RMNET_SHS_SWITCH_WQ_RATE]++; + } cpun = &rmnet_shs_cpu_node_tbl[node->map_cpu]; rmnet_shs_update_cpu_proc_q_all_cpus(); node->queue_head = cpun->qhead; rmnet_shs_cpu_node_move(node, &cpun->node_list_id); - trace_rmnet_shs_high(RMNET_SHS_FLUSH, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_NODE_CORE_SWITCH, node->map_cpu, prev_cpu, 0xDEF, 0xDEF, node, NULL); @@ -566,7 +638,7 @@ int rmnet_shs_node_can_flush_pkts(struct rmnet_shs_skbn_s *node, u8 force_flush) } } while (0); - trace_rmnet_shs_low(RMNET_SHS_FLUSH, + SHS_TRACE_LOW(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_CHK_NODE_CAN_FLUSH, ret, node->map_cpu, prev_cpu, 0xDEF, node, NULL); @@ -588,11 +660,11 @@ void rmnet_shs_flush_core(u8 cpu_num) * currently only use qtail for non TCP flows */ rmnet_shs_update_cpu_proc_q_all_cpus(); - trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_START, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_START, rmnet_shs_cfg.num_pkts_parked, rmnet_shs_cfg.num_bytes_parked, 0xDEF, 0xDEF, NULL, NULL); - + local_bh_disable(); spin_lock_irqsave(&rmnet_shs_ht_splock, ht_flags); cpu_tail = rmnet_shs_get_cpu_qtail(cpu_num); list_for_each_safe(ptr, next, @@ -602,7 +674,8 @@ void rmnet_shs_flush_core(u8 cpu_num) num_pkts_flush = n->skb_list.num_parked_skbs; num_bytes_flush = n->skb_list.num_parked_bytes; - rmnet_shs_chk_and_flush_node(n, 1); + rmnet_shs_chk_and_flush_node(n, 1, + RMNET_WQ_CTXT); total_pkts_flush += num_pkts_flush; total_bytes_flush += num_bytes_flush; @@ -620,8 +693,9 @@ void rmnet_shs_flush_core(u8 cpu_num) rmnet_shs_cpu_node_tbl[cpu_num].prio = 0; rmnet_shs_cpu_node_tbl[cpu_num].parkedlen = 0; spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); + local_bh_enable(); - trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_END, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_END, rmnet_shs_cfg.num_pkts_parked, rmnet_shs_cfg.num_bytes_parked, total_pkts_flush, total_bytes_flush, NULL, NULL); @@ -634,30 +708,35 @@ static void rmnet_shs_flush_core_work(struct work_struct *work) struct core_flush_s, work); rmnet_shs_flush_core(core_work->core); + rmnet_shs_flush_reason[RMNET_SHS_FLUSH_WQ_CORE_FLUSH]++; } + + /* Flushes all the packets parked in order for this flow */ -void rmnet_shs_flush_node(struct rmnet_shs_skbn_s *node) +void rmnet_shs_flush_node(struct rmnet_shs_skbn_s *node, u8 ctext) { struct sk_buff *skb; struct sk_buff *nxt_skb = NULL; - struct rps_map *map; u32 skbs_delivered = 0; u32 skb_bytes_delivered = 0; u32 hash2stamp; + u8 map, maplen; if (!node->skb_list.head) return; - map = rcu_dereference(node->dev->_rx->rps_map); + map = rmnet_shs_cfg.map_mask; + maplen = rmnet_shs_cfg.map_len; - if (!map) { + if (map) { hash2stamp = rmnet_shs_form_hash(node->map_index, - map->len, node->skb_list.head->hash); + maplen, + node->skb_list.head->hash); } else { node->is_shs_enabled = 0; } - trace_rmnet_shs_high(RMNET_SHS_FLUSH, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_NODE_START, node->hash, hash2stamp, node->skb_list.num_parked_skbs, @@ -673,8 +752,10 @@ void rmnet_shs_flush_node(struct rmnet_shs_skbn_s *node) skb->next = NULL; skbs_delivered += 1; skb_bytes_delivered += skb->len; - - rmnet_shs_deliver_skb(skb); + if (ctext == RMNET_RX_CTXT) + rmnet_shs_deliver_skb(skb); + else + rmnet_shs_deliver_skb_wq(skb); } @@ -683,7 +764,7 @@ void rmnet_shs_flush_node(struct rmnet_shs_skbn_s *node) node->skb_list.head = NULL; node->skb_list.tail = NULL; - trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_NODE_END, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_NODE_END, node->hash, hash2stamp, skbs_delivered, skb_bytes_delivered, node, NULL); } @@ -691,19 +772,50 @@ void rmnet_shs_flush_node(struct rmnet_shs_skbn_s *node) /* Evaluates if all the packets corresponding to a particular flow can * be flushed. */ -int rmnet_shs_chk_and_flush_node(struct rmnet_shs_skbn_s *node, u8 force_flush) +int rmnet_shs_chk_and_flush_node(struct rmnet_shs_skbn_s *node, + u8 force_flush, u8 ctxt) { int ret_val = 0; + /* Shoud stay int for error reporting*/ + int map = rmnet_shs_cfg.map_mask; + int map_idx; - trace_rmnet_shs_high(RMNET_SHS_FLUSH, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_CHK_AND_FLUSH_NODE_START, force_flush, 0xDEF, 0xDEF, 0xDEF, node, NULL); + /* Return saved cpu assignment if an entry found*/ + if (rmnet_shs_cpu_from_idx(node->map_index, map) != node->map_cpu) { + + /* Keep flow on the same core if possible + * or put Orphaned flow on the default 1st core + */ + map_idx = rmnet_shs_idx_from_cpu(node->map_cpu, + map); + if (map_idx >= 0) { + node->map_index = map_idx; + node->map_cpu = rmnet_shs_cpu_from_idx(map_idx, map); + + } else { + /*Put on default Core if no match*/ + node->map_index = MAIN_CORE; + node->map_cpu = rmnet_shs_cpu_from_idx(MAIN_CORE, map); + if (node->map_cpu < 0) + node->map_cpu = MAIN_CORE; + } + force_flush = 1; + rmnet_shs_crit_err[RMNET_SHS_RPS_MASK_CHANGE]++; + SHS_TRACE_ERR(RMNET_SHS_ASSIGN, + RMNET_SHS_ASSIGN_MASK_CHNG, + 0xDEF, 0xDEF, 0xDEF, 0xDEF, + NULL, NULL); + } + if (rmnet_shs_node_can_flush_pkts(node, force_flush)) { - rmnet_shs_flush_node(node); + rmnet_shs_flush_node(node, ctxt); ret_val = 1; } - trace_rmnet_shs_high(RMNET_SHS_FLUSH, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_CHK_AND_FLUSH_NODE_END, ret_val, force_flush, 0xDEF, 0xDEF, node, NULL); @@ -720,17 +832,20 @@ int rmnet_shs_chk_and_flush_node(struct rmnet_shs_skbn_s *node, u8 force_flush) * Each time a flushing is invoked we also keep track of the number of * packets waiting & have been processed by the next layers. */ -void rmnet_shs_flush_table(u8 flsh) + +void rmnet_shs_flush_lock_table(u8 flsh, u8 ctxt) { struct rmnet_shs_skbn_s *n; struct list_head *ptr, *next; - unsigned long ht_flags; int cpu_num; u32 cpu_tail; u32 num_pkts_flush = 0; u32 num_bytes_flush = 0; u32 total_pkts_flush = 0; u32 total_bytes_flush = 0; + u32 total_cpu_gro_flushed = 0; + u32 total_node_gro_flushed = 0; + u8 is_flushed = 0; u32 wait = (!rmnet_shs_max_core_wait) ? 1 : rmnet_shs_max_core_wait; @@ -738,76 +853,81 @@ void rmnet_shs_flush_table(u8 flsh) * currently only use qtail for non TCP flows */ rmnet_shs_update_cpu_proc_q_all_cpus(); - trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_START, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_START, rmnet_shs_cfg.num_pkts_parked, rmnet_shs_cfg.num_bytes_parked, 0xDEF, 0xDEF, NULL, NULL); - spin_lock_irqsave(&rmnet_shs_ht_splock, ht_flags); for (cpu_num = 0; cpu_num < MAX_CPUS; cpu_num++) { cpu_tail = rmnet_shs_get_cpu_qtail(cpu_num); - /* If core is loaded set core flows as priority and - * start a 10ms hard flush timer - */ - if (rmnet_shs_is_lpwr_cpu(cpu_num) && - !rmnet_shs_cpu_node_tbl[cpu_num].prio) - rmnet_shs_update_core_load(cpu_num, - rmnet_shs_cpu_node_tbl[cpu_num].parkedlen); - - if (rmnet_shs_is_core_loaded(cpu_num) && - rmnet_shs_is_lpwr_cpu(cpu_num) && - !rmnet_shs_cpu_node_tbl[cpu_num].prio) { - - rmnet_shs_cpu_node_tbl[cpu_num].prio = 1; - if (hrtimer_active(&GET_CTIMER(cpu_num))) - hrtimer_cancel(&GET_CTIMER(cpu_num)); - - hrtimer_start(&GET_CTIMER(cpu_num), - ns_to_ktime(wait * NS_IN_MS), - HRTIMER_MODE_REL); - - } - + total_cpu_gro_flushed = 0; list_for_each_safe(ptr, next, - &rmnet_shs_cpu_node_tbl[cpu_num].node_list_id) { + &rmnet_shs_cpu_node_tbl[cpu_num].node_list_id) { n = list_entry(ptr, struct rmnet_shs_skbn_s, node_id); if (n != NULL && n->skb_list.num_parked_skbs) { num_pkts_flush = n->skb_list.num_parked_skbs; num_bytes_flush = n->skb_list.num_parked_bytes; + total_node_gro_flushed = n->skb_list.skb_load; + is_flushed = rmnet_shs_chk_and_flush_node(n, - flsh); + flsh, + ctxt); if (is_flushed) { + total_cpu_gro_flushed += total_node_gro_flushed; total_pkts_flush += num_pkts_flush; total_bytes_flush += num_bytes_flush; rmnet_shs_cpu_node_tbl[n->map_cpu].parkedlen -= num_pkts_flush; - + n->skb_list.skb_load = 0; if (n->map_cpu == cpu_num) { - cpu_tail += num_pkts_flush; - n->queue_head = cpu_tail; + cpu_tail += num_pkts_flush; + n->queue_head = cpu_tail; } } } } + + /* If core is loaded set core flows as priority and + * start a 10ms hard flush timer + */ + if (rmnet_shs_inst_rate_switch) { + if (rmnet_shs_is_lpwr_cpu(cpu_num) && + !rmnet_shs_cpu_node_tbl[cpu_num].prio) + rmnet_shs_update_core_load(cpu_num, + total_cpu_gro_flushed); + + if (rmnet_shs_is_core_loaded(cpu_num) && + rmnet_shs_is_lpwr_cpu(cpu_num) && + !rmnet_shs_cpu_node_tbl[cpu_num].prio) { + + rmnet_shs_cpu_node_tbl[cpu_num].prio = 1; + if (hrtimer_active(&GET_CTIMER(cpu_num))) + hrtimer_cancel(&GET_CTIMER(cpu_num)); + + hrtimer_start(&GET_CTIMER(cpu_num), + ns_to_ktime(wait * NS_IN_MS), + HRTIMER_MODE_REL); + + } + } + if (rmnet_shs_cpu_node_tbl[cpu_num].parkedlen < 0) rmnet_shs_crit_err[RMNET_SHS_CPU_PKTLEN_ERR]++; if (rmnet_shs_get_cpu_qdiff(cpu_num) >= rmnet_shs_cpu_max_qdiff[cpu_num]) rmnet_shs_cpu_max_qdiff[cpu_num] = - rmnet_shs_get_cpu_qdiff(cpu_num); + rmnet_shs_get_cpu_qdiff(cpu_num); } - spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); - rmnet_shs_cfg.num_bytes_parked -= total_bytes_flush; rmnet_shs_cfg.num_pkts_parked -= total_pkts_flush; - trace_rmnet_shs_high(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_END, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_END, rmnet_shs_cfg.num_pkts_parked, rmnet_shs_cfg.num_bytes_parked, total_pkts_flush, total_bytes_flush, NULL, NULL); @@ -819,16 +939,83 @@ void rmnet_shs_flush_table(u8 flsh) rmnet_shs_cfg.num_pkts_parked = 0; rmnet_shs_cfg.is_pkt_parked = 0; rmnet_shs_cfg.force_flush_state = RMNET_SHS_FLUSH_DONE; + if (rmnet_shs_fall_back_timer) { + if (hrtimer_active(&rmnet_shs_cfg.hrtimer_shs)) { + hrtimer_cancel(&rmnet_shs_cfg.hrtimer_shs); + } + } + } } +void rmnet_shs_flush_table(u8 flsh, u8 ctxt) +{ + unsigned long ht_flags; + + spin_lock_irqsave(&rmnet_shs_ht_splock, ht_flags); + + rmnet_shs_flush_lock_table(flsh, ctxt); + + spin_unlock_irqrestore(&rmnet_shs_ht_splock, ht_flags); + +} + /* After we have decided to handle the incoming skb we park them in order * per flow */ void rmnet_shs_chain_to_skb_list(struct sk_buff *skb, struct rmnet_shs_skbn_s *node) { + u8 pushflush = 0; + struct napi_struct *napi = get_current_napi_context(); + /* UDP GRO should tell us how many packets make up a + * coalesced packet. Use that instead for stats for wq + * Node stats only used by WQ + * Parkedlen useful for cpu stats used by old IB + * skb_load used by IB + UDP coals + */ + + if ((skb->protocol == htons(ETH_P_IP) && + ip_hdr(skb)->protocol == IPPROTO_UDP) || + (skb->protocol == htons(ETH_P_IPV6) && + ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)) { + + if (skb_shinfo(skb)->gso_segs) { + node->num_skb += skb_shinfo(skb)->gso_segs; + rmnet_shs_cpu_node_tbl[node->map_cpu].parkedlen++; + node->skb_list.skb_load += skb_shinfo(skb)->gso_segs; + } else { + node->num_skb += 1; + rmnet_shs_cpu_node_tbl[node->map_cpu].parkedlen++; + node->skb_list.skb_load++; + + } + } else { + /* Early flush for TCP if PSH packet. + * Flush before parking PSH packet. + */ + if (skb->cb[SKB_FLUSH]){ + rmnet_shs_flush_lock_table(0, RMNET_RX_CTXT); + rmnet_shs_flush_reason[RMNET_SHS_FLUSH_PSH_PKT_FLUSH]++; + napi_gro_flush(napi, false); + pushflush = 1; + } + + /* TCP load tweaked for WQ since LRO changes load + * of each packet + */ + node->num_skb += ((skb->len / 4000) + 1); + rmnet_shs_cpu_node_tbl[node->map_cpu].parkedlen++; + node->skb_list.skb_load += ((skb->len / 4000) + 1); + + } + + node->num_skb_bytes += skb->len; + + node->skb_list.num_parked_bytes += skb->len; + rmnet_shs_cfg.num_bytes_parked += skb->len; + if (node->skb_list.num_parked_skbs > 0) { node->skb_list.tail->next = skb; node->skb_list.tail = node->skb_list.tail->next; @@ -837,13 +1024,18 @@ void rmnet_shs_chain_to_skb_list(struct sk_buff *skb, node->skb_list.tail = skb; } - node->skb_list.num_parked_bytes += skb->len; - rmnet_shs_cfg.num_bytes_parked += skb->len; - rmnet_shs_cpu_node_tbl[node->map_cpu].parkedlen++; - + /* skb_list.num_parked_skbs Number of packets are parked for this flow + */ node->skb_list.num_parked_skbs += 1; rmnet_shs_cfg.num_pkts_parked += 1; - trace_rmnet_shs_high(RMNET_SHS_ASSIGN, + + if (unlikely(pushflush)) { + rmnet_shs_flush_lock_table(0, RMNET_RX_CTXT); + napi_gro_flush(napi, false); + + } + + SHS_TRACE_HIGH(RMNET_SHS_ASSIGN, RMNET_SHS_ASSIGN_PARK_PKT_COMPLETE, node->skb_list.num_parked_skbs, node->skb_list.num_parked_bytes, @@ -858,18 +1050,21 @@ static void rmnet_flush_buffered(struct work_struct *work) { u8 is_force_flush = 0; - trace_rmnet_shs_high(RMNET_SHS_FLUSH, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_DELAY_WQ_START, is_force_flush, rmnet_shs_cfg.force_flush_state, 0xDEF, 0xDEF, NULL, NULL); - if (rmnet_shs_cfg.is_pkt_parked && + if (rmnet_shs_cfg.num_pkts_parked && rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_ON) { + local_bh_disable(); + rmnet_shs_flush_table(is_force_flush, + RMNET_WQ_CTXT); - rmnet_shs_flush_table(is_force_flush); - rmnet_shs_flush_reason[RMNET_SHS_FLUSH_TIMER_EXPIRY]++; + rmnet_shs_flush_reason[RMNET_SHS_FLUSH_WQ_FB_FLUSH]++; + local_bh_enable(); } - trace_rmnet_shs_high(RMNET_SHS_FLUSH, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_DELAY_WQ_END, is_force_flush, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); } @@ -884,31 +1079,32 @@ enum hrtimer_restart rmnet_shs_map_flush_queue(struct hrtimer *t) { enum hrtimer_restart ret = HRTIMER_NORESTART; - trace_rmnet_shs_high(RMNET_SHS_FLUSH, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_PARK_TMR_EXPIRY, rmnet_shs_cfg.force_flush_state, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); if (rmnet_shs_cfg.num_pkts_parked > 0) { - if (rmnet_shs_cfg.force_flush_state != RMNET_SHS_FLUSH_ON) { + if (rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_OFF) { rmnet_shs_cfg.force_flush_state = RMNET_SHS_FLUSH_ON; hrtimer_forward(t, hrtimer_cb_get_time(t), - ns_to_ktime(2000000)); + ns_to_ktime(WQ_DELAY)); ret = HRTIMER_RESTART; - trace_rmnet_shs_high(RMNET_SHS_FLUSH, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_PARK_TMR_RESTART, rmnet_shs_cfg.num_pkts_parked, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); } else if (rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_DONE) { - rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_OFF; + rmnet_shs_cfg.force_flush_state = RMNET_SHS_FLUSH_OFF; - } else { - trace_rmnet_shs_high(RMNET_SHS_FLUSH, + } else if (rmnet_shs_cfg.force_flush_state == + RMNET_SHS_FLUSH_ON) { + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_DELAY_WQ_TRIGGER, rmnet_shs_cfg.force_flush_state, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); - schedule_work((struct work_struct *)&shs_delayed_work); + schedule_work((struct work_struct *)&shs_rx_work); } } return ret; @@ -921,13 +1117,15 @@ enum hrtimer_restart rmnet_shs_queue_core(struct hrtimer *t) struct core_flush_s, core_timer); schedule_work(&core_work->work); + return ret; } -void rmnet_shs_aggregate_init(void) +void rmnet_shs_rx_wq_init(void) { int i; + /* Initialize a timer/work for each core for switching */ for (i = 0; i < MAX_CPUS; i++) { rmnet_shs_cfg.core_flush[i].core = i; INIT_WORK(&rmnet_shs_cfg.core_flush[i].work, @@ -938,10 +1136,21 @@ void rmnet_shs_aggregate_init(void) rmnet_shs_cfg.core_flush[i].core_timer.function = rmnet_shs_queue_core; } - hrtimer_init(&rmnet_shs_cfg.hrtimer_shs, - CLOCK_MONOTONIC, HRTIMER_MODE_REL); - rmnet_shs_cfg.hrtimer_shs.function = rmnet_shs_map_flush_queue; - INIT_WORK(&shs_delayed_work.work, rmnet_flush_buffered); + /* Initialize a fallback/failsafe work for when dl ind fails */ + hrtimer_init(&rmnet_shs_cfg.hrtimer_shs, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); + rmnet_shs_cfg.hrtimer_shs.function = rmnet_shs_map_flush_queue; + INIT_WORK(&shs_rx_work.work, rmnet_flush_buffered); +} + +void rmnet_shs_rx_wq_exit(void) +{ + int i; + + for (i = 0; i < MAX_CPUS; i++) + cancel_work_sync(&rmnet_shs_cfg.core_flush[i].work); + + cancel_work_sync(&shs_rx_work.work); } void rmnet_shs_ps_on_hdlr(void *port) @@ -956,9 +1165,17 @@ void rmnet_shs_ps_off_hdlr(void *port) void rmnet_shs_dl_hdr_handler(struct rmnet_map_dl_ind_hdr *dlhdr) { - trace_rmnet_shs_low(RMNET_SHS_DL_MRK, RMNET_SHS_DL_MRK_HDR_HDLR_START, + + SHS_TRACE_LOW(RMNET_SHS_DL_MRK, RMNET_SHS_DL_MRK_HDR_HDLR_START, dlhdr->le.seq, dlhdr->le.pkts, 0xDEF, 0xDEF, NULL, NULL); + if (rmnet_shs_cfg.num_pkts_parked > 0 && + rmnet_shs_cfg.dl_ind_state != RMNET_SHS_IND_COMPLETE) { + + rmnet_shs_flush_reason[RMNET_SHS_FLUSH_INV_DL_IND]++; + rmnet_shs_flush_table(0, RMNET_RX_CTXT); + } + rmnet_shs_cfg.dl_ind_state = RMNET_SHS_END_PENDING; } /* Triggers flushing of all packets upon DL trailer @@ -967,32 +1184,37 @@ void rmnet_shs_dl_hdr_handler(struct rmnet_map_dl_ind_hdr *dlhdr) void rmnet_shs_dl_trl_handler(struct rmnet_map_dl_ind_trl *dltrl) { - u8 is_force_flush = 0; - - trace_rmnet_shs_high(RMNET_SHS_DL_MRK, + SHS_TRACE_HIGH(RMNET_SHS_DL_MRK, RMNET_SHS_FLUSH_DL_MRK_TRLR_HDLR_START, - rmnet_shs_cfg.num_pkts_parked, is_force_flush, + rmnet_shs_cfg.num_pkts_parked, 0, dltrl->seq_le, 0xDEF, NULL, NULL); + rmnet_shs_cfg.dl_ind_state = RMNET_SHS_IND_COMPLETE; if (rmnet_shs_cfg.num_pkts_parked > 0) { rmnet_shs_flush_reason[RMNET_SHS_FLUSH_RX_DL_TRAILER]++; - rmnet_shs_flush_table(is_force_flush); + rmnet_shs_flush_table(0, RMNET_RX_CTXT); } } -void rmnet_shs_init(struct net_device *dev) +void rmnet_shs_init(struct net_device *dev, struct net_device *vnd) { + struct rps_map *map; u8 num_cpu; - if (rmnet_shs_init_complete) + if (rmnet_shs_cfg.rmnet_shs_init_complete) return; + map = rcu_dereference(vnd->_rx->rps_map); - rmnet_shs_cfg.port = rmnet_get_port(dev); + if (!map) + return; + rmnet_shs_cfg.port = rmnet_get_port(dev); + rmnet_shs_cfg.map_mask = rmnet_shs_mask_from_map(map); + rmnet_shs_cfg.map_len = rmnet_shs_get_mask_len(rmnet_shs_cfg.map_mask); for (num_cpu = 0; num_cpu < MAX_CPUS; num_cpu++) INIT_LIST_HEAD(&rmnet_shs_cpu_node_tbl[num_cpu].node_list_id); - rmnet_shs_init_complete = 1; + rmnet_shs_cfg.rmnet_shs_init_complete = 1; } /* Invoked during SHS module exit to gracefully consume all @@ -1058,7 +1280,7 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) struct rmnet_shs_skbn_s *node_p; struct hlist_node *tmp; struct net_device *dev = skb->dev; - struct rps_map *map = rcu_dereference(skb->dev->_rx->rps_map); + int map = rmnet_shs_cfg.map_mask; unsigned long ht_flags; int new_cpu; int map_cpu; @@ -1074,16 +1296,16 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) return; } - if ((unlikely(!map))|| !rmnet_shs_init_complete) { + if ((unlikely(!map))|| !rmnet_shs_cfg.rmnet_shs_init_complete) { rmnet_shs_deliver_skb(skb); - trace_rmnet_shs_err(RMNET_SHS_ASSIGN, + SHS_TRACE_ERR(RMNET_SHS_ASSIGN, RMNET_SHS_ASSIGN_CRIT_ERROR_NO_SHS_REQD, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); - rmnet_shs_crit_err[RMNET_SHS_MAIN_SHS_NOT_REQD]++; + rmnet_shs_crit_err[RMNET_SHS_MAIN_SHS_RPS_INIT_ERR]++; return; } - trace_rmnet_shs_high(RMNET_SHS_ASSIGN, RMNET_SHS_ASSIGN_START, + SHS_TRACE_HIGH(RMNET_SHS_ASSIGN, RMNET_SHS_ASSIGN_START, 0xDEF, 0xDEF, 0xDEF, 0xDEF, skb, NULL); hash = skb_get_hash(skb); @@ -1096,30 +1318,14 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) if (skb->hash != node_p->hash) continue; - /* Return saved cpu assignment if an entry found*/ - if ((node_p->map_index >= map->len) || - ((!node_p->hstats) && - (node_p->hstats->rps_config_msk != - rmnet_shs_wq_get_dev_rps_msk(dev)))) { - - map_cpu = rmnet_shs_new_flow_cpu(brate, dev); - node_p->map_cpu = map_cpu; - node_p->map_index = - rmnet_shs_map_idx_from_cpu(map_cpu, map); - - trace_rmnet_shs_err(RMNET_SHS_ASSIGN, - RMNET_SHS_ASSIGN_MASK_CHNG, - 0xDEF, 0xDEF, 0xDEF, 0xDEF, - NULL, NULL); - } - trace_rmnet_shs_low(RMNET_SHS_ASSIGN, + SHS_TRACE_LOW(RMNET_SHS_ASSIGN, RMNET_SHS_ASSIGN_MATCH_FLOW_COMPLETE, 0xDEF, 0xDEF, 0xDEF, 0xDEF, skb, NULL); - node_p->num_skb += 1; node_p->num_skb_bytes += skb->len; cpu_map_index = node_p->map_index; + rmnet_shs_chain_to_skb_list(skb, node_p); is_match_found = 1; is_shs_reqd = 1; @@ -1131,10 +1337,12 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) /* We haven't found a hash match upto this point */ new_cpu = rmnet_shs_new_flow_cpu(brate, dev); - if (new_cpu < 0) + if (new_cpu < 0) { + rmnet_shs_crit_err[RMNET_SHS_RPS_MASK_CHANGE]++; break; + } - node_p = kzalloc(sizeof(*node_p), 0); + node_p = kzalloc(sizeof(*node_p), GFP_ATOMIC); if (!node_p) { rmnet_shs_crit_err[RMNET_SHS_MAIN_MALLOC_ERR]++; @@ -1144,14 +1352,12 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) node_p->dev = skb->dev; node_p->hash = skb->hash; node_p->map_cpu = new_cpu; - cpu_map_index = rmnet_shs_map_idx_from_cpu(node_p->map_cpu, + node_p->map_index = rmnet_shs_idx_from_cpu(node_p->map_cpu, map); INIT_LIST_HEAD(&node_p->node_id); rmnet_shs_get_update_skb_proto(skb, node_p); - rmnet_shs_wq_inc_cpu_flow(map->cpus[cpu_map_index]); - node_p->map_index = cpu_map_index; - node_p->map_cpu = map->cpus[cpu_map_index]; - node_p->dev = skb->dev; + + rmnet_shs_wq_inc_cpu_flow(node_p->map_cpu); /* Workqueue utilizes some of the values from above * initializations . Therefore, we need to request * for memory (to workqueue) after the above initializations @@ -1176,7 +1382,7 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) if (!is_shs_reqd) { rmnet_shs_crit_err[RMNET_SHS_MAIN_SHS_NOT_REQD]++; rmnet_shs_deliver_skb(skb); - trace_rmnet_shs_err(RMNET_SHS_ASSIGN, + SHS_TRACE_ERR(RMNET_SHS_ASSIGN, RMNET_SHS_ASSIGN_CRIT_ERROR_NO_SHS_REQD, 0xDEF, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); return; @@ -1188,25 +1394,28 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) &rmnet_shs_cfg.rmnet_idl_ind_cb); rmnet_shs_cfg.is_reg_dl_mrk_ind = 1; - shs_delayed_work.port = port; + shs_rx_work.port = port; } /* We got the first packet after a previous successdul flush. Arm the * flushing timer. */ - if (!rmnet_shs_cfg.is_pkt_parked) { + if (!rmnet_shs_cfg.is_pkt_parked && + rmnet_shs_cfg.num_pkts_parked && + rmnet_shs_fall_back_timer) { rmnet_shs_cfg.is_pkt_parked = 1; rmnet_shs_cfg.force_flush_state = RMNET_SHS_FLUSH_OFF; if (hrtimer_active(&rmnet_shs_cfg.hrtimer_shs)) { - trace_rmnet_shs_low(RMNET_SHS_ASSIGN, + SHS_TRACE_LOW(RMNET_SHS_ASSIGN, RMNET_SHS_ASSIGN_PARK_TMR_CANCEL, RMNET_SHS_FORCE_FLUSH_TIME_NSEC, 0xDEF, 0xDEF, 0xDEF, skb, NULL); hrtimer_cancel(&rmnet_shs_cfg.hrtimer_shs); } hrtimer_start(&rmnet_shs_cfg.hrtimer_shs, - ns_to_ktime(2000000), HRTIMER_MODE_REL); - trace_rmnet_shs_low(RMNET_SHS_ASSIGN, + ns_to_ktime(rmnet_shs_timeout * NS_IN_MS), + HRTIMER_MODE_REL); + SHS_TRACE_LOW(RMNET_SHS_ASSIGN, RMNET_SHS_ASSIGN_PARK_TMR_START, RMNET_SHS_FORCE_FLUSH_TIME_NSEC, 0xDEF, 0xDEF, 0xDEF, skb, NULL); @@ -1218,10 +1427,10 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) if (rmnet_shs_stats_enabled) rmnet_shs_flush_reason[RMNET_SHS_FLUSH_PKT_LIMIT]++; - trace_rmnet_shs_high(RMNET_SHS_FLUSH, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_PKT_LIMIT_TRIGGER, 0, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); - rmnet_shs_flush_table(1); + rmnet_shs_flush_table(1, RMNET_RX_CTXT); } else if (rmnet_shs_cfg.num_bytes_parked > rmnet_shs_byte_store_limit) { @@ -1229,10 +1438,10 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) if (rmnet_shs_stats_enabled) rmnet_shs_flush_reason[RMNET_SHS_FLUSH_BYTE_LIMIT]++; - trace_rmnet_shs_high(RMNET_SHS_FLUSH, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_BYTE_LIMIT_TRIGGER, 0, 0xDEF, 0xDEF, 0xDEF, NULL, NULL); - rmnet_shs_flush_table(1); + rmnet_shs_flush_table(1, RMNET_RX_CTXT); } /* Flushing timer that was armed previously has successfully fired. @@ -1247,14 +1456,17 @@ void rmnet_shs_assign(struct sk_buff *skb, struct rmnet_port *port) */ else if (rmnet_shs_cfg.force_flush_state == RMNET_SHS_FLUSH_ON) { rmnet_shs_flush_reason[RMNET_SHS_FLUSH_TIMER_EXPIRY]++; - trace_rmnet_shs_high(RMNET_SHS_FLUSH, + SHS_TRACE_HIGH(RMNET_SHS_FLUSH, RMNET_SHS_FLUSH_FORCE_TRIGGER, 1, rmnet_shs_cfg.num_pkts_parked, 0xDEF, 0xDEF, NULL, NULL); - rmnet_shs_flush_table(0); + rmnet_shs_flush_table(0, RMNET_RX_CTXT); + } else if (rmnet_shs_cfg.num_pkts_parked && + rmnet_shs_cfg.dl_ind_state != RMNET_SHS_END_PENDING) { + rmnet_shs_flush_reason[RMNET_SHS_FLUSH_INV_DL_IND]++; + rmnet_shs_flush_table(0, RMNET_RX_CTXT); } - } /* Cancels the flushing timer if it has been armed @@ -1274,6 +1486,6 @@ void rmnet_shs_exit(void) hrtimer_cancel(&rmnet_shs_cfg.hrtimer_shs); memset(&rmnet_shs_cfg, 0, sizeof(rmnet_shs_cfg)); - rmnet_shs_init_complete = 0; + rmnet_shs_cfg.rmnet_shs_init_complete = 0; } diff --git a/drivers/rmnet/shs/rmnet_shs_wq.c b/drivers/rmnet/shs/rmnet_shs_wq.c index 42f316c..6f22589 100644 --- a/drivers/rmnet/shs/rmnet_shs_wq.c +++ b/drivers/rmnet/shs/rmnet_shs_wq.c @@ -1,4 +1,4 @@ -/* Copyright (c) 2018, The Linux Foundation. All rights reserved. +/* Copyright (c) 2018-2019 The Linux Foundation. All rights reserved. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 and @@ -101,6 +101,14 @@ unsigned long long rmnet_shs_cpu_rx_pps[MAX_CPUS]; module_param_array(rmnet_shs_cpu_rx_pps, ullong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_cpu_rx_pps, "SHS stamp pkt enq rate per CPU"); +unsigned long long rmnet_shs_cpu_qhead_diff[MAX_CPUS]; +module_param_array(rmnet_shs_cpu_qhead_diff, ullong, 0, 0444); +MODULE_PARM_DESC(rmnet_shs_cpu_qhead_diff, "SHS nw stack queue processed diff"); + +unsigned long long rmnet_shs_cpu_qhead_total[MAX_CPUS]; +module_param_array(rmnet_shs_cpu_qhead_total, ullong, 0, 0444); +MODULE_PARM_DESC(rmnet_shs_cpu_qhead_total, "SHS nw stack queue processed total"); + unsigned long rmnet_shs_flow_hash[MAX_SUPPORTED_FLOWS_DEBUG]; module_param_array(rmnet_shs_flow_hash, ulong, 0, 0444); MODULE_PARM_DESC(rmnet_shs_flow_hash, "SHS stamp hash flow"); @@ -297,7 +305,7 @@ void rmnet_shs_wq_hstat_alloc_nodes(u8 num_nodes_to_allocate, u8 is_store_perm) struct rmnet_shs_wq_hstat_s *hnode = NULL; while (num_nodes_to_allocate > 0) { - hnode = kzalloc(sizeof(*hnode), 0); + hnode = kzalloc(sizeof(*hnode), GFP_ATOMIC); if (hnode) { hnode->is_perm = is_store_perm; rmnet_shs_wq_hstat_reset_node(hnode); @@ -349,7 +357,8 @@ struct rmnet_shs_wq_hstat_s *rmnet_shs_wq_get_new_hstat_node(void) * However, this newly allocated memory will be released as soon as we * realize that this flow is inactive */ - ret_node = kzalloc(sizeof(*hnode), 0); + ret_node = kzalloc(sizeof(*hnode), GFP_ATOMIC); + if (!ret_node) { rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_HSTAT_ERR]++; return NULL; @@ -534,7 +543,8 @@ static void rmnet_shs_wq_refresh_cpu_rates_debug(u16 cpu, rmnet_shs_cpu_rx_flows[cpu] = cpu_p->flows; rmnet_shs_cpu_rx_bytes[cpu] = cpu_p->rx_bytes; rmnet_shs_cpu_rx_pkts[cpu] = cpu_p->rx_skbs; - + rmnet_shs_cpu_qhead_diff[cpu] = cpu_p->qhead_diff; + rmnet_shs_cpu_qhead_total[cpu] = cpu_p->qhead_total; } static void rmnet_shs_wq_refresh_dl_mrkr_stats(void) @@ -596,10 +606,21 @@ static void rmnet_shs_wq_refresh_cpu_stats(u16 cpu) struct rmnet_shs_wq_cpu_rx_pkt_q_s *cpu_p; time_t tdiff; u64 new_skbs, new_bytes; + u32 new_qhead; cpu_p = &rmnet_shs_rx_flow_tbl.cpu_list[cpu]; new_skbs = cpu_p->rx_skbs - cpu_p->last_rx_skbs; + new_qhead = rmnet_shs_get_cpu_qhead(cpu); + if (cpu_p->qhead_start == 0) { + cpu_p->qhead_start = new_qhead; + } + + cpu_p->last_qhead = cpu_p->qhead; + cpu_p->qhead = new_qhead; + cpu_p->qhead_diff = cpu_p->qhead - cpu_p->last_qhead; + cpu_p->qhead_total = cpu_p->qhead - cpu_p->qhead_start; + if (rmnet_shs_cpu_node_tbl[cpu].wqprio) rmnet_shs_cpu_node_tbl[cpu].wqprio = (rmnet_shs_cpu_node_tbl[cpu].wqprio + 1) % (PRIO_BACKOFF); @@ -661,16 +682,20 @@ void rmnet_shs_wq_update_cpu_rx_tbl(struct rmnet_shs_wq_hstat_s *hstat_p) if (hstat_p->inactive_duration > 0) return; + rcu_read_lock(); map = rcu_dereference(node_p->dev->_rx->rps_map); - if (!map) + if (!map || node_p->map_index > map->len || !map->len) { + rcu_read_unlock(); return; + } map_idx = node_p->map_index; cpu_num = map->cpus[map_idx]; skb_diff = hstat_p->rx_skb - hstat_p->last_rx_skb; byte_diff = hstat_p->rx_bytes - hstat_p->last_rx_bytes; + rcu_read_unlock(); if (hstat_p->is_new_flow) { rmnet_shs_wq_cpu_list_add(hstat_p, @@ -825,6 +850,7 @@ u16 rmnet_shs_wq_find_cpu_to_move_flows(u16 current_cpu, u16 cpu_to_move = current_cpu; u16 cpu_num; u8 is_core_in_msk; + u32 cpu_to_move_util = 0; if (!ep) { rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++; @@ -834,13 +860,15 @@ u16 rmnet_shs_wq_find_cpu_to_move_flows(u16 current_cpu, cur_cpu_list_p = &rx_flow_tbl_p->cpu_list[current_cpu]; cur_cpu_rx_pps = cur_cpu_list_p->rx_pps; pps_uthresh = rmnet_shs_cpu_rx_max_pps_thresh[current_cpu]; + pps_lthresh = rmnet_shs_cpu_rx_min_pps_thresh[current_cpu]; + /* If we are already on a perf core and required pps is beyond * beyond the capacity that even perf cores aren't sufficient * there is nothing much we can do. So we will continue to let flows * process packets on same perf core */ if (!rmnet_shs_is_lpwr_cpu(current_cpu) && - (cur_cpu_rx_pps > pps_uthresh)) { + (cur_cpu_rx_pps > pps_lthresh)) { return cpu_to_move; } /* If a core (should only be lpwr was marked prio we don't touch it @@ -874,10 +902,11 @@ u16 rmnet_shs_wq_find_cpu_to_move_flows(u16 current_cpu, current_cpu, cpu_num, reqd_pps, cpu_rx_pps, NULL, NULL); - /* Return the first available CPU */ - if ((reqd_pps > pps_lthresh) && (reqd_pps < pps_uthresh)) { + /* Return the most available valid CPU */ + if ((reqd_pps > pps_lthresh) && (reqd_pps < pps_uthresh) && + cpu_rx_pps <= cpu_to_move_util) { cpu_to_move = cpu_num; - break; + cpu_to_move_util = cpu_rx_pps; } } @@ -886,7 +915,6 @@ u16 rmnet_shs_wq_find_cpu_to_move_flows(u16 current_cpu, current_cpu, cpu_to_move, cur_cpu_rx_pps, rx_flow_tbl_p->cpu_list[cpu_to_move].rx_pps, NULL, NULL); - return cpu_to_move; } @@ -1206,14 +1234,14 @@ void rmnet_shs_wq_update_ep_rps_msk(struct rmnet_shs_wq_ep_s *ep) rmnet_shs_crit_err[RMNET_SHS_WQ_EP_ACCESS_ERR]++; return; } - + rcu_read_lock(); map = rcu_dereference(ep->ep->egress_dev->_rx->rps_map); ep->rps_config_msk = 0; if (map != NULL) { for (len = 0; len < map->len; len++) ep->rps_config_msk |= (1 << map->cpus[len]); } - + rcu_read_unlock(); ep->default_core_msk = ep->rps_config_msk & 0x0F; ep->pri_core_msk = ep->rps_config_msk & 0xF0; } @@ -1260,6 +1288,26 @@ void rmnet_shs_wq_refresh_ep_masks(void) rmnet_shs_wq_update_ep_rps_msk(ep); } } + +void rmnet_shs_update_cfg_mask(void) +{ + /* Start with most avaible mask all eps could share*/ + u8 mask = UPDATE_MASK; + struct rmnet_shs_wq_ep_s *ep; + + list_for_each_entry(ep, &rmnet_shs_wq_ep_tbl, ep_list_id) { + + if (!ep->is_ep_active) + continue; + /* Bitwise and to get common mask VNDs with different mask + * will have UNDEFINED behavior + */ + mask &= ep->rps_config_msk; + } + rmnet_shs_cfg.map_mask = mask; + rmnet_shs_cfg.map_len = rmnet_shs_get_mask_len(mask); +} + static void rmnet_shs_wq_update_stats(void) { struct timespec time; @@ -1268,6 +1316,7 @@ static void rmnet_shs_wq_update_stats(void) (void) getnstimeofday(&time); rmnet_shs_wq_tnsec = RMNET_SHS_SEC_TO_NSEC(time.tv_sec) + time.tv_nsec; rmnet_shs_wq_refresh_ep_masks(); + rmnet_shs_update_cfg_mask(); list_for_each_entry(hnode, &rmnet_shs_wq_hstat_tbl, hstat_node_id) { if (!hnode) @@ -1363,7 +1412,7 @@ void rmnet_shs_wq_gather_rmnet_ep(struct net_device *dev) trace_rmnet_shs_wq_high(RMNET_SHS_WQ_EP_TBL, RMNET_SHS_WQ_EP_TBL_INIT, 0xDEF, 0xDEF, 0xDEF, 0xDEF, ep, NULL); - ep_wq = kzalloc(sizeof(*ep_wq), 0); + ep_wq = kzalloc(sizeof(*ep_wq), GFP_ATOMIC); if (!ep_wq) { rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_EP_TBL_ERR]++; return; @@ -1423,7 +1472,7 @@ void rmnet_shs_wq_init(struct net_device *dev) } rmnet_shs_delayed_wq = kmalloc(sizeof(struct rmnet_shs_delay_wq_s), - GFP_ATOMIC); + GFP_ATOMIC); if (!rmnet_shs_delayed_wq) { rmnet_shs_crit_err[RMNET_SHS_WQ_ALLOC_DEL_WQ_ERR]++; diff --git a/drivers/rmnet/shs/rmnet_shs_wq.h b/drivers/rmnet/shs/rmnet_shs_wq.h index 96843f5..da85906 100644 --- a/drivers/rmnet/shs/rmnet_shs_wq.h +++ b/drivers/rmnet/shs/rmnet_shs_wq.h @@ -91,6 +91,11 @@ struct rmnet_shs_wq_cpu_rx_pkt_q_s { u64 last_rx_bps; /* bits per second*/ u64 avg_pps; u64 rx_bps_est; /*estimated bits per second*/ + u32 qhead; /* queue head */ + u32 last_qhead; /* last queue head */ + u32 qhead_diff; /* diff in pp in last tick*/ + u32 qhead_start; /* start mark of total pp*/ + u32 qhead_total; /* end mark of total pp*/ int flows; }; |