| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <u2h65634d661004190902lcd63436s5a817e6cf8ee27f5@mail.gmail.com>
Date: Mon, 19 Apr 2010 09:02:44 -0700
From: Tom Herbert <therbert@...gle.com>
To: Eric Dumazet <eric.dumazet@...il.com>
Cc: Changli Gao <xiaosuo@...il.com>,
David Miller <davem@...emloft.net>,
netdev <netdev@...r.kernel.org>
Subject: Re: [PATCH net-next-2.6] rps: shortcut net_rps_action()
>
> [PATCH net-next-2.6] rps: shortcut net_rps_action()
>
> net_rps_action() is a bit expensive on NR_CPUS=64..4096 kernels, even if
> RPS is not active.
>
> Tom Herbert used two bitmasks to hold information needed to send IPI,
> but a single LIFO list seems more appropriate.
>
Yes, this patch is an improvement over that.
> Move all RPS logic into net_rps_action() to cleanup net_rx_action() code
> (remove two ifdefs)
>
> Move rps_remote_softirq_cpus into softnet_data to share its first cache
> line, filling an existing hole.
>
> In a future patch, we could call net_rps_action() from process_backlog()
> to make sure we send IPI before handling this cpu backlog.
>
Yes. I did some quick experiments last night and there does seem to
be some gains in doing this.
> Signed-off-by: Eric Dumazet <eric.dumazet@...il.com>
> ---
> include/linux/netdevice.h | 9 ++--
> net/core/dev.c | 79 ++++++++++++++----------------------
> 2 files changed, 38 insertions(+), 50 deletions(-)
>
> diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
> index 649a025..83ab3da 100644
> --- a/include/linux/netdevice.h
> +++ b/include/linux/netdevice.h
> @@ -1381,17 +1381,20 @@ static inline int unregister_gifconf(unsigned int family)
> }
>
> /*
> - * Incoming packets are placed on per-cpu queues so that
> - * no locking is needed.
> + * Incoming packets are placed on per-cpu queues
> */
> struct softnet_data {
> struct Qdisc *output_queue;
> struct list_head poll_list;
> struct sk_buff *completion_queue;
>
> - /* Elements below can be accessed between CPUs for RPS */
> #ifdef CONFIG_RPS
> + struct softnet_data *rps_ipi_list;
> +
> + /* Elements below can be accessed between CPUs for RPS */
> struct call_single_data csd ____cacheline_aligned_in_smp;
> + struct softnet_data *rps_ipi_next;
> + unsigned int cpu;
> unsigned int input_queue_head;
> #endif
> struct sk_buff_head input_pkt_queue;
> diff --git a/net/core/dev.c b/net/core/dev.c
> index 7abf959..f6ff2cf 100644
> --- a/net/core/dev.c
> +++ b/net/core/dev.c
> @@ -2346,21 +2346,6 @@ done:
> return cpu;
> }
>
> -/*
> - * This structure holds the per-CPU mask of CPUs for which IPIs are scheduled
> - * to be sent to kick remote softirq processing. There are two masks since
> - * the sending of IPIs must be done with interrupts enabled. The select field
> - * indicates the current mask that enqueue_backlog uses to schedule IPIs.
> - * select is flipped before net_rps_action is called while still under lock,
> - * net_rps_action then uses the non-selected mask to send the IPIs and clears
> - * it without conflicting with enqueue_backlog operation.
> - */
> -struct rps_remote_softirq_cpus {
> - cpumask_t mask[2];
> - int select;
> -};
> -static DEFINE_PER_CPU(struct rps_remote_softirq_cpus, rps_remote_softirq_cpus);
> -
> /* Called from hardirq (IPI) context */
> static void trigger_softirq(void *data)
> {
> @@ -2403,10 +2388,12 @@ enqueue:
> if (napi_schedule_prep(&queue->backlog)) {
> #ifdef CONFIG_RPS
> if (cpu != smp_processor_id()) {
> - struct rps_remote_softirq_cpus *rcpus =
> - &__get_cpu_var(rps_remote_softirq_cpus);
> + struct softnet_data *myqueue;
> +
> + myqueue = &__get_cpu_var(softnet_data);
> + queue->rps_ipi_next = myqueue->rps_ipi_list;
> + myqueue->rps_ipi_list = queue;
>
> - cpu_set(cpu, rcpus->mask[rcpus->select]);
> __raise_softirq_irqoff(NET_RX_SOFTIRQ);
> goto enqueue;
> }
> @@ -2911,7 +2898,9 @@ int netif_receive_skb(struct sk_buff *skb)
> }
> EXPORT_SYMBOL(netif_receive_skb);
>
> -/* Network device is going away, flush any packets still pending */
> +/* Network device is going away, flush any packets still pending
> + * Called with irqs disabled.
> + */
> static void flush_backlog(void *arg)
> {
> struct net_device *dev = arg;
> @@ -3340,24 +3329,33 @@ void netif_napi_del(struct napi_struct *napi)
> }
> EXPORT_SYMBOL(netif_napi_del);
>
> -#ifdef CONFIG_RPS
> /*
> - * net_rps_action sends any pending IPI's for rps. This is only called from
> - * softirq and interrupts must be enabled.
> + * net_rps_action sends any pending IPI's for rps.
> + * Note: called with local irq disabled, but exits with local irq enabled.
> */
> -static void net_rps_action(cpumask_t *mask)
> +static void net_rps_action(void)
> {
> - int cpu;
> +#ifdef CONFIG_RPS
> + struct softnet_data *locqueue = &__get_cpu_var(softnet_data);
> + struct softnet_data *remqueue = locqueue->rps_ipi_list;
>
> - /* Send pending IPI's to kick RPS processing on remote cpus. */
> - for_each_cpu_mask_nr(cpu, *mask) {
> - struct softnet_data *queue = &per_cpu(softnet_data, cpu);
> - if (cpu_online(cpu))
> - __smp_call_function_single(cpu, &queue->csd, 0);
> - }
> - cpus_clear(*mask);
> -}
> + if (remqueue) {
> + locqueue->rps_ipi_list = NULL;
> +
> + local_irq_enable();
> +
> + /* Send pending IPI's to kick RPS processing on remote cpus. */
> + while (remqueue) {
> + struct softnet_data *next = remqueue->rps_ipi_next;
> + if (cpu_online(remqueue->cpu))
> + __smp_call_function_single(remqueue->cpu,
> + &remqueue->csd, 0);
> + remqueue = next;
> + }
> + } else
> #endif
> + local_irq_enable();
> +}
>
> static void net_rx_action(struct softirq_action *h)
> {
> @@ -3365,10 +3363,6 @@ static void net_rx_action(struct softirq_action *h)
> unsigned long time_limit = jiffies + 2;
> int budget = netdev_budget;
> void *have;
> -#ifdef CONFIG_RPS
> - int select;
> - struct rps_remote_softirq_cpus *rcpus;
> -#endif
>
> local_irq_disable();
>
> @@ -3431,17 +3425,7 @@ static void net_rx_action(struct softirq_action *h)
> netpoll_poll_unlock(have);
> }
> out:
> -#ifdef CONFIG_RPS
> - rcpus = &__get_cpu_var(rps_remote_softirq_cpus);
> - select = rcpus->select;
> - rcpus->select ^= 1;
> -
> - local_irq_enable();
> -
> - net_rps_action(&rcpus->mask[select]);
> -#else
> - local_irq_enable();
> -#endif
> + net_rps_action();
>
> #ifdef CONFIG_NET_DMA
> /*
> @@ -5841,6 +5825,7 @@ static int __init net_dev_init(void)
> queue->csd.func = trigger_softirq;
> queue->csd.info = queue;
> queue->csd.flags = 0;
> + queue->cpu = i;
> #endif
>
> queue->backlog.poll = process_backlog;
>
>
>
--
To unsubscribe from this list: send the line "unsubscribe netdev" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Powered by blists - more mailing lists