| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <1288892908.2659.175.camel@edumazet-laptop>
Date: Thu, 04 Nov 2010 18:48:28 +0100
From: Eric Dumazet <eric.dumazet@...il.com>
To: Andrew Morton <akpm@...ux-foundation.org>
Cc: linux-kernel <linux-kernel@...r.kernel.org>,
Christoph Lameter <cl@...ux-foundation.org>,
Ingo Molnar <mingo@...e.hu>, Andi Kleen <andi@...stfloor.org>,
Tejun Heo <tj@...nel.org>, Thomas Gleixner <tglx@...utronix.de>
Subject: [PATCH] irq: use per_cpu kstat_irqs
Use modern per_cpu API to increment {soft|hard}irq counters, and
use per_cpu allocation for (struct irq_desc)->kstats_irq instead of an
array.
This gives better SMP/NUMA locality and saves few instructions per irq.
With small nr_cpuids values (8 for example), kstats_irq was a small
array (less than L1_CACHE_BYTES), potentially source of false sharing.
Signed-off-by: Eric Dumazet <eric.dumazet@...il.com>
Cc: Christoph Lameter <cl@...ux-foundation.org>
Cc: Ingo Molnar <mingo@...e.hu>
Cc: Andi Kleen <andi@...stfloor.org>
Cc: Tejun Heo <tj@...nel.org>
Cc: Thomas Gleixner <tglx@...utronix.de>
---
include/linux/irqdesc.h | 2 +-
include/linux/kernel_stat.h | 21 ++++++++++-----------
kernel/irq/irqdesc.c | 16 +++++++++-------
3 files changed, 20 insertions(+), 19 deletions(-)
diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h
index 979c68c..6a64c6f 100644
--- a/include/linux/irqdesc.h
+++ b/include/linux/irqdesc.h
@@ -57,7 +57,7 @@ struct irq_desc {
#endif
struct timer_rand_state *timer_rand_state;
- unsigned int *kstat_irqs;
+ unsigned int __percpu *kstat_irqs;
irq_flow_handler_t handle_irq;
struct irqaction *action; /* IRQ action list */
unsigned int status; /* IRQ status */
diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h
index ad54c84..acd97a3 100644
--- a/include/linux/kernel_stat.h
+++ b/include/linux/kernel_stat.h
@@ -31,7 +31,7 @@ struct cpu_usage_stat {
struct kernel_stat {
struct cpu_usage_stat cpustat;
#ifndef CONFIG_GENERIC_HARDIRQS
- unsigned int irqs[NR_IRQS];
+ unsigned int irqs[NR_IRQS];
#endif
unsigned long irqs_sum;
unsigned int softirqs[NR_SOFTIRQS];
@@ -46,41 +46,40 @@ DECLARE_PER_CPU(struct kernel_stat, kstat);
extern unsigned long long nr_context_switches(void);
#ifndef CONFIG_GENERIC_HARDIRQS
-#define kstat_irqs_this_cpu(irq) \
- (kstat_this_cpu.irqs[irq])
+#define kstat_irqs_this_cpu(irq) __this_cpu_read(kstat.irqs[irq])
struct irq_desc;
static inline void kstat_incr_irqs_this_cpu(unsigned int irq,
struct irq_desc *desc)
{
- kstat_this_cpu.irqs[irq]++;
- kstat_this_cpu.irqs_sum++;
+ __this_cpu_inc(kstat.irqs[irq]);
+ __this_cpu_inc(kstat.irqs_sum);
}
static inline unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
{
- return kstat_cpu(cpu).irqs[irq];
+ return kstat_cpu(cpu).irqs[irq];
}
#else
#include <linux/irq.h>
extern unsigned int kstat_irqs_cpu(unsigned int irq, int cpu);
#define kstat_irqs_this_cpu(DESC) \
- ((DESC)->kstat_irqs[smp_processor_id()])
+ __this_cpu_read((DESC)->kstat_irqs)
#define kstat_incr_irqs_this_cpu(irqno, DESC) do {\
- ((DESC)->kstat_irqs[smp_processor_id()]++);\
- kstat_this_cpu.irqs_sum++; } while (0)
+ __this_cpu_inc((DESC)->kstat_irqs);\
+ __this_cpu_inc(kstat.irqs_sum); } while (0)
#endif
static inline void kstat_incr_softirqs_this_cpu(unsigned int irq)
{
- kstat_this_cpu.softirqs[irq]++;
+ __this_cpu_inc(kstat.softirqs[irq]);
}
static inline unsigned int kstat_softirqs_cpu(unsigned int irq, int cpu)
{
- return kstat_cpu(cpu).softirqs[irq];
+ return kstat_cpu(cpu).softirqs[irq];
}
/*
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index 9988d03..7683422 100644
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -72,6 +72,8 @@ static inline int desc_node(struct irq_desc *desc) { return 0; }
static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node)
{
+ int cpu;
+
desc->irq_data.irq = irq;
desc->irq_data.chip = &no_irq_chip;
desc->irq_data.chip_data = NULL;
@@ -83,7 +85,8 @@ static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node)
desc->irq_count = 0;
desc->irqs_unhandled = 0;
desc->name = NULL;
- memset(desc->kstat_irqs, 0, nr_cpu_ids * sizeof(*(desc->kstat_irqs)));
+ for_each_possible_cpu(cpu)
+ *per_cpu_ptr(desc->kstat_irqs, cpu) = 0;
desc_smp_init(desc, node);
}
@@ -133,8 +136,7 @@ static struct irq_desc *alloc_desc(int irq, int node)
if (!desc)
return NULL;
/* allocate based on nr_cpu_ids */
- desc->kstat_irqs = kzalloc_node(nr_cpu_ids * sizeof(*desc->kstat_irqs),
- gfp, node);
+ desc->kstat_irqs = alloc_percpu(unsigned int);
if (!desc->kstat_irqs)
goto err_desc;
@@ -149,7 +151,7 @@ static struct irq_desc *alloc_desc(int irq, int node)
return desc;
err_kstat:
- kfree(desc->kstat_irqs);
+ free_percpu(desc->kstat_irqs);
err_desc:
kfree(desc);
return NULL;
@@ -166,7 +168,7 @@ static void free_desc(unsigned int irq)
mutex_unlock(&sparse_irq_lock);
free_masks(desc);
- kfree(desc->kstat_irqs);
+ free_percpu(desc->kstat_irqs);
kfree(desc);
}
@@ -391,7 +393,7 @@ void dynamic_irq_cleanup(unsigned int irq)
unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
{
struct irq_desc *desc = irq_to_desc(irq);
- return desc ? desc->kstat_irqs[cpu] : 0;
+ return desc ? *per_cpu_ptr(desc->kstat_irqs, cpu) : 0;
}
#ifdef CONFIG_GENERIC_HARDIRQS
@@ -404,7 +406,7 @@ unsigned int kstat_irqs(unsigned int irq)
if (!desc)
return 0;
for_each_possible_cpu(cpu)
- sum += desc->kstat_irqs[cpu];
+ sum += *per_cpu_ptr(desc->kstat_irqs, cpu);
return sum;
}
#endif /* CONFIG_GENERIC_HARDIRQS */
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists