| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Mon, 30 Aug 2010 11:41:26 +0200
From: Peter Zijlstra <peterz@...radead.org>
To: Huang Ying <ying.huang@...el.com>
Cc: Ingo Molnar <mingo@...e.hu>, "H. Peter Anvin" <hpa@...or.com>,
paulus <paulus@...ba.org>, linux-kernel@...r.kernel.org,
Andi Kleen <andi@...stfloor.org>,
dhowells <dhowells@...hat.com>,
Russell King <rmk@....linux.org.uk>,
Kyle McMartin <kyle@...artin.ca>,
Martin Schwidefsky <schwidefsky@...ibm.com>,
davem <davem@...emloft.net>,
Linux-Arch <linux-arch@...r.kernel.org>
Subject: Re: [PATCH -v3 1/2] irq_work: generic hard-irq context callbacks
On Tue, 2010-07-13 at 12:59 +0800, Huang Ying wrote:
> From: Peter Zijlstra <a.p.zijlstra@...llo.nl>
>
> In order for other NMI context users that want to run things from
> hard-IRQ context, extract the perf_event callback mechanism.
>
> Huang Ying: some fixes
>
> This patch is only tested on x86 platform.
Right, looks ok, although it would require some acks from relevant
architecture maintainers, all of whoem you forgot to CC.
> Signed-off-by: Peter Zijlstra <a.p.zijlstra@...llo.nl>
> Signed-off-by: Huang Ying <ying.huang@...el.com>
> ---
> arch/alpha/Kconfig | 1
> arch/alpha/include/asm/perf_event.h | 9 -
> arch/arm/Kconfig | 1
> arch/arm/include/asm/perf_event.h | 12 --
> arch/arm/kernel/perf_event.c | 4
> arch/frv/Kconfig | 1
> arch/frv/lib/perf_event.c | 19 ----
> arch/parisc/Kconfig | 1
> arch/parisc/include/asm/perf_event.h | 7 -
> arch/powerpc/Kconfig | 1
> arch/powerpc/kernel/time.c | 42 ++++----
> arch/s390/Kconfig | 1
> arch/s390/include/asm/perf_event.h | 10 --
> arch/sh/Kconfig | 1
> arch/sh/include/asm/perf_event.h | 7 -
> arch/sparc/Kconfig | 2
> arch/sparc/include/asm/perf_event.h | 4
> arch/sparc/kernel/pcr.c | 8 -
> arch/x86/Kconfig | 1
> arch/x86/include/asm/entry_arch.h | 4
> arch/x86/include/asm/hardirq.h | 2
> arch/x86/include/asm/hw_irq.h | 2
> arch/x86/include/asm/irq_vectors.h | 4
> arch/x86/kernel/Makefile | 1
> arch/x86/kernel/cpu/perf_event.c | 19 ----
> arch/x86/kernel/entry_64.S | 6 -
> arch/x86/kernel/irq.c | 8 -
> arch/x86/kernel/irq_work.c | 30 ++++++
> arch/x86/kernel/irqinit.c | 6 -
> include/linux/irq_work.h | 20 ++++
> include/linux/perf_event.h | 11 --
> init/Kconfig | 8 +
> kernel/Makefile | 2
> kernel/irq_work.c | 164 +++++++++++++++++++++++++++++++++++
> kernel/perf_event.c | 104 +---------------------
> kernel/timer.c | 7 +
> 36 files changed, 290 insertions(+), 240 deletions(-)
>
> --- /dev/null
> +++ b/include/linux/irq_work.h
> @@ -0,0 +1,20 @@
> +#ifndef _LINUX_IRQ_WORK_H
> +#define _LINUX_IRQ_WORK_H
> +
> +struct irq_work {
> + struct irq_work *next;
> + void (*func)(struct irq_work *);
> +};
> +
> +static inline
> +void init_irq_work(struct irq_work *entry, void (*func)(struct irq_work *))
> +{
> + entry->next = NULL;
> + entry->func = func;
> +}
> +
> +bool irq_work_queue(struct irq_work *entry);
> +void irq_work_run(void);
> +void irq_work_sync(struct irq_work *entry);
> +
> +#endif /* _LINUX_IRQ_WORK_H */
> --- /dev/null
> +++ b/kernel/irq_work.c
> @@ -0,0 +1,164 @@
> +/*
> + * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@...hat.com>
> + *
> + * Provides a framework for enqueueing and running callbacks from hardirq
> + * context. The enqueueing is NMI-safe.
> + */
> +
> +#include <linux/kernel.h>
> +#include <linux/module.h>
> +#include <linux/irq_work.h>
> +#include <linux/hardirq.h>
> +
> +/*
> + * An entry can be in one of four states:
> + *
> + * free NULL, 0 -> {claimed} : free to be used
> + * claimed NULL, 3 -> {pending} : claimed to be enqueued
> + * pending next, 3 -> {busy} : queued, pending callback
> + * busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
> + *
> + * We use the lower two bits of the next pointer to keep PENDING and BUSY
> + * flags.
> + */
> +
> +#define IRQ_WORK_PENDING 1UL
> +#define IRQ_WORK_BUSY 2UL
> +#define IRQ_WORK_FLAGS 3UL
> +
> +static inline bool irq_work_is_set(struct irq_work *entry, int flags)
> +{
> + return (unsigned long)entry->next & flags;
> +}
> +
> +static inline struct irq_work *irq_work_next(struct irq_work *entry)
> +{
> + unsigned long next = (unsigned long)entry->next;
> + next &= ~IRQ_WORK_FLAGS;
> + return (struct irq_work *)next;
> +}
> +
> +static inline struct irq_work *next_flags(struct irq_work *entry, int flags)
> +{
> + unsigned long next = (unsigned long)entry;
> + next |= flags;
> + return (struct irq_work *)next;
> +}
> +
> +static DEFINE_PER_CPU(struct irq_work *, irq_work_list);
> +
> +/*
> + * Claim the entry so that no one else will poke at it.
> + */
> +static bool irq_work_claim(struct irq_work *entry)
> +{
> + struct irq_work *next, *nflags;
> +
> + do {
> + next = entry->next;
> + if ((unsigned long)next & IRQ_WORK_PENDING)
> + return false;
> + nflags = next_flags(next, IRQ_WORK_FLAGS);
> + } while (cmpxchg(&entry->next, next, nflags) != next);
> +
> + return true;
> +}
> +
> +
> +void __weak arch_irq_work_raise(void)
> +{
> + /*
> + * Lame architectures will get the timer tick callback
> + */
> +}
> +
> +/*
> + * Queue the entry and raise the IPI if needed.
> + */
> +static void __irq_work_queue(struct irq_work *entry)
> +{
> + struct irq_work **head, *next;
> +
> + head = &get_cpu_var(irq_work_list);
> +
> + do {
> + next = *head;
> + /* Can assign non-atomic because we keep the flags set. */
> + entry->next = next_flags(next, IRQ_WORK_FLAGS);
> + } while (cmpxchg(head, next, entry) != next);
> +
> + /* The list was empty, raise self-interrupt to start processing. */
> + if (!irq_work_next(entry))
> + arch_irq_work_raise();
> +
> + put_cpu_var(irq_work_list);
> +}
> +
> +/*
> + * Enqueue the irq_work @entry, returns true on success, failure when the
> + * @entry was already enqueued by someone else.
> + *
> + * Can be re-enqueued while the callback is still in progress.
> + */
> +bool irq_work_queue(struct irq_work *entry)
> +{
> + if (!irq_work_claim(entry)) {
> + /*
> + * Already enqueued, can't do!
> + */
> + return false;
> + }
> +
> + __irq_work_queue(entry);
> + return true;
> +}
> +EXPORT_SYMBOL_GPL(irq_work_queue);
> +
> +/*
> + * Run the irq_work entries on this cpu. Requires to be ran from hardirq
> + * context with local IRQs disabled.
> + */
> +void irq_work_run(void)
> +{
> + struct irq_work *list, **head;
> +
> + head = &__get_cpu_var(irq_work_list);
> + if (*head == NULL)
> + return;
> +
> + BUG_ON(!in_irq());
> + BUG_ON(!irqs_disabled());
> +
> + list = xchg(head, NULL);
> + while (list != NULL) {
> + struct irq_work *entry = list;
> +
> + list = irq_work_next(list);
> +
> + /*
> + * Clear the PENDING bit, after this point the @entry
> + * can be re-used.
> + */
> + entry->next = next_flags(NULL, IRQ_WORK_BUSY);
> + entry->func(entry);
> + /*
> + * Clear the BUSY bit and return to the free state if
> + * no-one else claimed it meanwhile.
> + */
> + cmpxchg(&entry->next, next_flags(NULL, IRQ_WORK_BUSY), NULL);
> + }
> +}
> +EXPORT_SYMBOL_GPL(irq_work_run);
> +
> +/*
> + * Synchronize against the irq_work @entry, ensures the entry is not
> + * currently in use.
> + */
> +void irq_work_sync(struct irq_work *entry)
> +{
> + WARN_ON_ONCE(irqs_disabled());
> +
> + while (irq_work_is_set(entry, IRQ_WORK_BUSY))
> + cpu_relax();
> +}
> +EXPORT_SYMBOL_GPL(irq_work_sync);
> --- a/arch/alpha/Kconfig
> +++ b/arch/alpha/Kconfig
> @@ -9,6 +9,7 @@ config ALPHA
> select HAVE_IDE
> select HAVE_OPROFILE
> select HAVE_SYSCALL_WRAPPERS
> + select HAVE_IRQ_WORK
> select HAVE_PERF_EVENTS
> select HAVE_DMA_ATTRS
> help
> --- a/arch/alpha/include/asm/perf_event.h
> +++ /dev/null
> @@ -1,9 +0,0 @@
> -#ifndef __ASM_ALPHA_PERF_EVENT_H
> -#define __ASM_ALPHA_PERF_EVENT_H
> -
> -/* Alpha only supports software events through this interface. */
> -static inline void set_perf_event_pending(void) { }
> -
> -#define PERF_EVENT_INDEX_OFFSET 0
> -
> -#endif /* __ASM_ALPHA_PERF_EVENT_H */
> --- a/arch/arm/Kconfig
> +++ b/arch/arm/Kconfig
> @@ -22,6 +22,7 @@ config ARM
> select HAVE_KERNEL_GZIP
> select HAVE_KERNEL_LZO
> select HAVE_KERNEL_LZMA
> + select HAVE_IRQ_WORK
> select HAVE_PERF_EVENTS
> select PERF_USE_VMALLOC
> help
> --- a/arch/arm/include/asm/perf_event.h
> +++ b/arch/arm/include/asm/perf_event.h
> @@ -12,18 +12,6 @@
> #ifndef __ARM_PERF_EVENT_H__
> #define __ARM_PERF_EVENT_H__
>
> -/*
> - * NOP: on *most* (read: all supported) ARM platforms, the performance
> - * counter interrupts are regular interrupts and not an NMI. This
> - * means that when we receive the interrupt we can call
> - * perf_event_do_pending() that handles all of the work with
> - * interrupts enabled.
> - */
> -static inline void
> -set_perf_event_pending(void)
> -{
> -}
> -
> /* ARM performance counters start from 1 (in the cp15 accesses) so use the
> * same indexes here for consistency. */
> #define PERF_EVENT_INDEX_OFFSET 1
> --- a/arch/frv/Kconfig
> +++ b/arch/frv/Kconfig
> @@ -7,6 +7,7 @@ config FRV
> default y
> select HAVE_IDE
> select HAVE_ARCH_TRACEHOOK
> + select HAVE_IRQ_WORK
> select HAVE_PERF_EVENTS
>
> config ZONE_DMA
> --- a/arch/frv/lib/perf_event.c
> +++ /dev/null
> @@ -1,19 +0,0 @@
> -/* Performance event handling
> - *
> - * Copyright (C) 2009 Red Hat, Inc. All Rights Reserved.
> - * Written by David Howells (dhowells@...hat.com)
> - *
> - * This program is free software; you can redistribute it and/or
> - * modify it under the terms of the GNU General Public Licence
> - * as published by the Free Software Foundation; either version
> - * 2 of the Licence, or (at your option) any later version.
> - */
> -
> -#include <linux/perf_event.h>
> -
> -/*
> - * mark the performance event as pending
> - */
> -void set_perf_event_pending(void)
> -{
> -}
> --- a/arch/parisc/Kconfig
> +++ b/arch/parisc/Kconfig
> @@ -16,6 +16,7 @@ config PARISC
> select RTC_DRV_GENERIC
> select INIT_ALL_POSSIBLE
> select BUG
> + select HAVE_IRQ_WORK
> select HAVE_PERF_EVENTS
> select GENERIC_ATOMIC64 if !64BIT
> help
> --- a/arch/parisc/include/asm/perf_event.h
> +++ /dev/null
> @@ -1,7 +0,0 @@
> -#ifndef __ASM_PARISC_PERF_EVENT_H
> -#define __ASM_PARISC_PERF_EVENT_H
> -
> -/* parisc only supports software events through this interface. */
> -static inline void set_perf_event_pending(void) { }
> -
> -#endif /* __ASM_PARISC_PERF_EVENT_H */
> --- a/arch/powerpc/Kconfig
> +++ b/arch/powerpc/Kconfig
> @@ -139,6 +139,7 @@ config PPC
> select HAVE_OPROFILE
> select HAVE_SYSCALL_WRAPPERS if PPC64
> select GENERIC_ATOMIC64 if PPC32
> + select HAVE_IRQ_WORK
> select HAVE_PERF_EVENTS
> select HAVE_REGS_AND_STACK_ACCESS_API
>
> --- a/arch/powerpc/kernel/time.c
> +++ b/arch/powerpc/kernel/time.c
> @@ -53,7 +53,7 @@
> #include <linux/posix-timers.h>
> #include <linux/irq.h>
> #include <linux/delay.h>
> -#include <linux/perf_event.h>
> +#include <linux/irq_work.h>
> #include <asm/trace.h>
>
> #include <asm/io.h>
> @@ -532,60 +532,60 @@ void __init iSeries_time_init_early(void
> }
> #endif /* CONFIG_PPC_ISERIES */
>
> -#ifdef CONFIG_PERF_EVENTS
> +#ifdef CONFIG_IRQ_WORK
>
> /*
> * 64-bit uses a byte in the PACA, 32-bit uses a per-cpu variable...
> */
> #ifdef CONFIG_PPC64
> -static inline unsigned long test_perf_event_pending(void)
> +static inline unsigned long test_irq_work_pending(void)
> {
> unsigned long x;
>
> asm volatile("lbz %0,%1(13)"
> : "=r" (x)
> - : "i" (offsetof(struct paca_struct, perf_event_pending)));
> + : "i" (offsetof(struct paca_struct, irq_work_pending)));
> return x;
> }
>
> -static inline void set_perf_event_pending_flag(void)
> +static inline void set_irq_work_pending_flag(void)
> {
> asm volatile("stb %0,%1(13)" : :
> "r" (1),
> - "i" (offsetof(struct paca_struct, perf_event_pending)));
> + "i" (offsetof(struct paca_struct, irq_work_pending)));
> }
>
> -static inline void clear_perf_event_pending(void)
> +static inline void clear_irq_work_pending(void)
> {
> asm volatile("stb %0,%1(13)" : :
> "r" (0),
> - "i" (offsetof(struct paca_struct, perf_event_pending)));
> + "i" (offsetof(struct paca_struct, irq_work_pending)));
> }
>
> #else /* 32-bit */
>
> -DEFINE_PER_CPU(u8, perf_event_pending);
> +DEFINE_PER_CPU(u8, irq_work_pending);
>
> -#define set_perf_event_pending_flag() __get_cpu_var(perf_event_pending) = 1
> -#define test_perf_event_pending() __get_cpu_var(perf_event_pending)
> -#define clear_perf_event_pending() __get_cpu_var(perf_event_pending) = 0
> +#define set_irq_work_pending_flag() __get_cpu_var(irq_work_pending) = 1
> +#define test_irq_work_pending() __get_cpu_var(irq_work_pending)
> +#define clear_irq_work_pending() __get_cpu_var(irq_work_pending) = 0
>
> #endif /* 32 vs 64 bit */
>
> -void set_perf_event_pending(void)
> +void set_irq_work_pending(void)
> {
> preempt_disable();
> - set_perf_event_pending_flag();
> + set_irq_work_pending_flag();
> set_dec(1);
> preempt_enable();
> }
>
> -#else /* CONFIG_PERF_EVENTS */
> +#else /* CONFIG_IRQ_WORK */
>
> -#define test_perf_event_pending() 0
> -#define clear_perf_event_pending()
> +#define test_irq_work_pending() 0
> +#define clear_irq_work_pending()
>
> -#endif /* CONFIG_PERF_EVENTS */
> +#endif /* CONFIG_IRQ_WORK */
>
> /*
> * For iSeries shared processors, we have to let the hypervisor
> @@ -635,9 +635,9 @@ void timer_interrupt(struct pt_regs * re
>
> calculate_steal_time();
>
> - if (test_perf_event_pending()) {
> - clear_perf_event_pending();
> - perf_event_do_pending();
> + if (test_irq_work_pending()) {
> + clear_irq_work_pending();
> + irq_work_run();
> }
>
> #ifdef CONFIG_PPC_ISERIES
> --- a/arch/s390/Kconfig
> +++ b/arch/s390/Kconfig
> @@ -98,6 +98,7 @@ config S390
> select HAVE_KVM if 64BIT
> select HAVE_ARCH_TRACEHOOK
> select INIT_ALL_POSSIBLE
> + select HAVE_IRQ_WORK
> select HAVE_PERF_EVENTS
> select HAVE_KERNEL_GZIP
> select HAVE_KERNEL_BZIP2
> --- a/arch/s390/include/asm/perf_event.h
> +++ /dev/null
> @@ -1,10 +0,0 @@
> -/*
> - * Performance event support - s390 specific definitions.
> - *
> - * Copyright 2009 Martin Schwidefsky, IBM Corporation.
> - */
> -
> -static inline void set_perf_event_pending(void) {}
> -static inline void clear_perf_event_pending(void) {}
> -
> -#define PERF_EVENT_INDEX_OFFSET 0
> --- a/arch/sh/Kconfig
> +++ b/arch/sh/Kconfig
> @@ -16,6 +16,7 @@ config SUPERH
> select HAVE_ARCH_TRACEHOOK
> select HAVE_DMA_API_DEBUG
> select HAVE_DMA_ATTRS
> + select HAVE_IRQ_WORK
> select HAVE_PERF_EVENTS
> select PERF_USE_VMALLOC
> select HAVE_KERNEL_GZIP
> --- a/arch/sh/include/asm/perf_event.h
> +++ b/arch/sh/include/asm/perf_event.h
> @@ -26,11 +26,4 @@ extern int register_sh_pmu(struct sh_pmu
> extern int reserve_pmc_hardware(void);
> extern void release_pmc_hardware(void);
>
> -static inline void set_perf_event_pending(void)
> -{
> - /* Nothing to see here, move along. */
> -}
> -
> -#define PERF_EVENT_INDEX_OFFSET 0
> -
> #endif /* __ASM_SH_PERF_EVENT_H */
> --- a/arch/sparc/Kconfig
> +++ b/arch/sparc/Kconfig
> @@ -25,6 +25,7 @@ config SPARC
> select ARCH_WANT_OPTIONAL_GPIOLIB
> select RTC_CLASS
> select RTC_DRV_M48T59
> + select HAVE_IRQ_WORK
> select HAVE_PERF_EVENTS
> select PERF_USE_VMALLOC
> select HAVE_DMA_ATTRS
> @@ -52,6 +53,7 @@ config SPARC64
> select RTC_DRV_BQ4802
> select RTC_DRV_SUN4V
> select RTC_DRV_STARFIRE
> + select HAVE_IRQ_WORK
> select HAVE_PERF_EVENTS
> select PERF_USE_VMALLOC
>
> --- a/arch/sparc/include/asm/perf_event.h
> +++ b/arch/sparc/include/asm/perf_event.h
> @@ -1,10 +1,6 @@
> #ifndef __ASM_SPARC_PERF_EVENT_H
> #define __ASM_SPARC_PERF_EVENT_H
>
> -extern void set_perf_event_pending(void);
> -
> -#define PERF_EVENT_INDEX_OFFSET 0
> -
> #ifdef CONFIG_PERF_EVENTS
> extern void init_hw_perf_events(void);
> #else
> --- a/arch/sparc/kernel/pcr.c
> +++ b/arch/sparc/kernel/pcr.c
> @@ -7,7 +7,7 @@
> #include <linux/init.h>
> #include <linux/irq.h>
>
> -#include <linux/perf_event.h>
> +#include <linux/irq_work.h>
> #include <linux/ftrace.h>
>
> #include <asm/pil.h>
> @@ -43,14 +43,14 @@ void __irq_entry deferred_pcr_work_irq(i
>
> old_regs = set_irq_regs(regs);
> irq_enter();
> -#ifdef CONFIG_PERF_EVENTS
> - perf_event_do_pending();
> +#ifdef CONFIG_IRQ_WORK
> + irq_work_run();
> #endif
> irq_exit();
> set_irq_regs(old_regs);
> }
>
> -void set_perf_event_pending(void)
> +void arch_irq_work_raise(void)
> {
> set_softint(1 << PIL_DEFERRED_PCR_WORK);
> }
> --- a/arch/x86/Kconfig
> +++ b/arch/x86/Kconfig
> @@ -25,6 +25,7 @@ config X86
> select HAVE_IDE
> select HAVE_OPROFILE
> select HAVE_PERF_EVENTS if (!M386 && !M486)
> + select HAVE_IRQ_WORK
> select HAVE_IOREMAP_PROT
> select HAVE_KPROBES
> select ARCH_WANT_OPTIONAL_GPIOLIB
> --- a/arch/x86/include/asm/entry_arch.h
> +++ b/arch/x86/include/asm/entry_arch.h
> @@ -49,8 +49,8 @@ BUILD_INTERRUPT(apic_timer_interrupt,LOC
> BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR)
> BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR)
>
> -#ifdef CONFIG_PERF_EVENTS
> -BUILD_INTERRUPT(perf_pending_interrupt, LOCAL_PENDING_VECTOR)
> +#ifdef CONFIG_IRQ_WORK
> +BUILD_INTERRUPT(irq_work_interrupt, IRQ_WORK_VECTOR)
> #endif
>
> #ifdef CONFIG_X86_THERMAL_VECTOR
> --- a/arch/x86/include/asm/hw_irq.h
> +++ b/arch/x86/include/asm/hw_irq.h
> @@ -29,7 +29,7 @@
> extern void apic_timer_interrupt(void);
> extern void x86_platform_ipi(void);
> extern void error_interrupt(void);
> -extern void perf_pending_interrupt(void);
> +extern void irq_work_interrupt(void);
>
> extern void spurious_interrupt(void);
> extern void thermal_interrupt(void);
> --- a/arch/x86/kernel/Makefile
> +++ b/arch/x86/kernel/Makefile
> @@ -33,6 +33,7 @@ obj-y := process_$(BITS).o signal.o en
> obj-y += traps.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
> obj-y += time.o ioport.o ldt.o dumpstack.o
> obj-y += setup.o x86_init.o i8259.o irqinit.o
> +obj-$(CONFIG_IRQ_WORK) += irq_work.o
> obj-$(CONFIG_X86_VISWS) += visws_quirks.o
> obj-$(CONFIG_X86_32) += probe_roms_32.o
> obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o
> --- a/arch/x86/kernel/cpu/perf_event.c
> +++ b/arch/x86/kernel/cpu/perf_event.c
> @@ -1160,25 +1160,6 @@ static int x86_pmu_handle_irq(struct pt_
> return handled;
> }
>
> -void smp_perf_pending_interrupt(struct pt_regs *regs)
> -{
> - irq_enter();
> - ack_APIC_irq();
> - inc_irq_stat(apic_pending_irqs);
> - perf_event_do_pending();
> - irq_exit();
> -}
> -
> -void set_perf_event_pending(void)
> -{
> -#ifdef CONFIG_X86_LOCAL_APIC
> - if (!x86_pmu.apic || !x86_pmu_initialized())
> - return;
> -
> - apic->send_IPI_self(LOCAL_PENDING_VECTOR);
> -#endif
> -}
> -
> void perf_events_lapic_init(void)
> {
> if (!x86_pmu.apic || !x86_pmu_initialized())
> --- a/arch/x86/kernel/entry_64.S
> +++ b/arch/x86/kernel/entry_64.S
> @@ -1023,9 +1023,9 @@ apicinterrupt ERROR_APIC_VECTOR \
> apicinterrupt SPURIOUS_APIC_VECTOR \
> spurious_interrupt smp_spurious_interrupt
>
> -#ifdef CONFIG_PERF_EVENTS
> -apicinterrupt LOCAL_PENDING_VECTOR \
> - perf_pending_interrupt smp_perf_pending_interrupt
> +#ifdef CONFIG_IRQ_WORK
> +apicinterrupt IRQ_WORK_VECTOR \
> + irq_work_interrupt smp_irq_work_interrupt
> #endif
>
> /*
> --- /dev/null
> +++ b/arch/x86/kernel/irq_work.c
> @@ -0,0 +1,30 @@
> +/*
> + * x86 specific code for irq_work
> + *
> + * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@...hat.com>
> + */
> +
> +#include <linux/kernel.h>
> +#include <linux/irq_work.h>
> +#include <linux/hardirq.h>
> +#include <asm/apic.h>
> +
> +void smp_irq_work_interrupt(struct pt_regs *regs)
> +{
> + irq_enter();
> + ack_APIC_irq();
> + inc_irq_stat(apic_irq_work_irqs);
> + irq_work_run();
> + irq_exit();
> +}
> +
> +void arch_irq_work_raise(void)
> +{
> +#ifdef CONFIG_X86_LOCAL_APIC
> + if (!cpu_has_apic)
> + return;
> +
> + apic->send_IPI_self(IRQ_WORK_VECTOR);
> + apic_wait_icr_idle();
> +#endif
> +}
> --- a/arch/x86/kernel/irqinit.c
> +++ b/arch/x86/kernel/irqinit.c
> @@ -224,9 +224,9 @@ static void __init apic_intr_init(void)
> alloc_intr_gate(SPURIOUS_APIC_VECTOR, spurious_interrupt);
> alloc_intr_gate(ERROR_APIC_VECTOR, error_interrupt);
>
> - /* Performance monitoring interrupts: */
> -# ifdef CONFIG_PERF_EVENTS
> - alloc_intr_gate(LOCAL_PENDING_VECTOR, perf_pending_interrupt);
> + /* IRQ work interrupts: */
> +# ifdef CONFIG_IRQ_WORK
> + alloc_intr_gate(IRQ_WORK_VECTOR, irq_work_interrupt);
> # endif
>
> #endif
> --- a/include/linux/perf_event.h
> +++ b/include/linux/perf_event.h
> @@ -484,6 +484,7 @@ struct perf_guest_info_callbacks {
> #include <linux/workqueue.h>
> #include <linux/ftrace.h>
> #include <linux/cpu.h>
> +#include <linux/irq_work.h>
> #include <asm/atomic.h>
> #include <asm/local.h>
>
> @@ -608,11 +609,6 @@ struct perf_mmap_data {
> void *data_pages[0];
> };
>
> -struct perf_pending_entry {
> - struct perf_pending_entry *next;
> - void (*func)(struct perf_pending_entry *);
> -};
> -
> struct perf_sample_data;
>
> typedef void (*perf_overflow_handler_t)(struct perf_event *, int,
> @@ -719,7 +715,7 @@ struct perf_event {
> int pending_wakeup;
> int pending_kill;
> int pending_disable;
> - struct perf_pending_entry pending;
> + struct irq_work pending;
>
> atomic_t event_limit;
>
> @@ -831,8 +827,6 @@ extern void perf_event_task_tick(struct
> extern int perf_event_init_task(struct task_struct *child);
> extern void perf_event_exit_task(struct task_struct *child);
> extern void perf_event_free_task(struct task_struct *task);
> -extern void set_perf_event_pending(void);
> -extern void perf_event_do_pending(void);
> extern void perf_event_print_debug(void);
> extern void __perf_disable(void);
> extern bool __perf_enable(void);
> @@ -1031,7 +1025,6 @@ perf_event_task_tick(struct task_struct
> static inline int perf_event_init_task(struct task_struct *child) { return 0; }
> static inline void perf_event_exit_task(struct task_struct *child) { }
> static inline void perf_event_free_task(struct task_struct *task) { }
> -static inline void perf_event_do_pending(void) { }
> static inline void perf_event_print_debug(void) { }
> static inline void perf_disable(void) { }
> static inline void perf_enable(void) { }
> --- a/init/Kconfig
> +++ b/init/Kconfig
> @@ -21,6 +21,13 @@ config CONSTRUCTORS
> depends on !UML
> default y
>
> +config HAVE_IRQ_WORK
> + bool
> +
> +config IRQ_WORK
> + bool
> + depends on HAVE_IRQ_WORK
> +
> menu "General setup"
>
> config EXPERIMENTAL
> @@ -983,6 +990,7 @@ config PERF_EVENTS
> default y if (PROFILING || PERF_COUNTERS)
> depends on HAVE_PERF_EVENTS
> select ANON_INODES
> + select IRQ_WORK
> help
> Enable kernel support for various performance events provided
> by software and hardware.
> --- a/kernel/Makefile
> +++ b/kernel/Makefile
> @@ -23,6 +23,7 @@ CFLAGS_REMOVE_rtmutex-debug.o = -pg
> CFLAGS_REMOVE_cgroup-debug.o = -pg
> CFLAGS_REMOVE_sched_clock.o = -pg
> CFLAGS_REMOVE_perf_event.o = -pg
> +CFLAGS_REMOVE_irq_work.o = -pg
> endif
>
> obj-$(CONFIG_FREEZER) += freezer.o
> @@ -101,6 +102,7 @@ obj-$(CONFIG_RING_BUFFER) += trace/
> obj-$(CONFIG_SMP) += sched_cpupri.o
> obj-$(CONFIG_SLOW_WORK) += slow-work.o
> obj-$(CONFIG_SLOW_WORK_DEBUG) += slow-work-debugfs.o
> +obj-$(CONFIG_IRQ_WORK) += irq_work.o
> obj-$(CONFIG_PERF_EVENTS) += perf_event.o
> obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
> obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
> --- a/kernel/perf_event.c
> +++ b/kernel/perf_event.c
> @@ -1882,12 +1882,11 @@ static void free_event_rcu(struct rcu_he
> kfree(event);
> }
>
> -static void perf_pending_sync(struct perf_event *event);
> static void perf_mmap_data_put(struct perf_mmap_data *data);
>
> static void free_event(struct perf_event *event)
> {
> - perf_pending_sync(event);
> + irq_work_sync(&event->pending);
>
> if (!event->parent) {
> atomic_dec(&nr_events);
> @@ -2824,16 +2823,7 @@ void perf_event_wakeup(struct perf_event
> }
> }
>
> -/*
> - * Pending wakeups
> - *
> - * Handle the case where we need to wakeup up from NMI (or rq->lock) context.
> - *
> - * The NMI bit means we cannot possibly take locks. Therefore, maintain a
> - * single linked list and use cmpxchg() to add entries lockless.
> - */
> -
> -static void perf_pending_event(struct perf_pending_entry *entry)
> +static void perf_pending_event(struct irq_work *entry)
> {
> struct perf_event *event = container_of(entry,
> struct perf_event, pending);
> @@ -2849,89 +2839,6 @@ static void perf_pending_event(struct pe
> }
> }
>
> -#define PENDING_TAIL ((struct perf_pending_entry *)-1UL)
> -
> -static DEFINE_PER_CPU(struct perf_pending_entry *, perf_pending_head) = {
> - PENDING_TAIL,
> -};
> -
> -static void perf_pending_queue(struct perf_pending_entry *entry,
> - void (*func)(struct perf_pending_entry *))
> -{
> - struct perf_pending_entry **head;
> -
> - if (cmpxchg(&entry->next, NULL, PENDING_TAIL) != NULL)
> - return;
> -
> - entry->func = func;
> -
> - head = &get_cpu_var(perf_pending_head);
> -
> - do {
> - entry->next = *head;
> - } while (cmpxchg(head, entry->next, entry) != entry->next);
> -
> - set_perf_event_pending();
> -
> - put_cpu_var(perf_pending_head);
> -}
> -
> -static int __perf_pending_run(void)
> -{
> - struct perf_pending_entry *list;
> - int nr = 0;
> -
> - list = xchg(&__get_cpu_var(perf_pending_head), PENDING_TAIL);
> - while (list != PENDING_TAIL) {
> - void (*func)(struct perf_pending_entry *);
> - struct perf_pending_entry *entry = list;
> -
> - list = list->next;
> -
> - func = entry->func;
> - entry->next = NULL;
> - /*
> - * Ensure we observe the unqueue before we issue the wakeup,
> - * so that we won't be waiting forever.
> - * -- see perf_not_pending().
> - */
> - smp_wmb();
> -
> - func(entry);
> - nr++;
> - }
> -
> - return nr;
> -}
> -
> -static inline int perf_not_pending(struct perf_event *event)
> -{
> - /*
> - * If we flush on whatever cpu we run, there is a chance we don't
> - * need to wait.
> - */
> - get_cpu();
> - __perf_pending_run();
> - put_cpu();
> -
> - /*
> - * Ensure we see the proper queue state before going to sleep
> - * so that we do not miss the wakeup. -- see perf_pending_handle()
> - */
> - smp_rmb();
> - return event->pending.next == NULL;
> -}
> -
> -static void perf_pending_sync(struct perf_event *event)
> -{
> - wait_event(event->waitq, perf_not_pending(event));
> -}
> -
> -void perf_event_do_pending(void)
> -{
> - __perf_pending_run();
> -}
> -
> /*
> * Callchain support -- arch specific
> */
> @@ -2996,8 +2903,7 @@ static void perf_output_wakeup(struct pe
>
> if (handle->nmi) {
> handle->event->pending_wakeup = 1;
> - perf_pending_queue(&handle->event->pending,
> - perf_pending_event);
> + irq_work_queue(&handle->event->pending);
> } else
> perf_event_wakeup(handle->event);
> }
> @@ -3976,8 +3882,7 @@ static int __perf_event_overflow(struct
> event->pending_kill = POLL_HUP;
> if (nmi) {
> event->pending_disable = 1;
> - perf_pending_queue(&event->pending,
> - perf_pending_event);
> + irq_work_queue(&event->pending);
> } else
> perf_event_disable(event);
> }
> @@ -4831,6 +4736,7 @@ perf_event_alloc(struct perf_event_attr
> INIT_LIST_HEAD(&event->event_entry);
> INIT_LIST_HEAD(&event->sibling_list);
> init_waitqueue_head(&event->waitq);
> + init_irq_work(&event->pending, perf_pending_event);
>
> mutex_init(&event->mmap_mutex);
>
> --- a/kernel/timer.c
> +++ b/kernel/timer.c
> @@ -37,7 +37,7 @@
> #include <linux/delay.h>
> #include <linux/tick.h>
> #include <linux/kallsyms.h>
> -#include <linux/perf_event.h>
> +#include <linux/irq_work.h>
> #include <linux/sched.h>
> #include <linux/slab.h>
>
> @@ -1264,7 +1264,10 @@ void update_process_times(int user_tick)
> run_local_timers();
> rcu_check_callbacks(cpu, user_tick);
> printk_tick();
> - perf_event_do_pending();
> +#ifdef CONFIG_IRQ_WORK
> + if (in_irq())
> + irq_work_run();
> +#endif
> scheduler_tick();
> run_posix_cpu_timers(p);
> }
> --- a/arch/arm/kernel/perf_event.c
> +++ b/arch/arm/kernel/perf_event.c
> @@ -1045,7 +1045,7 @@ armv6pmu_handle_irq(int irq_num,
> * platforms that can have the PMU interrupts raised as a PMI, this
> * will not work.
> */
> - perf_event_do_pending();
> + irq_work_run();
>
> return IRQ_HANDLED;
> }
> @@ -2021,7 +2021,7 @@ static irqreturn_t armv7pmu_handle_irq(i
> * platforms that can have the PMU interrupts raised as a PMI, this
> * will not work.
> */
> - perf_event_do_pending();
> + irq_work_run();
>
> return IRQ_HANDLED;
> }
> --- a/arch/x86/include/asm/irq_vectors.h
> +++ b/arch/x86/include/asm/irq_vectors.h
> @@ -114,9 +114,9 @@
> #define X86_PLATFORM_IPI_VECTOR 0xed
>
> /*
> - * Performance monitoring pending work vector:
> + * IRQ work vector:
> */
> -#define LOCAL_PENDING_VECTOR 0xec
> +#define IRQ_WORK_VECTOR 0xec
>
> #define UV_BAU_MESSAGE 0xea
>
> --- a/arch/x86/include/asm/hardirq.h
> +++ b/arch/x86/include/asm/hardirq.h
> @@ -14,7 +14,7 @@ typedef struct {
> #endif
> unsigned int x86_platform_ipis; /* arch dependent */
> unsigned int apic_perf_irqs;
> - unsigned int apic_pending_irqs;
> + unsigned int apic_irq_work_irqs;
> #ifdef CONFIG_SMP
> unsigned int irq_resched_count;
> unsigned int irq_call_count;
> --- a/arch/x86/kernel/irq.c
> +++ b/arch/x86/kernel/irq.c
> @@ -67,10 +67,10 @@ static int show_other_interrupts(struct
> for_each_online_cpu(j)
> seq_printf(p, "%10u ", irq_stats(j)->apic_perf_irqs);
> seq_printf(p, " Performance monitoring interrupts\n");
> - seq_printf(p, "%*s: ", prec, "PND");
> + seq_printf(p, "%*s: ", prec, "IWI");
> for_each_online_cpu(j)
> - seq_printf(p, "%10u ", irq_stats(j)->apic_pending_irqs);
> - seq_printf(p, " Performance pending work\n");
> + seq_printf(p, "%10u ", irq_stats(j)->apic_irq_work_irqs);
> + seq_printf(p, " IRQ work interrupts\n");
> #endif
> if (x86_platform_ipi_callback) {
> seq_printf(p, "%*s: ", prec, "PLT");
> @@ -185,7 +185,7 @@ u64 arch_irq_stat_cpu(unsigned int cpu)
> sum += irq_stats(cpu)->apic_timer_irqs;
> sum += irq_stats(cpu)->irq_spurious_count;
> sum += irq_stats(cpu)->apic_perf_irqs;
> - sum += irq_stats(cpu)->apic_pending_irqs;
> + sum += irq_stats(cpu)->apic_irq_work_irqs;
> #endif
> if (x86_platform_ipi_callback)
> sum += irq_stats(cpu)->x86_platform_ipis;
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists