| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20080919200645.GC6528@sgi.com>
Date: Fri, 19 Sep 2008 15:06:45 -0500
From: Dean Nelson <dcn@....com>
To: Ingo Molnar <mingo@...e.hu>
Cc: "Eric W. Biederman" <ebiederm@...ssion.com>,
"H. Peter Anvin" <hpa@...or.com>, Jack Steiner <steiner@....com>,
Alan Mayer <ajm@....com>, jeremy@...p.org,
rusty@...tcorp.com.au, suresh.b.siddha@...el.com,
torvalds@...ux-foundation.org, linux-kernel@...r.kernel.org,
Thomas Gleixner <tglx@...utronix.de>,
Yinghai Lu <Yinghai.lu@....com>
Subject: [PATCH 2/3] switch static system vector allocation to use vector_irq[] v2
Replace the current use of system_vectors[] for the allocation of static
system vectors by also using the per_cpu variable vector_irq[].
Signed-off-by: Dean Nelson <dcn@....com>
---
arch/x86/kernel/Makefile | 2
arch/x86/kernel/apic.c | 2
arch/x86/kernel/io_apic.c | 112 ++++++++++++++++++++-------------------
arch/x86/kernel/irq.c | 22 +++++++
arch/x86/kernel/irq_32.c | 2
arch/x86/kernel/irq_64.c | 2
arch/x86/kernel/irqinit_32.c | 11 ++-
arch/x86/kernel/irqinit_64.c | 11 ++-
include/asm-x86/desc.h | 21 +++----
include/asm-x86/hw_irq.h | 4 -
include/asm-x86/irq.h | 8 ++
11 files changed, 118 insertions(+), 79 deletions(-)
Index: linux/arch/x86/kernel/apic.c
===================================================================
--- linux.orig/arch/x86/kernel/apic.c 2008-09-19 14:12:25.000000000 -0500
+++ linux/arch/x86/kernel/apic.c 2008-09-19 14:13:17.000000000 -0500
@@ -118,8 +118,6 @@ EXPORT_SYMBOL_GPL(local_apic_timer_c2_ok
int first_system_vector = 0xfe;
-char system_vectors[NR_VECTORS] = { [0 ... NR_VECTORS-1] = SYS_VECTOR_FREE};
-
/*
* Debug level, exported for io_apic.c
*/
Index: linux/include/asm-x86/desc.h
===================================================================
--- linux.orig/include/asm-x86/desc.h 2008-09-19 14:12:25.000000000 -0500
+++ linux/include/asm-x86/desc.h 2008-09-19 14:13:17.000000000 -0500
@@ -5,6 +5,7 @@
#include <asm/desc_defs.h>
#include <asm/ldt.h>
#include <asm/mmu.h>
+#include <asm/irq.h>
#include <linux/smp.h>
static inline void fill_ldt(struct desc_struct *desc,
@@ -320,20 +321,20 @@ static inline void set_intr_gate(unsigne
_set_gate(n, GATE_INTERRUPT, addr, 0, 0, __KERNEL_CS);
}
-#define SYS_VECTOR_FREE 0
-#define SYS_VECTOR_ALLOCED 1
-
extern int first_system_vector;
-extern char system_vectors[];
static inline void alloc_system_vector(int vector)
{
- if (system_vectors[vector] == SYS_VECTOR_FREE) {
- system_vectors[vector] = SYS_VECTOR_ALLOCED;
- if (first_system_vector > vector)
- first_system_vector = vector;
- } else
- BUG();
+ unsigned long flags;
+ int ret;
+
+ spin_lock_irqsave(&vector_lock, flags);
+ ret = grab_irq_vector(NON_IRQ_DESC, vector, &cpu_possible_map);
+ BUG_ON(ret != vector);
+
+ if (first_system_vector > vector)
+ first_system_vector = vector;
+ spin_unlock_irqrestore(&vector_lock, flags);
}
static inline void alloc_intr_gate(unsigned int n, void *addr)
Index: linux/arch/x86/kernel/io_apic.c
===================================================================
--- linux.orig/arch/x86/kernel/io_apic.c 2008-09-19 14:13:09.000000000 -0500
+++ linux/arch/x86/kernel/io_apic.c 2008-09-19 14:13:17.000000000 -0500
@@ -72,7 +72,6 @@
int sis_apic_bug = -1;
static DEFINE_SPINLOCK(ioapic_lock);
-static DEFINE_SPINLOCK(vector_lock);
/*
* # of IRQ routing registers
@@ -546,7 +545,7 @@ static void __target_IO_APIC_irq(unsigne
}
}
-static int assign_irq_vector(int irq, cpumask_t mask);
+static int assign_irq_vector(int irq, cpumask_t *mask);
static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
{
@@ -561,7 +560,7 @@ static void set_ioapic_affinity_irq(unsi
return;
cfg = irq_cfg(irq);
- if (assign_irq_vector(irq, mask))
+ if (assign_irq_vector(irq, &mask))
return;
cpus_and(tmp, cfg->domain, mask);
@@ -1196,20 +1195,33 @@ static int pin_2_irq(int idx, int apic,
return irq;
}
-void lock_vector_lock(void)
+static int ioapic_grab_irq_vector(struct irq_desc *desc, unsigned int vector,
+ cpumask_t *domain, cpumask_t *domain_online)
{
- /* Used to the online set of cpus does not change
- * during assign_irq_vector.
- */
- spin_lock(&vector_lock);
-}
+ struct irq_cfg *cfg;
+ int ret;
-void unlock_vector_lock(void)
-{
- spin_unlock(&vector_lock);
+ BUG_ON(!spin_is_locked(&vector_lock));
+
+ ret = grab_irq_vector(desc, vector, domain_online);
+ if (ret == vector) {
+ cfg = irq_cfg(desc->irq);
+ if (cfg->vector) {
+ cfg->move_in_progress = 1;
+ cfg->old_domain = cfg->domain;
+ }
+ cfg->vector = vector;
+ cfg->domain = *domain;
+ }
+ return ret;
}
-static int __assign_irq_vector(int irq, cpumask_t mask)
+/* The following scratch cpumask_t variables are protected by vector lock. */
+static cpumask_t eligible_cpus;
+static cpumask_t domain;
+static cpumask_t domain_online;
+
+static int __assign_irq_vector(int irq, cpumask_t *mask)
{
/*
* NOTE! The local APIC isn't very good at handling
@@ -1223,36 +1235,34 @@ static int __assign_irq_vector(int irq,
* 0x80, because int 0x80 is hm, kind of importantish. ;)
*/
static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
- unsigned int old_vector;
int cpu;
struct irq_cfg *cfg;
struct irq_desc *desc;
+ int ret;
- cfg = irq_cfg(irq);
+ BUG_ON(!spin_is_locked(&vector_lock));
- /* Only try and allocate irqs on cpus that are present */
- cpus_and(mask, mask, cpu_online_map);
+ cfg = irq_cfg(irq);
if ((cfg->move_in_progress) || cfg->move_cleanup_count)
return -EBUSY;
- old_vector = cfg->vector;
- if (old_vector) {
- cpumask_t tmp;
- cpus_and(tmp, cfg->domain, mask);
- if (!cpus_empty(tmp))
+ /* Only try and allocate irqs on cpus that are present */
+ cpus_and(eligible_cpus, *mask, cpu_online_map);
+
+ if (cfg->vector) {
+ cpus_and(domain, cfg->domain, eligible_cpus);
+ if (!cpus_empty(domain))
return 0;
}
desc = irq_to_desc_alloc(irq);
- for_each_cpu_mask_nr(cpu, mask) {
- cpumask_t domain, new_mask;
- int new_cpu;
+ for_each_cpu_mask_nr(cpu, eligible_cpus) {
int vector, offset;
domain = vector_allocation_domain(cpu);
- cpus_and(new_mask, domain, cpu_online_map);
+ cpus_and(domain_online, domain, cpu_online_map);
vector = current_vector;
offset = current_offset;
@@ -1272,26 +1282,20 @@ next:
if (vector == SYSCALL_VECTOR)
goto next;
#endif
- for_each_cpu_mask_nr(new_cpu, new_mask)
- if (per_cpu(vector_irq, new_cpu)[vector] != NULL)
- goto next;
+ ret = ioapic_grab_irq_vector(desc, vector, &domain,
+ &domain_online);
+ if (ret != vector)
+ goto next;
+
/* Found one! */
current_vector = vector;
current_offset = offset;
- if (old_vector) {
- cfg->move_in_progress = 1;
- cfg->old_domain = cfg->domain;
- }
- for_each_cpu_mask_nr(new_cpu, new_mask)
- per_cpu(vector_irq, new_cpu)[vector] = desc;
- cfg->vector = vector;
- cfg->domain = domain;
return 0;
}
return -ENOSPC;
}
-static int assign_irq_vector(int irq, cpumask_t mask)
+static int assign_irq_vector(int irq, cpumask_t *mask)
{
int err;
unsigned long flags;
@@ -1334,13 +1338,13 @@ void __setup_vector_irq(int cpu)
continue;
vector = cfg->vector;
desc = irq_to_desc(irq);
- BUG_ON(desc == NULL);
+ BUG_ON(desc == NULL || desc == NON_IRQ_DESC);
per_cpu(vector_irq, cpu)[vector] = desc;
}
/* Mark the free vectors */
for (vector = 0; vector < NR_VECTORS; ++vector) {
desc = per_cpu(vector_irq, cpu)[vector];
- if (desc == NULL)
+ if (desc == NULL || desc == NON_IRQ_DESC)
continue;
cfg = irq_cfg(desc->irq);
@@ -1495,7 +1499,7 @@ static void setup_IO_APIC_irq(int apic,
cfg = irq_cfg(irq);
mask = TARGET_CPUS;
- if (assign_irq_vector(irq, mask))
+ if (assign_irq_vector(irq, &mask))
return;
cpus_and(mask, cfg->domain, mask);
@@ -2280,7 +2284,7 @@ static void migrate_ioapic_irq(int irq,
if (get_irte(irq, &irte))
return;
- if (assign_irq_vector(irq, mask))
+ if (assign_irq_vector(irq, &mask))
return;
cfg = irq_cfg(irq);
@@ -2399,7 +2403,7 @@ asmlinkage void smp_irq_move_cleanup_int
struct irq_cfg *cfg;
desc = __get_cpu_var(vector_irq)[vector];
- if (desc == NULL)
+ if (desc == NULL || desc == NON_IRQ_DESC)
continue;
cfg = irq_cfg(desc->irq);
@@ -2764,6 +2768,7 @@ static inline void __init check_timer(vo
unsigned long flags;
unsigned int ver;
int no_pin1 = 0;
+ cpumask_t mask;
local_irq_save(flags);
@@ -2774,7 +2779,8 @@ static inline void __init check_timer(vo
* get/set the timer IRQ vector:
*/
disable_8259A_irq(0);
- assign_irq_vector(0, TARGET_CPUS);
+ mask = TARGET_CPUS;
+ assign_irq_vector(0, &mask);
/*
* As IRQ0 is to be enabled in the 8259A, the virtual
@@ -3074,10 +3080,12 @@ unsigned int create_irq_nr(unsigned int
unsigned int new;
unsigned long flags;
struct irq_cfg *cfg_new;
+ cpumask_t mask;
#ifndef CONFIG_HAVE_SPARSE_IRQ
irq_want = nr_irqs - 1;
#endif
+ mask = TARGET_CPUS;
irq = 0;
spin_lock_irqsave(&vector_lock, flags);
@@ -3090,7 +3098,7 @@ unsigned int create_irq_nr(unsigned int
/* check if need to create one */
if (!cfg_new)
cfg_new = irq_cfg_alloc(new);
- if (__assign_irq_vector(new, TARGET_CPUS) == 0)
+ if (__assign_irq_vector(new, &mask) == 0)
irq = new;
break;
}
@@ -3140,7 +3148,7 @@ static int msi_compose_msg(struct pci_de
cpumask_t tmp;
tmp = TARGET_CPUS;
- err = assign_irq_vector(irq, tmp);
+ err = assign_irq_vector(irq, &tmp);
if (err)
return err;
@@ -3212,7 +3220,7 @@ static void set_msi_irq_affinity(unsigne
if (cpus_empty(tmp))
return;
- if (assign_irq_vector(irq, mask))
+ if (assign_irq_vector(irq, &mask))
return;
cfg = irq_cfg(irq);
@@ -3251,7 +3259,7 @@ static void ir_set_msi_irq_affinity(unsi
if (get_irte(irq, &irte))
return;
- if (assign_irq_vector(irq, mask))
+ if (assign_irq_vector(irq, &mask))
return;
cfg = irq_cfg(irq);
@@ -3491,7 +3499,7 @@ static void dmar_msi_set_affinity(unsign
if (cpus_empty(tmp))
return;
- if (assign_irq_vector(irq, mask))
+ if (assign_irq_vector(irq, &mask))
return;
cfg = irq_cfg(irq);
@@ -3552,7 +3560,7 @@ static void hpet_msi_set_affinity(unsign
if (cpus_empty(tmp))
return;
- if (assign_irq_vector(irq, mask))
+ if (assign_irq_vector(irq, &mask))
return;
cfg = irq_cfg(irq);
@@ -3632,7 +3640,7 @@ static void set_ht_irq_affinity(unsigned
if (cpus_empty(tmp))
return;
- if (assign_irq_vector(irq, mask))
+ if (assign_irq_vector(irq, &mask))
return;
cfg = irq_cfg(irq);
@@ -3663,7 +3671,7 @@ int arch_setup_ht_irq(unsigned int irq,
cpumask_t tmp;
tmp = TARGET_CPUS;
- err = assign_irq_vector(irq, tmp);
+ err = assign_irq_vector(irq, &tmp);
if (!err) {
struct ht_irq_msg msg;
unsigned dest;
Index: linux/arch/x86/kernel/irq_32.c
===================================================================
--- linux.orig/arch/x86/kernel/irq_32.c 2008-09-19 14:13:09.000000000 -0500
+++ linux/arch/x86/kernel/irq_32.c 2008-09-19 14:13:17.000000000 -0500
@@ -234,7 +234,7 @@ unsigned int do_IRQ(struct pt_regs *regs
overflow = check_stack_overflow();
desc = __get_cpu_var(vector_irq)[vector];
- if (unlikely(desc == NULL)) {
+ if (unlikely(desc == NULL || desc == NON_IRQ_DESC)) {
printk(KERN_EMERG "%s: cannot handle IRQ vector %#x cpu %d\n",
__func__, vector, smp_processor_id());
BUG();
Index: linux/arch/x86/kernel/irq_64.c
===================================================================
--- linux.orig/arch/x86/kernel/irq_64.c 2008-09-19 14:13:09.000000000 -0500
+++ linux/arch/x86/kernel/irq_64.c 2008-09-19 14:13:17.000000000 -0500
@@ -222,7 +222,7 @@ asmlinkage unsigned int do_IRQ(struct pt
#endif
desc = __get_cpu_var(vector_irq)[vector];
- if (likely(desc != NULL)) {
+ if (likely(desc != NULL && desc != NON_IRQ_DESC)) {
generic_handle_irq_desc(desc->irq, desc);
} else {
if (!disable_apic)
Index: linux/include/asm-x86/irq.h
===================================================================
--- linux.orig/include/asm-x86/irq.h 2008-09-19 14:12:25.000000000 -0500
+++ linux/include/asm-x86/irq.h 2008-09-19 14:13:17.000000000 -0500
@@ -47,4 +47,12 @@ extern void native_init_IRQ(void);
/* Interrupt vector management */
extern DECLARE_BITMAP(used_vectors, NR_VECTORS);
+struct irq_desc;
+extern int grab_irq_vector(struct irq_desc *desc, unsigned int vector,
+ cpumask_t *new_domain_mask);
+
+#define NON_IRQ_DESC ((struct irq_desc *)-1UL)
+
+extern spinlock_t vector_lock;
+
#endif /* ASM_X86__IRQ_H */
Index: linux/arch/x86/kernel/Makefile
===================================================================
--- linux.orig/arch/x86/kernel/Makefile 2008-09-19 14:12:25.000000000 -0500
+++ linux/arch/x86/kernel/Makefile 2008-09-19 14:13:17.000000000 -0500
@@ -24,7 +24,7 @@ CFLAGS_tsc.o := $(nostackp)
CFLAGS_paravirt.o := $(nostackp)
obj-y := process_$(BITS).o signal_$(BITS).o entry_$(BITS).o
-obj-y += traps_$(BITS).o irq_$(BITS).o
+obj-y += traps_$(BITS).o irq.o irq_$(BITS).o
obj-y += time_$(BITS).o ioport.o ldt.o
obj-y += setup.o i8259.o irqinit_$(BITS).o setup_percpu.o
obj-$(CONFIG_X86_VISWS) += visws_quirks.o
Index: linux/arch/x86/kernel/irq.c
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/arch/x86/kernel/irq.c 2008-09-19 14:13:17.000000000 -0500
@@ -0,0 +1,22 @@
+#include <linux/irq.h>
+
+DEFINE_SPINLOCK(vector_lock);
+
+int grab_irq_vector(struct irq_desc *desc, unsigned int vector,
+ cpumask_t *domain)
+{
+ int cpu;
+
+ BUG_ON(!spin_is_locked(&vector_lock));
+
+ for_each_cpu_mask_nr(cpu, *domain) {
+ if (per_cpu(vector_irq, cpu)[vector] != NULL)
+ return -EBUSY;
+ }
+
+ /* it's available, reserve it */
+ for_each_cpu_mask_nr(cpu, *domain)
+ per_cpu(vector_irq, cpu)[vector] = desc;
+
+ return vector;
+}
Index: linux/include/asm-x86/hw_irq.h
===================================================================
--- linux.orig/include/asm-x86/hw_irq.h 2008-09-19 14:13:09.000000000 -0500
+++ linux/include/asm-x86/hw_irq.h 2008-09-19 14:13:17.000000000 -0500
@@ -117,8 +117,8 @@ typedef struct irq_desc *vector_irq_t[NR
DECLARE_PER_CPU(vector_irq_t, vector_irq);
#ifdef CONFIG_X86_IO_APIC
-extern void lock_vector_lock(void);
-extern void unlock_vector_lock(void);
+static inline void lock_vector_lock(void) { spin_lock(&vector_lock); }
+static inline void unlock_vector_lock(void) { spin_unlock(&vector_lock); }
extern void __setup_vector_irq(int cpu);
#else
static inline void lock_vector_lock(void) {}
Index: linux/arch/x86/kernel/irqinit_32.c
===================================================================
--- linux.orig/arch/x86/kernel/irqinit_32.c 2008-09-19 14:13:09.000000000 -0500
+++ linux/arch/x86/kernel/irqinit_32.c 2008-09-19 14:13:17.000000000 -0500
@@ -59,8 +59,9 @@ static struct irqaction fpu_irq = {
void __init init_ISA_irqs (void)
{
int i;
- int cpu;
+ int ret;
unsigned int vector;
+ unsigned long flags;
#ifdef CONFIG_X86_LOCAL_APIC
init_bsp_APIC();
@@ -79,10 +80,10 @@ void __init init_ISA_irqs (void)
desc->depth = 1;
vector = IRQ0_VECTOR + i;
- for_each_possible_cpu(cpu) {
- BUG_ON(per_cpu(vector_irq, cpu)[vector] != NULL);
- per_cpu(vector_irq, cpu)[vector] = desc;
- }
+ spin_lock_irqsave(&vector_lock, flags);
+ ret = grab_irq_vector(desc, vector, &cpu_possible_map);
+ BUG_ON(ret != vector);
+ spin_unlock_irqrestore(&vector_lock, flags);
set_irq_chip_and_handler_name(i, &i8259A_chip,
handle_level_irq, "XT");
Index: linux/arch/x86/kernel/irqinit_64.c
===================================================================
--- linux.orig/arch/x86/kernel/irqinit_64.c 2008-09-19 14:13:09.000000000 -0500
+++ linux/arch/x86/kernel/irqinit_64.c 2008-09-19 14:13:17.000000000 -0500
@@ -119,8 +119,9 @@ DEFINE_PER_CPU(vector_irq_t, vector_irq)
static void __init init_ISA_irqs (void)
{
int i;
- int cpu;
+ int ret;
unsigned int vector;
+ unsigned long flags;
init_bsp_APIC();
init_8259A(0);
@@ -134,10 +135,10 @@ static void __init init_ISA_irqs (void)
desc->depth = 1;
vector = IRQ0_VECTOR + i;
- for_each_possible_cpu(cpu) {
- BUG_ON(per_cpu(vector_irq, cpu)[vector] != NULL);
- per_cpu(vector_irq, cpu)[vector] = desc;
- }
+ spin_lock_irqsave(&vector_lock, flags);
+ ret = grab_irq_vector(desc, vector, &cpu_possible_map);
+ BUG_ON(ret != vector);
+ spin_unlock_irqrestore(&vector_lock, flags);
/*
* 16 old-style INTA-cycle interrupts:
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists