--- arch/x86/kernel/apic/io_apic.c | 310 ++++++++++++++++++++++++++++++++++------ arch/x86/kernel/hpet.c | 2 +- drivers/pci/htirq.c | 2 +- include/linux/irq.h | 3 +- 4 files changed, 271 insertions(+), 46 deletions(-) diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index fadcd74..5e9decc 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c @@ -249,11 +249,6 @@ static struct irq_cfg *alloc_irq_and_cfg_at(unsigned int at, int node) return cfg; } -static int alloc_irq_from(unsigned int from, int node) -{ - return irq_alloc_desc_from(from, node); -} - static void free_irq_at(unsigned int at, struct irq_cfg *cfg) { free_irq_cfg(at, cfg); @@ -1037,6 +1032,39 @@ void unlock_vector_lock(void) raw_spin_unlock(&vector_lock); } +/* + * The P6 family and Pentium processors (presumably also earlier processors), + * can queue no more than two interrupts per priority level, and will ignore + * other interrupts that are received within the same priority level (the + * priority level is the vector number shifted right by 4), so we try to + * spread these out a bit to avoid this happening. + * + * Pentium 4, Xeon and later processors do not have this limitation. + * It is unknown what limitations AMD, Cyrix, Transmeta, VIA, IDT and + * other manufacturers have. + */ +static int many_vectors_per_prio(void) +{ + struct cpuinfo_x86 *c; + static char init, result; + if (init) + return result; + + c = &boot_cpu_data; + switch (c->x86_vendor) { + case X86_VENDOR_INTEL: + if (c->x86 > 6 || + ((c->x86 == 6) && (c->x86_model >= 13))) + result = 1; + break; + default: + break; + } + + init = 1; + return result; +} + static int __assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask) { @@ -1117,13 +1145,110 @@ next: return err; } +static int __assign_irq_vector_block(int irq, unsigned count, const struct cpumask *mask) +{ + static int current_vector = FIRST_EXTERNAL_VECTOR; + unsigned int old_vector; + unsigned i, cpu; + int err; + struct irq_cfg *cfg; + cpumask_var_t tmp_mask; + + BUG_ON(irq + count > NR_IRQS); + BUG_ON(count & (count - 1)); + + for (i = 0; i < count; i++) { + cfg = irq_cfg(irq + i); + if (cfg->move_in_progress) + return -EBUSY; + } + + if (!alloc_cpumask_var(&tmp_mask, GFP_ATOMIC)) + return -ENOMEM; + + cfg = irq_cfg(irq); + old_vector = cfg->vector; + if (old_vector) { + err = 0; + cpumask_and(tmp_mask, mask, cpu_online_mask); + cpumask_and(tmp_mask, cfg->domain, tmp_mask); + if (!cpumask_empty(tmp_mask)) + goto out; + } + + /* Only try and allocate irqs on cpus that are present */ + err = -ENOSPC; + for_each_cpu_and(cpu, mask, cpu_online_mask) { + int new_cpu; + int vector; + + apic->vector_allocation_domain(cpu, tmp_mask); + + vector = current_vector & ~(count - 1); +next: + vector += count; + if (vector + count >= first_system_vector) { + vector = FIRST_EXTERNAL_VECTOR & ~(count - 1); + if (vector < FIRST_EXTERNAL_VECTOR) + vector += count; + } + if (unlikely((current_vector & ~(count - 1)) == vector)) + continue; + + for (i = 0; i < count; i++) + if (test_bit(vector + i, used_vectors)) + goto next; + + for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask) { + for (i = 0; i < count; i++) { + if (per_cpu(vector_irq, new_cpu)[vector + i] != -1) + goto next; + } + } + /* Found one! */ + current_vector = vector + count - 1; + for (i = 0; i < count; i++) { + cfg = irq_cfg(irq + i); + if (old_vector) { + cfg->move_in_progress = 1; + cpumask_copy(cfg->old_domain, cfg->domain); + } + for_each_cpu_and(new_cpu, tmp_mask, cpu_online_mask) + per_cpu(vector_irq, new_cpu)[vector + i] = irq + i; + cfg->vector = vector + i; + cpumask_copy(cfg->domain, tmp_mask); + } + err = 0; + break; + } +out: + free_cpumask_var(tmp_mask); + return err; +} + int assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask) { int err; unsigned long flags; raw_spin_lock_irqsave(&vector_lock, flags); - err = __assign_irq_vector(irq, cfg, mask); + if (many_vectors_per_prio()) + err = __assign_irq_vector_block(irq, 1, mask); + else + err = __assign_irq_vector(irq, cfg, mask); + raw_spin_unlock_irqrestore(&vector_lock, flags); + return err; +} + +/* Assumes that count is a power of two and aligns to that power of two */ +static int +assign_irq_vector_block(int irq, unsigned count, const struct cpumask *mask) +{ + int err; + unsigned long flags; + + raw_spin_lock_irqsave(&vector_lock, flags); + err = __assign_irq_vector_block(irq, count, mask); raw_spin_unlock_irqrestore(&vector_lock, flags); return err; } @@ -2200,14 +2325,34 @@ int __ioapic_set_affinity(struct irq_data *data, const struct cpumask *mask, unsigned int *dest_id) { struct irq_cfg *cfg = data->chip_data; + unsigned irq; if (!cpumask_intersects(mask, cpu_online_mask)) return -1; - if (assign_irq_vector(data->irq, data->chip_data, mask)) - return -1; + irq = data->irq; + cfg = data->chip_data; - cpumask_copy(data->affinity, mask); + if (many_vectors_per_prio()) { + struct msi_desc *msi_desc = data->msi_desc; + unsigned i, count = 1; + + if (msi_desc) + count = 1 << msi_desc->msi_attrib.multiple; + + /* Multiple MSIs all go to the same destination */ + if (assign_irq_vector_block(irq, count, mask)) + return -1; + for (i = 0; i < count; i++) { + data = &irq_to_desc(irq + i)->irq_data; + cpumask_copy(data->affinity, mask); + } + } else { + if (assign_irq_vector(irq, cfg, mask)) + return BAD_APICID; + + cpumask_copy(data->affinity, mask); + } *dest_id = apic->cpu_mask_to_apicid_and(mask, cfg->domain); return 0; @@ -3053,7 +3198,7 @@ device_initcall(ioapic_init_sysfs); /* * Dynamic irq allocate and deallocation */ -unsigned int create_irq_nr(unsigned int from, int node) +unsigned int create_irq_nr(unsigned int from, unsigned count, int node) { struct irq_cfg *cfg; unsigned long flags; @@ -3063,25 +3208,31 @@ unsigned int create_irq_nr(unsigned int from, int node) if (from < nr_irqs_gsi) from = nr_irqs_gsi; - irq = alloc_irq_from(from, node); + irq = irq_alloc_descs(-1, from, count, node); if (irq < 0) return 0; cfg = alloc_irq_cfg(irq, node); if (!cfg) { - free_irq_at(irq, NULL); + irq_free_descs(irq, count); return 0; } raw_spin_lock_irqsave(&vector_lock, flags); - if (!__assign_irq_vector(irq, cfg, apic->target_cpus())) - ret = irq; + if (many_vectors_per_prio()) { + if (!__assign_irq_vector_block(irq, count, apic->target_cpus())) + ret = irq; + } else { + if (!__assign_irq_vector(irq, cfg, apic->target_cpus())) + ret = irq; + } raw_spin_unlock_irqrestore(&vector_lock, flags); if (ret) { set_irq_chip_data(irq, cfg); irq_clear_status_flags(irq, IRQ_NOREQUEST); } else { - free_irq_at(irq, cfg); + free_irq_cfg(irq, cfg); + irq_free_descs(irq, count); } return ret; } @@ -3093,7 +3244,7 @@ int create_irq(void) int irq; irq_want = nr_irqs_gsi; - irq = create_irq_nr(irq_want, node); + irq = create_irq_nr(irq_want, 1, node); if (irq == 0) irq = -1; @@ -3121,7 +3272,7 @@ void destroy_irq(unsigned int irq) */ #ifdef CONFIG_PCI_MSI static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, - struct msi_msg *msg, u8 hpet_id) + unsigned count, struct msi_msg *msg, u8 hpet_id) { struct irq_cfg *cfg; int err; @@ -3131,7 +3282,10 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, return -ENXIO; cfg = irq_cfg(irq); - err = assign_irq_vector(irq, cfg, apic->target_cpus()); + if (count == 1) + err = assign_irq_vector(irq, cfg, apic->target_cpus()); + else + err = assign_irq_vector_block(irq, count, apic->target_cpus()); if (err) return err; @@ -3307,47 +3461,99 @@ static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec) return index; } -static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq) +static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, + unsigned count, int base_irq) { struct msi_msg msg; + unsigned irq; int ret; - ret = msi_compose_msg(dev, irq, &msg, -1); + ret = msi_compose_msg(dev, base_irq, count, &msg, -1); if (ret < 0) return ret; - set_irq_msi(irq, msidesc); - write_msi_msg(irq, &msg); + msidesc->msi_attrib.multiple = order_base_2(count); - if (irq_remapped(get_irq_chip_data(irq))) { - irq_set_status_flags(irq, IRQ_MOVE_PCNTXT); - set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge"); - } else - set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge"); + /* perform loop backwards, so first irq has msidesc set */ + for (irq = base_irq + count - 1; irq >= base_irq; irq--) { + set_irq_msi(irq, msidesc); + if (irq_remapped(get_irq_chip_data(irq))) { + irq_set_status_flags(irq, IRQ_MOVE_PCNTXT); + set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge"); + } else + set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge"); + } - dev_printk(KERN_DEBUG, &dev->dev, "irq %d for MSI/MSI-X\n", irq); + write_msi_msg(base_irq, &msg); + dev_printk(KERN_DEBUG, &dev->dev, "irq %d-%d for MSI/MSI-X\n", + base_irq, base_irq + count - 1); return 0; } -int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) +static int setup_msi_irqs(struct pci_dev *dev, int nvec) +{ + unsigned base_irq, alloc, i; + int ret, node; + struct msi_desc *msidesc = list_first_entry(&dev->msi_list, + struct msi_desc, list); + struct intel_iommu *iommu = map_dev_to_ir(dev); + + if (intr_remapping_enabled && !iommu) + return -ENOENT; + if (nvec > 1 && !many_vectors_per_prio()) + return 1; + + /* + * MSI only lets you program the device with nvec that is a power + * of two. We could possibly trust the device driver that it'll + * only use the number it asked for, but to be safe, let's reserve + * all the interrupts we're telling the device it can use. + */ + alloc = roundup_pow_of_two(nvec); + node = dev_to_node(&dev->dev); + base_irq = create_irq_nr(nr_irqs_gsi, alloc, node); + if (base_irq == 0) + return (alloc > 1) ? alloc / 2 : -ENOSPC; + + if (intr_remapping_enabled) { + ret = msi_alloc_irte(dev, base_irq, alloc); + if (ret < 0) + goto error; + + for (i = 1; i < alloc; i++) + set_irte_irq(base_irq + i, iommu, ret, i); + } + + ret = setup_msi_irq(dev, msidesc, alloc, base_irq); + if (ret < 0) + goto error; + + return 0; + +error: + for (i = 0; i < alloc; i++) + destroy_irq(base_irq + i); + return ret; +} + +static int setup_msix_irqs(struct pci_dev *dev, int nvec) { int node, ret, sub_handle, index = 0; + struct intel_iommu *iommu = map_dev_to_ir(dev); unsigned int irq, irq_want; struct msi_desc *msidesc; - struct intel_iommu *iommu = NULL; - /* x86 doesn't support multiple MSI yet */ - if (type == PCI_CAP_ID_MSI && nvec > 1) - return 1; + if (intr_remapping_enabled && !iommu) + return -ENOENT; node = dev_to_node(&dev->dev); irq_want = nr_irqs_gsi; sub_handle = 0; list_for_each_entry(msidesc, &dev->msi_list, list) { - irq = create_irq_nr(irq_want, node); + irq = create_irq_nr(irq_want, 1, node); if (irq == 0) - return -1; + return -ENOSPC; irq_want = irq + 1; if (!intr_remapping_enabled) goto no_ir; @@ -3363,11 +3569,6 @@ int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) goto error; } } else { - iommu = map_dev_to_ir(dev); - if (!iommu) { - ret = -ENOENT; - goto error; - } /* * setup the mapping between the irq and the IRTE * base index, the sub_handle pointing to the @@ -3376,7 +3577,7 @@ int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) set_irte_irq(irq, iommu, index, sub_handle); } no_ir: - ret = setup_msi_irq(dev, msidesc, irq); + ret = setup_msi_irq(dev, msidesc, 1, irq); if (ret < 0) goto error; sub_handle++; @@ -3388,11 +3589,34 @@ error: return ret; } +int native_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) +{ + if (type == PCI_CAP_ID_MSI) { + return setup_msi_irqs(dev, nvec); + } else { + return setup_msix_irqs(dev, nvec); + } +} + void native_teardown_msi_irq(unsigned int irq) { destroy_irq(irq); } +void native_teardown_msi_irqs(struct pci_dev *dev) +{ + struct msi_desc *desc; + unsigned i; + + list_for_each_entry(desc, &dev->msi_list, list) { + if (desc->irq == 0) + continue; + for (i = 0; i < (1 << desc->msi_attrib.multiple); i++) { + destroy_irq(desc->irq + i); + } + } +} + #if defined (CONFIG_DMAR) || defined (CONFIG_INTR_REMAP) #ifdef CONFIG_SMP static int @@ -3437,7 +3661,7 @@ int arch_setup_dmar_msi(unsigned int irq) int ret; struct msi_msg msg; - ret = msi_compose_msg(NULL, irq, &msg, -1); + ret = msi_compose_msg(NULL, irq, 1, &msg, -1); if (ret < 0) return ret; dmar_msi_write(irq, &msg); @@ -3515,7 +3739,7 @@ int arch_setup_hpet_msi(unsigned int irq, unsigned int id) return -1; } - ret = msi_compose_msg(NULL, irq, &msg, id); + ret = msi_compose_msg(NULL, irq, 1, &msg, id); if (ret < 0) return ret; diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index 4ff5968..cce3afd 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c @@ -499,7 +499,7 @@ static int hpet_assign_irq(struct hpet_dev *dev) { unsigned int irq; - irq = create_irq_nr(0, -1); + irq = create_irq_nr(0, 1, -1); if (!irq) return -EINVAL; diff --git a/drivers/pci/htirq.c b/drivers/pci/htirq.c index 834842a..2b48cc3 100644 --- a/drivers/pci/htirq.c +++ b/drivers/pci/htirq.c @@ -120,7 +120,7 @@ int __ht_create_irq(struct pci_dev *dev, int idx, ht_irq_update_t *update) cfg->msg.address_hi = 0xffffffff; node = dev_to_node(&dev->dev); - irq = create_irq_nr(0, node); + irq = create_irq_nr(0, 1, node); if (irq <= 0) { kfree(cfg); diff --git a/include/linux/irq.h b/include/linux/irq.h index abde252..842a8c4 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -322,7 +322,8 @@ static inline void set_irq_probe(unsigned int irq) } /* Handle dynamic irq creation and destruction */ -extern unsigned int create_irq_nr(unsigned int irq_want, int node); +extern unsigned int create_irq_nr(unsigned int irq_want, unsigned count, + int node); extern int create_irq(void); extern void destroy_irq(unsigned int irq); -- 1.5.6.5