| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Tue, 25 Oct 2011 13:49:57 +0200
From: Eial Czerwacki <eial@...lemp.com>
To: linux-kernel@...r.kernel.org
CC: "Shai Fultheim (Shai@...leMP.com)" <Shai@...leMP.com>
Subject: [PATCH] APICID: Avoid false sharing on the read mostly x86_cpu_to_apicid
Avoid false sharing on the read mostly x86_cpu_to_apicid by moving it to
__read_mostly section.
The per-cpu area is write and read and this symbol shows up high on ipi
intensive/x86_cpu_to_apicid load.
Signed-off-by: Eial Czerwacki<eial@...lemp.com>
Signed-off-by: Shai Fultheim<shai@...lemp.com>
Author: Ravikiran Thirumalai<kiran@...lex86.org>
---
Index: b/arch/x86/include/asm/smp.h
===================================================================
--- a/arch/x86/include/asm/smp.h 2010-06-01 09:56:03.000000000 -0700
+++ b/arch/x86/include/asm/smp.h 2010-06-02 15:59:21.000000000 -0700
@@ -36,7 +36,8 @@ static inline struct cpumask *cpu_core_m
return per_cpu(cpu_core_map, cpu);
}
-DECLARE_EARLY_PER_CPU(u16, x86_cpu_to_apicid);
+extern u16 x86_cpu_to_apicid[NR_CPUS];
+
DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid);
/* Static state in head.S used to set up a CPU */
@@ -142,7 +143,7 @@ void native_send_call_func_ipi(const str
void native_send_call_func_single_ipi(int cpu);
void smp_store_cpu_info(int id);
-#define cpu_physical_id(cpu) per_cpu(x86_cpu_to_apicid, cpu)
+#define cpu_physical_id(cpu) x86_cpu_to_apicid[cpu]
/* We don't mark CPUs online until __cpu_up(), so we need another measure */
static inline int num_booting_cpus(void)
Index: b/arch/x86/kernel/acpi/boot.c
===================================================================
--- a/arch/x86/kernel/acpi/boot.c 2010-06-01 09:56:03.000000000 -0700
+++ b/arch/x86/kernel/acpi/boot.c 2010-06-02 15:59:21.000000000 -0700
@@ -568,7 +568,7 @@ EXPORT_SYMBOL(acpi_map_lsapic);
int acpi_unmap_lsapic(int cpu)
{
- per_cpu(x86_cpu_to_apicid, cpu) = -1;
+ x86_cpu_to_apicid[cpu] = -1;
set_cpu_present(cpu, false);
num_processors--;
Index: b/arch/x86/kernel/apic/apic.c
===================================================================
--- a/arch/x86/kernel/apic/apic.c 2010-06-01 09:56:03.000000000 -0700
+++ b/arch/x86/kernel/apic/apic.c 2010-06-02 15:59:21.000000000 -0700
@@ -78,9 +78,8 @@ physid_mask_t phys_cpu_present_map;
/*
* Map cpu index to physical APIC ID
*/
-DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
+u16 x86_cpu_to_apicid[NR_CPUS] __read_mostly = { [0 ... NR_CPUS-1] = BAD_APICID };
DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
-EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
#ifdef CONFIG_X86_32
@@ -1248,7 +1247,7 @@
* proper NUMA affinity.
*/
if (apic->x86_32_numa_cpu_node)
- set_apicid_to_node(early_per_cpu(x86_cpu_to_apicid, cpu),
+ set_apicid_to_node(x86_cpu_to_apicid[cpu],
apic->x86_32_numa_cpu_node(cpu));
#endif
@@ -1916,7 +1915,7 @@ void __cpuinit generic_processor_info(in
max_physical_apicid = apicid;
#if defined(CONFIG_SMP) || defined(CONFIG_X86_64)
- early_per_cpu(x86_cpu_to_apicid, cpu) = apicid;
+ x86_cpu_to_apicid[cpu] = apicid;
early_per_cpu(x86_bios_cpu_apicid, cpu) = apicid;
#endif
Index: b/arch/x86/kernel/apic/apic_flat_64.c
===================================================================
--- a/arch/x86/kernel/apic/apic_flat_64.c 2010-06-02 15:59:21.000000000 -0700
+++ b/arch/x86/kernel/apic/apic_flat_64.c 2010-06-02 16:49:32.000000000 -0700
@@ -294,7 +294,7 @@ static unsigned int physflat_cpu_mask_to
*/
cpu = cpumask_first(cpumask);
if ((unsigned)cpu< nr_cpu_ids)
- return per_cpu(x86_cpu_to_apicid, cpu);
+ return x86_cpu_to_apicid[cpu];
else
return BAD_APICID;
}
@@ -314,7 +314,7 @@ c(const st
if (cpumask_test_cpu(cpu, cpu_online_mask))
break;
}
- return per_cpu(x86_cpu_to_apicid, cpu);
+ return x86_cpu_to_apicid[cpu];
}
static int physflat_probe(void)
Index: b/arch/x86/kernel/apic/ipi.c
===================================================================
--- a/arch/x86/kernel/apic/ipi.c 2010-06-01 09:56:03.000000000 -0700
+++ b/arch/x86/kernel/apic/ipi.c 2010-06-02 15:59:21.000000000 -0700
@@ -31,8 +31,8 @@ void default_send_IPI_mask_sequence_phys
*/
local_irq_save(flags);
for_each_cpu(query_cpu, mask) {
- __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
- query_cpu), vector, APIC_DEST_PHYSICAL);
+ __default_send_IPI_dest_field(x86_cpu_to_apicid[query_cpu],
+ vector, APIC_DEST_PHYSICAL);
}
local_irq_restore(flags);
}
@@ -50,8 +50,8 @@ void default_send_IPI_mask_allbutself_ph
for_each_cpu(query_cpu, mask) {
if (query_cpu == this_cpu)
continue;
- __default_send_IPI_dest_field(per_cpu(x86_cpu_to_apicid,
- query_cpu), vector, APIC_DEST_PHYSICAL);
+ __default_send_IPI_dest_field(x86_cpu_to_apicid[query_cpu],
+ vector, APIC_DEST_PHYSICAL);
}
local_irq_restore(flags);
}
@@ -143,7 +143,7 @@ static int convert_apicid_to_cpu(int api
int i;
for_each_possible_cpu(i) {
- if (per_cpu(x86_cpu_to_apicid, i) == apic_id)
+ if (x86_cpu_to_apicid[i] == apic_id)
return i;
}
return -1;
Index: b/arch/x86/kernel/apic/x2apic_phys.c
===================================================================
--- a/arch/x86/kernel/apic/x2apic_phys.c 2010-06-02 15:59:21.000000000 -0700
+++ b/arch/x86/kernel/apic/x2apic_phys.c 2010-06-02 15:59:21.000000000 -0700
@@ -103,7 +103,7 @@ static void x2apic_send_IPI_allbutself(i
for_each_cpu(query_cpu) {
if (apic_dest == APIC_DEST_ALLBUT&& query_cpu == this_cpu)
continue;
- __x2apic_send_IPI_dest(per_cpu(x86_cpu_to_apicid, query_cpu),
+ __x2apic_send_IPI_dest(x86_cpu_to_apicid[query_cpu],
vector, APIC_DEST_PHYSICAL);
}
local_irq_restore(flags);
@@ -128,7 +128,7 @@ static unsigned int x2apic_cpu_mask_to_a
int cpu = cpumask_first(cpumask);
if ((unsigned)cpu< nr_cpu_ids)
- return per_cpu(x86_cpu_to_apicid, cpu);
+ return x86_cpu_to_apicid[cpu];
else
return BAD_APICID;
}
@@ -149,7 +149,7 @@ x2apic_cpu_mask_to_apicid_and(const stru
break;
}
- return per_cpu(x86_cpu_to_apicid, cpu);
+ return x86_cpu_to_apicid[cpu];
}
static void init_x2apic_ldr(void)
Index: b/arch/x86/kernel/apic/x2apic_uv_x.c
===================================================================
--- a/arch/x86/kernel/apic/x2apic_uv_x.c 2010-06-01 09:56:03.000000000 -0700
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c 2010-06-02 15:59:21.000000000 -0700
@@ -135,7 +135,7 @@ static void uv_send_IPI_one(int cpu, int
unsigned long apicid;
int pnode;
- apicid = per_cpu(x86_cpu_to_apicid, cpu);
+ apicid = x86_cpu_to_apicid[cpu];
pnode = uv_apicid_to_pnode(apicid);
uv_hub_send_ipi(pnode, apicid, vector);
}
@@ -193,7 +193,7 @@ static unsigned int uv_cpu_mask_to_apici
int cpu = cpumask_first(cpumask);
if ((unsigned)cpu< nr_cpu_ids)
- return per_cpu(x86_cpu_to_apicid, cpu) | uv_apicid_hibits;
+ return x86_cpu_to_apicid[cpu] | uv_apicid_hibits;
else
return BAD_APICID;
}
@@ -213,6 +213,6 @@ uv_cpu_mask_to_apicid_and(const struct c
if (cpumask_test_cpu(cpu, cpu_online_mask))
break;
}
- return per_cpu(x86_cpu_to_apicid, cpu) | uv_apicid_hibits;
+ return return x86_cpu_to_apicid[cpu] | uv_apicid_hibits;
}
@@ -608,7 +608,7 @@ void __init uv_system_init(void)
uv_rtc_init();
for_each_present_cpu(cpu) {
- int apicid = per_cpu(x86_cpu_to_apicid, cpu);
+ int apicid = x86_cpu_to_apicid[cpu];
nid = cpu_to_node(cpu);
/*
Index: b/arch/x86/kernel/setup_percpu.c
===================================================================
--- a/arch/x86/kernel/setup_percpu.c 2010-06-01 09:56:03.000000000 -0700
+++ b/arch/x86/kernel/setup_percpu.c 2010-06-02 15:59:21.000000000 -0700
@@ -225,8 +225,6 @@ void __init setup_per_cpu_areas(void)
* gone.
*/
#ifdef CONFIG_X86_LOCAL_APIC
- per_cpu(x86_cpu_to_apicid, cpu) =
- early_per_cpu_map(x86_cpu_to_apicid, cpu);
per_cpu(x86_bios_cpu_apicid, cpu) =
early_per_cpu_map(x86_bios_cpu_apicid, cpu);
#endif
@@ -249,7 +247,6 @@ void __init setup_per_cpu_areas(void)
/* indicate the early static arrays will soon be gone */
#ifdef CONFIG_X86_LOCAL_APIC
- early_per_cpu_ptr(x86_cpu_to_apicid) = NULL;
early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL;
#endif
#if defined(CONFIG_X86_64)&& defined(CONFIG_NUMA)
Index: b/arch/x86/kernel/smpboot.c
===================================================================
--- a/arch/x86/kernel/smpboot.c 2010-06-01 09:56:03.000000000 -0700
+++ b/arch/x86/kernel/smpboot.c 2010-06-02 15:59:21.000000000 -0700
@@ -818,7 +818,7 @@ do_rest:
cpumask_clear_cpu(cpu, cpu_initialized_mask);
set_cpu_present(cpu, false);
- per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
+ x86_cpu_to_apicid[cpu] = BAD_APICID;
}
/* mark "stuck" area as not stuck */
Index: b/arch/x86/platform/uv/tlb_uv.c
===================================================================
--- a/arch/x86/platform/uv/tlb_uv.c 2010-06-01 09:56:03.000000000 -0700
+++ b/arch/x86/platform/uv/tlb_uv.c 2010-06-02 15:59:21.000000000 -0700
@@ -55,7 +55,7 @@ static int __init blade_to_first_uvhub(
for_each_present_cpu(cpu)
if (uvhub == uv_cpu_to_blade_id(cpu))
- return per_cpu(x86_cpu_to_apicid, cpu);
+ return x86_cpu_to_apicid[cpu];
return -1;
}
Index: b/arch/x86/mm/numa.c
===================================================================
--- a/arch/x86/mm/numa.c 2010-06-01 09:56:03.000000000 -0700
+++ b/arch/x86/mm/numa.c 2010-06-02 16:19:04.000000000 -0700
@@ -62,7 +62,7 @@ s16 __apicid_to_node[MAX_LOCAL_APIC] __c
int __cpuinit numa_cpu_node(int cpu)
{
- int apicid = early_per_cpu(x86_cpu_to_apicid, cpu);
+ int apicid = x86_cpu_to_apicid[cpu];
if (apicid != BAD_APICID)
return __apicid_to_node[apicid];
@@ -694,9 +694,6 @@ static __init int find_near_online_node(
void __init init_cpu_to_node(void)
{
int cpu;
- u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid);
-
- BUG_ON(cpu_to_apicid == NULL);
for_each_possible_cpu(cpu) {
int node = numa_cpu_node(cpu);
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists