| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Mon, 22 Sep 2008 20:13:55 +0900
From: KAMEZAWA Hiroyuki <kamezawa.hiroyu@...fujitsu.com>
To: KAMEZAWA Hiroyuki <kamezawa.hiroyu@...fujitsu.com>
Cc: "linux-mm@...ck.org" <linux-mm@...ck.org>,
"balbir@...ux.vnet.ibm.com" <balbir@...ux.vnet.ibm.com>,
"nishimura@....nes.nec.co.jp" <nishimura@....nes.nec.co.jp>,
"xemul@...nvz.org" <xemul@...nvz.org>,
LKML <linux-kernel@...r.kernel.org>
Subject: [PATCH 10/13] memcg: page_cgroup look aside table
Use per-cpu cache for fast access to page_cgroup.
This patch is for making fastpath faster.
Because page_cgroup is accessed when the page is allocated/freed,
we can assume several of continuous page_cgroup will be accessed soon.
(If not interleaved on NUMA...but in such case, alloc/free itself is slow.)
We cache some set of page_cgroup's base pointer on per-cpu area and
use it when we hit.
Changelong: v3 -> v4
- rewrite noinline -> noinline_for_stack.
- added cpu hotplug support.
Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@...fujitsu.com>
mm/page_cgroup.c | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 70 insertions(+), 3 deletions(-)
Index: mmotm-2.6.27-rc6+/mm/page_cgroup.c
===================================================================
--- mmotm-2.6.27-rc6+.orig/mm/page_cgroup.c
+++ mmotm-2.6.27-rc6+/mm/page_cgroup.c
@@ -6,7 +6,7 @@
#include <linux/page_cgroup.h>
#include <linux/hash.h>
#include <linux/memory.h>
-
+#include <linux/cpu.h>
struct pcg_hash_head {
@@ -44,15 +44,26 @@ static int pcg_hashmask __read_mostly;
#define PCG_HASHMASK (pcg_hashmask)
#define PCG_HASHSIZE (1 << pcg_hashshift)
+#define PCG_CACHE_MAX_SLOT (32)
+#define PCG_CACHE_MASK (PCG_CACHE_MAX_SLOT - 1)
+struct percpu_page_cgroup_cache {
+ struct {
+ unsigned long index;
+ struct page_cgroup *base;
+ } slots[PCG_CACHE_MAX_SLOT];
+};
+DEFINE_PER_CPU(struct percpu_page_cgroup_cache, pcg_cache);
+
static int pcg_hashfun(unsigned long index)
{
return hash_long(index, pcg_hashshift);
}
-struct page_cgroup *lookup_page_cgroup(struct page *page)
+noinline_for_stack static struct page_cgroup *
+__lookup_page_cgroup(struct percpu_page_cgroup_cache *pcc,unsigned long pfn)
{
- unsigned long pfn = page_to_pfn(page);
unsigned long index = pfn >> ENTS_PER_CHUNK_SHIFT;
+ int s = index & PCG_CACHE_MASK;
struct pcg_hash *ent;
struct pcg_hash_head *head;
struct hlist_node *node;
@@ -65,6 +76,8 @@ struct page_cgroup *lookup_page_cgroup(s
hlist_for_each_entry(ent, node, &head->head, node) {
if (ent->index == index) {
pc = ent->map + pfn;
+ pcc->slots[s].index = ent->index;
+ pcc->slots[s].base = ent->map;
break;
}
}
@@ -123,6 +136,56 @@ static int __meminit alloc_page_cgroup(i
return 0;
}
+struct page_cgroup *lookup_page_cgroup(struct page *page)
+{
+ unsigned long pfn = page_to_pfn(page);
+ struct percpu_page_cgroup_cache *pcc;
+ struct page_cgroup *ret;
+ unsigned long index = pfn >> ENTS_PER_CHUNK_SHIFT;
+ int hnum = index & PCG_CACHE_MASK;
+
+ pcc = &get_cpu_var(pcg_cache);
+ if (likely(pcc->slots[hnum].index == index))
+ ret = pcc->slots[hnum].base + pfn;
+ else
+ ret = __lookup_page_cgroup(pcc, pfn);
+ put_cpu_var(pcg_cache);
+ return ret;
+}
+
+
+
+void __cpuinit clear_page_cgroup_cache_pcg(int cpu)
+{
+ struct percpu_page_cgroup_cache *pcc;
+ int i;
+
+ pcc = &per_cpu(pcg_cache, cpu);
+ for (i = 0; i < PCG_CACHE_MAX_SLOT; i++)
+ pcc->slots[i].index = -1;
+}
+
+static int __cpuinit cpu_page_cgroup_callback(struct notifier_block *nb,
+ unsigned long action, void *hcpu)
+{
+ int cpu = (long)hcpu;
+
+ switch(action){
+ case CPU_UP_PREPARE:
+ clear_page_cgroup_cache_pcg(cpu);
+ break;
+ default:
+ break;
+ }
+ return NOTIFY_OK;
+}
+
+struct notifier_block cpu_page_cgroup_nb = {
+ .notifier_call = cpu_page_cgroup_callback,
+};
+
+
+
#ifdef CONFIG_MEMORY_HOTPLUG
int online_page_cgroup(unsigned long start_pfn,
@@ -227,6 +290,10 @@ void __init page_cgroup_init(void)
if (fail)
break;
}
+ cpu_page_cgroup_callback(&cpu_page_cgroup_nb,
+ (unsigned long)CPU_UP_PREPARE,
+ (void *)(long)smp_processor_id());
+ register_hotcpu_notifier(&cpu_page_cgroup_nb);
hotplug_memory_notifier(pcg_memory_callback, 0);
nomem:
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists