| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Date: Fri, 8 Oct 2021 14:39:33 +0800
From: ultrachin@....com
To: akpm@...ux-foundation.org, linux-mm@...ck.org,
linux-kernel@...r.kernel.org
Cc: brookxu.cn@...il.com, chen xiaoguang <xiaoggchen@...cent.com>,
zeng jingxiang <linuszeng@...cent.com>,
lu yihui <yihuilu@...cent.com>
Subject: [PATCH] mm: Free per cpu pages async to shorten program exit time
From: chen xiaoguang <xiaoggchen@...cent.com>
The exit time is long when program allocated big memory and
the most time consuming part is free memory which takes 99.9%
of the total exit time. By using async free we can save 25% of
exit time.
Signed-off-by: chen xiaoguang <xiaoggchen@...cent.com>
Signed-off-by: zeng jingxiang <linuszeng@...cent.com>
Signed-off-by: lu yihui <yihuilu@...cent.com>
---
include/linux/mm.h | 1 +
kernel/exit.c | 2 ++
mm/page_alloc.c | 89 +++++++++++++++++++++++++++++++++++++++++++---
3 files changed, 87 insertions(+), 5 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 73a52aba448f..2add3b635eee 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -908,6 +908,7 @@ void put_pages_list(struct list_head *pages);
void split_page(struct page *page, unsigned int order);
void copy_huge_page(struct page *dst, struct page *src);
+void kfreepcp_set_run(unsigned int cpu);
/*
* Compound pages have a destructor function. Provide a
diff --git a/kernel/exit.c b/kernel/exit.c
index 91a43e57a32e..269eb81acbe9 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -167,10 +167,12 @@ static void __exit_signal(struct task_struct *tsk)
static void delayed_put_task_struct(struct rcu_head *rhp)
{
struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
+ unsigned int cpu = tsk->cpu;
perf_event_delayed_put(tsk);
trace_sched_process_free(tsk);
put_task_struct(tsk);
+ kfreepcp_set_run(cpu);
}
void put_task_struct_rcu_user(struct task_struct *task)
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index b37435c274cf..8a748ea9156b 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -72,6 +72,7 @@
#include <linux/padata.h>
#include <linux/khugepaged.h>
#include <linux/buffer_head.h>
+#include <linux/smpboot.h>
#include <asm/sections.h>
#include <asm/tlbflush.h>
#include <asm/div64.h>
@@ -147,6 +148,12 @@ DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
EXPORT_PER_CPU_SYMBOL(_numa_mem_);
#endif
+struct freepcp_stat {
+ struct task_struct *thread;
+ bool should_run;
+};
+DEFINE_PER_CPU(struct freepcp_stat, kfreepcp);
+
/* work_structs for global per-cpu drains */
struct pcpu_drain {
struct zone *zone;
@@ -3361,6 +3368,81 @@ static int nr_pcp_high(struct per_cpu_pages *pcp, struct zone *zone)
return min(READ_ONCE(pcp->batch) << 2, high);
}
+void kfreepcp_set_run(unsigned int cpu)
+{
+ struct task_struct *tsk;
+ struct freepcp_stat *stat = this_cpu_ptr(&kfreepcp);
+
+ tsk = stat->thread;
+ per_cpu(kfreepcp.should_run, cpu) = true;
+
+ if (tsk && !task_is_running(tsk))
+ wake_up_process(tsk);
+}
+EXPORT_SYMBOL_GPL(kfreepcp_set_run);
+
+static int kfreepcp_should_run(unsigned int cpu)
+{
+ struct freepcp_stat *stat = this_cpu_ptr(&kfreepcp);
+
+ return stat->should_run;
+}
+
+static void run_kfreepcp(unsigned int cpu)
+{
+ struct zone *zone;
+ struct per_cpu_pages *pcp;
+ unsigned long flags;
+ struct freepcp_stat *stat = this_cpu_ptr(&kfreepcp);
+ bool need_free_more = false;
+
+
+
+again:
+ need_free_more = false;
+ for_each_populated_zone(zone) {
+ pcp = per_cpu_ptr(zone->per_cpu_pageset, cpu);
+ if (pcp->count && pcp->high && pcp->count > pcp->high) {
+ unsigned long batch = READ_ONCE(pcp->batch);
+ int high;
+
+ high = nr_pcp_high(pcp, zone);
+ local_irq_save(flags);
+ free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch),
+ pcp);
+ local_irq_restore(flags);
+ if (pcp->count > pcp->high)
+ need_free_more = true;
+ }
+
+ cond_resched();
+ }
+ if (need_free_more)
+ goto again;
+
+ stat->should_run = false;
+}
+
+static struct smp_hotplug_thread freepcp_threads = {
+ .store = &kfreepcp.thread,
+ .thread_should_run = kfreepcp_should_run,
+ .thread_fn = run_kfreepcp,
+ .thread_comm = "kfreepcp/%u",
+};
+
+static int __init freepcp_init(void)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ per_cpu(kfreepcp.should_run, cpu) = false;
+
+ BUG_ON(smpboot_register_percpu_thread(&freepcp_threads));
+
+ return 0;
+}
+late_initcall(freepcp_init);
+
static void free_unref_page_commit(struct page *page, unsigned long pfn,
int migratetype, unsigned int order)
{
@@ -3375,11 +3457,8 @@ static void free_unref_page_commit(struct page *page, unsigned long pfn,
list_add(&page->lru, &pcp->lists[pindex]);
pcp->count += 1 << order;
high = nr_pcp_high(pcp, zone);
- if (pcp->count >= high) {
- int batch = READ_ONCE(pcp->batch);
-
- free_pcppages_bulk(zone, nr_pcp_free(pcp, high, batch), pcp);
- }
+ if (pcp->count >= high)
+ this_cpu_ptr(&kfreepcp)->should_run = false;
}
/*
--
2.27.0
Powered by blists - more mailing lists