| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20260123150108.43443-2-wujianyue000@gmail.com>
Date: Fri, 23 Jan 2026 23:01:08 +0800
From: Jianyue Wu <wujianyue000@...il.com>
To: akpm@...ux-foundation.org
Cc: shakeel.butt@...ux.dev,
hannes@...xchg.org,
mhocko@...nel.org,
roman.gushchin@...ux.dev,
muchun.song@...ux.dev,
linux-mm@...ck.org,
cgroups@...r.kernel.org,
linux-kernel@...r.kernel.org,
inwardvessel@...il.com,
Jianyue Wu <wujianyue000@...il.com>
Subject: [PATCH v4 1/1] mm: optimize stat output for 11% sys time reduce
Replace seq_buf_printf() calls in memcg stat formatting with a
lightweight helper function and replace seq_printf() in numa stat
output with direct seq operations to avoid printf format parsing
overhead.
Key changes:
- Add memcg_seq_buf_print_stat() helper that embeds separator and
newline directly in the number buffer to reduce function calls
- Replace seq_buf_printf() in memcg_stat_format() with
memcg_seq_buf_print_stat()
- Replace seq_printf() in memory_numa_stat_show() and
memcg_numa_stat_show() with seq_puts() and seq_put_decimal_ull()
- Update memory_stat_format() and related v1 stats formatting
Performance improvement (1M reads of memory.stat + memory.numa_stat):
- Before: real 0m9.663s, user 0m4.840s, sys 0m4.823s
- After: real 0m8.909s, user 0m4.661s, sys 0m4.247s
- Result: ~11% sys time reduction
Test script:
for ((i=1; i<=1000000; i++)); do
: > /dev/null < /sys/fs/cgroup/memory.stat
: > /dev/null < /sys/fs/cgroup/memory.numa_stat
done
Suggested-by: JP Kobryn <inwardvessel@...il.com>
Acked-by: Shakeel Butt <shakeel.butt@...ux.dev>
Tested-by: JP Kobryn <inwardvessel@...il.com>
Signed-off-by: Jianyue Wu <wujianyue000@...il.com>
---
mm/memcontrol-v1.c | 84 ++++++++++++++++++++++++-------------------
mm/memcontrol-v1.h | 4 +++
mm/memcontrol.c | 88 +++++++++++++++++++++++++++++++++++++---------
3 files changed, 123 insertions(+), 53 deletions(-)
diff --git a/mm/memcontrol-v1.c b/mm/memcontrol-v1.c
index 6eed14bff742..5efea38da69e 100644
--- a/mm/memcontrol-v1.c
+++ b/mm/memcontrol-v1.c
@@ -10,7 +10,7 @@
#include <linux/poll.h>
#include <linux/sort.h>
#include <linux/file.h>
-#include <linux/seq_buf.h>
+#include <linux/string.h>
#include "internal.h"
#include "swap.h"
@@ -1794,26 +1794,39 @@ static int memcg_numa_stat_show(struct seq_file *m, void *v)
mem_cgroup_flush_stats(memcg);
+ /*
+ * Output format: "stat_name=value N0=value0 N1=value1 ...\n"
+ * Use seq_puts and seq_put_decimal_ull to avoid printf format
+ * parsing overhead in this hot path.
+ */
for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
- seq_printf(m, "%s=%lu", stat->name,
- mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
- false));
- for_each_node_state(nid, N_MEMORY)
- seq_printf(m, " N%d=%lu", nid,
- mem_cgroup_node_nr_lru_pages(memcg, nid,
- stat->lru_mask, false));
+ seq_puts(m, stat->name);
+ seq_put_decimal_ull(m, "=",
+ mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
+ false));
+ for_each_node_state(nid, N_MEMORY) {
+ seq_put_decimal_ull(m, " N", nid);
+ seq_put_decimal_ull(m, "=",
+ mem_cgroup_node_nr_lru_pages(memcg, nid,
+ stat->lru_mask,
+ false));
+ }
seq_putc(m, '\n');
}
for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
-
- seq_printf(m, "hierarchical_%s=%lu", stat->name,
- mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
- true));
- for_each_node_state(nid, N_MEMORY)
- seq_printf(m, " N%d=%lu", nid,
- mem_cgroup_node_nr_lru_pages(memcg, nid,
- stat->lru_mask, true));
+ seq_puts(m, "hierarchical_");
+ seq_puts(m, stat->name);
+ seq_put_decimal_ull(m, "=",
+ mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
+ true));
+ for_each_node_state(nid, N_MEMORY) {
+ seq_put_decimal_ull(m, " N", nid);
+ seq_put_decimal_ull(m, "=",
+ mem_cgroup_node_nr_lru_pages(memcg, nid,
+ stat->lru_mask,
+ true));
+ }
seq_putc(m, '\n');
}
@@ -1879,17 +1892,17 @@ void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
unsigned long nr;
nr = memcg_page_state_local_output(memcg, memcg1_stats[i]);
- seq_buf_printf(s, "%s %lu\n", memcg1_stat_names[i], nr);
+ memcg_seq_buf_print_stat(s, NULL, memcg1_stat_names[i], ' ', (u64)nr);
}
for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
- seq_buf_printf(s, "%s %lu\n", vm_event_name(memcg1_events[i]),
- memcg_events_local(memcg, memcg1_events[i]));
+ memcg_seq_buf_print_stat(s, NULL, vm_event_name(memcg1_events[i]),
+ ' ', memcg_events_local(memcg, memcg1_events[i]));
for (i = 0; i < NR_LRU_LISTS; i++)
- seq_buf_printf(s, "%s %lu\n", lru_list_name(i),
- memcg_page_state_local(memcg, NR_LRU_BASE + i) *
- PAGE_SIZE);
+ memcg_seq_buf_print_stat(s, NULL, lru_list_name(i), ' ',
+ memcg_page_state_local(memcg, NR_LRU_BASE + i) *
+ PAGE_SIZE);
/* Hierarchical information */
memory = memsw = PAGE_COUNTER_MAX;
@@ -1897,28 +1910,27 @@ void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
memory = min(memory, READ_ONCE(mi->memory.max));
memsw = min(memsw, READ_ONCE(mi->memsw.max));
}
- seq_buf_printf(s, "hierarchical_memory_limit %llu\n",
- (u64)memory * PAGE_SIZE);
- seq_buf_printf(s, "hierarchical_memsw_limit %llu\n",
- (u64)memsw * PAGE_SIZE);
+ memcg_seq_buf_print_stat(s, NULL, "hierarchical_memory_limit", ' ',
+ (u64)memory * PAGE_SIZE);
+ memcg_seq_buf_print_stat(s, NULL, "hierarchical_memsw_limit", ' ',
+ (u64)memsw * PAGE_SIZE);
for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
unsigned long nr;
nr = memcg_page_state_output(memcg, memcg1_stats[i]);
- seq_buf_printf(s, "total_%s %llu\n", memcg1_stat_names[i],
- (u64)nr);
+ memcg_seq_buf_print_stat(s, "total_", memcg1_stat_names[i], ' ',
+ (u64)nr);
}
for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
- seq_buf_printf(s, "total_%s %llu\n",
- vm_event_name(memcg1_events[i]),
- (u64)memcg_events(memcg, memcg1_events[i]));
+ memcg_seq_buf_print_stat(s, "total_", vm_event_name(memcg1_events[i]),
+ ' ', (u64)memcg_events(memcg, memcg1_events[i]));
for (i = 0; i < NR_LRU_LISTS; i++)
- seq_buf_printf(s, "total_%s %llu\n", lru_list_name(i),
- (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
- PAGE_SIZE);
+ memcg_seq_buf_print_stat(s, "total_", lru_list_name(i), ' ',
+ (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
+ PAGE_SIZE);
#ifdef CONFIG_DEBUG_VM
{
@@ -1933,8 +1945,8 @@ void memcg1_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
anon_cost += mz->lruvec.anon_cost;
file_cost += mz->lruvec.file_cost;
}
- seq_buf_printf(s, "anon_cost %lu\n", anon_cost);
- seq_buf_printf(s, "file_cost %lu\n", file_cost);
+ memcg_seq_buf_print_stat(s, NULL, "anon_cost", ' ', (u64)anon_cost);
+ memcg_seq_buf_print_stat(s, NULL, "file_cost", ' ', (u64)file_cost);
}
#endif
}
diff --git a/mm/memcontrol-v1.h b/mm/memcontrol-v1.h
index 6358464bb416..b1015cbe858f 100644
--- a/mm/memcontrol-v1.h
+++ b/mm/memcontrol-v1.h
@@ -4,6 +4,7 @@
#define __MM_MEMCONTROL_V1_H
#include <linux/cgroup-defs.h>
+#include <linux/seq_buf.h>
/* Cgroup v1 and v2 common declarations */
@@ -33,6 +34,9 @@ int memory_stat_show(struct seq_file *m, void *v);
void mem_cgroup_id_get_many(struct mem_cgroup *memcg, unsigned int n);
struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg);
+void memcg_seq_buf_print_stat(struct seq_buf *s, const char *prefix,
+ const char *name, char sep, u64 val);
+
/* Cgroup v1-specific declarations */
#ifdef CONFIG_MEMCG_V1
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 86f43b7e5f71..136c08462cd1 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -42,6 +42,7 @@
#include <linux/bit_spinlock.h>
#include <linux/rcupdate.h>
#include <linux/limits.h>
+#include <linux/sprintf.h>
#include <linux/export.h>
#include <linux/list.h>
#include <linux/mutex.h>
@@ -1460,6 +1461,53 @@ static bool memcg_accounts_hugetlb(void)
}
#endif /* CONFIG_HUGETLB_PAGE */
+/* Max 2^64 - 1 = 18446744073709551615 (20 digits) */
+#define MEMCG_DEC_U64_MAX_LEN 20
+
+/**
+ * memcg_seq_buf_print_stat - Write a name-value pair to a seq_buf with newline
+ * @s: The seq_buf to write to
+ * @prefix: Optional prefix string (can be NULL or "")
+ * @name: The name string to write
+ * @sep: Separator character between name and value (typically ' ' or '=')
+ * @val: The u64 value to write
+ *
+ * This helper efficiently formats and writes "<prefix><name><sep><value>\n"
+ * to a seq_buf. It manually converts the value to a string using num_to_str
+ * and embeds the separator and newline in the buffer to minimize function
+ * calls for better performance.
+ *
+ * The function checks for overflow at each step and returns early if any
+ * operation would cause the buffer to overflow.
+ *
+ * Example: memcg_seq_buf_print_stat(s, "total_", "cache", ' ', 1048576)
+ * Output: "total_cache 1048576\n"
+ */
+void memcg_seq_buf_print_stat(struct seq_buf *s, const char *prefix,
+ const char *name, char sep, u64 val)
+{
+ char num_buf[MEMCG_DEC_U64_MAX_LEN + 2]; /* +2 for separator and newline */
+ int num_len;
+
+ /* Embed separator at the beginning */
+ num_buf[0] = sep;
+
+ /* Convert number starting at offset 1 */
+ num_len = num_to_str(num_buf + 1, sizeof(num_buf) - 2, val, 0);
+ if (num_len <= 0)
+ return;
+
+ /* Embed newline at the end */
+ num_buf[num_len + 1] = '\n';
+
+ if (prefix && *prefix && seq_buf_puts(s, prefix))
+ return;
+ if (seq_buf_puts(s, name))
+ return;
+ /* Output separator, value, and newline in one call */
+ seq_buf_putmem(s, num_buf, num_len + 2);
+}
+
static void memcg_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
{
int i;
@@ -1485,26 +1533,26 @@ static void memcg_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
continue;
#endif
size = memcg_page_state_output(memcg, memory_stats[i].idx);
- seq_buf_printf(s, "%s %llu\n", memory_stats[i].name, size);
+ memcg_seq_buf_print_stat(s, NULL, memory_stats[i].name, ' ', size);
if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) {
size += memcg_page_state_output(memcg,
NR_SLAB_RECLAIMABLE_B);
- seq_buf_printf(s, "slab %llu\n", size);
+ memcg_seq_buf_print_stat(s, NULL, "slab", ' ', size);
}
}
/* Accumulated memory events */
- seq_buf_printf(s, "pgscan %lu\n",
- memcg_events(memcg, PGSCAN_KSWAPD) +
- memcg_events(memcg, PGSCAN_DIRECT) +
- memcg_events(memcg, PGSCAN_PROACTIVE) +
- memcg_events(memcg, PGSCAN_KHUGEPAGED));
- seq_buf_printf(s, "pgsteal %lu\n",
- memcg_events(memcg, PGSTEAL_KSWAPD) +
- memcg_events(memcg, PGSTEAL_DIRECT) +
- memcg_events(memcg, PGSTEAL_PROACTIVE) +
- memcg_events(memcg, PGSTEAL_KHUGEPAGED));
+ memcg_seq_buf_print_stat(s, NULL, "pgscan", ' ',
+ memcg_events(memcg, PGSCAN_KSWAPD) +
+ memcg_events(memcg, PGSCAN_DIRECT) +
+ memcg_events(memcg, PGSCAN_PROACTIVE) +
+ memcg_events(memcg, PGSCAN_KHUGEPAGED));
+ memcg_seq_buf_print_stat(s, NULL, "pgsteal", ' ',
+ memcg_events(memcg, PGSTEAL_KSWAPD) +
+ memcg_events(memcg, PGSTEAL_DIRECT) +
+ memcg_events(memcg, PGSTEAL_PROACTIVE) +
+ memcg_events(memcg, PGSTEAL_KHUGEPAGED));
for (i = 0; i < ARRAY_SIZE(memcg_vm_event_stat); i++) {
#ifdef CONFIG_MEMCG_V1
@@ -1512,9 +1560,9 @@ static void memcg_stat_format(struct mem_cgroup *memcg, struct seq_buf *s)
memcg_vm_event_stat[i] == PGPGOUT)
continue;
#endif
- seq_buf_printf(s, "%s %lu\n",
- vm_event_name(memcg_vm_event_stat[i]),
- memcg_events(memcg, memcg_vm_event_stat[i]));
+ memcg_seq_buf_print_stat(s, NULL,
+ vm_event_name(memcg_vm_event_stat[i]), ' ',
+ memcg_events(memcg, memcg_vm_event_stat[i]));
}
}
@@ -4544,7 +4592,12 @@ static int memory_numa_stat_show(struct seq_file *m, void *v)
if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS)
continue;
- seq_printf(m, "%s", memory_stats[i].name);
+ /*
+ * Output format: "stat_name N0=value0 N1=value1 ...\n"
+ * Use seq_puts and seq_put_decimal_ull to avoid printf
+ * format parsing overhead in this hot path.
+ */
+ seq_puts(m, memory_stats[i].name);
for_each_node_state(nid, N_MEMORY) {
u64 size;
struct lruvec *lruvec;
@@ -4552,7 +4605,8 @@ static int memory_numa_stat_show(struct seq_file *m, void *v)
lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
size = lruvec_page_state_output(lruvec,
memory_stats[i].idx);
- seq_printf(m, " N%d=%llu", nid, size);
+ seq_put_decimal_ull(m, " N", nid);
+ seq_put_decimal_ull(m, "=", size);
}
seq_putc(m, '\n');
}
--
2.43.0
Powered by blists - more mailing lists