| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20251223102124.738818-4-ptikhomirov@virtuozzo.com>
Date: Tue, 23 Dec 2025 18:20:09 +0800
From: Pavel Tikhomirov <ptikhomirov@...tuozzo.com>
To: Tejun Heo <tj@...nel.org>
Cc: Johannes Weiner <hannes@...xchg.org>,
Michal Koutný <mkoutny@...e.com>,
cgroups@...r.kernel.org,
linux-kernel@...r.kernel.org,
Pavel Tikhomirov <ptikhomirov@...tuozzo.com>
Subject: [PATCH 2/2] cgroup-v2/freezer: Print information about unfreezable process
There can be a situation when freezer cgroup can not freeze for a long
time, e.g. we saw some nfs related hangs (due to lost connection) when
stop and suspend (CRIU) of containers (we use freezer cgroup in both)
were failing with timeout (waiting for frozen status of cgroup).
So we came up with this debugging infrastructure for freezer cgroup
which points to the stack of the unfreezable task, so that later one can
identify the problem location in dmesg.
When one reads from cgroup.events cgroup file, and freeze is in progress
and time since freeze start is over the timeout we trigger the warning.
It walks over all the tasks in the cgroup sub-tree which is freezing and
report the first task which is not frozen.
This patch also adds kernel.freeze_timeout_us sysctl to control the
timeout for reporting unfreezable tasks. Default is 0, which means
it is disabled.
Example output:
I used the (https://github.com/Snorch/proc-hang-module) test module
which introduces proc file, reading from which hangs in kernel, to
emulate unfreezable process and it produces this stack:
[ 220.994136] Freeze of /test took 10 sec, due to unfreezable process 6192:cat.
[ 220.994326] Call Trace:
[ 220.994418] <TASK>
[ 220.994507] ? proc_hang_read+0x35/0x60 [proc_hang]
[ 220.994680] ? proc_hang_read+0x3a/0x60 [proc_hang]
[ 220.994861] ? proc_reg_read+0x5a/0xa0
[ 220.995021] ? vfs_read+0xc1/0x370
[ 220.995176] ? auditd_test_task+0x3d/0x50
[ 220.995344] ? __audit_syscall_entry+0xf1/0x140
[ 220.995514] ? ksys_read+0x6b/0xe0
[ 220.995667] ? do_syscall_64+0x7f/0x6d0
...
[ 220.998999] </TASK>
Signed-off-by: Pavel Tikhomirov <ptikhomirov@...tuozzo.com>
---
kernel/cgroup/cgroup-internal.h | 5 ++
kernel/cgroup/cgroup.c | 2 +
kernel/cgroup/freezer.c | 118 ++++++++++++++++++++++++++++++++
3 files changed, 125 insertions(+)
diff --git a/kernel/cgroup/cgroup-internal.h b/kernel/cgroup/cgroup-internal.h
index 22051b4f1ccb..7e2f729996c8 100644
--- a/kernel/cgroup/cgroup-internal.h
+++ b/kernel/cgroup/cgroup-internal.h
@@ -283,6 +283,11 @@ void cgroup_base_stat_cputime_show(struct seq_file *seq);
*/
extern const struct proc_ns_operations cgroupns_operations;
+/*
+ * freezer.c
+ */
+void check_freeze_timeout(struct cgroup *cgrp);
+
/*
* cgroup-v1.c
*/
diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c
index e717208cfb18..097cebbeed1b 100644
--- a/kernel/cgroup/cgroup.c
+++ b/kernel/cgroup/cgroup.c
@@ -3822,6 +3822,8 @@ static int cgroup_events_show(struct seq_file *seq, void *v)
seq_printf(seq, "populated %d\n", cgroup_is_populated(cgrp));
seq_printf(seq, "frozen %d\n", test_bit(CGRP_FROZEN, &cgrp->flags));
+ check_freeze_timeout(cgrp);
+
return 0;
}
diff --git a/kernel/cgroup/freezer.c b/kernel/cgroup/freezer.c
index 02a1db180b70..3880ed400879 100644
--- a/kernel/cgroup/freezer.c
+++ b/kernel/cgroup/freezer.c
@@ -1,8 +1,11 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/cgroup.h>
+#include <linux/ratelimit.h>
+#include <linux/sysctl.h>
#include <linux/sched.h>
#include <linux/sched/task.h>
#include <linux/sched/signal.h>
+#include <linux/sched/debug.h>
#include "cgroup-internal.h"
@@ -349,3 +352,118 @@ void cgroup_freeze(struct cgroup *cgrp, bool freeze)
cgroup_file_notify(&cgrp->events_file);
}
}
+
+#define MAX_STACK_TRACE_DEPTH 64
+
+static void warn_freeze_timeout_task(struct cgroup *cgrp, int timeout,
+ struct task_struct *task)
+{
+ char *buf __free(kfree) = NULL;
+ pid_t tgid;
+
+ buf = kmalloc(PATH_MAX, GFP_KERNEL);
+ if (!buf)
+ return;
+
+ if (cgroup_path(cgrp, buf, PATH_MAX) < 0)
+ return;
+
+ tgid = task_pid_nr_ns(task, &init_pid_ns);
+ pr_warn("Freeze of %s took %ld sec, due to unfreezable process %d:%s.\n",
+ buf, timeout / USEC_PER_SEC, tgid, task->comm);
+ if (!try_get_task_stack(task))
+ return;
+ show_stack(task, NULL, KERN_WARNING);
+ put_task_stack(task);
+}
+
+static void warn_freeze_timeout(struct cgroup *cgrp, int timeout)
+{
+ char *buf __free(kfree) = NULL;
+ struct cgroup_subsys_state *css;
+
+ guard(rcu)();
+ css_for_each_descendant_post(css, &cgrp->self) {
+ struct task_struct *task;
+ struct css_task_iter it;
+
+ css_task_iter_start(css, 0, &it);
+ while ((task = css_task_iter_next(&it))) {
+ if (task->flags & PF_KTHREAD)
+ continue;
+ if (task->frozen)
+ continue;
+
+ warn_freeze_timeout_task(cgrp, timeout, task);
+ css_task_iter_end(&it);
+ return;
+ }
+ css_task_iter_end(&it);
+ }
+
+ buf = kmalloc(PATH_MAX, GFP_KERNEL);
+ if (!buf)
+ return;
+
+ if (cgroup_path(cgrp, buf, PATH_MAX) < 0)
+ return;
+
+ pr_warn("Freeze of %s took %ld sec, but no unfreezable process detected.\n",
+ buf, timeout / USEC_PER_SEC);
+}
+
+#define DEFAULT_FREEZE_RATELIMIT (30 * HZ)
+int sysctl_freeze_timeout_us;
+
+void check_freeze_timeout(struct cgroup *cgrp)
+{
+ static DEFINE_RATELIMIT_STATE(freeze_timeout_rs,
+ DEFAULT_FREEZE_RATELIMIT, 1);
+ unsigned int sequence;
+ u64 last_freeze_start = 0;
+ u64 last_freeze_time;
+ int timeout;
+
+ timeout = READ_ONCE(sysctl_freeze_timeout_us);
+ if (!timeout)
+ return;
+
+ do {
+ sequence = read_seqcount_begin(&cgrp->freezer.freeze_seq);
+ if (test_bit(CGRP_FREEZE, &cgrp->flags) &&
+ !test_bit(CGRP_FROZEN, &cgrp->flags))
+ last_freeze_start = cgrp->freezer.freeze_start_nsec;
+ } while (read_seqcount_retry(&cgrp->freezer.freeze_seq, sequence));
+
+ if (!last_freeze_start)
+ return;
+
+ last_freeze_time = ktime_get_ns() - last_freeze_start;
+ do_div(last_freeze_time, NSEC_PER_USEC);
+
+ if (last_freeze_time < timeout)
+ return;
+
+ if (!__ratelimit(&freeze_timeout_rs))
+ return;
+
+ warn_freeze_timeout(cgrp, timeout);
+}
+
+static const struct ctl_table freezer_sysctls[] = {
+ {
+ .procname = "freeze_timeout_us",
+ .data = &sysctl_freeze_timeout_us,
+ .maxlen = sizeof(int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_minmax,
+ .extra1 = SYSCTL_ZERO,
+ },
+};
+
+static int __init freezer_sysctls_init(void)
+{
+ register_sysctl_init("kernel", freezer_sysctls);
+ return 0;
+}
+late_initcall(freezer_sysctls_init);
--
2.52.0
Powered by blists - more mailing lists