[<prev] [next>] [<thread-prev] [thread-next>] [day] [month] [year] [list]
Message-Id: <20220128131006.67712-23-michel@lespinasse.org>
Date: Fri, 28 Jan 2022 05:09:53 -0800
From: Michel Lespinasse <michel@...pinasse.org>
To: Linux-MM <linux-mm@...ck.org>, linux-kernel@...r.kernel.org,
Andrew Morton <akpm@...ux-foundation.org>
Cc: kernel-team@...com, Laurent Dufour <ldufour@...ux.ibm.com>,
Jerome Glisse <jglisse@...gle.com>,
Peter Zijlstra <peterz@...radead.org>,
Michal Hocko <mhocko@...e.com>,
Vlastimil Babka <vbabka@...e.cz>,
Davidlohr Bueso <dave@...olabs.net>,
Matthew Wilcox <willy@...radead.org>,
Liam Howlett <liam.howlett@...cle.com>,
Rik van Riel <riel@...riel.com>,
Paul McKenney <paulmck@...nel.org>,
Song Liu <songliubraving@...com>,
Suren Baghdasaryan <surenb@...gle.com>,
Minchan Kim <minchan@...gle.com>,
Joel Fernandes <joelaf@...gle.com>,
David Rientjes <rientjes@...gle.com>,
Axel Rasmussen <axelrasmussen@...gle.com>,
Andy Lutomirski <luto@...nel.org>,
Michel Lespinasse <michel@...pinasse.org>
Subject: [PATCH v2 22/35] percpu-rwsem: enable percpu_sem destruction in atomic context
From: Suren Baghdasaryan <surenb@...gle.com>
Calling percpu_free_rwsem in atomic context results in "scheduling while
atomic" bug being triggered:
BUG: scheduling while atomic: klogd/158/0x00000002
...
__schedule_bug+0x191/0x290
schedule_debug+0x97/0x180
__schedule+0xdc/0xba0
schedule+0xda/0x250
schedule_timeout+0x92/0x2d0
__wait_for_common+0x25b/0x430
wait_for_completion+0x1f/0x30
rcu_barrier+0x440/0x4f0
rcu_sync_dtor+0xaa/0x190
percpu_free_rwsem+0x41/0x80
Introduce percpu_rwsem_destroy function to perform semaphore destruction
in a worker thread.
Signed-off-by: Suren Baghdasaryan <surenb@...gle.com>
Signed-off-by: Michel Lespinasse <michel@...pinasse.org>
---
include/linux/percpu-rwsem.h | 13 ++++++++++++-
kernel/locking/percpu-rwsem.c | 32 ++++++++++++++++++++++++++++++++
2 files changed, 44 insertions(+), 1 deletion(-)
diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h
index 5fda40f97fe9..bf1668fc9c5e 100644
--- a/include/linux/percpu-rwsem.h
+++ b/include/linux/percpu-rwsem.h
@@ -13,7 +13,14 @@ struct percpu_rw_semaphore {
struct rcu_sync rss;
unsigned int __percpu *read_count;
struct rcuwait writer;
- wait_queue_head_t waiters;
+ /*
+ * destroy_list_entry is used during object destruction when waiters
+ * can't be used, therefore reusing the same space.
+ */
+ union {
+ wait_queue_head_t waiters;
+ struct list_head destroy_list_entry;
+ };
atomic_t block;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map;
@@ -127,8 +134,12 @@ extern void percpu_up_write(struct percpu_rw_semaphore *);
extern int __percpu_init_rwsem(struct percpu_rw_semaphore *,
const char *, struct lock_class_key *);
+/* Can't be called in atomic context. */
extern void percpu_free_rwsem(struct percpu_rw_semaphore *);
+/* Invokes percpu_free_rwsem and frees the semaphore from a worker thread. */
+extern void percpu_rwsem_async_destroy(struct percpu_rw_semaphore *sem);
+
#define percpu_init_rwsem(sem) \
({ \
static struct lock_class_key rwsem_key; \
diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c
index 70a32a576f3f..a3d37bf83c60 100644
--- a/kernel/locking/percpu-rwsem.c
+++ b/kernel/locking/percpu-rwsem.c
@@ -7,6 +7,7 @@
#include <linux/rcupdate.h>
#include <linux/sched.h>
#include <linux/sched/task.h>
+#include <linux/slab.h>
#include <linux/errno.h>
int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
@@ -268,3 +269,34 @@ void percpu_up_write(struct percpu_rw_semaphore *sem)
rcu_sync_exit(&sem->rss);
}
EXPORT_SYMBOL_GPL(percpu_up_write);
+
+static LIST_HEAD(destroy_list);
+static DEFINE_SPINLOCK(destroy_list_lock);
+
+static void destroy_list_workfn(struct work_struct *work)
+{
+ struct percpu_rw_semaphore *sem, *sem2;
+ LIST_HEAD(to_destroy);
+
+ spin_lock(&destroy_list_lock);
+ list_splice_init(&destroy_list, &to_destroy);
+ spin_unlock(&destroy_list_lock);
+
+ if (list_empty(&to_destroy))
+ return;
+
+ list_for_each_entry_safe(sem, sem2, &to_destroy, destroy_list_entry) {
+ percpu_free_rwsem(sem);
+ kfree(sem);
+ }
+}
+
+static DECLARE_WORK(destroy_list_work, destroy_list_workfn);
+
+void percpu_rwsem_async_destroy(struct percpu_rw_semaphore *sem)
+{
+ spin_lock(&destroy_list_lock);
+ list_add_tail(&sem->destroy_list_entry, &destroy_list);
+ spin_unlock(&destroy_list_lock);
+ schedule_work(&destroy_list_work);
+}
--
2.20.1
Powered by blists - more mailing lists