| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <1358896415-28569-4-git-send-email-walken@google.com>
Date: Tue, 22 Jan 2013 15:13:32 -0800
From: Michel Lespinasse <walken@...gle.com>
To: Rik van Riel <riel@...hat.com>, Ingo Molnar <mingo@...hat.com>,
"Paul E. McKenney" <paulmck@...ibm.com>,
David Howells <dhowells@...hat.com>,
Thomas Gleixner <tglx@...utronix.de>,
Eric Dumazet <edumazet@...gle.com>,
"Eric W. Biederman" <ebiederm@...ssion.com>,
Manfred Spraul <manfred@...orfullife.com>
Cc: linux-kernel@...r.kernel.org
Subject: [RFC PATCH 3/6] ipc: convert ipc objects to use queue spinlock API
This change converts the IPC message queues, semaphores and shm segments
to use the queue spinlock API.
This is relatively large, but mostly mechanical, mostly adding
struct q_spinlock_node variables before locking ipc objects and passing
them all the way to the actual locking and unlocking functions.
In most cases the struct q_spinlock_node is allocated on stack, but
there are exceptions such as the sysvipc_proc_seqops allocating it
within their iterator structure.
Signed-off-by: Michel Lespinasse <walken@...gle.com>
---
include/linux/ipc.h | 4 +-
ipc/msg.c | 61 ++++++++++++++++------------
ipc/namespace.c | 8 ++-
ipc/sem.c | 112 ++++++++++++++++++++++++++++----------------------
ipc/shm.c | 95 ++++++++++++++++++++++++-------------------
ipc/util.c | 55 ++++++++++++++----------
ipc/util.h | 25 +++++++----
7 files changed, 207 insertions(+), 153 deletions(-)
diff --git a/include/linux/ipc.h b/include/linux/ipc.h
index 8d861b2651f7..81693a8a5177 100644
--- a/include/linux/ipc.h
+++ b/include/linux/ipc.h
@@ -1,7 +1,7 @@
#ifndef _LINUX_IPC_H
#define _LINUX_IPC_H
-#include <linux/spinlock.h>
+#include <linux/queue_spinlock.h>
#include <linux/uidgid.h>
#include <uapi/linux/ipc.h>
@@ -10,7 +10,7 @@
/* used by in-kernel data structures */
struct kern_ipc_perm
{
- spinlock_t lock;
+ struct q_spinlock lock;
int deleted;
int id;
key_t key;
diff --git a/ipc/msg.c b/ipc/msg.c
index a71af5a65abf..aeb32d539f6e 100644
--- a/ipc/msg.c
+++ b/ipc/msg.c
@@ -69,9 +69,10 @@ struct msg_sender {
#define msg_ids(ns) ((ns)->ids[IPC_MSG_IDS])
-#define msg_unlock(msq) ipc_unlock(&(msq)->q_perm)
+#define msg_unlock(msq,node) ipc_unlock(&(msq)->q_perm, node)
-static void freeque(struct ipc_namespace *, struct kern_ipc_perm *);
+static void freeque(struct ipc_namespace *, struct kern_ipc_perm *,
+ struct q_spinlock_node *);
static int newque(struct ipc_namespace *, struct ipc_params *);
#ifdef CONFIG_PROC_FS
static int sysvipc_msg_proc_show(struct seq_file *s, void *it);
@@ -144,9 +145,10 @@ void __init msg_init(void)
* msg_lock_(check_) routines are called in the paths where the rw_mutex
* is not held.
*/
-static inline struct msg_queue *msg_lock(struct ipc_namespace *ns, int id)
+static inline struct msg_queue *msg_lock(struct ipc_namespace *ns, int id,
+ struct q_spinlock_node *node)
{
- struct kern_ipc_perm *ipcp = ipc_lock(&msg_ids(ns), id);
+ struct kern_ipc_perm *ipcp = ipc_lock(&msg_ids(ns), id, node);
if (IS_ERR(ipcp))
return (struct msg_queue *)ipcp;
@@ -155,9 +157,10 @@ static inline struct msg_queue *msg_lock(struct ipc_namespace *ns, int id)
}
static inline struct msg_queue *msg_lock_check(struct ipc_namespace *ns,
- int id)
+ int id,
+ struct q_spinlock_node *node)
{
- struct kern_ipc_perm *ipcp = ipc_lock_check(&msg_ids(ns), id);
+ struct kern_ipc_perm *ipcp = ipc_lock_check(&msg_ids(ns), id, node);
if (IS_ERR(ipcp))
return (struct msg_queue *)ipcp;
@@ -183,6 +186,7 @@ static int newque(struct ipc_namespace *ns, struct ipc_params *params)
int id, retval;
key_t key = params->key;
int msgflg = params->flg;
+ struct q_spinlock_node node;
msq = ipc_rcu_alloc(sizeof(*msq));
if (!msq)
@@ -201,7 +205,7 @@ static int newque(struct ipc_namespace *ns, struct ipc_params *params)
/*
* ipc_addid() locks msq
*/
- id = ipc_addid(&msg_ids(ns), &msq->q_perm, ns->msg_ctlmni);
+ id = ipc_addid(&msg_ids(ns), &msq->q_perm, ns->msg_ctlmni, &node);
if (id < 0) {
security_msg_queue_free(msq);
ipc_rcu_putref(msq);
@@ -217,7 +221,7 @@ static int newque(struct ipc_namespace *ns, struct ipc_params *params)
INIT_LIST_HEAD(&msq->q_receivers);
INIT_LIST_HEAD(&msq->q_senders);
- msg_unlock(msq);
+ msg_unlock(msq, &node);
return msq->q_perm.id;
}
@@ -276,7 +280,8 @@ static void expunge_all(struct msg_queue *msq, int res)
* msg_ids.rw_mutex (writer) and the spinlock for this message queue are held
* before freeque() is called. msg_ids.rw_mutex remains locked on exit.
*/
-static void freeque(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
+static void freeque(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp,
+ struct q_spinlock_node *node)
{
struct list_head *tmp;
struct msg_queue *msq = container_of(ipcp, struct msg_queue, q_perm);
@@ -284,7 +289,7 @@ static void freeque(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
expunge_all(msq, -EIDRM);
ss_wakeup(&msq->q_senders, 1);
msg_rmid(ns, msq);
- msg_unlock(msq);
+ msg_unlock(msq, node);
tmp = msq->q_messages.next;
while (tmp != &msq->q_messages) {
@@ -415,6 +420,7 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd,
struct msqid64_ds uninitialized_var(msqid64);
struct msg_queue *msq;
int err;
+ struct q_spinlock_node node;
if (cmd == IPC_SET) {
if (copy_msqid_from_user(&msqid64, buf, version))
@@ -422,7 +428,7 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd,
}
ipcp = ipcctl_pre_down(ns, &msg_ids(ns), msqid, cmd,
- &msqid64.msg_perm, msqid64.msg_qbytes);
+ &msqid64.msg_perm, msqid64.msg_qbytes, &node);
if (IS_ERR(ipcp))
return PTR_ERR(ipcp);
@@ -434,7 +440,7 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd,
switch (cmd) {
case IPC_RMID:
- freeque(ns, ipcp);
+ freeque(ns, ipcp, &node);
goto out_up;
case IPC_SET:
if (msqid64.msg_qbytes > ns->msg_ctlmnb &&
@@ -463,7 +469,7 @@ static int msgctl_down(struct ipc_namespace *ns, int msqid, int cmd,
err = -EINVAL;
}
out_unlock:
- msg_unlock(msq);
+ msg_unlock(msq, &node);
out_up:
up_write(&msg_ids(ns).rw_mutex);
return err;
@@ -474,6 +480,7 @@ SYSCALL_DEFINE3(msgctl, int, msqid, int, cmd, struct msqid_ds __user *, buf)
struct msg_queue *msq;
int err, version;
struct ipc_namespace *ns;
+ struct q_spinlock_node node;
if (msqid < 0 || cmd < 0)
return -EINVAL;
@@ -531,12 +538,12 @@ SYSCALL_DEFINE3(msgctl, int, msqid, int, cmd, struct msqid_ds __user *, buf)
return -EFAULT;
if (cmd == MSG_STAT) {
- msq = msg_lock(ns, msqid);
+ msq = msg_lock(ns, msqid, &node);
if (IS_ERR(msq))
return PTR_ERR(msq);
success_return = msq->q_perm.id;
} else {
- msq = msg_lock_check(ns, msqid);
+ msq = msg_lock_check(ns, msqid, &node);
if (IS_ERR(msq))
return PTR_ERR(msq);
success_return = 0;
@@ -560,7 +567,7 @@ SYSCALL_DEFINE3(msgctl, int, msqid, int, cmd, struct msqid_ds __user *, buf)
tbuf.msg_qbytes = msq->q_qbytes;
tbuf.msg_lspid = msq->q_lspid;
tbuf.msg_lrpid = msq->q_lrpid;
- msg_unlock(msq);
+ msg_unlock(msq, &node);
if (copy_msqid_to_user(buf, &tbuf, version))
return -EFAULT;
return success_return;
@@ -574,7 +581,7 @@ SYSCALL_DEFINE3(msgctl, int, msqid, int, cmd, struct msqid_ds __user *, buf)
}
out_unlock:
- msg_unlock(msq);
+ msg_unlock(msq, &node);
return err;
}
@@ -642,6 +649,7 @@ long do_msgsnd(int msqid, long mtype, void __user *mtext,
struct msg_msg *msg;
int err;
struct ipc_namespace *ns;
+ struct q_spinlock_node node;
ns = current->nsproxy->ipc_ns;
@@ -657,7 +665,7 @@ long do_msgsnd(int msqid, long mtype, void __user *mtext,
msg->m_type = mtype;
msg->m_ts = msgsz;
- msq = msg_lock_check(ns, msqid);
+ msq = msg_lock_check(ns, msqid, &node);
if (IS_ERR(msq)) {
err = PTR_ERR(msq);
goto out_free;
@@ -686,10 +694,10 @@ long do_msgsnd(int msqid, long mtype, void __user *mtext,
}
ss_add(msq, &s);
ipc_rcu_getref(msq);
- msg_unlock(msq);
+ msg_unlock(msq, &node);
schedule();
- ipc_lock_by_ptr(&msq->q_perm);
+ ipc_lock_by_ptr(&msq->q_perm, &node);
ipc_rcu_putref(msq);
if (msq->q_perm.deleted) {
err = -EIDRM;
@@ -719,7 +727,7 @@ long do_msgsnd(int msqid, long mtype, void __user *mtext,
msg = NULL;
out_unlock_free:
- msg_unlock(msq);
+ msg_unlock(msq, &node);
out_free:
if (msg != NULL)
free_msg(msg);
@@ -762,13 +770,14 @@ long do_msgrcv(int msqid, long *pmtype, void __user *mtext,
struct msg_msg *msg;
int mode;
struct ipc_namespace *ns;
+ struct q_spinlock_node node;
if (msqid < 0 || (long) msgsz < 0)
return -EINVAL;
mode = convert_mode(&msgtyp, msgflg);
ns = current->nsproxy->ipc_ns;
- msq = msg_lock_check(ns, msqid);
+ msq = msg_lock_check(ns, msqid, &node);
if (IS_ERR(msq))
return PTR_ERR(msq);
@@ -819,7 +828,7 @@ long do_msgrcv(int msqid, long *pmtype, void __user *mtext,
atomic_sub(msg->m_ts, &ns->msg_bytes);
atomic_dec(&ns->msg_hdrs);
ss_wakeup(&msq->q_senders, 0);
- msg_unlock(msq);
+ msg_unlock(msq, &node);
break;
}
/* No message waiting. Wait for a message */
@@ -837,7 +846,7 @@ long do_msgrcv(int msqid, long *pmtype, void __user *mtext,
msr_d.r_maxsize = msgsz;
msr_d.r_msg = ERR_PTR(-EAGAIN);
current->state = TASK_INTERRUPTIBLE;
- msg_unlock(msq);
+ msg_unlock(msq, &node);
schedule();
@@ -876,7 +885,7 @@ long do_msgrcv(int msqid, long *pmtype, void __user *mtext,
/* Lockless receive, part 3:
* Acquire the queue spinlock.
*/
- ipc_lock_by_ptr(&msq->q_perm);
+ ipc_lock_by_ptr(&msq->q_perm, &node);
rcu_read_unlock();
/* Lockless receive, part 4:
@@ -890,7 +899,7 @@ long do_msgrcv(int msqid, long *pmtype, void __user *mtext,
if (signal_pending(current)) {
msg = ERR_PTR(-ERESTARTNOHAND);
out_unlock:
- msg_unlock(msq);
+ msg_unlock(msq, &node);
break;
}
}
diff --git a/ipc/namespace.c b/ipc/namespace.c
index f362298c5ce4..ba4f87c18870 100644
--- a/ipc/namespace.c
+++ b/ipc/namespace.c
@@ -70,11 +70,13 @@ struct ipc_namespace *copy_ipcs(unsigned long flags,
* Called for each kind of ipc when an ipc_namespace exits.
*/
void free_ipcs(struct ipc_namespace *ns, struct ipc_ids *ids,
- void (*free)(struct ipc_namespace *, struct kern_ipc_perm *))
+ void (*free)(struct ipc_namespace *, struct kern_ipc_perm *,
+ struct q_spinlock_node *))
{
struct kern_ipc_perm *perm;
int next_id;
int total, in_use;
+ struct q_spinlock_node node;
down_write(&ids->rw_mutex);
@@ -84,8 +86,8 @@ void free_ipcs(struct ipc_namespace *ns, struct ipc_ids *ids,
perm = idr_find(&ids->ipcs_idr, next_id);
if (perm == NULL)
continue;
- ipc_lock_by_ptr(perm);
- free(ns, perm);
+ ipc_lock_by_ptr(perm, &node);
+ free(ns, perm, &node);
total++;
}
up_write(&ids->rw_mutex);
diff --git a/ipc/sem.c b/ipc/sem.c
index 58d31f1c1eb5..84b7f3b2c632 100644
--- a/ipc/sem.c
+++ b/ipc/sem.c
@@ -138,11 +138,12 @@ struct sem_undo_list {
#define sem_ids(ns) ((ns)->ids[IPC_SEM_IDS])
-#define sem_unlock(sma) ipc_unlock(&(sma)->sem_perm)
+#define sem_unlock(sma,node) ipc_unlock(&(sma)->sem_perm, node)
#define sem_checkid(sma, semid) ipc_checkid(&sma->sem_perm, semid)
static int newary(struct ipc_namespace *, struct ipc_params *);
-static void freeary(struct ipc_namespace *, struct kern_ipc_perm *);
+static void freeary(struct ipc_namespace *, struct kern_ipc_perm *,
+ struct q_spinlock_node *);
#ifdef CONFIG_PROC_FS
static int sysvipc_sem_proc_show(struct seq_file *s, void *it);
#endif
@@ -194,9 +195,10 @@ void __init sem_init (void)
* sem_lock_(check_) routines are called in the paths where the rw_mutex
* is not held.
*/
-static inline struct sem_array *sem_lock(struct ipc_namespace *ns, int id)
+static inline struct sem_array *sem_lock(struct ipc_namespace *ns, int id,
+ struct q_spinlock_node *node)
{
- struct kern_ipc_perm *ipcp = ipc_lock(&sem_ids(ns), id);
+ struct kern_ipc_perm *ipcp = ipc_lock(&sem_ids(ns), id, node);
if (IS_ERR(ipcp))
return (struct sem_array *)ipcp;
@@ -205,9 +207,10 @@ static inline struct sem_array *sem_lock(struct ipc_namespace *ns, int id)
}
static inline struct sem_array *sem_lock_check(struct ipc_namespace *ns,
- int id)
+ int id,
+ struct q_spinlock_node *node)
{
- struct kern_ipc_perm *ipcp = ipc_lock_check(&sem_ids(ns), id);
+ struct kern_ipc_perm *ipcp = ipc_lock_check(&sem_ids(ns), id, node);
if (IS_ERR(ipcp))
return (struct sem_array *)ipcp;
@@ -215,23 +218,26 @@ static inline struct sem_array *sem_lock_check(struct ipc_namespace *ns,
return container_of(ipcp, struct sem_array, sem_perm);
}
-static inline void sem_lock_and_putref(struct sem_array *sma)
+static inline void sem_lock_and_putref(struct sem_array *sma,
+ struct q_spinlock_node *node)
{
- ipc_lock_by_ptr(&sma->sem_perm);
+ ipc_lock_by_ptr(&sma->sem_perm, node);
ipc_rcu_putref(sma);
}
-static inline void sem_getref_and_unlock(struct sem_array *sma)
+static inline void sem_getref_and_unlock(struct sem_array *sma,
+ struct q_spinlock_node *node)
{
ipc_rcu_getref(sma);
- ipc_unlock(&(sma)->sem_perm);
+ ipc_unlock(&(sma)->sem_perm, node);
}
static inline void sem_putref(struct sem_array *sma)
{
- ipc_lock_by_ptr(&sma->sem_perm);
+ struct q_spinlock_node node;
+ ipc_lock_by_ptr(&sma->sem_perm, &node);
ipc_rcu_putref(sma);
- ipc_unlock(&(sma)->sem_perm);
+ ipc_unlock(&(sma)->sem_perm, &node);
}
static inline void sem_rmid(struct ipc_namespace *ns, struct sem_array *s)
@@ -291,6 +297,7 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params)
int nsems = params->u.nsems;
int semflg = params->flg;
int i;
+ struct q_spinlock_node node;
if (!nsems)
return -EINVAL;
@@ -314,7 +321,7 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params)
return retval;
}
- id = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni);
+ id = ipc_addid(&sem_ids(ns), &sma->sem_perm, ns->sc_semmni, &node);
if (id < 0) {
security_sem_free(sma);
ipc_rcu_putref(sma);
@@ -332,7 +339,7 @@ static int newary(struct ipc_namespace *ns, struct ipc_params *params)
INIT_LIST_HEAD(&sma->list_id);
sma->sem_nsems = nsems;
sma->sem_ctime = get_seconds();
- sem_unlock(sma);
+ sem_unlock(sma, &node);
return sma->sem_perm.id;
}
@@ -739,7 +746,8 @@ static int count_semzcnt (struct sem_array * sma, ushort semnum)
* as a writer and the spinlock for this semaphore set hold. sem_ids.rw_mutex
* remains locked on exit.
*/
-static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
+static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp,
+ struct q_spinlock_node *node)
{
struct sem_undo *un, *tu;
struct sem_queue *q, *tq;
@@ -747,7 +755,7 @@ static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
struct list_head tasks;
/* Free the existing undo structures for this semaphore set. */
- assert_spin_locked(&sma->sem_perm.lock);
+ assert_q_spin_locked(&sma->sem_perm.lock);
list_for_each_entry_safe(un, tu, &sma->list_id, list_id) {
list_del(&un->list_id);
spin_lock(&un->ulp->lock);
@@ -766,7 +774,7 @@ static void freeary(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
/* Remove the semaphore set from the IDR */
sem_rmid(ns, sma);
- sem_unlock(sma);
+ sem_unlock(sma, node);
wake_up_sem_queue_do(&tasks);
ns->used_sems -= sma->sem_nsems;
@@ -803,6 +811,7 @@ static int semctl_nolock(struct ipc_namespace *ns, int semid,
{
int err;
struct sem_array *sma;
+ struct q_spinlock_node node;
switch(cmd) {
case IPC_INFO:
@@ -845,12 +854,12 @@ static int semctl_nolock(struct ipc_namespace *ns, int semid,
int id;
if (cmd == SEM_STAT) {
- sma = sem_lock(ns, semid);
+ sma = sem_lock(ns, semid, &node);
if (IS_ERR(sma))
return PTR_ERR(sma);
id = sma->sem_perm.id;
} else {
- sma = sem_lock_check(ns, semid);
+ sma = sem_lock_check(ns, semid, &node);
if (IS_ERR(sma))
return PTR_ERR(sma);
id = 0;
@@ -870,7 +879,7 @@ static int semctl_nolock(struct ipc_namespace *ns, int semid,
tbuf.sem_otime = sma->sem_otime;
tbuf.sem_ctime = sma->sem_ctime;
tbuf.sem_nsems = sma->sem_nsems;
- sem_unlock(sma);
+ sem_unlock(sma, &node);
if (copy_semid_to_user (arg.buf, &tbuf, version))
return -EFAULT;
return id;
@@ -879,7 +888,7 @@ static int semctl_nolock(struct ipc_namespace *ns, int semid,
return -EINVAL;
}
out_unlock:
- sem_unlock(sma);
+ sem_unlock(sma, &node);
return err;
}
@@ -893,8 +902,9 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
ushort* sem_io = fast_sem_io;
int nsems;
struct list_head tasks;
+ struct q_spinlock_node node;
- sma = sem_lock_check(ns, semid);
+ sma = sem_lock_check(ns, semid, &node);
if (IS_ERR(sma))
return PTR_ERR(sma);
@@ -918,7 +928,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
int i;
if(nsems > SEMMSL_FAST) {
- sem_getref_and_unlock(sma);
+ sem_getref_and_unlock(sma, &node);
sem_io = ipc_alloc(sizeof(ushort)*nsems);
if(sem_io == NULL) {
@@ -926,9 +936,9 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
return -ENOMEM;
}
- sem_lock_and_putref(sma);
+ sem_lock_and_putref(sma, &node);
if (sma->sem_perm.deleted) {
- sem_unlock(sma);
+ sem_unlock(sma, &node);
err = -EIDRM;
goto out_free;
}
@@ -936,7 +946,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
for (i = 0; i < sma->sem_nsems; i++)
sem_io[i] = sma->sem_base[i].semval;
- sem_unlock(sma);
+ sem_unlock(sma, &node);
err = 0;
if(copy_to_user(array, sem_io, nsems*sizeof(ushort)))
err = -EFAULT;
@@ -947,7 +957,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
int i;
struct sem_undo *un;
- sem_getref_and_unlock(sma);
+ sem_getref_and_unlock(sma, &node);
if(nsems > SEMMSL_FAST) {
sem_io = ipc_alloc(sizeof(ushort)*nsems);
@@ -970,9 +980,9 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
goto out_free;
}
}
- sem_lock_and_putref(sma);
+ sem_lock_and_putref(sma, &node);
if (sma->sem_perm.deleted) {
- sem_unlock(sma);
+ sem_unlock(sma, &node);
err = -EIDRM;
goto out_free;
}
@@ -980,7 +990,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
for (i = 0; i < nsems; i++)
sma->sem_base[i].semval = sem_io[i];
- assert_spin_locked(&sma->sem_perm.lock);
+ assert_q_spin_locked(&sma->sem_perm.lock);
list_for_each_entry(un, &sma->list_id, list_id) {
for (i = 0; i < nsems; i++)
un->semadj[i] = 0;
@@ -1021,7 +1031,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
if (val > SEMVMX || val < 0)
goto out_unlock;
- assert_spin_locked(&sma->sem_perm.lock);
+ assert_q_spin_locked(&sma->sem_perm.lock);
list_for_each_entry(un, &sma->list_id, list_id)
un->semadj[semnum] = 0;
@@ -1035,7 +1045,7 @@ static int semctl_main(struct ipc_namespace *ns, int semid, int semnum,
}
}
out_unlock:
- sem_unlock(sma);
+ sem_unlock(sma, &node);
wake_up_sem_queue_do(&tasks);
out_free:
@@ -1082,6 +1092,7 @@ static int semctl_down(struct ipc_namespace *ns, int semid,
int err;
struct semid64_ds semid64;
struct kern_ipc_perm *ipcp;
+ struct q_spinlock_node node;
if(cmd == IPC_SET) {
if (copy_semid_from_user(&semid64, arg.buf, version))
@@ -1089,7 +1100,7 @@ static int semctl_down(struct ipc_namespace *ns, int semid,
}
ipcp = ipcctl_pre_down(ns, &sem_ids(ns), semid, cmd,
- &semid64.sem_perm, 0);
+ &semid64.sem_perm, 0, &node);
if (IS_ERR(ipcp))
return PTR_ERR(ipcp);
@@ -1101,7 +1112,7 @@ static int semctl_down(struct ipc_namespace *ns, int semid,
switch(cmd){
case IPC_RMID:
- freeary(ns, ipcp);
+ freeary(ns, ipcp, &node);
goto out_up;
case IPC_SET:
err = ipc_update_perm(&semid64.sem_perm, ipcp);
@@ -1114,7 +1125,7 @@ static int semctl_down(struct ipc_namespace *ns, int semid,
}
out_unlock:
- sem_unlock(sma);
+ sem_unlock(sma, &node);
out_up:
up_write(&sem_ids(ns).rw_mutex);
return err;
@@ -1237,6 +1248,7 @@ static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid)
struct sem_undo *un, *new;
int nsems;
int error;
+ struct q_spinlock_node node;
error = get_undo_list(&ulp);
if (error)
@@ -1252,12 +1264,12 @@ static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid)
/* no undo structure around - allocate one. */
/* step 1: figure out the size of the semaphore array */
- sma = sem_lock_check(ns, semid);
+ sma = sem_lock_check(ns, semid, &node);
if (IS_ERR(sma))
return ERR_CAST(sma);
nsems = sma->sem_nsems;
- sem_getref_and_unlock(sma);
+ sem_getref_and_unlock(sma, &node);
/* step 2: allocate new undo structure */
new = kzalloc(sizeof(struct sem_undo) + sizeof(short)*nsems, GFP_KERNEL);
@@ -1267,9 +1279,9 @@ static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid)
}
/* step 3: Acquire the lock on semaphore array */
- sem_lock_and_putref(sma);
+ sem_lock_and_putref(sma, &node);
if (sma->sem_perm.deleted) {
- sem_unlock(sma);
+ sem_unlock(sma, &node);
kfree(new);
un = ERR_PTR(-EIDRM);
goto out;
@@ -1290,14 +1302,14 @@ static struct sem_undo *find_alloc_undo(struct ipc_namespace *ns, int semid)
new->semid = semid;
assert_spin_locked(&ulp->lock);
list_add_rcu(&new->list_proc, &ulp->list_proc);
- assert_spin_locked(&sma->sem_perm.lock);
+ assert_q_spin_locked(&sma->sem_perm.lock);
list_add(&new->list_id, &sma->list_id);
un = new;
success:
spin_unlock(&ulp->lock);
rcu_read_lock();
- sem_unlock(sma);
+ sem_unlock(sma, &node);
out:
return un;
}
@@ -1342,6 +1354,7 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
unsigned long jiffies_left = 0;
struct ipc_namespace *ns;
struct list_head tasks;
+ struct q_spinlock_node node;
ns = current->nsproxy->ipc_ns;
@@ -1392,7 +1405,7 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
INIT_LIST_HEAD(&tasks);
- sma = sem_lock_check(ns, semid);
+ sma = sem_lock_check(ns, semid, &node);
if (IS_ERR(sma)) {
if (un)
rcu_read_unlock();
@@ -1477,7 +1490,7 @@ SYSCALL_DEFINE4(semtimedop, int, semid, struct sembuf __user *, tsops,
sleep_again:
current->state = TASK_INTERRUPTIBLE;
- sem_unlock(sma);
+ sem_unlock(sma, &node);
if (timeout)
jiffies_left = schedule_timeout(jiffies_left);
@@ -1499,7 +1512,7 @@ sleep_again:
goto out_free;
}
- sma = sem_lock(ns, semid);
+ sma = sem_lock(ns, semid, &node);
/*
* Wait until it's guaranteed that no wakeup_sem_queue_do() is ongoing.
@@ -1538,7 +1551,7 @@ sleep_again:
unlink_queue(sma, &queue);
out_unlock_free:
- sem_unlock(sma);
+ sem_unlock(sma, &node);
wake_up_sem_queue_do(&tasks);
out_free:
@@ -1604,6 +1617,7 @@ void exit_sem(struct task_struct *tsk)
struct list_head tasks;
int semid;
int i;
+ struct q_spinlock_node node;
rcu_read_lock();
un = list_entry_rcu(ulp->list_proc.next,
@@ -1617,7 +1631,7 @@ void exit_sem(struct task_struct *tsk)
if (semid == -1)
break;
- sma = sem_lock_check(tsk->nsproxy->ipc_ns, un->semid);
+ sma = sem_lock_check(tsk->nsproxy->ipc_ns, un->semid, &node);
/* exit_sem raced with IPC_RMID, nothing to do */
if (IS_ERR(sma))
@@ -1628,12 +1642,12 @@ void exit_sem(struct task_struct *tsk)
/* exit_sem raced with IPC_RMID+semget() that created
* exactly the same semid. Nothing to do.
*/
- sem_unlock(sma);
+ sem_unlock(sma, &node);
continue;
}
/* remove un from the linked lists */
- assert_spin_locked(&sma->sem_perm.lock);
+ assert_q_spin_locked(&sma->sem_perm.lock);
list_del(&un->list_id);
spin_lock(&ulp->lock);
@@ -1668,7 +1682,7 @@ void exit_sem(struct task_struct *tsk)
/* maybe some queued-up processes were waiting for this */
INIT_LIST_HEAD(&tasks);
do_smart_update(sma, NULL, 0, 1, &tasks);
- sem_unlock(sma);
+ sem_unlock(sma, &node);
wake_up_sem_queue_do(&tasks);
kfree_rcu(un, rcu);
diff --git a/ipc/shm.c b/ipc/shm.c
index dff40c9f73c9..67645b4de436 100644
--- a/ipc/shm.c
+++ b/ipc/shm.c
@@ -58,13 +58,14 @@ static const struct vm_operations_struct shm_vm_ops;
#define shm_ids(ns) ((ns)->ids[IPC_SHM_IDS])
-#define shm_unlock(shp) \
- ipc_unlock(&(shp)->shm_perm)
+#define shm_unlock(shp,node) \
+ ipc_unlock(&(shp)->shm_perm,node)
static int newseg(struct ipc_namespace *, struct ipc_params *);
static void shm_open(struct vm_area_struct *vma);
static void shm_close(struct vm_area_struct *vma);
-static void shm_destroy (struct ipc_namespace *ns, struct shmid_kernel *shp);
+static void shm_destroy (struct ipc_namespace *ns, struct shmid_kernel *shp,
+ struct q_spinlock_node *node);
#ifdef CONFIG_PROC_FS
static int sysvipc_shm_proc_show(struct seq_file *s, void *it);
#endif
@@ -83,7 +84,8 @@ void shm_init_ns(struct ipc_namespace *ns)
* Called with shm_ids.rw_mutex (writer) and the shp structure locked.
* Only shm_ids.rw_mutex remains locked on exit.
*/
-static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
+static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp,
+ struct q_spinlock_node *node)
{
struct shmid_kernel *shp;
shp = container_of(ipcp, struct shmid_kernel, shm_perm);
@@ -92,9 +94,9 @@ static void do_shm_rmid(struct ipc_namespace *ns, struct kern_ipc_perm *ipcp)
shp->shm_perm.mode |= SHM_DEST;
/* Do not find it any more */
shp->shm_perm.key = IPC_PRIVATE;
- shm_unlock(shp);
+ shm_unlock(shp, node);
} else
- shm_destroy(ns, shp);
+ shm_destroy(ns, shp, node);
}
#ifdef CONFIG_IPC_NS
@@ -128,9 +130,10 @@ void __init shm_init (void)
* shm_lock_(check_) routines are called in the paths where the rw_mutex
* is not necessarily held.
*/
-static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id)
+static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id,
+ struct q_spinlock_node *node)
{
- struct kern_ipc_perm *ipcp = ipc_lock(&shm_ids(ns), id);
+ struct kern_ipc_perm *ipcp = ipc_lock(&shm_ids(ns), id, node);
if (IS_ERR(ipcp))
return (struct shmid_kernel *)ipcp;
@@ -138,16 +141,17 @@ static inline struct shmid_kernel *shm_lock(struct ipc_namespace *ns, int id)
return container_of(ipcp, struct shmid_kernel, shm_perm);
}
-static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp)
+static inline void shm_lock_by_ptr(struct shmid_kernel *ipcp,
+ struct q_spinlock_node *node)
{
- rcu_read_lock();
- spin_lock(&ipcp->shm_perm.lock);
+ ipc_lock_by_ptr(&ipcp->shm_perm, node);
}
static inline struct shmid_kernel *shm_lock_check(struct ipc_namespace *ns,
- int id)
+ int id,
+ struct q_spinlock_node *node)
{
- struct kern_ipc_perm *ipcp = ipc_lock_check(&shm_ids(ns), id);
+ struct kern_ipc_perm *ipcp = ipc_lock_check(&shm_ids(ns), id, node);
if (IS_ERR(ipcp))
return (struct shmid_kernel *)ipcp;
@@ -167,13 +171,14 @@ static void shm_open(struct vm_area_struct *vma)
struct file *file = vma->vm_file;
struct shm_file_data *sfd = shm_file_data(file);
struct shmid_kernel *shp;
+ struct q_spinlock_node node;
- shp = shm_lock(sfd->ns, sfd->id);
+ shp = shm_lock(sfd->ns, sfd->id, &node);
BUG_ON(IS_ERR(shp));
shp->shm_atim = get_seconds();
shp->shm_lprid = task_tgid_vnr(current);
shp->shm_nattch++;
- shm_unlock(shp);
+ shm_unlock(shp, &node);
}
/*
@@ -185,11 +190,12 @@ static void shm_open(struct vm_area_struct *vma)
* It has to be called with shp and shm_ids.rw_mutex (writer) locked,
* but returns with shp unlocked and freed.
*/
-static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp)
+static void shm_destroy(struct ipc_namespace *ns, struct shmid_kernel *shp,
+ struct q_spinlock_node *node)
{
ns->shm_tot -= (shp->shm_segsz + PAGE_SIZE - 1) >> PAGE_SHIFT;
shm_rmid(ns, shp);
- shm_unlock(shp);
+ shm_unlock(shp, node);
if (!is_file_hugepages(shp->shm_file))
shmem_lock(shp->shm_file, 0, shp->mlock_user);
else if (shp->mlock_user)
@@ -229,18 +235,19 @@ static void shm_close(struct vm_area_struct *vma)
struct shm_file_data *sfd = shm_file_data(file);
struct shmid_kernel *shp;
struct ipc_namespace *ns = sfd->ns;
+ struct q_spinlock_node node;
down_write(&shm_ids(ns).rw_mutex);
/* remove from the list of attaches of the shm segment */
- shp = shm_lock(ns, sfd->id);
+ shp = shm_lock(ns, sfd->id, &node);
BUG_ON(IS_ERR(shp));
shp->shm_lprid = task_tgid_vnr(current);
shp->shm_dtim = get_seconds();
shp->shm_nattch--;
if (shm_may_destroy(ns, shp))
- shm_destroy(ns, shp);
+ shm_destroy(ns, shp, &node);
else
- shm_unlock(shp);
+ shm_unlock(shp, &node);
up_write(&shm_ids(ns).rw_mutex);
}
@@ -269,8 +276,9 @@ static int shm_try_destroy_current(int id, void *p, void *data)
return 0;
if (shm_may_destroy(ns, shp)) {
- shm_lock_by_ptr(shp);
- shm_destroy(ns, shp);
+ struct q_spinlock_node node;
+ shm_lock_by_ptr(shp, &node);
+ shm_destroy(ns, shp, &node);
}
return 0;
}
@@ -292,8 +300,9 @@ static int shm_try_destroy_orphaned(int id, void *p, void *data)
return 0;
if (shm_may_destroy(ns, shp)) {
- shm_lock_by_ptr(shp);
- shm_destroy(ns, shp);
+ struct q_spinlock_node node;
+ shm_lock_by_ptr(shp, &node);
+ shm_destroy(ns, shp, &node);
}
return 0;
}
@@ -467,6 +476,7 @@ static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
char name[13];
int id;
vm_flags_t acctflag = 0;
+ struct q_spinlock_node node;
if (size < SHMMIN || size > ns->shm_ctlmax)
return -EINVAL;
@@ -510,7 +520,7 @@ static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
if (IS_ERR(file))
goto no_file;
- id = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni);
+ id = ipc_addid(&shm_ids(ns), &shp->shm_perm, ns->shm_ctlmni, &node);
if (id < 0) {
error = id;
goto no_id;
@@ -532,7 +542,7 @@ static int newseg(struct ipc_namespace *ns, struct ipc_params *params)
ns->shm_tot += numpages;
error = shp->shm_perm.id;
- shm_unlock(shp);
+ shm_unlock(shp, &node);
return error;
no_id:
@@ -737,6 +747,7 @@ static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
struct shmid64_ds shmid64;
struct shmid_kernel *shp;
int err;
+ struct q_spinlock_node node;
if (cmd == IPC_SET) {
if (copy_shmid_from_user(&shmid64, buf, version))
@@ -744,7 +755,7 @@ static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
}
ipcp = ipcctl_pre_down(ns, &shm_ids(ns), shmid, cmd,
- &shmid64.shm_perm, 0);
+ &shmid64.shm_perm, 0, &node);
if (IS_ERR(ipcp))
return PTR_ERR(ipcp);
@@ -755,7 +766,7 @@ static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
goto out_unlock;
switch (cmd) {
case IPC_RMID:
- do_shm_rmid(ns, ipcp);
+ do_shm_rmid(ns, ipcp, &node);
goto out_up;
case IPC_SET:
err = ipc_update_perm(&shmid64.shm_perm, ipcp);
@@ -767,7 +778,7 @@ static int shmctl_down(struct ipc_namespace *ns, int shmid, int cmd,
err = -EINVAL;
}
out_unlock:
- shm_unlock(shp);
+ shm_unlock(shp, &node);
out_up:
up_write(&shm_ids(ns).rw_mutex);
return err;
@@ -778,6 +789,7 @@ SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
struct shmid_kernel *shp;
int err, version;
struct ipc_namespace *ns;
+ struct q_spinlock_node node;
if (cmd < 0 || shmid < 0) {
err = -EINVAL;
@@ -845,14 +857,14 @@ SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
int result;
if (cmd == SHM_STAT) {
- shp = shm_lock(ns, shmid);
+ shp = shm_lock(ns, shmid, &node);
if (IS_ERR(shp)) {
err = PTR_ERR(shp);
goto out;
}
result = shp->shm_perm.id;
} else {
- shp = shm_lock_check(ns, shmid);
+ shp = shm_lock_check(ns, shmid, &node);
if (IS_ERR(shp)) {
err = PTR_ERR(shp);
goto out;
@@ -874,7 +886,7 @@ SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
tbuf.shm_cpid = shp->shm_cprid;
tbuf.shm_lpid = shp->shm_lprid;
tbuf.shm_nattch = shp->shm_nattch;
- shm_unlock(shp);
+ shm_unlock(shp, &node);
if(copy_shmid_to_user (buf, &tbuf, version))
err = -EFAULT;
else
@@ -886,7 +898,7 @@ SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
{
struct file *shm_file;
- shp = shm_lock_check(ns, shmid);
+ shp = shm_lock_check(ns, shmid, &node);
if (IS_ERR(shp)) {
err = PTR_ERR(shp);
goto out;
@@ -929,7 +941,7 @@ SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
shp->shm_perm.mode &= ~SHM_LOCKED;
shp->mlock_user = NULL;
get_file(shm_file);
- shm_unlock(shp);
+ shm_unlock(shp, &node);
shmem_unlock_mapping(shm_file->f_mapping);
fput(shm_file);
goto out;
@@ -943,7 +955,7 @@ SYSCALL_DEFINE3(shmctl, int, shmid, int, cmd, struct shmid_ds __user *, buf)
}
out_unlock:
- shm_unlock(shp);
+ shm_unlock(shp, &node);
out:
return err;
}
@@ -971,6 +983,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, ulong *raddr,
struct shm_file_data *sfd;
struct path path;
fmode_t f_mode;
+ struct q_spinlock_node node;
err = -EINVAL;
if (shmid < 0)
@@ -1012,7 +1025,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, ulong *raddr,
* additional creator id...
*/
ns = current->nsproxy->ipc_ns;
- shp = shm_lock_check(ns, shmid);
+ shp = shm_lock_check(ns, shmid, &node);
if (IS_ERR(shp)) {
err = PTR_ERR(shp);
goto out;
@@ -1030,7 +1043,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, ulong *raddr,
path_get(&path);
shp->shm_nattch++;
size = i_size_read(path.dentry->d_inode);
- shm_unlock(shp);
+ shm_unlock(shp, &node);
err = -ENOMEM;
sfd = kzalloc(sizeof(*sfd), GFP_KERNEL);
@@ -1082,20 +1095,20 @@ out_fput:
out_nattch:
down_write(&shm_ids(ns).rw_mutex);
- shp = shm_lock(ns, shmid);
+ shp = shm_lock(ns, shmid, &node);
BUG_ON(IS_ERR(shp));
shp->shm_nattch--;
if (shm_may_destroy(ns, shp))
- shm_destroy(ns, shp);
+ shm_destroy(ns, shp, &node);
else
- shm_unlock(shp);
+ shm_unlock(shp, &node);
up_write(&shm_ids(ns).rw_mutex);
out:
return err;
out_unlock:
- shm_unlock(shp);
+ shm_unlock(shp, &node);
goto out;
out_free:
diff --git a/ipc/util.c b/ipc/util.c
index 72fd0785ac94..b4232f0cb473 100644
--- a/ipc/util.c
+++ b/ipc/util.c
@@ -178,7 +178,8 @@ void __init ipc_init_proc_interface(const char *path, const char *header,
* If key is found ipc points to the owning ipc structure
*/
-static struct kern_ipc_perm *ipc_findkey(struct ipc_ids *ids, key_t key)
+static struct kern_ipc_perm *ipc_findkey(struct ipc_ids *ids, key_t key,
+ struct q_spinlock_node *node)
{
struct kern_ipc_perm *ipc;
int next_id;
@@ -195,7 +196,7 @@ static struct kern_ipc_perm *ipc_findkey(struct ipc_ids *ids, key_t key)
continue;
}
- ipc_lock_by_ptr(ipc);
+ ipc_lock_by_ptr(ipc, node);
return ipc;
}
@@ -247,7 +248,8 @@ int ipc_get_maxid(struct ipc_ids *ids)
* Called with ipc_ids.rw_mutex held as a writer.
*/
-int ipc_addid(struct ipc_ids* ids, struct kern_ipc_perm* new, int size)
+int ipc_addid(struct ipc_ids* ids, struct kern_ipc_perm* new, int size,
+ struct q_spinlock_node *node)
{
kuid_t euid;
kgid_t egid;
@@ -259,14 +261,14 @@ int ipc_addid(struct ipc_ids* ids, struct kern_ipc_perm* new, int size)
if (ids->in_use >= size)
return -ENOSPC;
- spin_lock_init(&new->lock);
+ q_spin_lock_init(&new->lock);
new->deleted = 0;
rcu_read_lock();
- spin_lock(&new->lock);
+ q_spin_lock(&new->lock, node);
err = idr_get_new(&ids->ipcs_idr, new, &id);
if (err) {
- spin_unlock(&new->lock);
+ q_spin_unlock(&new->lock, node);
rcu_read_unlock();
return err;
}
@@ -368,6 +370,7 @@ static int ipcget_public(struct ipc_namespace *ns, struct ipc_ids *ids,
struct kern_ipc_perm *ipcp;
int flg = params->flg;
int err;
+ struct q_spinlock_node node;
retry:
err = idr_pre_get(&ids->ipcs_idr, GFP_KERNEL);
@@ -376,7 +379,7 @@ retry:
* a new entry + read locks are not "upgradable"
*/
down_write(&ids->rw_mutex);
- ipcp = ipc_findkey(ids, params->key);
+ ipcp = ipc_findkey(ids, params->key, &node);
if (ipcp == NULL) {
/* key not used */
if (!(flg & IPC_CREAT))
@@ -401,7 +404,7 @@ retry:
*/
err = ipc_check_perms(ns, ipcp, ops, params);
}
- ipc_unlock(ipcp);
+ ipc_unlock(ipcp, &node);
}
up_write(&ids->rw_mutex);
@@ -681,7 +684,8 @@ void ipc64_perm_to_ipc_perm (struct ipc64_perm *in, struct ipc_perm *out)
* The ipc object is locked on exit.
*/
-struct kern_ipc_perm *ipc_lock(struct ipc_ids *ids, int id)
+struct kern_ipc_perm *ipc_lock(struct ipc_ids *ids, int id,
+ struct q_spinlock_node *node)
{
struct kern_ipc_perm *out;
int lid = ipcid_to_idx(id);
@@ -693,13 +697,13 @@ struct kern_ipc_perm *ipc_lock(struct ipc_ids *ids, int id)
return ERR_PTR(-EINVAL);
}
- spin_lock(&out->lock);
+ q_spin_lock(&out->lock, node);
/* ipc_rmid() may have already freed the ID while ipc_lock
* was spinning: here verify that the structure is still valid
*/
if (out->deleted) {
- spin_unlock(&out->lock);
+ q_spin_unlock(&out->lock, node);
rcu_read_unlock();
return ERR_PTR(-EINVAL);
}
@@ -707,16 +711,17 @@ struct kern_ipc_perm *ipc_lock(struct ipc_ids *ids, int id)
return out;
}
-struct kern_ipc_perm *ipc_lock_check(struct ipc_ids *ids, int id)
+struct kern_ipc_perm *ipc_lock_check(struct ipc_ids *ids, int id,
+ struct q_spinlock_node *node)
{
struct kern_ipc_perm *out;
- out = ipc_lock(ids, id);
+ out = ipc_lock(ids, id, node);
if (IS_ERR(out))
return out;
if (ipc_checkid(out, id)) {
- ipc_unlock(out);
+ ipc_unlock(out, node);
return ERR_PTR(-EIDRM);
}
@@ -781,14 +786,15 @@ int ipc_update_perm(struct ipc64_perm *in, struct kern_ipc_perm *out)
*/
struct kern_ipc_perm *ipcctl_pre_down(struct ipc_namespace *ns,
struct ipc_ids *ids, int id, int cmd,
- struct ipc64_perm *perm, int extra_perm)
+ struct ipc64_perm *perm, int extra_perm,
+ struct q_spinlock_node *node)
{
struct kern_ipc_perm *ipcp;
kuid_t euid;
int err;
down_write(&ids->rw_mutex);
- ipcp = ipc_lock_check(ids, id);
+ ipcp = ipc_lock_check(ids, id, node);
if (IS_ERR(ipcp)) {
err = PTR_ERR(ipcp);
goto out_up;
@@ -805,7 +811,7 @@ struct kern_ipc_perm *ipcctl_pre_down(struct ipc_namespace *ns,
return ipcp;
err = -EPERM;
- ipc_unlock(ipcp);
+ ipc_unlock(ipcp, node);
out_up:
up_write(&ids->rw_mutex);
return ERR_PTR(err);
@@ -839,13 +845,15 @@ int ipc_parse_version (int *cmd)
struct ipc_proc_iter {
struct ipc_namespace *ns;
struct ipc_proc_iface *iface;
+ struct q_spinlock_node node;
};
/*
* This routine locks the ipc structure found at least at position pos.
*/
static struct kern_ipc_perm *sysvipc_find_ipc(struct ipc_ids *ids, loff_t pos,
- loff_t *new_pos)
+ loff_t *new_pos,
+ struct q_spinlock_node *node)
{
struct kern_ipc_perm *ipc;
int total, id;
@@ -864,7 +872,7 @@ static struct kern_ipc_perm *sysvipc_find_ipc(struct ipc_ids *ids, loff_t pos,
ipc = idr_find(&ids->ipcs_idr, pos);
if (ipc != NULL) {
*new_pos = pos + 1;
- ipc_lock_by_ptr(ipc);
+ ipc_lock_by_ptr(ipc, node);
return ipc;
}
}
@@ -881,9 +889,10 @@ static void *sysvipc_proc_next(struct seq_file *s, void *it, loff_t *pos)
/* If we had an ipc id locked before, unlock it */
if (ipc && ipc != SEQ_START_TOKEN)
- ipc_unlock(ipc);
+ ipc_unlock(ipc, &iter->node);
- return sysvipc_find_ipc(&iter->ns->ids[iface->ids], *pos, pos);
+ return sysvipc_find_ipc(&iter->ns->ids[iface->ids], *pos, pos,
+ &iter->node);
}
/*
@@ -913,7 +922,7 @@ static void *sysvipc_proc_start(struct seq_file *s, loff_t *pos)
return SEQ_START_TOKEN;
/* Find the (pos-1)th ipc */
- return sysvipc_find_ipc(ids, *pos - 1, pos);
+ return sysvipc_find_ipc(ids, *pos - 1, pos, &iter->node);
}
static void sysvipc_proc_stop(struct seq_file *s, void *it)
@@ -925,7 +934,7 @@ static void sysvipc_proc_stop(struct seq_file *s, void *it)
/* If we had a locked structure, release it */
if (ipc && ipc != SEQ_START_TOKEN)
- ipc_unlock(ipc);
+ ipc_unlock(ipc, &iter->node);
ids = &iter->ns->ids[iface->ids];
/* Release the lock we took in start() */
diff --git a/ipc/util.h b/ipc/util.h
index c8fe2f7631e9..f03c36188cc3 100644
--- a/ipc/util.h
+++ b/ipc/util.h
@@ -94,7 +94,8 @@ void __init ipc_init_proc_interface(const char *path, const char *header,
#define ipcid_to_idx(id) ((id) % SEQ_MULTIPLIER)
/* must be called with ids->rw_mutex acquired for writing */
-int ipc_addid(struct ipc_ids *, struct kern_ipc_perm *, int);
+int ipc_addid(struct ipc_ids *, struct kern_ipc_perm *, int,
+ struct q_spinlock_node *);
/* must be called with ids->rw_mutex acquired for reading */
int ipc_get_maxid(struct ipc_ids *);
@@ -121,14 +122,16 @@ void* ipc_rcu_alloc(int size);
void ipc_rcu_getref(void *ptr);
void ipc_rcu_putref(void *ptr);
-struct kern_ipc_perm *ipc_lock(struct ipc_ids *, int);
+struct kern_ipc_perm *ipc_lock(struct ipc_ids *, int,
+ struct q_spinlock_node *);
void kernel_to_ipc64_perm(struct kern_ipc_perm *in, struct ipc64_perm *out);
void ipc64_perm_to_ipc_perm(struct ipc64_perm *in, struct ipc_perm *out);
int ipc_update_perm(struct ipc64_perm *in, struct kern_ipc_perm *out);
struct kern_ipc_perm *ipcctl_pre_down(struct ipc_namespace *ns,
struct ipc_ids *ids, int id, int cmd,
- struct ipc64_perm *perm, int extra_perm);
+ struct ipc64_perm *perm, int extra_perm,
+ struct q_spinlock_node *node);
#ifndef CONFIG_ARCH_WANT_IPC_PARSE_VERSION
/* On IA-64, we always use the "64-bit version" of the IPC structures. */
@@ -158,21 +161,25 @@ static inline int ipc_checkid(struct kern_ipc_perm *ipcp, int uid)
return 0;
}
-static inline void ipc_lock_by_ptr(struct kern_ipc_perm *perm)
+static inline void ipc_lock_by_ptr(struct kern_ipc_perm *perm,
+ struct q_spinlock_node *node)
{
rcu_read_lock();
- spin_lock(&perm->lock);
+ q_spin_lock(&perm->lock, node);
}
-static inline void ipc_unlock(struct kern_ipc_perm *perm)
+static inline void ipc_unlock(struct kern_ipc_perm *perm,
+ struct q_spinlock_node *node)
{
- spin_unlock(&perm->lock);
+ q_spin_unlock(&perm->lock, node);
rcu_read_unlock();
}
-struct kern_ipc_perm *ipc_lock_check(struct ipc_ids *ids, int id);
+struct kern_ipc_perm *ipc_lock_check(struct ipc_ids *ids, int id,
+ struct q_spinlock_node *node);
int ipcget(struct ipc_namespace *ns, struct ipc_ids *ids,
struct ipc_ops *ops, struct ipc_params *params);
void free_ipcs(struct ipc_namespace *ns, struct ipc_ids *ids,
- void (*free)(struct ipc_namespace *, struct kern_ipc_perm *));
+ void (*free)(struct ipc_namespace *, struct kern_ipc_perm *,
+ struct q_spinlock_node *));
#endif
--
1.7.7.3
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@...r.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
Powered by blists - more mailing lists