| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-Id: <20181101095844.24462-14-mathieu.desnoyers@efficios.com>
Date: Thu, 1 Nov 2018 10:58:41 +0100
From: Mathieu Desnoyers <mathieu.desnoyers@...icios.com>
To: Peter Zijlstra <peterz@...radead.org>,
"Paul E . McKenney" <paulmck@...ux.vnet.ibm.com>,
Boqun Feng <boqun.feng@...il.com>
Cc: linux-kernel@...r.kernel.org, linux-api@...r.kernel.org,
Thomas Gleixner <tglx@...utronix.de>,
Andy Lutomirski <luto@...capital.net>,
Dave Watson <davejwatson@...com>, Paul Turner <pjt@...gle.com>,
Andrew Morton <akpm@...ux-foundation.org>,
Russell King <linux@....linux.org.uk>,
Ingo Molnar <mingo@...hat.com>,
"H . Peter Anvin" <hpa@...or.com>,
Andi Kleen <andi@...stfloor.org>, Chris Lameter <cl@...ux.com>,
Ben Maurer <bmaurer@...com>,
Steven Rostedt <rostedt@...dmis.org>,
Josh Triplett <josh@...htriplett.org>,
Linus Torvalds <torvalds@...ux-foundation.org>,
Catalin Marinas <catalin.marinas@....com>,
Will Deacon <will.deacon@....com>,
Michael Kerrisk <mtk.manpages@...il.com>,
Joel Fernandes <joelaf@...gle.com>,
Mathieu Desnoyers <mathieu.desnoyers@...icios.com>,
Shuah Khan <shuah@...nel.org>, linux-kselftest@...r.kernel.org
Subject: [RFC PATCH for 4.21 13/16] cpu-opv/selftests: Provide percpu_op API
Introduce percpu-op.h API. It uses rseq internally as fast-path if
invoked from the right CPU, else cpu_opv as slow-path if called
from the wrong CPU or if rseq fails.
This allows acting on per-cpu data from various CPUs transparently from
user-space: cpu_opv will take care of migrating the thread to the
requested CPU. Use-cases such as rebalancing memory across per-cpu
memory pools, or migrating tasks for a user-space scheduler, are thus
facilitated. This also handles debugger single-stepping.
The use from userspace is, e.g. for a counter increment:
int cpu, ret;
cpu = percpu_current_cpu();
ret = percpu_addv(&data->c[cpu].count, 1, cpu);
if (unlikely(ret)) {
perror("percpu_addv");
return -1;
}
return 0;
Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@...icios.com>
CC: Shuah Khan <shuah@...nel.org>
CC: Russell King <linux@....linux.org.uk>
CC: Catalin Marinas <catalin.marinas@....com>
CC: Will Deacon <will.deacon@....com>
CC: Thomas Gleixner <tglx@...utronix.de>
CC: Paul Turner <pjt@...gle.com>
CC: Peter Zijlstra <peterz@...radead.org>
CC: Andy Lutomirski <luto@...capital.net>
CC: Andi Kleen <andi@...stfloor.org>
CC: Dave Watson <davejwatson@...com>
CC: Chris Lameter <cl@...ux.com>
CC: Ingo Molnar <mingo@...hat.com>
CC: "H. Peter Anvin" <hpa@...or.com>
CC: Ben Maurer <bmaurer@...com>
CC: Steven Rostedt <rostedt@...dmis.org>
CC: "Paul E. McKenney" <paulmck@...ux.vnet.ibm.com>
CC: Josh Triplett <josh@...htriplett.org>
CC: Linus Torvalds <torvalds@...ux-foundation.org>
CC: Andrew Morton <akpm@...ux-foundation.org>
CC: Boqun Feng <boqun.feng@...il.com>
CC: linux-kselftest@...r.kernel.org
CC: linux-api@...r.kernel.org
---
tools/testing/selftests/cpu-opv/percpu-op.h | 151 ++++++++++++++++++++++++++++
1 file changed, 151 insertions(+)
create mode 100644 tools/testing/selftests/cpu-opv/percpu-op.h
diff --git a/tools/testing/selftests/cpu-opv/percpu-op.h b/tools/testing/selftests/cpu-opv/percpu-op.h
new file mode 100644
index 000000000000..918171e585d7
--- /dev/null
+++ b/tools/testing/selftests/cpu-opv/percpu-op.h
@@ -0,0 +1,151 @@
+/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
+/*
+ * percpu-op.h
+ *
+ * (C) Copyright 2017-2018 - Mathieu Desnoyers <mathieu.desnoyers@...icios.com>
+ */
+
+#ifndef PERCPU_OP_H
+#define PERCPU_OP_H
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <stdlib.h>
+#include "rseq.h"
+#include "cpu-op.h"
+
+static inline uint32_t percpu_current_cpu(void)
+{
+ return rseq_current_cpu();
+}
+
+static inline __attribute__((always_inline))
+int percpu_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv,
+ int cpu)
+{
+ int ret;
+
+ ret = rseq_cmpeqv_storev(v, expect, newv, cpu);
+ if (rseq_unlikely(ret)) {
+ if (ret > 0)
+ return ret;
+ return cpu_op_cmpeqv_storev(v, expect, newv, cpu);
+ }
+ return 0;
+}
+
+static inline __attribute__((always_inline))
+int percpu_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
+ off_t voffp, intptr_t *load, int cpu)
+{
+ int ret;
+
+ ret = rseq_cmpnev_storeoffp_load(v, expectnot, voffp, load, cpu);
+ if (rseq_unlikely(ret)) {
+ if (ret > 0)
+ return ret;
+ return cpu_op_cmpnev_storeoffp_load(v, expectnot, voffp,
+ load, cpu);
+ }
+ return 0;
+}
+
+static inline __attribute__((always_inline))
+int percpu_addv(intptr_t *v, intptr_t count, int cpu)
+{
+ if (rseq_unlikely(rseq_addv(v, count, cpu)))
+ return cpu_op_addv(v, count, cpu);
+ return 0;
+}
+
+static inline __attribute__((always_inline))
+int percpu_cmpeqv_storev_storev(intptr_t *v, intptr_t expect,
+ intptr_t *v2, intptr_t newv2,
+ intptr_t newv, int cpu)
+{
+ int ret;
+
+ ret = rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2,
+ newv, cpu);
+ if (rseq_unlikely(ret)) {
+ if (ret > 0)
+ return ret;
+ return cpu_op_cmpeqv_storev_storev(v, expect, v2, newv2,
+ newv, cpu);
+ }
+ return 0;
+}
+
+static inline __attribute__((always_inline))
+int percpu_cmpeqv_storev_storev_release(intptr_t *v, intptr_t expect,
+ intptr_t *v2, intptr_t newv2,
+ intptr_t newv, int cpu)
+{
+ int ret;
+
+ ret = rseq_cmpeqv_trystorev_storev_release(v, expect, v2, newv2,
+ newv, cpu);
+ if (rseq_unlikely(ret)) {
+ if (ret > 0)
+ return ret;
+ return cpu_op_cmpeqv_storev_storev_release(v, expect, v2, newv2,
+ newv, cpu);
+ }
+ return 0;
+}
+
+static inline __attribute__((always_inline))
+int percpu_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
+ intptr_t *v2, intptr_t expect2,
+ intptr_t newv, int cpu)
+{
+ int ret;
+
+ ret = rseq_cmpeqv_cmpeqv_storev(v, expect, v2, expect2, newv, cpu);
+ if (rseq_unlikely(ret)) {
+ if (ret > 0)
+ return ret;
+ return cpu_op_cmpeqv_cmpeqv_storev(v, expect, v2, expect2,
+ newv, cpu);
+ }
+ return 0;
+}
+
+static inline __attribute__((always_inline))
+int percpu_cmpeqv_memcpy_storev(intptr_t *v, intptr_t expect,
+ void *dst, void *src, size_t len,
+ intptr_t newv, int cpu)
+{
+ int ret;
+
+ ret = rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
+ newv, cpu);
+ if (rseq_unlikely(ret)) {
+ if (ret > 0)
+ return ret;
+ return cpu_op_cmpeqv_memcpy_storev(v, expect, dst, src, len,
+ newv, cpu);
+ }
+ return 0;
+}
+
+static inline __attribute__((always_inline))
+int percpu_cmpeqv_memcpy_storev_release(intptr_t *v, intptr_t expect,
+ void *dst, void *src, size_t len,
+ intptr_t newv, int cpu)
+{
+ int ret;
+
+ ret = rseq_cmpeqv_trymemcpy_storev_release(v, expect, dst, src, len,
+ newv, cpu);
+ if (rseq_unlikely(ret)) {
+ if (ret > 0)
+ return ret;
+ return cpu_op_cmpeqv_memcpy_storev_release(v, expect, dst, src,
+ len, newv, cpu);
+ }
+ return 0;
+}
+
+#endif /* PERCPU_OP_H_ */
--
2.11.0
Powered by blists - more mailing lists