| lists.openwall.net | lists / announce owl-users owl-dev john-users john-dev passwdqc-users yescrypt popa3d-users / oss-security kernel-hardening musl sabotage tlsify passwords / crypt-dev xvendor / Bugtraq Full-Disclosure linux-kernel linux-netdev linux-ext4 linux-hardening linux-cve-announce PHC | |
|
Open Source and information security mailing list archives
| ||
|
Message-ID: <20250923174752.35701-6-shivankg@amd.com>
Date: Tue, 23 Sep 2025 17:47:40 +0000
From: Shivank Garg <shivankg@....com>
To: <akpm@...ux-foundation.org>, <david@...hat.com>
CC: <ziy@...dia.com>, <willy@...radead.org>, <matthew.brost@...el.com>,
<joshua.hahnjy@...il.com>, <rakie.kim@...com>, <byungchul@...com>,
<gourry@...rry.net>, <ying.huang@...ux.alibaba.com>, <apopple@...dia.com>,
<lorenzo.stoakes@...cle.com>, <Liam.Howlett@...cle.com>, <vbabka@...e.cz>,
<rppt@...nel.org>, <surenb@...gle.com>, <mhocko@...e.com>,
<vkoul@...nel.org>, <lucas.demarchi@...el.com>, <rdunlap@...radead.org>,
<jgg@...pe.ca>, <kuba@...nel.org>, <justonli@...omium.org>,
<ivecera@...hat.com>, <dave.jiang@...el.com>, <Jonathan.Cameron@...wei.com>,
<dan.j.williams@...el.com>, <rientjes@...gle.com>,
<Raghavendra.KodsaraThimmappa@....com>, <bharata@....com>,
<shivankg@....com>, <alirad.malek@...corp.com>, <yiannis@...corp.com>,
<weixugc@...gle.com>, <linux-kernel@...r.kernel.org>, <linux-mm@...ck.org>
Subject: [RFC V3 5/9] mm: add support for copy offload for folio Migration
From: Mike Day <michael.day@....com>
Offload-Copy drivers should implement following functions to enable folio
migration offloading:
migrate_offc() - This function takes src and dst folios list undergoing
migration. It is responsible for transfer of page content between the
src and dst folios.
can_migrate_offc() - It performs necessary checks if offload copying
migration is supported for the give src and dst folios.
Offload-Copy driver should include a mechanism to call start_offloading and
stop_offloading for enabling and disabling migration offload respectively.
Signed-off-by: Mike Day <michael.day@....com>
Co-developed-by: Shivank Garg <shivankg@....com>
Signed-off-by: Shivank Garg <shivankg@....com>
---
include/linux/migrate_offc.h | 34 +++++++++++++++++++++
mm/Kconfig | 8 +++++
mm/Makefile | 1 +
mm/migrate.c | 49 +++++++++++++++++++++++++++++-
mm/migrate_offc.c | 58 ++++++++++++++++++++++++++++++++++++
5 files changed, 149 insertions(+), 1 deletion(-)
create mode 100644 include/linux/migrate_offc.h
create mode 100644 mm/migrate_offc.c
diff --git a/include/linux/migrate_offc.h b/include/linux/migrate_offc.h
new file mode 100644
index 000000000000..e9e8a30f40f0
--- /dev/null
+++ b/include/linux/migrate_offc.h
@@ -0,0 +1,34 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _MIGRATE_OFFC_H
+#define _MIGRATE_OFFC_H
+#include <linux/migrate_mode.h>
+
+#define MIGRATOR_NAME_LEN 32
+struct migrator {
+ char name[MIGRATOR_NAME_LEN];
+ int (*migrate_offc)(struct list_head *dst_list, struct list_head *src_list,
+ unsigned int folio_cnt);
+ struct rcu_head srcu_head;
+ struct module *owner;
+};
+
+extern struct migrator migrator;
+extern struct mutex migrator_mut;
+extern struct srcu_struct mig_srcu;
+
+#ifdef CONFIG_OFFC_MIGRATION
+void srcu_mig_cb(struct rcu_head *head);
+int offc_update_migrator(struct migrator *mig);
+unsigned char *get_active_migrator_name(void);
+int start_offloading(struct migrator *migrator);
+int stop_offloading(void);
+#else
+static inline void srcu_mig_cb(struct rcu_head *head) { };
+static inline int offc_update_migrator(struct migrator *mig) { return 0; };
+static inline unsigned char *get_active_migrator_name(void) { return NULL; };
+static inline void start_offloading(struct migrator *migrator) { return 0; };
+static inline void stop_offloading(void) { return 0; };
+#endif /* CONFIG_OFFC_MIGRATION */
+
+#endif /* _MIGRATE_OFFC_H */
diff --git a/mm/Kconfig b/mm/Kconfig
index e443fe8cd6cf..a9cbb8d1f1f6 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -689,6 +689,14 @@ config MIGRATION
config DEVICE_MIGRATION
def_bool MIGRATION && ZONE_DEVICE
+config OFFC_MIGRATION
+ bool "Migrate Pages offloading copy"
+ def_bool n
+ depends on MIGRATION
+ help
+ An interface allowing external modules or driver to offload
+ page copying in page migration.
+
config ARCH_ENABLE_HUGEPAGE_MIGRATION
bool
diff --git a/mm/Makefile b/mm/Makefile
index ef54aa615d9d..f609d3899992 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -96,6 +96,7 @@ obj-$(CONFIG_FAILSLAB) += failslab.o
obj-$(CONFIG_FAIL_PAGE_ALLOC) += fail_page_alloc.o
obj-$(CONFIG_MEMTEST) += memtest.o
obj-$(CONFIG_MIGRATION) += migrate.o
+obj-$(CONFIG_OFFC_MIGRATION) += migrate_offc.o
obj-$(CONFIG_NUMA) += memory-tiers.o
obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o
obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o
diff --git a/mm/migrate.c b/mm/migrate.c
index ce94e73a930d..41bea48d823c 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -43,6 +43,7 @@
#include <linux/sched/sysctl.h>
#include <linux/memory-tiers.h>
#include <linux/pagewalk.h>
+#include <linux/migrate_offc.h>
#include <asm/tlbflush.h>
@@ -834,6 +835,52 @@ void folio_migrate_flags(struct folio *newfolio, struct folio *folio)
}
EXPORT_SYMBOL(folio_migrate_flags);
+#ifdef CONFIG_HAVE_STATIC_CALL
+DEFINE_STATIC_CALL(_folios_copy, folios_mc_copy);
+
+#ifdef CONFIG_OFFC_MIGRATION
+void srcu_mig_cb(struct rcu_head *head)
+{
+ static_call_query(_folios_copy);
+}
+
+int offc_update_migrator(struct migrator *mig)
+{
+ struct module *old_owner, *new_owner;
+ int index;
+ int ret = 0;
+
+ mutex_lock(&migrator_mut);
+ index = srcu_read_lock(&mig_srcu);
+ old_owner = READ_ONCE(migrator.owner);
+ new_owner = mig ? mig->owner : NULL;
+
+ if (new_owner && !try_module_get(new_owner)) {
+ ret = -ENODEV;
+ goto out_unlock;
+ }
+
+ strscpy(migrator.name, mig ? mig->name : "kernel", MIGRATOR_NAME_LEN);
+ static_call_update(_folios_copy, mig ? mig->migrate_offc : folios_mc_copy);
+ xchg(&migrator.owner, mig ? mig->owner : NULL);
+ if (old_owner)
+ module_put(old_owner);
+
+out_unlock:
+ WARN_ON(ret < 0);
+ srcu_read_unlock(&mig_srcu, index);
+ mutex_unlock(&migrator_mut);
+
+ if (ret == 0) {
+ call_srcu(&mig_srcu, &migrator.srcu_head, srcu_mig_cb);
+ srcu_barrier(&mig_srcu);
+ }
+ return ret;
+}
+
+#endif /* CONFIG_OFFC_MIGRATION */
+#endif /* CONFIG_HAVE_STATIC_CALL */
+
/************************************************************
* Migration functions
***********************************************************/
@@ -1870,7 +1917,7 @@ static void migrate_folios_batch_move(struct list_head *src_folios,
goto out;
/* Batch copy the folios */
- rc = folios_mc_copy(dst_folios, src_folios, nr_batched_folios);
+ rc = static_call(_folios_copy)(dst_folios, src_folios, nr_batched_folios);
/* TODO: Is there a better way of handling the poison
* recover for batch copy, instead of falling back to serial copy?
diff --git a/mm/migrate_offc.c b/mm/migrate_offc.c
new file mode 100644
index 000000000000..a6530658a3f7
--- /dev/null
+++ b/mm/migrate_offc.c
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <linux/migrate.h>
+#include <linux/migrate_offc.h>
+#include <linux/rculist.h>
+#include <linux/static_call.h>
+
+atomic_t dispatch_to_offc = ATOMIC_INIT(0);
+EXPORT_SYMBOL_GPL(dispatch_to_offc);
+
+DEFINE_MUTEX(migrator_mut);
+DEFINE_SRCU(mig_srcu);
+
+struct migrator migrator = {
+ .name = "kernel",
+ .migrate_offc = folios_mc_copy,
+ .srcu_head.func = srcu_mig_cb,
+ .owner = NULL,
+};
+
+int start_offloading(struct migrator *m)
+{
+ int offloading = 0;
+ int ret;
+
+ pr_info("starting migration offload by %s\n", m->name);
+ ret = offc_update_migrator(m);
+ if (ret < 0) {
+ pr_err("failed to start migration offload by %s, err=%d\n",
+ m->name, ret);
+ return ret;
+ }
+ atomic_try_cmpxchg(&dispatch_to_offc, &offloading, 1);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(start_offloading);
+
+int stop_offloading(void)
+{
+ int offloading = 1;
+ int ret;
+
+ pr_info("stopping migration offload by %s\n", migrator.name);
+ ret = offc_update_migrator(NULL);
+ if (ret < 0) {
+ pr_err("failed to stop migration offload by %s, err=%d\n",
+ migrator.name, ret);
+ return ret;
+ }
+ atomic_try_cmpxchg(&dispatch_to_offc, &offloading, 0);
+ return 0;
+}
+EXPORT_SYMBOL_GPL(stop_offloading);
+
+unsigned char *get_active_migrator_name(void)
+{
+ return migrator.name;
+}
+EXPORT_SYMBOL_GPL(get_active_migrator_name);
--
2.43.0
Powered by blists - more mailing lists