[v3 06/11] mm/memremap: add folio_split support
Balbir Singh
balbirs at nvidia.com
Tue Aug 12 02:40:31 UTC 2025
When a zone device page is split (via huge pmd folio split). The
driver callback for folio_split is invoked to let the device driver
know that the folio size has been split into a smaller order.
Provide a default implementation for drivers that do not provide
this callback that copies the pgmap and mapping fields for the
split folios.
Update the HMM test driver to handle the split.
Cc: Andrew Morton <akpm at linux-foundation.org>
Cc: David Hildenbrand <david at redhat.com>
Cc: Zi Yan <ziy at nvidia.com>
Cc: Joshua Hahn <joshua.hahnjy at gmail.com>
Cc: Rakie Kim <rakie.kim at sk.com>
Cc: Byungchul Park <byungchul at sk.com>
Cc: Gregory Price <gourry at gourry.net>
Cc: Ying Huang <ying.huang at linux.alibaba.com>
Cc: Alistair Popple <apopple at nvidia.com>
Cc: Oscar Salvador <osalvador at suse.de>
Cc: Lorenzo Stoakes <lorenzo.stoakes at oracle.com>
Cc: Baolin Wang <baolin.wang at linux.alibaba.com>
Cc: "Liam R. Howlett" <Liam.Howlett at oracle.com>
Cc: Nico Pache <npache at redhat.com>
Cc: Ryan Roberts <ryan.roberts at arm.com>
Cc: Dev Jain <dev.jain at arm.com>
Cc: Barry Song <baohua at kernel.org>
Cc: Lyude Paul <lyude at redhat.com>
Cc: Danilo Krummrich <dakr at kernel.org>
Cc: David Airlie <airlied at gmail.com>
Cc: Simona Vetter <simona at ffwll.ch>
Cc: Ralph Campbell <rcampbell at nvidia.com>
Cc: Mika Penttilä <mpenttil at redhat.com>
Cc: Matthew Brost <matthew.brost at intel.com>
Cc: Francois Dugast <francois.dugast at intel.com>
Signed-off-by: Balbir Singh <balbirs at nvidia.com>
---
include/linux/memremap.h | 29 +++++++++++++++++++++++++++++
include/linux/mm.h | 1 +
lib/test_hmm.c | 35 +++++++++++++++++++++++++++++++++++
mm/huge_memory.c | 2 +-
4 files changed, 66 insertions(+), 1 deletion(-)
diff --git a/include/linux/memremap.h b/include/linux/memremap.h
index 0c5141a7d58c..20f4b5ebbc93 100644
--- a/include/linux/memremap.h
+++ b/include/linux/memremap.h
@@ -100,6 +100,13 @@ struct dev_pagemap_ops {
*/
int (*memory_failure)(struct dev_pagemap *pgmap, unsigned long pfn,
unsigned long nr_pages, int mf_flags);
+
+ /*
+ * Used for private (un-addressable) device memory only.
+ * This callback is used when a folio is split into
+ * a smaller folio
+ */
+ void (*folio_split)(struct folio *head, struct folio *tail);
};
#define PGMAP_ALTMAP_VALID (1 << 0)
@@ -229,6 +236,23 @@ static inline void zone_device_page_init(struct page *page)
zone_device_folio_init(folio, 0);
}
+static inline void zone_device_private_split_cb(struct folio *original_folio,
+ struct folio *new_folio)
+{
+ if (folio_is_device_private(original_folio)) {
+ if (!original_folio->pgmap->ops->folio_split) {
+ if (new_folio) {
+ new_folio->pgmap = original_folio->pgmap;
+ new_folio->page.mapping =
+ original_folio->page.mapping;
+ }
+ } else {
+ original_folio->pgmap->ops->folio_split(original_folio,
+ new_folio);
+ }
+ }
+}
+
#else
static inline void *devm_memremap_pages(struct device *dev,
struct dev_pagemap *pgmap)
@@ -263,6 +287,11 @@ static inline unsigned long memremap_compat_align(void)
{
return PAGE_SIZE;
}
+
+static inline void zone_device_private_split_cb(struct folio *original_folio,
+ struct folio *new_folio)
+{
+}
#endif /* CONFIG_ZONE_DEVICE */
static inline void put_dev_pagemap(struct dev_pagemap *pgmap)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index ceaa780a703a..f755afe533e5 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1185,6 +1185,7 @@ static inline struct folio *virt_to_folio(const void *x)
void __folio_put(struct folio *folio);
void split_page(struct page *page, unsigned int order);
+void prep_compound_page(struct page *page, unsigned int order);
void folio_copy(struct folio *dst, struct folio *src);
int folio_mc_copy(struct folio *dst, struct folio *src);
diff --git a/lib/test_hmm.c b/lib/test_hmm.c
index d814056151d0..14dbce719896 100644
--- a/lib/test_hmm.c
+++ b/lib/test_hmm.c
@@ -1653,9 +1653,44 @@ static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
return ret;
}
+static void dmirror_devmem_folio_split(struct folio *head, struct folio *tail)
+{
+ struct page *rpage = BACKING_PAGE(folio_page(head, 0));
+ struct page *rpage_tail;
+ struct folio *rfolio;
+ unsigned long offset = 0;
+
+ if (!rpage) {
+ tail->page.zone_device_data = NULL;
+ return;
+ }
+
+ rfolio = page_folio(rpage);
+
+ if (tail == NULL) {
+ folio_reset_order(rfolio);
+ rfolio->mapping = NULL;
+ folio_set_count(rfolio, 1);
+ return;
+ }
+
+ offset = folio_pfn(tail) - folio_pfn(head);
+
+ rpage_tail = folio_page(rfolio, offset);
+ tail->page.zone_device_data = rpage_tail;
+ rpage_tail->zone_device_data = rpage->zone_device_data;
+ clear_compound_head(rpage_tail);
+ rpage_tail->mapping = NULL;
+
+ folio_page(tail, 0)->mapping = folio_page(head, 0)->mapping;
+ tail->pgmap = head->pgmap;
+ folio_set_count(page_folio(rpage_tail), 1);
+}
+
static const struct dev_pagemap_ops dmirror_devmem_ops = {
.page_free = dmirror_devmem_free,
.migrate_to_ram = dmirror_devmem_fault,
+ .folio_split = dmirror_devmem_folio_split,
};
static int dmirror_device_init(struct dmirror_device *mdevice, int id)
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 8888140e57a3..dc58081b661c 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -3922,7 +3922,6 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
ret = __split_unmapped_folio(folio, new_order, split_at, &xas,
mapping, uniform_split);
-
/*
* Unfreeze after-split folios and put them back to the right
* list. @folio should be kept frozon until page cache
@@ -3973,6 +3972,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
__filemap_remove_folio(new_folio, NULL);
folio_put_refs(new_folio, nr_pages);
}
+
/*
* Unfreeze @folio only after all page cache entries, which
* used to point to it, have been updated with new folios.
--
2.50.1
More information about the dri-devel
mailing list