summaryrefslogtreecommitdiff
path: root/drivers
diff options
context:
space:
mode:
Diffstat (limited to 'drivers')
-rw-r--r--drivers/gpu/drm/drm_pagemap.c128
1 files changed, 103 insertions, 25 deletions
diff --git a/drivers/gpu/drm/drm_pagemap.c b/drivers/gpu/drm/drm_pagemap.c
index 6aa745682270..a0546955d0b9 100644
--- a/drivers/gpu/drm/drm_pagemap.c
+++ b/drivers/gpu/drm/drm_pagemap.c
@@ -384,6 +384,58 @@ next:
}
/**
+ * drm_pagemap_migrate_populate_src_pages() - Populate the source page array
+ * @pages: Array of source pages to populate
+ * @src_mpfn: Source array of migrate PFNs
+ * @dst_mpfn: Destination array of migrate PFNs
+ * @npages: Number of pages in the arrays
+ *
+ * Populate @pages with the device pages the copy callback is to read from.
+ *
+ * Entries are normally only populated at the head of each source folio, with
+ * the copy callback deriving the rest of the folio from the order recorded in
+ * the corresponding drm_pagemap_addr. That does not work where
+ * drm_pagemap_migrate_populate_ram_pfn() had to demote a higher-order source
+ * folio to order-0 destination folios: the drm_pagemap_addr entries are then
+ * per-page, and the copy callback needs a source page for each of them.
+ * Populate every entry for those ranges.
+ *
+ * Note that the source folio itself is only split later, by
+ * migrate_vma_pages() / migrate_device_pages(), so its order cannot be used to
+ * detect the demotion - the destination has to be inspected instead.
+ */
+static void drm_pagemap_migrate_populate_src_pages(struct page **pages,
+ unsigned long *src_mpfn,
+ unsigned long *dst_mpfn,
+ unsigned long npages)
+{
+ unsigned long i;
+
+ for (i = 0; i < npages;) {
+ struct page *page = migrate_pfn_to_page(src_mpfn[i]);
+ unsigned int order = 0;
+ unsigned long j, nr;
+
+ if (!page) {
+ i++;
+ continue;
+ }
+
+ order = folio_order(page_folio(page));
+ nr = NR_PAGES(order);
+
+ if (order && !(dst_mpfn[i] & MIGRATE_PFN_COMPOUND)) {
+ for (j = 0; j < nr && i + j < npages; j++)
+ pages[i + j] = folio_page(page_folio(page), j);
+ } else {
+ pages[i] = page;
+ }
+
+ i += nr;
+ }
+}
+
+/**
* drm_pagemap_migrate_unmap_pages() - Unmap pages previously mapped for GPU SVM migration
* @dev: The device for which the pages were mapped
* @migrate_pfn: Array of migrate pfns set up for the mapped pages. Used to
@@ -875,6 +927,7 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas,
struct page *page = NULL, *src_page;
struct folio *folio;
unsigned int order = 0;
+ gfp_t gfp = GFP_HIGHUSER;
if (!(src_mpfn[i] & MIGRATE_PFN_MIGRATE))
goto next;
@@ -891,11 +944,51 @@ static int drm_pagemap_migrate_populate_ram_pfn(struct vm_area_struct *vas,
order = folio_order(page_folio(src_page));
- /* TODO: Support fallback to single pages if THP allocation fails */
+ /*
+ * A large source folio is always collected whole, at its head
+ * page, PMD aligned and flagged MIGRATE_PFN_COMPOUND: anything
+ * else is split before it reaches us, either by
+ * migrate_vma_collect_pmd() or, for the eviction path, by
+ * migrate_device_pfns(). Both the order-0 fallback below and
+ * drm_pagemap_migrate_populate_src_pages() rely on that, as
+ * they index the folio from @i.
+ */
+ WARN_ON_ONCE(order &&
+ (src_page != folio_page(page_folio(src_page), 0) ||
+ !(src_mpfn[i] & MIGRATE_PFN_COMPOUND)));
+
+ if (order)
+ gfp |= __GFP_NOWARN;
+
if (vas)
- folio = vma_alloc_folio(GFP_HIGHUSER, order, vas, addr);
+ folio = vma_alloc_folio(gfp, order, vas, addr);
else
- folio = folio_alloc(GFP_HIGHUSER, order);
+ folio = folio_alloc(gfp, order);
+
+ if (!folio && order) {
+ /*
+ * Higher-order allocation failed, fall back to
+ * order-0 allocations for the entire range covered
+ * by the original higher-order allocation, without
+ * setting MIGRATE_PFN_COMPOUND, until we move past
+ * that range.
+ */
+ unsigned long nr = NR_PAGES(order);
+ unsigned long j;
+
+ gfp &= ~__GFP_NOWARN;
+ for (j = 0; j < nr && i < npages; j++, i++, addr += PAGE_SIZE) {
+ folio = vas ?
+ vma_alloc_folio(gfp, 0, vas, addr) :
+ folio_alloc(gfp, 0);
+ if (!folio)
+ goto free_pages;
+
+ page = folio_page(folio, 0);
+ mpfn[i] = migrate_pfn(page_to_pfn(page));
+ }
+ continue;
+ }
if (!folio)
goto free_pages;
@@ -940,11 +1033,11 @@ free_pages:
if (!page)
goto next_put;
+ order = folio_order(page_folio(page));
+
put_page(page);
mpfn[i] = 0;
- order = folio_order(page_folio(page));
-
next_put:
i += NR_PAGES(order);
}
@@ -1225,7 +1318,7 @@ int drm_pagemap_evict_to_ram(struct drm_pagemap_devmem *devmem_allocation)
unsigned long *src, *dst;
struct drm_pagemap_addr *pagemap_addr;
void *buf;
- int i, err = 0;
+ int err = 0;
unsigned int retry_count = 2;
npages = devmem_allocation->size >> PAGE_SHIFT;
@@ -1268,15 +1361,7 @@ retry:
if (err)
goto err_finalize;
- for (i = 0; i < npages;) {
- unsigned int order = 0;
-
- pages[i] = migrate_pfn_to_page(src[i]);
- if (pages[i])
- order = folio_order(page_folio(pages[i]));
-
- i += NR_PAGES(order);
- }
+ drm_pagemap_migrate_populate_src_pages(pages, src, dst, npages);
err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL);
if (err)
@@ -1344,7 +1429,7 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas,
struct drm_pagemap_addr *pagemap_addr;
unsigned long start, end;
void *buf;
- int i, err = 0;
+ int err = 0;
zdd = drm_pagemap_page_zone_device_data(page);
if (time_before64(get_jiffies_64(), zdd->devmem_allocation->timeslice_expiration))
@@ -1401,15 +1486,8 @@ static int __drm_pagemap_migrate_to_ram(struct vm_area_struct *vas,
if (err)
goto err_finalize;
- for (i = 0; i < npages;) {
- unsigned int order = 0;
-
- pages[i] = migrate_pfn_to_page(migrate.src[i]);
- if (pages[i])
- order = folio_order(page_folio(pages[i]));
-
- i += NR_PAGES(order);
- }
+ drm_pagemap_migrate_populate_src_pages(pages, migrate.src, migrate.dst,
+ npages);
err = ops->copy_to_ram(pages, pagemap_addr, npages, NULL);
if (err)
generated by cgit (git 2.54.0.dirty) at 2026-09-20 11:37:28 +0000