summaryrefslogtreecommitdiff
path: root/mm/memory.c
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-08-27 09:17:06 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2026-08-27 09:17:06 -0700
commit18fbf5151d2c0bfe433c7428eef03cabf5fdb2fa (patch)
tree6e151850bf46f900bbf9dcc682e0c75d91924865 /mm/memory.c
parent5e6ff28676dd92a608eb00eeb8d1319ad34024dc (diff)
parent0685630fdccb62dcb0e3f44525a40578da5f6dc8 (diff)
Merge tag 'mm-stable-2026-08-26-15-22' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm
Pull more MM updates from Andrew Morton: - "mm/rmap: index MAP_PRIVATE file-backed folios by anonymous pgoff" (Lorenzo Stoakes) Index MAP_PRIVATE file-backed folios by their anonymous page offset to resolve confusion around reverse mapping for zeroed and CoW'd file-backed memory. Use this new VMA anonymous page offset tracking to eliminate index conflicts and lay the foundation for scalable CoW performance improvements. - "promote mapped executable folios after first usage for MGLRU" (Baolin Wang) Make MGLRU's protection of mapped executable file folios more reliable. Follow the classical LRU's logic, promoting mapped executable file folios after their first usage to give executable code a better chance to stay in memory and improve workload performance. - "mm: vmscan: fix node reclaim ignoring swappiness parameter" (Ridong Chen) Fix per-node proactive reclaim interface's ignoring the swappiness parameter when CONFIG_MEMCG is disabled by consolidating sc_swappiness() into a single function that checks proactive_swappiness regardless of kernel configuration. - "mm/vmscan: reduce lru_lock contention via vmstat-derived scan-balance cost" (Usama Arif) Reduce lru_lock contention in the reclaim path by deriving scan-balance costs from vmstat counters rather than lock-acquired producer updates. Read and decay these cost signals on the reclaim side under a dedicated per-lruvec lock, reducing total LRU lock wait time by over 60% without impacting scan throughput. - "zram: fix zram issues reported by sashiko" (Sergey Senozhatsky) Fix two low-risk zram bugs which Sashiko spotted in drive-by review. - "Honor XA_FLAGS_ACCOUNT in xas_split_alloc() and charge to folio's memcg" (Zi Yan) Fix xas_split_alloc() by enabling target folio memcg charging during splits and adding the missing __GFP_ACCOUNT flag for proper XArray node memory accounting. - "selftests/mm: use pattern matching in .gitignore" (Pratyush Mallick) Replace hardcoded binary names in selftests/mm/.gitignore with a generic pattern-matching rule to automatically ignore generated test files and avoid manual updates when adding new tests. - "mm/page_ext: remove pgdat_page_ext_init()" (Sang-Heon Jeon) Make the incompatibility between FLATMEM and NUMA explicit in mm/Kconfig and remove the unused pgdat_page_ext_init() function. - "zram: fix zstd error paths and add parameter validation" (Haoqin Huang) Clean up zram compression backends by removing redundant error cleanup, adding parameter and dictionary validation, auto-prefixing algorithm error logs, and resetting parameters prior to reinitialization. - "zram: fix stale scan bounds after reinitialization" (Longlong Xia) Prevent out-of-bounds slot accesses during concurrent zram resets by moving table scan bound calculations under dev_lock in writeback_store() and read_block_state(). - "add anon mTHP collapse test cases" (Baolin Wang) Extend selftests helper functions to support arbitrary page orders and add new test cases and options for mTHP collapse in khugepaged. - "selftests/mm: Handle unsupported and transient test conditions" (Muhammad Usama Anjum) Update MM selftests to report a SKIP status instead of a failure when required kernel or filesystem features are unsupported, while adding retry logic for transient page migration errors. - "mm/zswap: Fixes and improves the zswap shrink" (Hao Jia) Fix the missing zswap global shrinker when CONFIG_MEMCG is disabled and extend shrink_memcg() to support batch writeback for improved writeback efficiency. - "alloc_tag: introduce IOCTL-based filtering for MAP" (Suren Baghdasaryan) Introduce an IOCTL-based binary interface for memory allocation profiling that enables kernel-side filtering before per-CPU counter aggregation. This eliminates the text-parsing overhead of /proc/allocinfo and provides up to a 20x speedup by transferring only filtered allocation data to userspace. - "better block swap batching and a different take on swap_ops v5" (Christoph Hellwig) Refactor block swap I/O to use swap_iocb for batching instead of single-bio requests and rebase the swap_ops interface, achieving faster swap throughput during kernel builds. - "mm: kmemleak: reduce transient false positives by confirming leaks" (Catalin Marinas) Reduce false-positive kmemleak reports by combining two kmemleak enhancements that add a second confirmation scan and a configurable minimum unreferenced scan count module parameter. - "mm: kmemleak: default min_unref_scans to 2 for verbose kernels" (Breno Leitao) Auto-scanning kernels can generate false-positive memory leak reports on single scans, so this patch defaults min_unref_scans to 2 when CONFIG_DEBUG_KMEMLEAK_VERBOSE is enabled to require a second confirming scan. - "swap_ops updates" (Christoph Hellwig) Batching I/O for synchronous swap devices causes performance regressions and filesystem-based swap suffers from double-indirection overhead. This series resolves both issues by reintroducing per-folio writes for synchronous swap and allowing filesystems to directly export their own swap_ops. - "mm/khugepaged: several cleanups" (Nico Pache) khugepaged accumulated redundant state-checking patterns and outdated comments following mTHP integration. Introduce dedicated helpers for PTE validation and event counting while refreshing the internal documentation. - "maple_tree: lock checking and clean ups" (Liam Howlett) Syzbot reports incorrectly blame memory management exit paths for locking bugs, maple tree erase operations risk allocation failures without gfp flags and internal documentation lacks clarity. Improve lock error detection, update docs, fix race and allocation edge cases and optimize erase allocations using a fallback to GFP_KERNEL | GFP_NOFAIL. * tag 'mm-stable-2026-08-26-15-22' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm: (172 commits) selftests/proc: make proc-maps-race work with READ_IMPLIES_EXEC memcg: move LRU size accounting on reparenting instead of copying it mm/vmscan: fix comment logic in balance_pgdat maple_tree: add helper mas_make_walkable() maple_tree: avoid extra gap calculation maple_tree: fix argument name in header maple_tree: change two GFP flags in tests maple_tree: document erase and allocations better maple_tree: avoid mas_erase() and mtree_erase() failures maple_tree: document that erase may use GFP_KERNEL for allocations maple_tree: catch race in mas_alloc_cyclic() maple_tree: add bulk parent set helper maple_tree: micro optimisation of mas_wr_store_type() maple_tree: optimise mas_wr_node_store() when not in rcu mode maple_tree: use prefetched value in mas_wr_store_type() maple_tree: clarify comments on mas_nomem() maple_tree: drop MAPLE_ALLOC_SLOTS maple_tree: drop dead code from mas_extend_spanning_null() maple_tree: documentation fix maple_tree: add write lock checking with lockdep sequence numbers ...
Diffstat (limited to 'mm/memory.c')
-rw-r--r--mm/memory.c140
1 files changed, 101 insertions, 39 deletions
diff --git a/mm/memory.c b/mm/memory.c
index 8da0f945141b..8b0c2c735d3d 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -519,9 +519,52 @@ static bool is_bad_page_map_ratelimited(void)
return false;
}
+static void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size)
+{
+ if (WARN_ON_ONCE(buf_size < entry_size * 2 + 1)) {
+ snprintf(buf, buf_size, "overflow");
+ return;
+ }
+
+ switch (entry_size) {
+ case sizeof(u32):
+ snprintf(buf, buf_size, "%08x", *(const u32 *)entry);
+ break;
+ case sizeof(u64):
+ snprintf(buf, buf_size, "%016llx", *(const u64 *)entry);
+ break;
+#if defined(__SIZEOF_INT128__)
+ case sizeof(u128):
+ snprintf(buf, buf_size, "%016llx%016llx",
+ (unsigned long long)(*(const u128 *)entry >> 64),
+ (unsigned long long)*(const u128 *)entry);
+ break;
+#endif
+ default:
+ snprintf(buf, buf_size, "unsupported");
+ break;
+ }
+}
+
+#define ptval_to_str(buf, val) \
+ do { \
+ auto __val = (val); \
+ \
+ ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val)); \
+ } while (0)
+
+#if defined(__SIZEOF_INT128__)
+#define PTVAL_STR_MAX (32 + 1) /* Max 128-bit value in hex + NUL */
+#else
+#define PTVAL_STR_MAX (16 + 1) /* Max 64-bit value in hex + NUL */
+#endif
+
static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long addr)
{
- unsigned long long pgdv, p4dv, pudv, pmdv;
+ char pgd_str[PTVAL_STR_MAX];
+ char p4d_str[PTVAL_STR_MAX];
+ char pud_str[PTVAL_STR_MAX];
+ char pmd_str[PTVAL_STR_MAX];
p4d_t p4d, *p4dp;
pud_t pud, *pudp;
pmd_t pmd, *pmdp;
@@ -532,34 +575,34 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add
* see locking requirements for print_bad_page_map().
*/
pgdp = pgd_offset(mm, addr);
- pgdv = pgd_val(*pgdp);
+ ptval_to_str(pgd_str, pgd_val(*pgdp));
if (!pgd_present(*pgdp) || pgd_leaf(*pgdp)) {
- pr_alert("pgd:%08llx\n", pgdv);
+ pr_alert("pgd:%s\n", pgd_str);
return;
}
p4dp = p4d_offset(pgdp, addr);
p4d = p4dp_get(p4dp);
- p4dv = p4d_val(p4d);
+ ptval_to_str(p4d_str, p4d_val(p4d));
if (!p4d_present(p4d) || p4d_leaf(p4d)) {
- pr_alert("pgd:%08llx p4d:%08llx\n", pgdv, p4dv);
+ pr_alert("pgd:%s p4d:%s\n", pgd_str, p4d_str);
return;
}
pudp = pud_offset(p4dp, addr);
pud = pudp_get(pudp);
- pudv = pud_val(pud);
+ ptval_to_str(pud_str, pud_val(pud));
if (!pud_present(pud) || pud_leaf(pud)) {
- pr_alert("pgd:%08llx p4d:%08llx pud:%08llx\n", pgdv, p4dv, pudv);
+ pr_alert("pgd:%s p4d:%s pud:%s\n", pgd_str, p4d_str, pud_str);
return;
}
pmdp = pmd_offset(pudp, addr);
pmd = pmdp_get(pmdp);
- pmdv = pmd_val(pmd);
+ ptval_to_str(pmd_str, pmd_val(pmd));
/*
* Dumping the PTE would be nice, but it's tricky with CONFIG_HIGHPTE,
@@ -567,8 +610,7 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add
* doing another map would be bad. print_bad_page_map() should
* already take care of printing the PTE.
*/
- pr_alert("pgd:%08llx p4d:%08llx pud:%08llx pmd:%08llx\n", pgdv,
- p4dv, pudv, pmdv);
+ pr_alert("pgd:%s p4d:%s pud:%s pmd:%s\n", pgd_str, p4d_str, pud_str, pmd_str);
}
/*
@@ -584,25 +626,34 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add
* page table lock.
*/
static void print_bad_page_map(struct vm_area_struct *vma,
- unsigned long addr, unsigned long long entry, struct page *page,
- enum pgtable_level level)
+ unsigned long addr, const void *entry, size_t entry_size,
+ struct page *page, enum pgtable_level level)
{
struct address_space *mapping;
- pgoff_t index;
+ char entry_str[PTVAL_STR_MAX];
+ pgoff_t index, anon_index;
if (is_bad_page_map_ratelimited())
return;
mapping = vma->vm_file ? vma->vm_file->f_mapping : NULL;
index = linear_page_index(vma, addr);
+ anon_index = __linear_anon_page_index(vma, addr);
- pr_alert("BUG: Bad page map in process %s %s:%08llx", current->comm,
- pgtable_level_to_str(level), entry);
+ ptval_bytes_to_hex_str(entry_str, sizeof(entry_str), entry, entry_size);
+ pr_alert("BUG: Bad page map in process %s %s:%s", current->comm,
+ pgtable_level_to_str(level), entry_str);
__print_bad_page_map_pgtable(vma->vm_mm, addr);
if (page)
dump_page(page, "bad page map");
- pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px index:%lx\n",
- (void *)addr, vma->vm_flags, vma->anon_vma, mapping, index);
+ pr_alert("addr:%px vm_flags:%08lx anon_vma:%px mapping:%px",
+ (void *)addr, vma->vm_flags, vma->anon_vma, mapping);
+ if (!vma_is_cow_mapping(vma) || index == anon_index) {
+ pr_cont(" index:%lx\n", index);
+ } else {
+ pr_cont(" index:%lx (file) %lx (anon)\n", index, anon_index);
+ }
+
pr_alert("file:%pD fault:%ps mmap:%ps mmap_prepare: %ps read_folio:%ps\n",
vma->vm_file,
vma->vm_ops ? vma->vm_ops->fault : NULL,
@@ -627,8 +678,13 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level)
}
}
-#define print_bad_pte(vma, addr, pte, page) \
- print_bad_page_map(vma, addr, pte_val(pte), page, PGTABLE_LEVEL_PTE)
+static void print_bad_pte(struct vm_area_struct *vma, unsigned long addr,
+ pte_t pte, struct page *page)
+{
+ auto entry = pte_val(pte);
+
+ print_bad_page_map(vma, addr, &entry, sizeof(entry), page, PGTABLE_LEVEL_PTE);
+}
/**
* __vm_normal_page() - Get the "struct page" associated with a page table entry.
@@ -636,8 +692,9 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level)
* @addr: The address where the page table entry is mapped.
* @pfn: The PFN stored in the page table entry.
* @special: Whether the page table entry is marked "special".
- * @level: The page table level for error reporting purposes only.
* @entry: The page table entry value for error reporting purposes only.
+ * @entry_size: The size of @entry.
+ * @level: The page table level for error reporting purposes only.
*
* "Special" mappings do not wish to be associated with a "struct page" (either
* it doesn't exist, or it exists but they don't want to touch it). In this
@@ -697,7 +754,7 @@ static inline bool pgtable_level_has_pxx_special(enum pgtable_level level)
*/
static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
unsigned long addr, unsigned long pfn, bool special,
- unsigned long long entry, enum pgtable_level level)
+ const void *entry, size_t entry_size, enum pgtable_level level)
{
if (pgtable_level_has_pxx_special(level)) {
if (unlikely(special)) {
@@ -710,7 +767,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
if (is_zero_pfn(pfn) || is_huge_zero_pfn(pfn))
return NULL;
- print_bad_page_map(vma, addr, entry, NULL, level);
+ print_bad_page_map(vma, addr, entry, entry_size, NULL, level);
return NULL;
}
/*
@@ -730,7 +787,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
/* Only CoW'ed anon folios are "normal". */
if (pfn == index)
return NULL;
- if (!is_cow_mapping(vma->vm_flags))
+ if (!vma_is_cow_mapping(vma))
return NULL;
}
}
@@ -741,7 +798,7 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
if (unlikely(pfn > highest_memmap_pfn)) {
/* Corrupted page table entry. */
- print_bad_page_map(vma, addr, entry, NULL, level);
+ print_bad_page_map(vma, addr, entry, entry_size, NULL, level);
return NULL;
}
/*
@@ -767,8 +824,10 @@ static inline struct page *__vm_normal_page(struct vm_area_struct *vma,
struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
pte_t pte)
{
+ auto entry = pte_val(pte);
+
return __vm_normal_page(vma, addr, pte_pfn(pte), pte_special(pte),
- pte_val(pte), PGTABLE_LEVEL_PTE);
+ &entry, sizeof(entry), PGTABLE_LEVEL_PTE);
}
/**
@@ -809,8 +868,10 @@ struct folio *vm_normal_folio(struct vm_area_struct *vma, unsigned long addr,
struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr,
pmd_t pmd)
{
+ auto entry = pmd_val(pmd);
+
return __vm_normal_page(vma, addr, pmd_pfn(pmd), pmd_special(pmd),
- pmd_val(pmd), PGTABLE_LEVEL_PMD);
+ &entry, sizeof(entry), PGTABLE_LEVEL_PMD);
}
/**
@@ -850,8 +911,10 @@ struct folio *vm_normal_folio_pmd(struct vm_area_struct *vma,
struct page *vm_normal_page_pud(struct vm_area_struct *vma,
unsigned long addr, pud_t pud)
{
+ auto entry = pud_val(pud);
+
return __vm_normal_page(vma, addr, pud_pfn(pud), pud_special(pud),
- pud_val(pud), PGTABLE_LEVEL_PUD);
+ &entry, sizeof(entry), PGTABLE_LEVEL_PUD);
}
#endif
@@ -946,7 +1009,6 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma,
struct vm_area_struct *src_vma, unsigned long addr, int *rss)
{
- vm_flags_t vm_flags = dst_vma->vm_flags;
pte_t orig_pte = ptep_get(src_pte);
softleaf_t entry = softleaf_from_pte(orig_pte);
pte_t pte = orig_pte;
@@ -970,7 +1032,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
rss[mm_counter(folio)]++;
if (!softleaf_is_migration_read(entry) &&
- is_cow_mapping(vm_flags)) {
+ vma_is_cow_mapping(dst_vma)) {
/*
* COW mappings require pages in both parent and child
* to be set to read. A previously exclusive entry is
@@ -1011,7 +1073,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
* save and restore device driver state).
*/
if (softleaf_is_device_private_write(entry) &&
- is_cow_mapping(vm_flags)) {
+ vma_is_cow_mapping(dst_vma)) {
entry = make_readable_device_private_entry(
swp_offset(entry));
pte = swp_entry_to_pte(entry);
@@ -1026,7 +1088,7 @@ copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
* exclusive entries currently only support private writable
* (ie. COW) mappings.
*/
- VM_BUG_ON(!is_cow_mapping(src_vma->vm_flags));
+ VM_BUG_ON(!vma_is_cow_mapping(src_vma));
if (try_restore_exclusive_pte(src_vma, addr, src_pte, orig_pte))
return -EBUSY;
return -ENOENT;
@@ -1125,7 +1187,7 @@ static __always_inline void __copy_present_ptes(struct vm_area_struct *dst_vma,
}
/* If it's a COW mapping, write protect it both processes. */
- if (is_cow_mapping(src_vma->vm_flags) && writable) {
+ if (vma_is_cow_mapping(src_vma) && writable) {
wrprotect_ptes(src_mm, addr, src_pte, nr);
pte = pte_wrprotect(pte);
}
@@ -1546,9 +1608,9 @@ copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
* We need to invalidate the secondary MMU mappings only when
* there could be a permission downgrade on the ptes of the
* parent mm. And a permission downgrade will only happen if
- * is_cow_mapping() returns true.
+ * vma_is_cow_mapping() returns true.
*/
- is_cow = is_cow_mapping(src_vma->vm_flags);
+ is_cow = vma_is_cow_mapping(src_vma);
if (is_cow) {
mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
@@ -2381,7 +2443,7 @@ static bool vm_mixed_zeropage_allowed(struct vm_area_struct *vma)
if (mm_forbids_zeropage(vma->vm_mm))
return false;
/* zeropages in COW mappings are common and unproblematic. */
- if (is_cow_mapping(vma->vm_flags))
+ if (vma_is_cow_mapping(vma))
return true;
/* Mappings that do not allow for writable PTEs are unproblematic. */
if (!(vma->vm_flags & (VM_WRITE | VM_MAYWRITE)))
@@ -2832,7 +2894,7 @@ vm_fault_t vmf_insert_pfn_prot(struct vm_area_struct *vma, unsigned long addr,
BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
(VM_PFNMAP|VM_MIXEDMAP));
- BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
+ BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma));
BUG_ON((vma->vm_flags & VM_MIXEDMAP) && pfn_valid(pfn));
if (addr < vma->vm_start || addr >= vma->vm_end)
@@ -3244,7 +3306,7 @@ static int remap_pfn_range_prepare_vma(struct vm_area_struct *vma,
unsigned long size)
{
const unsigned long end = addr + PAGE_ALIGN(size);
- const bool is_cow = is_cow_mapping(vma->vm_flags);
+ const bool is_cow = vma_is_cow_mapping(vma);
int err;
err = get_remap_pgoff(is_cow, addr, end, vma->vm_start, vma->vm_end,
@@ -6744,7 +6806,7 @@ static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
* FAULT_FLAG_UNSHARE only applies to COW mappings. Let's
* just treat it like an ordinary read-fault otherwise.
*/
- if (!is_cow_mapping(vma->vm_flags))
+ if (!vma_is_cow_mapping(vma))
*flags &= ~FAULT_FLAG_UNSHARE;
} else if (*flags & FAULT_FLAG_WRITE) {
/* Write faults on read-only mappings are impossible ... */
@@ -6752,7 +6814,7 @@ static vm_fault_t sanitize_fault_flags(struct vm_area_struct *vma,
return VM_FAULT_SIGSEGV;
/* ... and FOLL_FORCE only applies to COW mappings. */
if (WARN_ON_ONCE(!(vma->vm_flags & VM_WRITE) &&
- !is_cow_mapping(vma->vm_flags)))
+ !vma_is_cow_mapping(vma)))
return VM_FAULT_SIGSEGV;
}
#ifdef CONFIG_PER_VMA_LOCK