From e34be29ecd47f5dcc27a90bf1563ae1da96555a7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:15 -0700 Subject: KVM: Move nVMX's lockdep logic for vcpu->mutex to a common helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract nVMX's lockdep assertion that a vCPU is locked or otherwise unreachable into a common helper, as KVM x86 is about to gain another user, but there is nothing x86-specific about the logic, i.e. the assertion may be useful for other architectures. No functional change intended. Reviewed-by: Philippe Mathieu-Daudé Link: https://patch.msgid.link/20260630225619.511632-9-seanjc@google.com Signed-off-by: Sean Christopherson --- include/linux/kvm_host.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include') diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index ab8cfaec82d3..b10814f99a50 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -989,6 +989,12 @@ static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx) lockdep_is_held(&kvm->slots_lock)); } +static inline void kvm_lockdep_assert_vcpu_is_locked_or_unreachable(struct kvm_vcpu *vcpu) +{ + lockdep_assert_once(lockdep_is_held(&vcpu->mutex) || + !refcount_read(&vcpu->kvm->users_count)); +} + static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) { int num_vcpus = atomic_read(&kvm->online_vcpus); -- cgit From c33aef581703cb41e2a4900c258018bc7aa1ff54 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:16 -0700 Subject: KVM: x86: Treat a vCPU as unreachable if its index is invalid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the "vCPU locked or unreachable" lockdep assertion, treat a vCPU as unreachable if its index is invalid, i.e. if the vCPU is in the process of being created. Until the vCPU is inserted into the array of vCPUs, the only way to get at the vCPU is via kvm_vm_ioctl_create_vcpu(). Note, the actual index is set _before_ adding the vCPU to the array, i.e. there's no risk of a false negative on the lockdep assertion. Reviewed-by: Philippe Mathieu-Daudé Link: https://patch.msgid.link/20260630225619.511632-10-seanjc@google.com Signed-off-by: Sean Christopherson --- include/linux/kvm_host.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index b10814f99a50..0bdfa3699352 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -992,6 +992,7 @@ static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx) static inline void kvm_lockdep_assert_vcpu_is_locked_or_unreachable(struct kvm_vcpu *vcpu) { lockdep_assert_once(lockdep_is_held(&vcpu->mutex) || + vcpu->vcpu_idx < 0 || !refcount_read(&vcpu->kvm->users_count)); } -- cgit From d1a3c216233413f57f5341a9b878b7e2dde7e785 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:40 -0700 Subject: KVM: SEV: Forcefully invalidate SNP VMSA if its backing gmem page is zapped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire up a gmem_invalidate_range() call for SNP VMs, and use it to force vCPUs to reload/recheck their guest-provided VMSA if the backing gmem page is being invalidated, e.g. is being PUNCH_HOLE'd. Use the same core logic to handle invalidations as VMX does for the APIC-access page, as the two concepts are nearly identical: shove the physical address of a page into the vCPU's control structure: 1. Snapshot the invalidation sequence counter 2. Grab the pfn (from guest_memfd in this case) 3. Acquire mmu_lock for read 4. Re-request reload if retry is needed, otherwise commit the change. Note, the re-request action in #4 is necessary as KVM's retry logic is fuzzy, i.e. can get false positives. If the guest_memfd page has been dropped, at some point a subsequent reload will fail to get a PFN from guest_memfd, and KVM will fail KVM_RUN. If the retry was due to a false positive, KVM will retry until there are no relevant MMU notifier events (and will retry in the "outer" loop, i.e. will drop locks and resched as needed). Note #2! Take care to invalidate the VMSA when a relevant memslot is DELETED or MOVED, as invalidations in response to PUNCH_HOLE are predicated on memslot bindings (KVM doesn't know what GFN range(s) to invalidate without a binding). And more importantly, the VMSA mapping requires a memslot, i.e. must be invalidated if its memslots disappears, regardless of the state of the underlying guest_memfd inode. Failure to invalidate the vCPU's control.vmsa_pa (which is checked by pre_sev_run()) can prevent KVM from properly freeing the page as firmware will reject the RMPUPDATE to reclaim the page with FAIL_INUSE if the vCPU is actively running, i.e. if VMSA page is in-use. That in turn leads to an RMP #PF on the next use, as the page will still be assigned to the SNP VM. SEV-SNP: RMPUPDATE failed for PFN 78d198, pg_level: 1, ret: 3 SEV-SNP: PFN 0x78d198, RMP entry: [0xfff0000000144001 - 0x000000000000000f] CPU: 3 UID: 0 PID: 31345 Comm: sev_snp_vmsa_pu Tainted: G U O Tainted: [U]=USER, [O]=OOT_MODULE Hardware name: Google, Inc. Arcadia_IT_80/Arcadia_IT_80, BIOS 34.86.0-102 01/25/2026 Call Trace: dump_stack_lvl+0x54/0x70 rmpupdate+0x12c/0x140 rmp_make_shared+0x3b/0x60 sev_gmem_invalidate+0xe0/0x170 [kvm_amd] delete_from_page_cache_batch+0x1d8/0x220 truncate_inode_pages_range+0x120/0x3d0 kvm_gmem_fallocate+0x19a/0x270 [kvm] vfs_fallocate+0x1bc/0x1f0 __x64_sys_fallocate+0x48/0x70 do_syscall_64+0x10a/0x480 entry_SYSCALL_64_after_hwframe+0x4b/0x53 RIP: 0033:0x496c7e ------------[ cut here ]------------ SEV: Failed to update RMP entry for PFN 0x78d198 error -14 WARNING: arch/x86/kvm/svm/sev.c:5160 at sev_gmem_invalidate+0x126/0x170 [kvm_amd], CPU#3: sev_snp_vmsa_pu/31345 CPU: 3 UID: 0 PID: 31345 Comm: sev_snp_vmsa_pu Tainted: G U O Tainted: [U]=USER, [O]=OOT_MODULE Hardware name: Google, Inc. Arcadia_IT_80/Arcadia_IT_80, BIOS 34.86.0-102 01/25/2026 RIP: 0010:sev_gmem_invalidate+0x12b/0x170 [kvm_amd] Call Trace: delete_from_page_cache_batch+0x1d8/0x220 truncate_inode_pages_range+0x120/0x3d0 kvm_gmem_fallocate+0x19a/0x270 [kvm] vfs_fallocate+0x1bc/0x1f0 __x64_sys_fallocate+0x48/0x70 do_syscall_64+0x10a/0x480 entry_SYSCALL_64_after_hwframe+0x4b/0x53 RIP: 0033:0x496c7e irq event stamp: 20689 hardirqs last enabled at (20699): [] __console_unlock+0x5c/0x60 hardirqs last disabled at (20708): [] __console_unlock+0x41/0x60 softirqs last enabled at (20722): [] __irq_exit_rcu+0x7e/0x140 softirqs last disabled at (20717): [] __irq_exit_rcu+0x7e/0x140 ---[ end trace 0000000000000000 ]--- BUG: unable to handle page fault for address: ffff99a64d198000 #PF: supervisor write access in kernel mode #PF: error_code(0x80000003) - RMP violation PGD 13eb001067 P4D 13eb001067 PUD 78d1d1063 PMD 1184e0063 PTE 800000078d198163 SEV-SNP: PFN 0x78d198, RMP entry: [0x6030000000144001 - 0x000000000000000f] Oops: Oops: 0003 [#1] SMP CPU: 3 UID: 0 PID: 31407 Comm: highlanderd_hea Tainted: G U W O Tainted: [U]=USER, [W]=WARN, [O]=OOT_MODULE Hardware name: Google, Inc. Arcadia_IT_80/Arcadia_IT_80, BIOS 34.86.0-102 01/25/2026 RIP: 0010:prep_new_page+0x67/0x220 Call Trace: get_page_from_freelist+0x1c40/0x1c70 __alloc_frozen_pages_noprof+0xca/0x1f0 alloc_pages_mpol+0x10b/0x1b0 alloc_pages_noprof+0x81/0x90 pte_alloc_one+0x1b/0xd0 do_pte_missing+0xdf/0x1020 handle_mm_fault+0x7c7/0xb20 do_user_addr_fault+0x268/0x6b0 exc_page_fault+0x67/0xa0 asm_exc_page_fault+0x26/0x30 RIP: 0033:0x4a6b1e gsmi: Log Shutdown Reason 0x03 CR2: ffff99a64d198000 ---[ end trace 0000000000000000 ]--- RIP: 0010:prep_new_page+0x67/0x220 Drop the pseudo-TODO comment about needing to pin the page if guest_memfd every supports migration, as integrating with invalidations events means KVM will Just Work if/when page migration is ever supported (assuming SNP hardware supports migrating VMSA pages). Note #3, invalidate() and invalidate_range() have _completely_ different semantics; the new invalidate_range() is a true invalidation, whereas the existing invalidate() is really a "make shared" operation. Ignore the confusing naming and poor Kconfig bundling for the moment to minimize the delta for LTS kernels, the mess will be cleaned up shortly. Reported-by: Hyunwoo Kim Closes: https://lore.kernel.org/all/aimMWzAf5b3luM0b@v4bel Fixes: e366f92ea99e ("KVM: SEV: Support SEV-SNP AP Creation NAE event") Cc: stable@vger.kernel.org Cc: Tom Lendacky Cc: Michael Roth Cc: Jörg Rödel Cc: Fuad Tabba Cc: Ackerley Tng Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-11-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 4 ++ arch/x86/include/asm/kvm_host.h | 6 +++ arch/x86/kvm/mmu/mmu.c | 5 +++ arch/x86/kvm/svm/sev.c | 80 ++++++++++++++++++++++++++++++++------ arch/x86/kvm/svm/svm.c | 2 + arch/x86/kvm/svm/svm.h | 2 + arch/x86/kvm/x86.c | 6 +++ include/linux/kvm_host.h | 1 + virt/kvm/guest_memfd.c | 4 ++ 9 files changed, 99 insertions(+), 11 deletions(-) (limited to 'include') diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 83dc5086138b..ccf23b3f0e1c 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -134,6 +134,7 @@ KVM_X86_OP_OPTIONAL(mem_enc_unregister_region) KVM_X86_OP_OPTIONAL(vm_copy_enc_context_from) KVM_X86_OP_OPTIONAL(vm_move_enc_context_from) KVM_X86_OP_OPTIONAL(guest_memory_reclaimed) +KVM_X86_OP_OPTIONAL(reload_vmsa) KVM_X86_OP(get_feature_msr) KVM_X86_OP(check_emulate_instruction) KVM_X86_OP(apic_init_signal_blocked) @@ -148,6 +149,9 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level) KVM_X86_OP_OPTIONAL(gmem_invalidate) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE +KVM_X86_OP_OPTIONAL(gmem_invalidate_range) +#endif #endif #undef KVM_X86_OP diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index b517257a6315..1c598b40e0c3 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -122,6 +122,8 @@ KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_HV_TLB_FLUSH \ KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) +#define KVM_REQ_VMSA_PAGE_RELOAD \ + KVM_ARCH_REQ_FLAGS(33, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \ KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT) @@ -1878,6 +1880,7 @@ struct kvm_x86_ops { int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd); int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd); void (*guest_memory_reclaimed)(struct kvm *kvm); + void (*reload_vmsa)(struct kvm_vcpu *vcpu); int (*get_feature_msr)(u32 msr, u64 *data); @@ -1902,6 +1905,9 @@ struct kvm_x86_ops { void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE + void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); +#endif int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); }; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 3eb1f86593b1..e2978e9a1731 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -7589,6 +7589,11 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, write_lock(&kvm->mmu_lock); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE + if (slot->gmem.file) + kvm_arch_gmem_invalidate_range(kvm, &range); +#endif + if (zap_all) { __kvm_mmu_zap_all_fast_front_half(kvm); } else { diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 30792adcfc8e..62c6126d52c0 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -3979,19 +3979,25 @@ static int snp_begin_psc(struct vcpu_svm *svm) return snp_do_psc(svm); } -static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) +static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_memory_slot *slot; + struct kvm *kvm = vcpu->kvm; gfn_t gfn = gpa_to_gfn(gpa); + unsigned long mmu_seq; struct page *page; kvm_pfn_t pfn; lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex); - /* Clear use of the VMSA. */ + /* + * Clear use of the VMSA. Ensure snp_guest_vmsa_gpa is written exactly + * once, as it is read locklessly when responding to gfn invalidations. + * Pairs with the READ_ONCE() in sev_gmem_invalidate_range(). + */ svm->vmcb->control.vmsa_pa = INVALID_PAGE; - svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; + WRITE_ONCE(svm->sev_es.snp_guest_vmsa_gpa, INVALID_PAGE); /* * When replacing the VMSA during SEV-SNP AP creation, @@ -4006,6 +4012,9 @@ static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) if (!slot) return; + mmu_seq = kvm->mmu_invalidate_seq; + smp_rmb(); + /* * The new VMSA will be private memory guest memory, so retrieve the * PFN from the gmem backend. @@ -4024,15 +4033,20 @@ static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) */ svm->sev_es.snp_has_guest_vmsa = true; - /* Use the new VMSA */ - svm->sev_es.snp_guest_vmsa_gpa = gpa; - svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); - + read_lock(&kvm->mmu_lock); /* - * gmem pages aren't currently migratable, but if this ever changes - * then care should be taken to ensure svm->sev_es.vmsa is pinned - * through some other means. + * Save the guest-provided GPA. If retry is needed, then KVM will try + * again with the same GPA. If the VMSA is usable, then KVM needs to + * track the GPA so that the VMSA can be reloaded if the backing page + * for the GPA is invalidated. */ + svm->sev_es.snp_guest_vmsa_gpa = gpa; + if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) + kvm_make_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu); + else + svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); + read_unlock(&kvm->mmu_lock); + kvm_release_page_clean(page); } @@ -4058,7 +4072,7 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) gpa = svm->sev_es.snp_pending_vmsa_gpa; svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; - sev_snp_reload_vmsa(vcpu, gpa); + __sev_snp_reload_vmsa(vcpu, gpa); /* * Mark the vCPU as runnable for CREATE requests, indicated by a valid @@ -4070,6 +4084,15 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); } +void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu) +{ + struct vcpu_sev_es_state *sev_es = &to_svm(vcpu)->sev_es; + + guard(mutex)(&sev_es->snp_vmsa_mutex); + + __sev_snp_reload_vmsa(vcpu, sev_es->snp_guest_vmsa_gpa); +} + static int sev_snp_ap_creation(struct vcpu_svm *svm) { struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm); @@ -5199,6 +5222,41 @@ next_pfn: } } +void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + struct kvm_vcpu *vcpu; + unsigned long i; + + lockdep_assert_held_write(&kvm->mmu_lock); + + /* + * An unstable result for "is SNP" is a-ok here, thanks to mmu_lock. + * The vCPU's VMSA GPA is invalidated before the vCPU is made visible + * to other tasks, and can only become valid while holding mmu_lock, + * after the VM is fully committed to being an SNP VM. + */ + if (!____sev_snp_guest(kvm)) + return; + + kvm_for_each_vcpu(i, vcpu, kvm) { + /* + * Read snp_guest_vmsa_gpa without taking the vCPU's VMSA mutex + * (or its generic mutex) as mmu_lock is held, i.e. this task + * can't sleep. The VMSA is invalidated outside of mmu_lock, + * but can only become valid inside of mmu_lock, i.e. the below + * can get false positives, but not false negatives. A false + * positive is benign, as a spurious request simply forces the + * vCPU to re-establish its VMSA. + */ + gpa_t gpa = READ_ONCE(to_svm(vcpu)->sev_es.snp_guest_vmsa_gpa); + + if (VALID_PAGE(gpa) && + gpa_to_gfn(gpa) >= range->start && + gpa_to_gfn(gpa) < range->end) + kvm_make_request_and_kick(KVM_REQ_VMSA_PAGE_RELOAD, vcpu); + } +} + int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) { int level, rc; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 79c818d91dda..dd51df74c2dc 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5445,12 +5445,14 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .mem_enc_register_region = sev_mem_enc_register_region, .mem_enc_unregister_region = sev_mem_enc_unregister_region, .guest_memory_reclaimed = sev_guest_memory_reclaimed, + .reload_vmsa = sev_snp_reload_vmsa, .vm_copy_enc_context_from = sev_vm_copy_enc_context_from, .vm_move_enc_context_from = sev_vm_move_enc_context_from, .gmem_prepare = sev_gmem_prepare, .gmem_invalidate = sev_gmem_invalidate, + .gmem_invalidate_range = sev_gmem_invalidate_range, .gmem_max_mapping_level = sev_gmem_max_mapping_level, #endif .check_emulate_instruction = svm_check_emulate_instruction, diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index effca6372e15..130205defffa 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -996,6 +996,7 @@ static inline struct page *snp_safe_alloc_page(void) { return snp_safe_alloc_page_node(numa_node_id(), GFP_KERNEL_ACCOUNT); } +void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu); int sev_vcpu_create(struct kvm_vcpu *vcpu); void sev_free_vcpu(struct kvm_vcpu *vcpu); @@ -1010,6 +1011,7 @@ extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu); void sev_free_decrypted_vmsa(struct kvm_vcpu *vcpu, struct vmcb_save_area *vmsa); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 8abd733d5173..5a5fd6211d23 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8170,6 +8170,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) goto out; } } + if (kvm_check_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu)) + kvm_x86_call(reload_vmsa)(vcpu); } if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win || @@ -10599,6 +10601,10 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) { kvm_x86_call(gmem_invalidate)(start, end); } +void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + kvm_x86_call(gmem_invalidate_range)(kvm, range); +} #endif #endif diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index ab8cfaec82d3..c00fc1740ce5 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2608,6 +2608,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); #endif #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 86690683b2fe..659b8dbe0b30 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -185,6 +185,10 @@ static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start, } flush |= kvm_mmu_unmap_gfn_range(kvm, &gfn_range); + +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE + kvm_arch_gmem_invalidate_range(kvm, &gfn_range); +#endif } if (flush) -- cgit From 36d32222fcab099cf07b032aa5f022136bfa91be Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:54 +0100 Subject: KVM: arm64: vgic-v3: Kill kvm_vgic_global_state.ich_vtr_el2 kvm_vgic_global_state.ich_vtr_el2 is the last bit of caching that we can get rid of. Not as bad as a sysreg access, but still worse than a constant. Move over to the inlined stuff and remove the cached value. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-7-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 3 ++- arch/arm64/kvm/vgic-sys-reg-v3.c | 8 ++++---- arch/arm64/kvm/vgic/vgic-v3.c | 5 ++--- arch/arm64/kvm/vgic/vgic-v5.c | 5 +---- arch/arm64/kvm/vgic/vgic.h | 2 +- include/kvm/arm_vgic.h | 2 -- 6 files changed, 10 insertions(+), 15 deletions(-) (limited to 'include') diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index fb54f6dad995..54defcd8240d 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -16,6 +16,7 @@ #include #include "sys_regs.h" +#include "vgic/vgic.h" struct vncr_tlb { /* The guest's VNCR_EL2 */ @@ -1904,7 +1905,7 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) /* ICH_HCR_EL2 */ resx.res0 = ICH_HCR_EL2_RES0; resx.res1 = ICH_HCR_EL2_RES1; - if (!(kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_EL2_TDS)) + if (!(vgic_ich_vtr() & ICH_VTR_EL2_TDS)) resx.res0 |= ICH_HCR_EL2_TDIR; /* No GICv4 is presented to the guest */ resx.res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount; diff --git a/arch/arm64/kvm/vgic-sys-reg-v3.c b/arch/arm64/kvm/vgic-sys-reg-v3.c index bdc2d57370b2..89315336b9d3 100644 --- a/arch/arm64/kvm/vgic-sys-reg-v3.c +++ b/arch/arm64/kvm/vgic-sys-reg-v3.c @@ -35,12 +35,12 @@ static int set_gic_ctlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, vgic_v3_cpu->num_id_bits = host_id_bits; - host_seis = FIELD_GET(ICH_VTR_EL2_SEIS, kvm_vgic_global_state.ich_vtr_el2); + host_seis = FIELD_GET(ICH_VTR_EL2_SEIS, vgic_ich_vtr()); seis = FIELD_GET(ICC_CTLR_EL1_SEIS_MASK, val); if (host_seis != seis) return -EINVAL; - host_a3v = FIELD_GET(ICH_VTR_EL2_A3V, kvm_vgic_global_state.ich_vtr_el2); + host_a3v = FIELD_GET(ICH_VTR_EL2_A3V, vgic_ich_vtr()); a3v = FIELD_GET(ICC_CTLR_EL1_A3V_MASK, val); if (host_a3v != a3v) return -EINVAL; @@ -69,9 +69,9 @@ static int get_gic_ctlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, val |= FIELD_PREP(ICC_CTLR_EL1_ID_BITS_MASK, vgic_v3_cpu->num_id_bits); val |= FIELD_PREP(ICC_CTLR_EL1_SEIS_MASK, FIELD_GET(ICH_VTR_EL2_SEIS, - kvm_vgic_global_state.ich_vtr_el2)); + vgic_ich_vtr())); val |= FIELD_PREP(ICC_CTLR_EL1_A3V_MASK, - FIELD_GET(ICH_VTR_EL2_A3V, kvm_vgic_global_state.ich_vtr_el2)); + FIELD_GET(ICH_VTR_EL2_A3V, vgic_ich_vtr())); /* * The VMCR.CTLR value is in ICC_CTLR_EL1 layout. * Extract it directly using ICC_CTLR_EL1 reg definitions. diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 734fa3138723..91514d6faf74 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -490,9 +490,9 @@ void vgic_v3_reset(struct kvm_vcpu *vcpu) } vcpu->arch.vgic_cpu.num_id_bits = FIELD_GET(ICH_VTR_EL2_IDbits, - kvm_vgic_global_state.ich_vtr_el2); + vgic_ich_vtr()); vcpu->arch.vgic_cpu.num_pri_bits = FIELD_GET(ICH_VTR_EL2_PRIbits, - kvm_vgic_global_state.ich_vtr_el2) + 1; + vgic_ich_vtr()) + 1; } void vcpu_set_ich_hcr(struct kvm_vcpu *vcpu) @@ -949,7 +949,6 @@ int vgic_v3_probe(const struct gic_kvm_info *info) */ kvm_vgic_global_state.nr_lr = (ich_vtr_el2 & 0xf) + 1; kvm_vgic_global_state.can_emulate_gicv2 = false; - kvm_vgic_global_state.ich_vtr_el2 = ich_vtr_el2; /* GICv4 support? */ if (info->has_v4) { diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 16bc0a670d3e..4e3e4db24e77 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -40,7 +40,6 @@ static void vgic_v5_get_implemented_ppis(void) int vgic_v5_probe(const struct gic_kvm_info *info) { bool v5_registered = false; - u64 ich_vtr_el2; int ret; kvm_vgic_global_state.type = VGIC_V5; @@ -83,14 +82,12 @@ skip_v5: } kvm_vgic_global_state.has_gcie_v3_compat = true; - ich_vtr_el2 = vgic_ich_vtr(); - kvm_vgic_global_state.ich_vtr_el2 = (u32)ich_vtr_el2; /* * The ListRegs field is 5 bits, but there is an architectural * maximum of 16 list registers. Just ignore bit 4... */ - kvm_vgic_global_state.nr_lr = (ich_vtr_el2 & 0xf) + 1; + kvm_vgic_global_state.nr_lr = (vgic_ich_vtr() & 0xf) + 1; ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3); if (ret) { diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index b76bb6e56de3..b71d486ae514 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -92,7 +92,7 @@ static inline u64 kvm_get_guest_vtr_el2(void) { u64 vtr; - vtr = kvm_vgic_global_state.ich_vtr_el2; + vtr = vgic_ich_vtr(); vtr &= ~KVM_ICH_VTR_EL2_RES0; vtr |= KVM_ICH_VTR_EL2_RES1; diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index fe49fb56dc3c..bd1bb03500b3 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -176,8 +176,6 @@ struct vgic_global { /* GICv3 compat mode on a GICv5 host */ bool has_gcie_v3_compat; - u32 ich_vtr_el2; - /* GICv5 PPI capabilities */ struct { DECLARE_BITMAP(impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); -- cgit From 0f0893ac5af3ac9d18440955f302bd20a35ccab2 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:03 -0700 Subject: KVM: guest_memfd: Pass the number of pages instead of the end pfn into .invalidate() Pass the number of pages to "invalidate", i.e. reclaim, instead of the end pfn, as a first step towards aligning the function prototypes between the de facto "to private" and "to shared" arch hooks. Eventually, the goal is to end up with kvm_gmem_arch_make_{private,shared}(), and in both cases, providing the number of pages makes the call sites slightly nicer, and also avoids any confusion over whether the end pfn is inclusive or exclusive. Opportunistically rename "start" to "pfn", again to align with the expected signature of make_private() (which needs to pass a starting gfn as well, at which point the "start" becomes noise). No functional change intended. Cc: Fuad Tabba Cc: Ackerley Tng Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723210811.72720-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/svm/sev.c | 8 ++++---- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/x86.c | 4 ++-- include/linux/kvm_host.h | 2 +- virt/kvm/guest_memfd.c | 6 +----- 6 files changed, 10 insertions(+), 14 deletions(-) (limited to 'include') diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 1f8ef0e1566a..ef20f0e12193 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1906,7 +1906,7 @@ struct kvm_x86_ops { #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); #endif - void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end); + void (*gmem_invalidate)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); #endif diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index a7584b7ed6dc..5a5e8342c4fe 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5159,16 +5159,16 @@ int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) return 0; } -void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) +void sev_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { - kvm_pfn_t pfn; + kvm_pfn_t end = pfn + nr_pages; if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) return; - pr_debug("%s: PFN start 0x%llx PFN end 0x%llx\n", __func__, start, end); + pr_debug("%s: PFN start 0x%llx PFN end 0x%llx\n", __func__, pfn, end); - for (pfn = start; pfn < end;) { + while (pfn < end) { bool use_2m_update = false; int rc, rmp_level; bool assigned; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 130205defffa..2180d03bb0a6 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1010,7 +1010,7 @@ int sev_dev_get_attr(u32 group, u64 attr, u64 *val); extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); -void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void sev_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 5a5fd6211d23..3519fca53cfc 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10597,9 +10597,9 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) +void kvm_arch_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { - kvm_x86_call(gmem_invalidate)(start, end); + kvm_x86_call(gmem_invalidate)(pfn, nr_pages); } void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) { diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index c00fc1740ce5..79868ebfc113 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2607,7 +2607,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void kvm_arch_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); #endif diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 659b8dbe0b30..c96a2ef5f2ce 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -530,11 +530,7 @@ static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *fol #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE static void kvm_gmem_free_folio(struct folio *folio) { - struct page *page = folio_page(folio, 0); - kvm_pfn_t pfn = page_to_pfn(page); - int order = folio_order(folio); - - kvm_arch_gmem_invalidate(pfn, pfn + (1ul << order)); + kvm_arch_gmem_invalidate(folio_file_pfn(folio, 0), folio_nr_pages(folio)); } #endif -- cgit From 7b8529e70a11ed110a15568f6b391a5f32eef67f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:04 -0700 Subject: KVM: guest_memfd: Rename invalidate() arch hook to reclaim() and isolate it Rename guest_memfd's invalidate() hook to reclaim() and isolate it via its own RECLAIM Kconfig, as the hook is called when a folio is freed, which is far too late and lacks sufficient information for KVM to actually invalidate its usage of the memory. E.g. SNP uses the hook to convert memory back to SHARED so that it can be safely accessed by the host, there is no invalidation of guest mappings anywhere. Isolating the hook will also allow pKVM on arm64 to opt-in to reclaim() without also having to differentiate between reclaim and conversions to shared for active VMs. Keep guest_memfd's trampoline, even though it would be trivial to wire up .free_folio() directly to an arch callback, to avoid bleeding guest_memfd internals into arch code (specifically, avoid referencing folios in arch code). Leave the kvm_x86_ops hook as-is for the moment, as "reclaim" on SNP is the same as convert-to-shared, i.e. using a different name for the x86 hook will allow reusing it for in-place conversion. Reviewed-by: Xiaoyao Li Reviewed-by: Fuad Tabba Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260723210811.72720-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/Kconfig | 1 + arch/x86/kvm/x86.c | 7 +++++-- include/linux/kvm_host.h | 5 ++++- virt/kvm/Kconfig | 4 ++++ virt/kvm/guest_memfd.c | 6 +++--- 5 files changed, 17 insertions(+), 6 deletions(-) (limited to 'include') diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 801bf9e520db..e0e7ad015839 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -161,6 +161,7 @@ config KVM_AMD_SEV select ARCH_HAS_CC_PLATFORM select KVM_GENERIC_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_PREPARE + select HAVE_KVM_ARCH_GMEM_RECLAIM select HAVE_KVM_ARCH_GMEM_INVALIDATE select HAVE_KVM_ARCH_GMEM_POPULATE help diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 3519fca53cfc..befba27672c7 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10596,11 +10596,14 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord } #endif -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { kvm_x86_call(gmem_invalidate)(pfn, nr_pages); } +#endif + +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) { kvm_x86_call(gmem_invalidate_range)(kvm, range); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 79868ebfc113..46c8d18fd043 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2606,8 +2606,11 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, kvm_gmem_populate_cb post_populate, void *opaque); #endif +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#endif + #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); #endif diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 794976b88c6f..617876993225 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -111,6 +111,10 @@ config HAVE_KVM_ARCH_GMEM_PREPARE bool depends on KVM_GUEST_MEMFD +config HAVE_KVM_ARCH_GMEM_RECLAIM + bool + depends on KVM_GUEST_MEMFD + config HAVE_KVM_ARCH_GMEM_INVALIDATE bool depends on KVM_GUEST_MEMFD diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index c96a2ef5f2ce..c086cfcd0257 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -527,10 +527,10 @@ static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *fol return MF_DELAYED; } -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM static void kvm_gmem_free_folio(struct folio *folio) { - kvm_arch_gmem_invalidate(folio_file_pfn(folio, 0), folio_nr_pages(folio)); + kvm_arch_gmem_reclaim(folio_file_pfn(folio, 0), folio_nr_pages(folio)); } #endif @@ -538,7 +538,7 @@ static const struct address_space_operations kvm_gmem_aops = { .dirty_folio = noop_dirty_folio, .migrate_folio = kvm_gmem_migrate_folio, .error_remove_folio = kvm_gmem_error_folio, -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM .free_folio = kvm_gmem_free_folio, #endif }; -- cgit From 2131c4f763d2e4cbbe5c227a406488c38e23f806 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:09 -0700 Subject: KVM: guest_memfd: Rename prepare() hook and Kconfig to make_private() / CONVERT Rework guest_memfd's prepare() hook into a more accurate make_private(), and rework its Kconfig from PREPARE to a more generic CONVERT. This will allow x86 to share (pun intended) a kvm_x86_ops.gmem_make_shared() hook between the "convert to shared" and "reclaim" flows, which are one and the same for SNP. No functional change intended. Reviewed-by: Ackerley Tng Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723210811.72720-8-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 4 ++-- arch/x86/include/asm/kvm_host.h | 4 ++-- arch/x86/kvm/Kconfig | 2 +- arch/x86/kvm/svm/sev.c | 2 +- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/x86.c | 6 +++--- include/linux/kvm_host.h | 5 +++-- virt/kvm/Kconfig | 2 +- virt/kvm/guest_memfd.c | 4 ++-- 10 files changed, 17 insertions(+), 16 deletions(-) (limited to 'include') diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 210cb95d0a0b..a4d872ddef9d 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -146,8 +146,8 @@ KVM_X86_OP(vcpu_deliver_sipi_vector) KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons); KVM_X86_OP_OPTIONAL(get_untagged_addr) KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE -KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +KVM_X86_OP_OPTIONAL_RET0(gmem_make_private) #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM KVM_X86_OP_OPTIONAL(gmem_make_shared) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 693be16abd95..ae9a229c6b11 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1903,8 +1903,8 @@ struct kvm_x86_ops { gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE - int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT + int (*gmem_make_private)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index e0e7ad015839..538ed1e80332 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -160,7 +160,7 @@ config KVM_AMD_SEV depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m) select ARCH_HAS_CC_PLATFORM select KVM_GENERIC_MEMORY_ATTRIBUTES - select HAVE_KVM_ARCH_GMEM_PREPARE + select HAVE_KVM_ARCH_GMEM_CONVERT select HAVE_KVM_ARCH_GMEM_RECLAIM select HAVE_KVM_ARCH_GMEM_INVALIDATE select HAVE_KVM_ARCH_GMEM_POPULATE diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 4c46081889a6..779889f5e994 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5112,7 +5112,7 @@ static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order) return false; } -int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) +int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) { struct kvm_sev_info *sev = to_kvm_sev_info(kvm); kvm_pfn_t pfn_aligned; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 44c3af6f71d8..d9f632ea8057 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5450,7 +5450,7 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vm_copy_enc_context_from = sev_vm_copy_enc_context_from, .vm_move_enc_context_from = sev_vm_move_enc_context_from, - .gmem_prepare = sev_gmem_prepare, + .gmem_make_private = sev_gmem_make_private, .gmem_make_shared = sev_gmem_make_shared, .gmem_invalidate_range = sev_gmem_invalidate_range, .gmem_max_mapping_level = sev_gmem_max_mapping_level, diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 51e3494e7802..b5cd8437988f 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1009,7 +1009,7 @@ int sev_cpu_init(struct svm_cpu_data *sd); int sev_dev_get_attr(u32 group, u64 attr, u64 *val); extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); -int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); +int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0075815acdd5..512bcc35507a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10589,10 +10589,10 @@ bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm) return !kvm_arch_has_private_mem(kvm); } -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE -int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order) { - return kvm_x86_call(gmem_prepare)(kvm, pfn, gfn, max_order); + return kvm_x86_call(gmem_make_private)(kvm, pfn, gfn, max_order); } #endif diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 46c8d18fd043..ecdda1f00c6c 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2572,8 +2572,9 @@ static inline int kvm_gmem_get_pfn(struct kvm *kvm, } #endif /* CONFIG_KVM_GUEST_MEMFD */ -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE -int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + int max_order); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 617876993225..c3c0ee253fc7 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -107,7 +107,7 @@ config KVM_GUEST_MEMFD select XARRAY_MULTI bool -config HAVE_KVM_ARCH_GMEM_PREPARE +config HAVE_KVM_ARCH_GMEM_CONVERT bool depends on KVM_GUEST_MEMFD diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 25da7778af01..6635ed05f411 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -73,11 +73,11 @@ static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, pgoff_t index, struct folio *folio) { -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT kvm_pfn_t pfn = folio_file_pfn(folio, index); gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff; - return kvm_arch_gmem_prepare(kvm, gfn, pfn, folio_order(folio)); + return kvm_arch_gmem_make_private(kvm, gfn, pfn, folio_order(folio)); #else return 0; #endif -- cgit From fb50ca77b672fd359453728fc59730105f2bf7eb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:10 -0700 Subject: KVM: guest_memfd: Explicitly pass number of pages to make_private() hook Tweak the guest_memfd make_private() hook to explicitly pass the number of pages to align with the signature of the make_shared() hook, and because the existing code is outright broken if a guest_memfd folio is comprised of more than one page (which can't happen, yet). The SNP code *tries* to create a corresponding huge entry, but if the RMP must use 4KiB entries for whatever reason, KVM will only convert the first pfn, and not the entire range of pfns that will be mapped into the guest. Alternatively, @max_order could simply be repurposed as _the_ @order, but that will fall apart when in-place conversion comes along, at which point KVM will need to deal with conversions that aren't bound 1:1 to a folio. I.e. the number of pages to convert may not be exactly a power-of-2 (and folios don't strictly guarantee power-of-2 pages anyways). WARN in the SNP code if the number of pages to prepare is anything other than '1', i.e. if guest_memfd is trying to prepare/convert more than a single 4KiB page, as sev_gmem_prepare() doesn't actually handle conversion greater than order-0 folios. Opportunistically swap the ordering of @pfn and @gfn params for kvm_x86_ops.gmem_make_private() to match kvm_arch_gmem_make_private(). Fixes: b85524314a3d ("KVM: guest_memfd: delay kvm_gmem_prepare_folio() until the memory is passed to the guest") Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260723210811.72720-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 3 ++- arch/x86/kvm/svm/sev.c | 27 +++++++++++---------------- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/x86.c | 5 +++-- include/linux/kvm_host.h | 2 +- virt/kvm/guest_memfd.c | 2 +- 6 files changed, 19 insertions(+), 22 deletions(-) (limited to 'include') diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index ae9a229c6b11..7643e078ba36 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1904,7 +1904,8 @@ struct kvm_x86_ops { gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT - int (*gmem_make_private)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); + int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + kvm_pfn_t nr_pages); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 779889f5e994..5dd45b7a09f2 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5088,15 +5088,7 @@ static bool is_pfn_range_shared(kvm_pfn_t start, kvm_pfn_t end) return true; } -static u8 max_level_for_order(int order) -{ - if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M)) - return PG_LEVEL_2M; - - return PG_LEVEL_4K; -} - -static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order) +static bool is_large_rmp_possible(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { kvm_pfn_t pfn_aligned = ALIGN_DOWN(pfn, PTRS_PER_PMD); @@ -5105,14 +5097,14 @@ static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order) * PFN is currently shared, then the entire 2M-aligned range can be * set to private via a single 2M RMP entry. */ - if (max_level_for_order(order) > PG_LEVEL_4K && + if (nr_pages >= KVM_PAGES_PER_HPAGE(PG_LEVEL_2M) && is_pfn_range_shared(pfn_aligned, pfn_aligned + PTRS_PER_PMD)) return true; return false; } -int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) +int sev_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages) { struct kvm_sev_info *sev = to_kvm_sev_info(kvm); kvm_pfn_t pfn_aligned; @@ -5123,6 +5115,9 @@ int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_ord if (!sev_snp_guest(kvm)) return 0; + if (WARN_ON_ONCE(nr_pages != 1)) + return -EIO; + rc = snp_lookup_rmpentry(pfn, &assigned, &level); if (rc) { pr_err_ratelimited("SEV: Failed to look up RMP entry: GFN %llx PFN %llx error %d\n", @@ -5131,12 +5126,12 @@ int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_ord } if (assigned) { - pr_debug("%s: already assigned: gfn %llx pfn %llx max_order %d level %d\n", - __func__, gfn, pfn, max_order, level); + pr_debug("%s: already assigned: gfn %llx pfn %llx nr_pages %llx level %d\n", + __func__, gfn, pfn, nr_pages, level); return 0; } - if (is_large_rmp_possible(kvm, pfn, max_order)) { + if (is_large_rmp_possible(pfn, nr_pages)) { level = PG_LEVEL_2M; pfn_aligned = ALIGN_DOWN(pfn, PTRS_PER_PMD); gfn_aligned = ALIGN_DOWN(gfn, PTRS_PER_PMD); @@ -5153,8 +5148,8 @@ int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_ord return -EINVAL; } - pr_debug("%s: updated: gfn %llx pfn %llx pfn_aligned %llx max_order %d level %d\n", - __func__, gfn, pfn, pfn_aligned, max_order, level); + pr_debug("%s: updated: gfn %llx pfn %llx pfn_aligned %llx nr_pages %llx level %d\n", + __func__, gfn, pfn, pfn_aligned, nr_pages, level); return 0; } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index b5cd8437988f..da4c66eb8d70 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1009,7 +1009,7 @@ int sev_cpu_init(struct svm_cpu_data *sd); int sev_dev_get_attr(u32 group, u64 attr, u64 *val); extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); -int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); +int sev_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 512bcc35507a..9390e0d4c1e5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10590,9 +10590,10 @@ bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm) } #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT -int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order) +int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + kvm_pfn_t nr_pages) { - return kvm_x86_call(gmem_make_private)(kvm, pfn, gfn, max_order); + return kvm_x86_call(gmem_make_private)(kvm, gfn, pfn, nr_pages); } #endif diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index ecdda1f00c6c..b24a090eb34d 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2574,7 +2574,7 @@ static inline int kvm_gmem_get_pfn(struct kvm *kvm, #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, - int max_order); + kvm_pfn_t nr_pages); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 6635ed05f411..0c1ee3e49176 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -77,7 +77,7 @@ static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slo kvm_pfn_t pfn = folio_file_pfn(folio, index); gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff; - return kvm_arch_gmem_make_private(kvm, gfn, pfn, folio_order(folio)); + return kvm_arch_gmem_make_private(kvm, gfn, pfn, folio_nr_pages(folio)); #else return 0; #endif -- cgit From d196e4d91e7845b9e65436574bff63234a6f4d1d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 17:47:53 -0700 Subject: KVM: nSVM: Add CLASS()es for automagically handling local kvm_vcpu_map() usage Add CLASS() definitions for locally mapping a PFN using kvm_vcpu_map() given a vCPU+gfn pair. In addition to eliminating the need to manually do unmap(), e.g. in error paths, this will allow hardening KVM against double-mapping without having to manually ensure every on-stack declaration is zero-initialized. Use "map local" as the primary terminology as the basic concept is more or less the same as kmap_local(): ensure the current context has a kernel mapping to the underlying memory. Immediately convert the relatively straightforward nested SVM flows, and defer converting the more involved SMM flows to a separate change. No functional change intended. Cc: Yosry Ahmed Link: https://patch.msgid.link/20260724004757.131420-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 18 +++++++----------- arch/x86/kvm/svm/svm.c | 8 +++----- include/linux/kvm_host.h | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 16 deletions(-) (limited to 'include') diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 5f6d9971a3f2..4deb8c75df3d 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -1086,14 +1086,14 @@ int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, bool from_vmrun) static int nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu *vcpu, u64 vmcb12_gpa) { struct vcpu_svm *svm = to_svm(vcpu); - struct kvm_host_map map; struct vmcb *vmcb12; int r = 0; - if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map)) + CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(vmcb12_gpa)); + if (m.ret) return -EFAULT; - vmcb12 = map.hva; + vmcb12 = m.map.hva; nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); @@ -1107,7 +1107,6 @@ static int nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu *vcpu, u64 vmcb12_gpa r = -EINVAL; } - kvm_vcpu_unmap(vcpu, &map); return r; } @@ -1251,15 +1250,13 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; - struct kvm_host_map map; struct vmcb *vmcb12; - int rc; - rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map); - if (rc) - return rc; + CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa)); + if (m.ret) + return m.ret; - vmcb12 = map.hva; + vmcb12 = m.map.hva; vmcb12->save.es = vmcb02->save.es; vmcb12->save.cs = vmcb02->save.cs; @@ -1314,7 +1311,6 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) vmcb12->control.exit_int_info_err, KVM_ISA_SVM); - kvm_vcpu_unmap(vcpu, &map); return 0; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 1b7e613fde7b..10401330a6b1 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2221,7 +2221,6 @@ static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) u64 vmcb12_gpa = kvm_rax_read(vcpu); struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb12; - struct kvm_host_map map; int ret; if (nested_svm_check_permissions(vcpu)) @@ -2232,10 +2231,11 @@ static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) return 1; } - if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map)) + CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(vmcb12_gpa)); + if (m.ret) return kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL); - vmcb12 = map.hva; + vmcb12 = m.map.hva; ret = kvm_skip_emulated_instruction(vcpu); @@ -2248,8 +2248,6 @@ static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) svm_copy_vmloadsave_state(vmcb12, svm->vmcb01.ptr); } - kvm_vcpu_unmap(vcpu, &map); - return ret; } diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 0bdfa3699352..ef851de0392b 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1420,6 +1420,25 @@ static inline void kvm_vcpu_map_mark_dirty(struct kvm_vcpu *vcpu, kvm_vcpu_mark_page_dirty(vcpu, map->gfn); } +typedef struct { + struct kvm_vcpu *vcpu; + struct kvm_host_map map; + int ret; +} kvm_vcpu_local_map_t; + +#define DEFINE_VCPU_MAP_CLASS(ro) \ +DEFINE_CLASS(kvm_vcpu_map_local##ro, kvm_vcpu_local_map_t, \ + if (!_T.ret) kvm_vcpu_unmap(_T.vcpu, &_T.map), \ + ({ \ + kvm_vcpu_local_map_t m = { .vcpu = vcpu }; \ + \ + m.ret = kvm_vcpu_map##ro(vcpu, gfn, &m.map); \ + \ + m; \ + }), struct kvm_vcpu *vcpu, gfn_t gfn); +DEFINE_VCPU_MAP_CLASS(); +DEFINE_VCPU_MAP_CLASS(_readonly); + unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn); unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable); int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset, -- cgit From 453f6ff04b8d28991f94ccac89350951252aa947 Mon Sep 17 00:00:00 2001 From: Congkai Tan Date: Wed, 22 Jul 2026 20:26:59 +0000 Subject: KVM: arm64: Expose PMMIR_EL1.SLOTS under strict PMUv3 UAPI Introduce a new field pmmir_slots in struct kvm_arch to store PMMIR_EL1.SLOTS. It only saves the actual hardware PMU value when the VMM explicitly selects a PMU under KVM_ARM_VCPU_PMU_V3_STRICT. Otherwise, it stays 0 after allocation. Use this field to implement guest access, userspace get, and userspace set for PMMIR_EL1: - access_pmmir(): uses the value in kvm->arch.pmmir_slots directly. If the VMM selected a PMU and KVM_ARM_VCPU_PMU_V3_STRICT is set, the guest can correctly read the underlying core's SLOTS. Otherwise, it continues to read 0 since the true SLOTS value can be nondeterministic. - get_pmmir(): same as access_pmmir(). - set_pmmir(): only the SLOTS field is writable; a value setting any other bit is rejected with -EINVAL, since get_pmmir() returns SLOTS zero-extended. A value of 0 resets kvm->arch.pmmir_slots to 0 for backward compatibility, as the register is RAZ in older KVM, a value matching the current SLOTS is accepted as a no-op, and anything else is rejected with -EINVAL. Once the VM has run PMMIR_EL1 is immutable, so a mismatching write then returns -EBUSY. The register is now exposed via KVM_GET_REG_LIST for PMUv3 vCPUs, so add it to the get-reg-list selftest's PMU register list. Signed-off-by: Congkai Tan Reviewed-by: Geoff Blake Reviewed-by: Haris Okanovic Reviewed-by: Stanislav Spassov Co-developed-by: Oliver Upton Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260722202702.4165917-2-congkai@amazon.com Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_host.h | 3 ++ arch/arm64/include/uapi/asm/kvm.h | 1 + arch/arm64/kvm/pmu-emul.c | 11 +++++ arch/arm64/kvm/sys_regs.c | 63 +++++++++++++++++++++++- include/kvm/arm_pmu.h | 4 ++ tools/testing/selftests/kvm/arm64/get-reg-list.c | 1 + 6 files changed, 81 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index bae2c4f92ef5..9e035d587970 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -387,6 +387,9 @@ struct kvm_arch { /* Maximum number of counters for the guest */ u8 nr_pmu_counters; + /* PMMIR_EL1.SLOTS value exposed to the guest. */ + u8 pmmir_slots; + /* Hypercall features firmware registers' descriptor */ struct kvm_smccc_features smccc_feat; struct maple_tree smccc_filter; diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 1c13bfa2d38a..019e5e3d892e 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -106,6 +106,7 @@ struct kvm_regs { #define KVM_ARM_VCPU_PTRAUTH_GENERIC 6 /* VCPU uses generic authentication */ #define KVM_ARM_VCPU_HAS_EL2 7 /* Support nested virtualization */ #define KVM_ARM_VCPU_HAS_EL2_E2H0 8 /* Limit NV support to E2H RES0 */ +#define KVM_ARM_VCPU_PMU_V3_STRICT 9 /* No default PMU creation */ struct kvm_vcpu_init { __u32 target; diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index 98305bbfc095..7ac00423fa8d 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -1092,6 +1092,17 @@ static int kvm_arm_pmu_v3_set_pmu(struct kvm_vcpu *vcpu, int pmu_id) kvm_arm_set_pmu(kvm, arm_pmu); cpumask_copy(kvm->arch.supported_cpus, &arm_pmu->supported_cpus); + + /* + * Since a specific PMU is explicitly selected, + * PMMIR_EL1.SLOTS is deterministic to the guest. + * If KVM_ARM_VCPU_PMU_V3_STRICT is set, snapshot + * the value to allow the guest to read it. + */ + if (kvm_vcpu_has_pmuv3_strict(vcpu)) + kvm->arch.pmmir_slots = + FIELD_GET(ARMV8_PMU_SLOTS, + arm_pmu->reg_pmmir); ret = 0; break; } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5d5c579d4579..4c6f608c4fe5 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1367,6 +1367,64 @@ static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p, return true; } +static bool access_pmmir(struct kvm_vcpu *vcpu, struct sys_reg_params *p, + const struct sys_reg_desc *r) +{ + if (p->is_write) + return write_to_read_only(vcpu, p, r); + + /* + * If KVM_ARM_VCPU_PMU_V3_STRICT is set and PMU was explicitly + * selected, the underlying hardware SLOTS value was read into this + * field. Otherwise, it stays 0. All other PMMIR_EL1 fields are RAZ. + */ + p->regval = FIELD_PREP(ARMV8_PMU_SLOTS, vcpu->kvm->arch.pmmir_slots); + return true; +} + +static int get_pmmir(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + *val = FIELD_PREP(ARMV8_PMU_SLOTS, vcpu->kvm->arch.pmmir_slots); + return 0; +} + +static int set_pmmir(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct kvm *kvm = vcpu->kvm; + u8 slots = FIELD_GET(ARMV8_PMU_SLOTS, val); + + /* + * Only the SLOTS field is exposed (get_pmmir returns just that field), + * so reject a write that sets any other bit rather than silently + * masking it. + */ + if (val & ~(u64)ARMV8_PMU_SLOTS) + return -EINVAL; + + guard(mutex)(&kvm->arch.config_lock); + + /* + * Once the VM has started PMMIR_EL1 is immutable. Reject any write + * that does not match the current value. + */ + if (kvm_vm_has_ran_once(kvm)) + return slots == kvm->arch.pmmir_slots ? 0 : -EBUSY; + + /* + * Only SLOTS = 0 is honored for backwards compatibility with the + * old RAZ behavior. Reject any non-zero write that does not match + * the current value. + */ + if (!slots) + kvm->arch.pmmir_slots = 0; + else if (slots != kvm->arch.pmmir_slots) + return -EINVAL; + + return 0; +} + static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p, const struct sys_reg_desc *r) { @@ -3448,7 +3506,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { { PMU_SYS_REG(PMINTENCLR_EL1), .access = access_pminten, .reg = PMINTENSET_EL1, .get_user = get_pmreg, .set_user = set_pmreg }, - { SYS_DESC(SYS_PMMIR_EL1), trap_raz_wi }, + { PMU_SYS_REG(PMMIR_EL1), .access = access_pmmir, .reset = NULL, + .get_user = get_pmmir, .set_user = set_pmmir }, { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 }, { SYS_DESC(SYS_PIRE0_EL1), NULL, reset_unknown, PIRE0_EL1, @@ -4593,7 +4652,7 @@ static const struct sys_reg_desc cp15_regs[] = { { CP15_PMU_SYS_REG(HI, 0, 9, 14, 4), .access = access_pmceid }, { CP15_PMU_SYS_REG(HI, 0, 9, 14, 5), .access = access_pmceid }, /* PMMIR */ - { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 6), .access = trap_raz_wi }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 6), .access = access_pmmir }, /* PRRR/MAIR0 */ { AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 }, diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h index b5e5942204fc..6b4a118d17ca 100644 --- a/include/kvm/arm_pmu.h +++ b/include/kvm/arm_pmu.h @@ -75,6 +75,9 @@ void kvm_vcpu_pmu_resync_el0(void); #define kvm_vcpu_has_pmu(vcpu) \ (vcpu_has_feature(vcpu, KVM_ARM_VCPU_PMU_V3)) +#define kvm_vcpu_has_pmuv3_strict(vcpu) \ + (vcpu_has_feature(vcpu, KVM_ARM_VCPU_PMU_V3_STRICT)) + /* * Updates the vcpu's view of the pmu events for this cpu. * Must be called before every vcpu run after disabling interrupts, to ensure @@ -160,6 +163,7 @@ static inline u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) } #define kvm_vcpu_has_pmu(vcpu) ({ false; }) +#define kvm_vcpu_has_pmuv3_strict(vcpu) ({ false; }) static inline void kvm_pmu_update_vcpu_events(struct kvm_vcpu *vcpu) {} static inline void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) {} static inline void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) {} diff --git a/tools/testing/selftests/kvm/arm64/get-reg-list.c b/tools/testing/selftests/kvm/arm64/get-reg-list.c index 0a3a94c4cca1..cfa99979d57c 100644 --- a/tools/testing/selftests/kvm/arm64/get-reg-list.c +++ b/tools/testing/selftests/kvm/arm64/get-reg-list.c @@ -532,6 +532,7 @@ static __u64 base_regs[] = { static __u64 pmu_regs[] = { ARM64_SYS_REG(3, 0, 9, 14, 1), /* PMINTENSET_EL1 */ ARM64_SYS_REG(3, 0, 9, 14, 2), /* PMINTENCLR_EL1 */ + ARM64_SYS_REG(3, 0, 9, 14, 6), /* PMMIR_EL1 */ ARM64_SYS_REG(3, 3, 9, 12, 0), /* PMCR_EL0 */ ARM64_SYS_REG(3, 3, 9, 12, 1), /* PMCNTENSET_EL0 */ ARM64_SYS_REG(3, 3, 9, 12, 2), /* PMCNTENCLR_EL0 */ -- cgit From ad220e275c3987aea865d4c05ad66ba0295c494d Mon Sep 17 00:00:00 2001 From: Congkai Tan Date: Wed, 22 Jul 2026 20:27:02 +0000 Subject: KVM: arm64: Add KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature Introduce a new vCPU feature KVM_ARM_VCPU_PMU_V3_STRICT. When set, KVM does not create a default PMU when initializing the vCPU, and userspace must select one explicitly via KVM_ARM_VCPU_PMU_V3_SET_PMU before the first KVM_RUN. The flag forces the VMM to be aware of the PMU implementation of the guest to be created, so that certain information about the PMU becomes deterministic (if on a heterogeneous system) and becomes safe to be exposed to the guest. It can be used as an umbrella flag to gate future PMUv3 UAPI changes. When no default PMU is created, kvm->arch.arm_pmu stays NULL until SET_PMU runs, so kvm_arm_pmu_v3_init() now refuses to run if kvm->arch.arm_pmu is NULL. Signed-off-by: Congkai Tan Reviewed-by: Geoff Blake Reviewed-by: Haris Okanovic Reviewed-by: Stanislav Spassov Co-developed-by: Oliver Upton Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260722202702.4165917-5-congkai@amazon.com Signed-off-by: Oliver Upton --- Documentation/virt/kvm/api.rst | 11 +++++++++++ Documentation/virt/kvm/devices/vcpu.rst | 11 +++++++++-- arch/arm64/include/asm/kvm_host.h | 2 +- arch/arm64/kvm/arm.c | 19 +++++++++++++++---- arch/arm64/kvm/pmu-emul.c | 18 +++++++++++++++++- include/uapi/linux/kvm.h | 1 + 6 files changed, 54 insertions(+), 8 deletions(-) (limited to 'include') diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index a5f9ee92f43e..4988c32df4bf 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -3515,6 +3515,17 @@ Possible features: Depends on KVM_CAP_ARM_PSCI_0_2. - KVM_ARM_VCPU_PMU_V3: Emulate PMUv3 for the CPU. Depends on KVM_CAP_ARM_PMU_V3. + - KVM_ARM_VCPU_PMU_V3_STRICT: Enable strict PMUv3 UAPI. + Requires KVM_ARM_VCPU_PMU_V3. Depends on KVM_CAP_ARM_PMU_V3_STRICT. + When enabled: + + * Userspace must explicitly select a PMU implementation before + initializing the PMU or configuring a PMU event filter + + * If the PMU implements FEAT_PMUv3p4, PMMIR_EL1.SLOTS provides the + hardware value of the underlying implementation + + * Writes to PMCR_EL0.N via KVM_SET_ONE_REG are ignored - KVM_ARM_VCPU_PTRAUTH_ADDRESS: Enables Address Pointer authentication for arm64 only. diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst index 66e714f2fcfa..deb5c51bc00c 100644 --- a/Documentation/virt/kvm/devices/vcpu.rst +++ b/Documentation/virt/kvm/devices/vcpu.rst @@ -53,8 +53,9 @@ Returns: ======= ====================================================== -EEXIST Interrupt number already used -ENODEV PMUv3 not supported or GIC not initialized - -ENXIO PMUv3 not supported, missing VCPU feature or interrupt - number not set (non-GICv5 guests, only) + -ENXIO PMUv3 not supported, missing VCPU feature, missing + hardware PMU, or interrupt number not set (non-GICv5 + guests, only) -EBUSY PMUv3 already initialized ======= ====================================================== @@ -62,6 +63,9 @@ Request the initialization of the PMUv3. If using the PMUv3 with an in-kernel virtual GIC implementation, this must be done after initializing the in-kernel irqchip. +When the KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature is enabled this must be done +after selecting a hardware PMU. + 1.3 ATTRIBUTE: KVM_ARM_VCPU_PMU_V3_FILTER ----------------------------------------- @@ -108,6 +112,9 @@ hardware event. Filtering event 0x1E (CHAIN) has no effect either, as it isn't strictly speaking an event. Filtering the cycle counter is possible using event 0x11 (CPU_CYCLES). +When the KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature is enabled this must be done +after selecting a hardware PMU. + 1.4 ATTRIBUTE: KVM_ARM_VCPU_PMU_V3_SET_PMU ------------------------------------------ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 9e035d587970..39f7fc740d07 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -39,7 +39,7 @@ #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS -#define KVM_VCPU_MAX_FEATURES 9 +#define KVM_VCPU_MAX_FEATURES 10 #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) #define KVM_REQ_SLEEP \ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 50adfff75be8..7dbefdd846aa 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -452,6 +452,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = get_num_wrps(); break; case KVM_CAP_ARM_PMU_V3: + case KVM_CAP_ARM_PMU_V3_STRICT: r = kvm_supports_guest_pmuv3(); break; case KVM_CAP_ARM_INJECT_SERROR_ESR: @@ -1563,8 +1564,10 @@ static unsigned long system_supported_vcpu_features(void) if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features); - if (!kvm_supports_guest_pmuv3()) + if (!kvm_supports_guest_pmuv3()) { clear_bit(KVM_ARM_VCPU_PMU_V3, &features); + clear_bit(KVM_ARM_VCPU_PMU_V3_STRICT, &features); + } if (!system_supports_sve()) clear_bit(KVM_ARM_VCPU_SVE, &features); @@ -1605,6 +1608,11 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features)) return -EINVAL; + /* Strict PMUv3 UAPI requires PMUv3. */ + if (test_bit(KVM_ARM_VCPU_PMU_V3_STRICT, &features) && + !test_bit(KVM_ARM_VCPU_PMU_V3, &features)) + return -EINVAL; + if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features)) return 0; @@ -1634,10 +1642,13 @@ static int kvm_setup_vcpu(struct kvm_vcpu *vcpu) int ret = 0; /* - * When the vCPU has a PMU, but no PMU is set for the guest - * yet, set the default one. + * When the vCPU has a PMU, but no PMU is set for the guest yet, set + * the default one. If KVM_ARM_VCPU_PMU_V3_STRICT is set, no default + * PMU is created, and userspace must select a PMU via + * KVM_ARM_VCPU_PMU_V3_SET_PMU. */ - if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu) + if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu && + !kvm_vcpu_has_pmuv3_strict(vcpu)) ret = kvm_arm_set_default_pmu(kvm); /* Prepare for nested if required */ diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index bcfd0a91e114..5b1af7e2176f 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -939,6 +939,10 @@ int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu) static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu) { + /* Only possible when using KVM_ARM_VCPU_PMU_V3_STRICT */ + if (!vcpu->kvm->arch.arm_pmu) + return -ENXIO; + if (irqchip_in_kernel(vcpu->kvm)) { int ret; @@ -1009,6 +1013,14 @@ u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm) { struct arm_pmu *arm_pmu = kvm->arch.arm_pmu; + /* + * Under KVM_ARM_VCPU_PMU_V3_STRICT no PMU exists until userspace sets + * one, so this can be reached before arm_pmu is set. Report no + * counters in that case. + */ + if (!arm_pmu) + return 0; + /* * PMUv3 requires that all event counters are capable of counting any * event, though the same may not be true of non-PMUv3 hardware. @@ -1050,7 +1062,8 @@ static void kvm_arm_set_pmu(struct kvm *kvm, struct arm_pmu *arm_pmu) } /** - * kvm_arm_set_default_pmu - No PMU set, get the default one. + * kvm_arm_set_default_pmu - No PMU set and KVM_ARM_VCPU_PMU_V3_STRICT not + * set, get the default one. * @kvm: The kvm pointer * * The observant among you will notice that the supported_cpus @@ -1190,6 +1203,9 @@ int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) if (kvm_vm_has_ran_once(kvm)) return -EBUSY; + if (!kvm->arch.arm_pmu) + return -ENXIO; + if (!kvm->arch.pmu_filter) { kvm->arch.pmu_filter = bitmap_alloc(nr_events, GFP_KERNEL_ACCOUNT); if (!kvm->arch.pmu_filter) diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 419011097fa8..9fc8dfdfd65f 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -997,6 +997,7 @@ struct kvm_enable_cap { #define KVM_CAP_S390_KEYOP 247 #define KVM_CAP_S390_VSIE_ESAMODE 248 #define KVM_CAP_S390_HPAGE_2G 249 +#define KVM_CAP_ARM_PMU_V3_STRICT 250 struct kvm_irq_routing_irqchip { __u32 irqchip; -- cgit From 86db2bd7f5f07e4b169345d291bdd5c72730061c Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 29 Jul 2026 14:18:19 +0100 Subject: KVM: arm64: Move PSCI helper functions to a shared header Move kvm_psci_valid_affinity() and kvm_psci_narrow_to_32bit() from psci.c to include/kvm/arm_psci.h, and move psci_affinity_mask() there too, renaming it kvm_psci_affinity_mask() now that it is no longer file-local. A follow-up series handles some protected-guest PSCI calls at EL2 using these helpers. No functional change intended. Reviewed-by: Vincent Donnefort Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-5-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/psci.c | 30 +----------------------------- include/kvm/arm_psci.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 29 deletions(-) (limited to 'include') diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index 3b5dbe9a0a0e..e3db84400d1f 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -21,16 +21,6 @@ * as described in ARM document number ARM DEN 0022A. */ -#define AFFINITY_MASK(level) ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1) - -static unsigned long psci_affinity_mask(unsigned long affinity_level) -{ - if (affinity_level <= 3) - return MPIDR_HWID_BITMASK & AFFINITY_MASK(affinity_level); - - return 0; -} - static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu) { /* @@ -51,12 +41,6 @@ static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu) return PSCI_RET_SUCCESS; } -static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu, - unsigned long affinity) -{ - return !(affinity & ~MPIDR_HWID_BITMASK); -} - static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) { struct vcpu_reset_state *reset_state; @@ -135,7 +119,7 @@ static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu) return PSCI_RET_INVALID_PARAMS; /* Determine target affinity mask */ - target_affinity_mask = psci_affinity_mask(lowest_affinity_level); + target_affinity_mask = kvm_psci_affinity_mask(lowest_affinity_level); if (!target_affinity_mask) return PSCI_RET_INVALID_PARAMS; @@ -220,18 +204,6 @@ static void kvm_psci_system_suspend(struct kvm_vcpu *vcpu) run->exit_reason = KVM_EXIT_SYSTEM_EVENT; } -static void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu) -{ - int i; - - /* - * Zero the input registers' upper 32 bits. They will be fully - * zeroed on exit, so we're fine changing them in place. - */ - for (i = 1; i < 4; i++) - vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i))); -} - static unsigned long kvm_psci_check_allowed_function(struct kvm_vcpu *vcpu, u32 fn) { /* diff --git a/include/kvm/arm_psci.h b/include/kvm/arm_psci.h index cbaec804eb83..f86a006d6713 100644 --- a/include/kvm/arm_psci.h +++ b/include/kvm/arm_psci.h @@ -38,6 +38,33 @@ static inline int kvm_psci_version(struct kvm_vcpu *vcpu) return KVM_ARM_PSCI_0_1; } +/* Narrow the PSCI register arguments (r1 to r3) to 32 bits. */ +static inline void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu) +{ + int i; + + /* + * Zero the input registers' upper 32 bits. They will be fully + * zeroed on exit, so we're fine changing them in place. + */ + for (i = 1; i < 4; i++) + vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i))); +} + +static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu, + unsigned long affinity) +{ + return !(affinity & ~MPIDR_HWID_BITMASK); +} + +static inline unsigned long kvm_psci_affinity_mask(unsigned long affinity_level) +{ + if (affinity_level <= 3) + return MPIDR_HWID_BITMASK & + ~((0x1UL << (affinity_level * MPIDR_LEVEL_BITS)) - 1); + + return 0; +} int kvm_psci_call(struct kvm_vcpu *vcpu); -- cgit From 97d65b544f48b2ee49f6aea32145e3e7969955dc Mon Sep 17 00:00:00 2001 From: Dmytro Maluka Date: Wed, 29 Jul 2026 17:06:20 +0000 Subject: KVM: Check for duplicate vcpu_id as early as possible If userspace tries to create a vCPU with the same vcpu_id as an existing one, kvm_vm_ioctl_create_vcpu() checks for that and fails with -EEXIST only after it already created the vCPU via kvm_arch_vcpu_create(). As a result, even though this newly created vCPU is destroyed in the failure path, the fact that it is temporarily created with an invalid vcpu_id and that there are temporarily two vCPUs with the same vcpu_id is a potential source of subtle issues. In particular, this prevents fixing an VMX IPIv issue where a stale entry left in the VM's PI descriptor table after the vCPU is destroyed in the failure path. The right way to fix that issue is to clear that entry when destroying the vCPU, however right now that would have a nasty side effect: since the same entry is used for the other, previously created vCPU with same vcpu_id, clearing it would mean effectively disabling IPIv for that existing good vCPU. So to avoid this and similar problems, check for duplicate vcpu_id as early in the vCPU creation path as possible, before kvm_arch_vcpu_create() and even before kvm_arch_vcpu_precreate(). Simply moving the existing kvm_get_vcpu_by_id() check earlier doesn't work, as kvm->lock is dropped and reacquired, i.e. moving kvm_get_vcpu_by_id() would introduce a race: 1. vCPU A is being created but not installed in kvm->vcpu_array yet. 2. vCPU B with the same vcpu_id is being created. It passes the duplicated vcpu_id check, since the check doesn't find vCPU A in kvm->vcpu_array. 3. vCPU A is installed in kvm->vcpu_array, vCPU creation succeeds. 4. vCPU B with the same vcpu_id is installed in kvm->vcpu_array, vCPU creation succeeds. So introduce the bitmap of vcpu_ids used by the VM, in order to safely check if the given vcpu_id is used and mark is as used before releasing kvm->lock first time. Alternatively, KVM could use another Xarray[*] for roughly the same code complexity, which would minimize KVM's steady state memory footprint at the cost of higher runtime latency (to allocate and free entries). Given that the worst case scenario is 256 bytes per-VM (on x86, which allows up to 16KiB vCPU IDs), go with the slightly simpler approach until there's a need to save memory. Suggested-by: Sean Christopherson Link: https://lore.kernel.org/kvm/al6eg7C-2sDBEAFD@google.com [*] Signed-off-by: Dmytro Maluka Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260729170621.308809-2-dmaluka@chromium.org [sean: massage changelog] Signed-off-by: Sean Christopherson --- include/linux/kvm_host.h | 1 + virt/kvm/kvm_main.c | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index ab8cfaec82d3..6f883ed82581 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -791,6 +791,7 @@ struct kvm { /* The current active memslot set for each address space */ struct kvm_memslots __rcu *memslots[KVM_MAX_NR_ADDRESS_SPACES]; struct xarray vcpu_array; + DECLARE_BITMAP(vcpu_ids, KVM_MAX_VCPU_IDS); /* * Protected by slots_lock, but can be read outside if an * incorrect answer is acceptable. diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index d4420ebfd972..53f593f54288 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4173,6 +4173,11 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) return -EINVAL; } + if (test_bit(id, kvm->vcpu_ids)) { + mutex_unlock(&kvm->lock); + return -EEXIST; + } + r = kvm_arch_vcpu_precreate(kvm, id); if (r) { mutex_unlock(&kvm->lock); @@ -4180,6 +4185,7 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) } kvm->created_vcpus++; + __set_bit(id, kvm->vcpu_ids); mutex_unlock(&kvm->lock); vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL_ACCOUNT); @@ -4211,7 +4217,7 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) mutex_lock(&kvm->lock); - if (kvm_get_vcpu_by_id(kvm, id)) { + if (WARN_ON_ONCE(kvm_get_vcpu_by_id(kvm, id))) { r = -EEXIST; goto unlock_vcpu_destroy; } @@ -4265,6 +4271,7 @@ vcpu_free: vcpu_decrement: mutex_lock(&kvm->lock); kvm->created_vcpus--; + __clear_bit(id, kvm->vcpu_ids); mutex_unlock(&kvm->lock); return r; } -- cgit From 2858600ecd014c4465008b7c39a9799d7e37722b Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sat, 8 Aug 2026 08:58:22 +0000 Subject: KVM: arm64: Make timer_get_offset() work in all contexts We currently have two implementations of get_timer offset(), one in arm_arch_timer.h, and another one in switch.h. These two only differ by a pair of kern_hyp_va(), which seems a pretty weak reason to open-code it. Turn this function into a macro to avoid the include dependency hell on kern_hyp_va(), and make it work correctly in all contexts. Signed-off-by: Marc Zyngier Signed-off-by: Mostafa Saleh Link: https://patch.msgid.link/20260808085824.732659-2-smostafa@google.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/include/hyp/switch.h | 15 +-------------- include/kvm/arm_arch_timer.h | 34 ++++++++++++++++++++------------- 2 files changed, 22 insertions(+), 27 deletions(-) (limited to 'include') diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h index 18131e395e24..4057773aa40b 100644 --- a/arch/arm64/kvm/hyp/include/hyp/switch.h +++ b/arch/arm64/kvm/hyp/include/hyp/switch.h @@ -705,22 +705,9 @@ static inline bool handle_tx2_tvm(struct kvm_vcpu *vcpu) return true; } -/* Open-coded version of timer_get_offset() to allow for kern_hyp_va() */ -static inline u64 hyp_timer_get_offset(struct arch_timer_context *ctxt) -{ - u64 offset = 0; - - if (ctxt->offset.vm_offset) - offset += *kern_hyp_va(ctxt->offset.vm_offset); - if (ctxt->offset.vcpu_offset) - offset += *kern_hyp_va(ctxt->offset.vcpu_offset); - - return offset; -} - static inline u64 compute_counter_value(struct arch_timer_context *ctxt) { - return arch_timer_read_cntpct_el0() - hyp_timer_get_offset(ctxt); + return arch_timer_read_cntpct_el0() - timer_get_offset(ctxt); } static bool kvm_handle_cntxct(struct kvm_vcpu *vcpu) diff --git a/include/kvm/arm_arch_timer.h b/include/kvm/arm_arch_timer.h index 15a4f97f8105..bc6f2fdd7ad3 100644 --- a/include/kvm/arm_arch_timer.h +++ b/include/kvm/arm_arch_timer.h @@ -162,20 +162,28 @@ static inline bool has_cntpoff(void) return (has_vhe() && cpus_have_final_cap(ARM64_HAS_ECV_CNTPOFF)); } -static inline u64 timer_get_offset(struct arch_timer_context *ctxt) -{ - u64 offset = 0; - - if (!ctxt) - return 0; - - if (ctxt->offset.vm_offset) - offset += *ctxt->offset.vm_offset; - if (ctxt->offset.vcpu_offset) - offset += *ctxt->offset.vcpu_offset; +#ifdef __KVM_NVHE_HYPERVISOR__ +#define KERN_HYP_VA(x) kern_hyp_va(x) +#else +#define KERN_HYP_VA(x) x +#endif - return offset; -} +#define timer_get_offset(ctxt) \ + ({ \ + struct arch_timer_context *__ctxt = (ctxt); \ + u64 off = 0; \ + \ + if (__ctxt) { \ + struct arch_timer_offset *ato = &__ctxt->offset;\ + \ + if (ato->vm_offset) \ + off += *KERN_HYP_VA(ato->vm_offset); \ + if (ato->vcpu_offset) \ + off += *KERN_HYP_VA(ato->vcpu_offset); \ + } \ + \ + off; \ + }) static inline void timer_set_offset(struct arch_timer_context *ctxt, u64 offset) { -- cgit From 2962174fdfa63e272082ea7ae989a8fdef29bd0e Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Tue, 11 Aug 2026 15:11:18 +0000 Subject: KVM: arm64: vgic: Reject out-of-range GICv5 PPI IDs GICv5 supports up to 128 PPIs, but KVM currently implements only the first 64, which contain the architected PPIs it supports. An encoded PPI with an ID outside that range passes irq_is_ppi(), which only checks the encoded interrupt type. vgic_get_vcpu_irq() therefore looks it up in private_irqs[], where array_index_nospec() clamps the out-of-range index to zero and aliases PPI 0. Include the supported PPI range in irq_is_ppi() so that KVM interfaces reject unsupported PPIs. Also reject an out-of-range PPI in the lookup as a safeguard against callers bypassing the predicate. Fixes: 4d591252bacb ("KVM: arm64: gic-v5: Implement PPI interrupt injection") Fixes: eb8bce08ecb1 ("KVM: arm64: gic: Introduce interrupt type helpers") Link: https://sashiko.dev/#/patchset/20260724104819.1296803-1-sascha.bischoff@arm.com?part=27 Signed-off-by: Sascha Bischoff Reviewed-by: Joey Gouly Reviewed-by: Marc Zyngier Link: https://patch.msgid.link/20260811150941.941295-4-sascha.bischoff@arm.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/vgic/vgic.c | 2 ++ include/kvm/arm_vgic.h | 2 ++ 2 files changed, 4 insertions(+) (limited to 'include') diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 3077cfdaa146..d0c91f0c5269 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -118,6 +118,8 @@ struct vgic_irq *vgic_get_vcpu_irq(struct kvm_vcpu *vcpu, u32 intid) switch (type) { case KVM_DEV_TYPE_ARM_VGIC_V5: intid = vgic_v5_get_hwirq_id(intid); + if (intid >= VGIC_V5_NR_PRIVATE_IRQS) + return NULL; intid = array_index_nospec(intid, VGIC_V5_NR_PRIVATE_IRQS); break; default: diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index fe49fb56dc3c..41012457841c 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -65,6 +65,8 @@ switch (t) { \ case KVM_DEV_TYPE_ARM_VGIC_V5: \ __ret = is_v5_type(GICV5_HWIRQ_TYPE_PPI, (i)); \ + __ret &= FIELD_GET(GICV5_HWIRQ_ID, (i)) < \ + VGIC_V5_NR_PRIVATE_IRQS; \ break; \ default: \ __ret = (i) >= VGIC_NR_SGIS; \ -- cgit