2018-12-03 13:53:08 -08:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0 */
|
|
|
|
|
#ifndef __KVM_X86_VMX_H
|
|
|
|
|
#define __KVM_X86_VMX_H
|
|
|
|
|
|
|
|
|
|
#include <linux/kvm_host.h>
|
|
|
|
|
|
|
|
|
|
#include <asm/kvm.h>
|
2018-10-24 16:05:10 +08:00
|
|
|
#include <asm/intel_pt.h>
|
2022-07-27 23:34:23 +00:00
|
|
|
#include <asm/perf_event.h>
|
2024-04-23 10:41:03 -07:00
|
|
|
#include <asm/posted_intr.h>
|
2018-12-03 13:53:08 -08:00
|
|
|
|
|
|
|
|
#include "capabilities.h"
|
2022-06-14 23:16:16 +02:00
|
|
|
#include "../kvm_cache_regs.h"
|
2024-10-30 12:00:33 -07:00
|
|
|
#include "pmu_intel.h"
|
2018-12-03 13:53:08 -08:00
|
|
|
#include "vmcs.h"
|
2020-09-23 11:31:12 -07:00
|
|
|
#include "vmx_ops.h"
|
2022-06-14 23:16:16 +02:00
|
|
|
#include "../cpuid.h"
|
2022-06-14 23:16:12 +02:00
|
|
|
#include "run_flags.h"
|
2024-03-19 11:11:11 +08:00
|
|
|
#include "../mmu.h"
|
2025-03-14 14:06:48 -04:00
|
|
|
#include "common.h"
|
2018-12-03 13:53:08 -08:00
|
|
|
|
2019-12-03 16:24:42 -08:00
|
|
|
#ifdef CONFIG_X86_64
|
2020-09-23 11:03:58 -07:00
|
|
|
#define MAX_NR_USER_RETURN_MSRS 7
|
2019-12-03 16:24:42 -08:00
|
|
|
#else
|
2020-09-23 11:03:58 -07:00
|
|
|
#define MAX_NR_USER_RETURN_MSRS 4
|
2019-12-03 16:24:42 -08:00
|
|
|
#endif
|
|
|
|
|
|
2020-09-23 11:03:56 -07:00
|
|
|
#define MAX_NR_LOADSTORE_MSRS 8
|
2018-12-03 13:53:08 -08:00
|
|
|
|
|
|
|
|
struct vmx_msrs {
|
|
|
|
|
unsigned int nr;
|
2020-09-23 11:03:56 -07:00
|
|
|
struct vmx_msr_entry val[MAX_NR_LOADSTORE_MSRS];
|
2018-12-03 13:53:08 -08:00
|
|
|
};
|
|
|
|
|
|
2020-09-23 11:03:58 -07:00
|
|
|
struct vmx_uret_msr {
|
2021-05-04 10:17:28 -07:00
|
|
|
bool load_into_hardware;
|
2018-12-03 13:53:08 -08:00
|
|
|
u64 data;
|
|
|
|
|
u64 mask;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
enum segment_cache_field {
|
|
|
|
|
SEG_FIELD_SEL = 0,
|
|
|
|
|
SEG_FIELD_BASE = 1,
|
|
|
|
|
SEG_FIELD_LIMIT = 2,
|
|
|
|
|
SEG_FIELD_AR = 3,
|
|
|
|
|
|
|
|
|
|
SEG_FIELD_NR = 4
|
|
|
|
|
};
|
|
|
|
|
|
2018-10-24 16:05:12 +08:00
|
|
|
#define RTIT_ADDR_RANGE 4
|
|
|
|
|
|
|
|
|
|
struct pt_ctx {
|
|
|
|
|
u64 ctl;
|
|
|
|
|
u64 status;
|
|
|
|
|
u64 output_base;
|
|
|
|
|
u64 output_mask;
|
|
|
|
|
u64 cr3_match;
|
|
|
|
|
u64 addr_a[RTIT_ADDR_RANGE];
|
|
|
|
|
u64 addr_b[RTIT_ADDR_RANGE];
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct pt_desc {
|
|
|
|
|
u64 ctl_bitmask;
|
2021-08-27 15:02:45 +08:00
|
|
|
u32 num_address_ranges;
|
2018-10-24 16:05:12 +08:00
|
|
|
u32 caps[PT_CPUID_REGS_NUM * PT_CPUID_LEAVES];
|
|
|
|
|
struct pt_ctx host;
|
|
|
|
|
struct pt_ctx guest;
|
|
|
|
|
};
|
2018-12-03 13:53:08 -08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* The nested_vmx structure is part of vcpu_vmx, and holds information we need
|
|
|
|
|
* for correct emulation of VMX (i.e., nested VMX) on this vcpu.
|
|
|
|
|
*/
|
|
|
|
|
struct nested_vmx {
|
|
|
|
|
/* Has the level1 guest done vmxon? */
|
|
|
|
|
bool vmxon;
|
|
|
|
|
gpa_t vmxon_ptr;
|
|
|
|
|
bool pml_full;
|
|
|
|
|
|
|
|
|
|
/* The guest-physical address of the current VMCS L1 keeps for L2 */
|
|
|
|
|
gpa_t current_vmptr;
|
|
|
|
|
/*
|
|
|
|
|
* Cache of the guest's VMCS, existing outside of guest memory.
|
|
|
|
|
* Loaded from guest memory during VMPTRLD. Flushed to guest
|
|
|
|
|
* memory during VMCLEAR and VMPTRLD.
|
|
|
|
|
*/
|
|
|
|
|
struct vmcs12 *cached_vmcs12;
|
|
|
|
|
/*
|
|
|
|
|
* Cache of the guest's shadow VMCS, existing outside of guest
|
|
|
|
|
* memory. Loaded from guest memory during VM entry. Flushed
|
|
|
|
|
* to guest memory during VM exit.
|
|
|
|
|
*/
|
|
|
|
|
struct vmcs12 *cached_shadow_vmcs12;
|
2019-05-07 08:36:29 -07:00
|
|
|
|
2021-11-15 16:50:24 +00:00
|
|
|
/*
|
|
|
|
|
* GPA to HVA cache for accessing vmcs12->vmcs_link_pointer
|
|
|
|
|
*/
|
|
|
|
|
struct gfn_to_hva_cache shadow_vmcs12_cache;
|
|
|
|
|
|
2021-11-15 16:50:26 +00:00
|
|
|
/*
|
|
|
|
|
* GPA to HVA cache for VMCS12
|
|
|
|
|
*/
|
|
|
|
|
struct gfn_to_hva_cache vmcs12_cache;
|
|
|
|
|
|
2018-12-03 13:53:08 -08:00
|
|
|
/*
|
|
|
|
|
* Indicates if the shadow vmcs or enlightened vmcs must be updated
|
|
|
|
|
* with the data held by struct vmcs12.
|
|
|
|
|
*/
|
2019-05-07 08:36:27 -07:00
|
|
|
bool need_vmcs12_to_shadow_sync;
|
2018-12-03 13:53:08 -08:00
|
|
|
bool dirty_vmcs12;
|
|
|
|
|
|
2021-11-29 10:47:03 +01:00
|
|
|
/*
|
|
|
|
|
* Indicates whether MSR bitmap for L2 needs to be rebuilt due to
|
|
|
|
|
* changes in MSR bitmap for L1 or switching to a different L2. Note,
|
|
|
|
|
* this flag can only be used reliably in conjunction with a paravirt L1
|
|
|
|
|
* which informs L0 whether any changes to MSR bitmap for L2 were done
|
|
|
|
|
* on its side.
|
|
|
|
|
*/
|
|
|
|
|
bool force_msr_bitmap_recalc;
|
|
|
|
|
|
2019-05-07 08:36:29 -07:00
|
|
|
/*
|
|
|
|
|
* Indicates lazily loaded guest state has not yet been decached from
|
|
|
|
|
* vmcs02.
|
|
|
|
|
*/
|
|
|
|
|
bool need_sync_vmcs02_to_vmcs12_rare;
|
|
|
|
|
|
2018-12-03 13:53:08 -08:00
|
|
|
/*
|
|
|
|
|
* vmcs02 has been initialized, i.e. state that is constant for
|
|
|
|
|
* vmcs02 has been written to the backing VMCS. Initialization
|
|
|
|
|
* is delayed until L1 actually attempts to run a nested VM.
|
|
|
|
|
*/
|
|
|
|
|
bool vmcs02_initialized;
|
|
|
|
|
|
|
|
|
|
bool change_vmcs01_virtual_apic_mode;
|
2020-03-20 14:28:23 -07:00
|
|
|
bool reload_vmcs01_apic_access_page;
|
2021-02-12 16:50:12 -08:00
|
|
|
bool update_vmcs01_cpu_dirty_logging;
|
KVM: nVMX: Defer APICv updates while L2 is active until L1 is active
Defer APICv updates that occur while L2 is active until nested VM-Exit,
i.e. until L1 regains control. vmx_refresh_apicv_exec_ctrl() assumes L1
is active and (a) stomps all over vmcs02 and (b) neglects to ever updated
vmcs01. E.g. if vmcs12 doesn't enable the TPR shadow for L2 (and thus no
APICv controls), L1 performs nested VM-Enter APICv inhibited, and APICv
becomes unhibited while L2 is active, KVM will set various APICv controls
in vmcs02 and trigger a failed VM-Entry. The kicker is that, unless
running with nested_early_check=1, KVM blames L1 and chaos ensues.
In all cases, ignoring vmcs02 and always deferring the inhibition change
to vmcs01 is correct (or at least acceptable). The ABSENT and DISABLE
inhibitions cannot truly change while L2 is active (see below).
IRQ_BLOCKING can change, but it is firmly a best effort debug feature.
Furthermore, only L2's APIC is accelerated/virtualized to the full extent
possible, e.g. even if L1 passes through its APIC to L2, normal MMIO/MSR
interception will apply to the virtual APIC managed by KVM.
The exception is the SELF_IPI register when x2APIC is enabled, but that's
an acceptable hole.
Lastly, Hyper-V's Auto EOI can technically be toggled if L1 exposes the
MSRs to L2, but for that to work in any sane capacity, L1 would need to
pass through IRQs to L2 as well, and IRQs must be intercepted to enable
virtual interrupt delivery. I.e. exposing Auto EOI to L2 and enabling
VID for L2 are, for all intents and purposes, mutually exclusive.
Lack of dynamic toggling is also why this scenario is all but impossible
to encounter in KVM's current form. But a future patch will pend an
APICv update request _during_ vCPU creation to plug a race where a vCPU
that's being created doesn't get included in the "all vCPUs request"
because it's not yet visible to other vCPUs. If userspaces restores L2
after VM creation (hello, KVM selftests), the first KVM_RUN will occur
while L2 is active and thus service the APICv update request made during
VM creation.
Cc: stable@vger.kernel.org
Signed-off-by: Sean Christopherson <seanjc@google.com>
Message-Id: <20220420013732.3308816-3-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2022-04-20 01:37:30 +00:00
|
|
|
bool update_vmcs01_apicv_status;
|
KVM: nVMX: Defer SVI update to vmcs01 on EOI when L2 is active w/o VID
If KVM emulates an EOI for L1's virtual APIC while L2 is active, defer
updating GUEST_INTERUPT_STATUS.SVI, i.e. the VMCS's cache of the highest
in-service IRQ, until L1 is active, as vmcs01, not vmcs02, needs to track
vISR. The missed SVI update for vmcs01 can result in L1 interrupts being
incorrectly blocked, e.g. if there is a pending interrupt with lower
priority than the interrupt that was EOI'd.
This bug only affects use cases where L1's vAPIC is effectively passed
through to L2, e.g. in a pKVM scenario where L2 is L1's depriveleged host,
as KVM will only emulate an EOI for L1's vAPIC if Virtual Interrupt
Delivery (VID) is disabled in vmc12, and L1 isn't intercepting L2 accesses
to its (virtual) APIC page (or if x2APIC is enabled, the EOI MSR).
WARN() if KVM updates L1's ISR while L2 is active with VID enabled, as an
EOI from L2 is supposed to affect L2's vAPIC, but still defer the update,
to try to keep L1 alive. Specifically, KVM forwards all APICv-related
VM-Exits to L1 via nested_vmx_l1_wants_exit():
case EXIT_REASON_APIC_ACCESS:
case EXIT_REASON_APIC_WRITE:
case EXIT_REASON_EOI_INDUCED:
/*
* The controls for "virtualize APIC accesses," "APIC-
* register virtualization," and "virtual-interrupt
* delivery" only come from vmcs12.
*/
return true;
Fixes: c7c9c56ca26f ("x86, apicv: add virtual interrupt delivery support")
Cc: stable@vger.kernel.org
Link: https://lore.kernel.org/kvm/20230312180048.1778187-1-jason.cj.chen@intel.com
Reported-by: Markku Ahvenjärvi <mankku@gmail.com>
Closes: https://lore.kernel.org/all/20240920080012.74405-1-mankku@gmail.com
Cc: Janne Karhunen <janne.karhunen@gmail.com>
Signed-off-by: Chao Gao <chao.gao@intel.com>
[sean: drop request, handle in VMX, write changelog]
Tested-by: Chao Gao <chao.gao@intel.com>
Link: https://lore.kernel.org/r/20241128000010.4051275-3-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>
2024-11-27 16:00:10 -08:00
|
|
|
bool update_vmcs01_hwapic_isr;
|
2018-12-03 13:53:08 -08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Enlightened VMCS has been enabled. It does not mean that L1 has to
|
|
|
|
|
* use it. However, VMX features available to L1 will be limited based
|
|
|
|
|
* on what the enlightened VMCS supports.
|
|
|
|
|
*/
|
|
|
|
|
bool enlightened_vmcs_enabled;
|
|
|
|
|
|
|
|
|
|
/* L2 must run next, and mustn't decide to exit to L1. */
|
|
|
|
|
bool nested_run_pending;
|
|
|
|
|
|
2020-02-07 02:36:07 -08:00
|
|
|
/* Pending MTF VM-exit into L1. */
|
|
|
|
|
bool mtf_pending;
|
|
|
|
|
|
2018-12-03 13:53:08 -08:00
|
|
|
struct loaded_vmcs vmcs02;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Guest pages referred to in the vmcs02 with host-physical
|
|
|
|
|
* pointers, so we must keep them pinned while L2 runs.
|
|
|
|
|
*/
|
2022-04-29 01:04:11 +00:00
|
|
|
struct kvm_host_map apic_access_page_map;
|
2019-01-31 21:24:37 +01:00
|
|
|
struct kvm_host_map virtual_apic_map;
|
2019-01-31 21:24:38 +01:00
|
|
|
struct kvm_host_map pi_desc_map;
|
2019-01-31 21:24:36 +01:00
|
|
|
|
2018-12-03 13:53:08 -08:00
|
|
|
struct pi_desc *pi_desc;
|
|
|
|
|
bool pi_pending;
|
|
|
|
|
u16 posted_intr_nv;
|
|
|
|
|
|
|
|
|
|
struct hrtimer preemption_timer;
|
2020-05-26 14:51:06 -07:00
|
|
|
u64 preemption_timer_deadline;
|
|
|
|
|
bool has_preemption_timer_deadline;
|
2018-12-03 13:53:08 -08:00
|
|
|
bool preemption_timer_expired;
|
|
|
|
|
|
2022-06-14 21:58:29 +00:00
|
|
|
/*
|
|
|
|
|
* Used to snapshot MSRs that are conditionally loaded on VM-Enter in
|
|
|
|
|
* order to propagate the guest's pre-VM-Enter value into vmcs02. For
|
|
|
|
|
* emulation of VMLAUNCH/VMRESUME, the snapshot will be of L1's value.
|
|
|
|
|
* For KVM_SET_NESTED_STATE, the snapshot is of L2's value, _if_
|
|
|
|
|
* userspace restores MSRs before nested state. If userspace restores
|
|
|
|
|
* MSRs after nested state, the snapshot holds garbage, but KVM can't
|
|
|
|
|
* detect that, and the garbage value in vmcs02 will be overwritten by
|
|
|
|
|
* MSR restoration in any case.
|
|
|
|
|
*/
|
|
|
|
|
u64 pre_vmenter_debugctl;
|
|
|
|
|
u64 pre_vmenter_bndcfgs;
|
2018-12-03 13:53:08 -08:00
|
|
|
|
2019-11-11 14:30:55 +02:00
|
|
|
/* to migrate it to L1 if L2 writes to L1's CR8 directly */
|
|
|
|
|
int l1_tpr_threshold;
|
|
|
|
|
|
2018-12-03 13:53:08 -08:00
|
|
|
u16 vpid02;
|
|
|
|
|
u16 last_vpid;
|
|
|
|
|
|
|
|
|
|
struct nested_vmx_msrs msrs;
|
|
|
|
|
|
|
|
|
|
/* SMM related state */
|
|
|
|
|
struct {
|
|
|
|
|
/* in VMX operation on SMM entry? */
|
|
|
|
|
bool vmxon;
|
|
|
|
|
/* in guest mode on SMM entry? */
|
|
|
|
|
bool guest_mode;
|
|
|
|
|
} smm;
|
|
|
|
|
|
2023-12-05 11:36:29 +01:00
|
|
|
#ifdef CONFIG_KVM_HYPERV
|
2018-12-03 13:53:08 -08:00
|
|
|
gpa_t hv_evmcs_vmptr;
|
2019-01-31 21:24:42 +01:00
|
|
|
struct kvm_host_map hv_evmcs_map;
|
2018-12-03 13:53:08 -08:00
|
|
|
struct hv_enlightened_vmcs *hv_evmcs;
|
2023-12-05 11:36:29 +01:00
|
|
|
#endif
|
2018-12-03 13:53:08 -08:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct vcpu_vmx {
|
|
|
|
|
struct kvm_vcpu vcpu;
|
2025-03-14 14:06:48 -04:00
|
|
|
struct vcpu_vt vt;
|
2018-12-03 13:53:08 -08:00
|
|
|
u8 fail;
|
2021-07-13 09:33:19 -07:00
|
|
|
u8 x2apic_msr_bitmap_mode;
|
2019-06-07 19:00:14 +02:00
|
|
|
|
2018-12-03 13:53:08 -08:00
|
|
|
u32 idt_vectoring_info;
|
|
|
|
|
ulong rflags;
|
2019-05-07 12:17:54 -07:00
|
|
|
|
2021-05-04 10:17:27 -07:00
|
|
|
/*
|
|
|
|
|
* User return MSRs are always emulated when enabled in the guest, but
|
|
|
|
|
* only loaded into hardware when necessary, e.g. SYSCALL #UDs outside
|
|
|
|
|
* of 64-bit mode or if EFER.SCE=1, thus the SYSCALL MSRs don't need to
|
|
|
|
|
* be loaded into hardware if those conditions aren't met.
|
|
|
|
|
*/
|
2020-09-23 11:03:58 -07:00
|
|
|
struct vmx_uret_msr guest_uret_msrs[MAX_NR_USER_RETURN_MSRS];
|
2020-09-23 11:04:01 -07:00
|
|
|
bool guest_uret_msrs_loaded;
|
2018-12-03 13:53:08 -08:00
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
|
u64 msr_guest_kernel_gs_base;
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
u64 spec_ctrl;
|
2019-07-16 14:55:50 +08:00
|
|
|
u32 msr_ia32_umwait_control;
|
2018-12-03 13:53:08 -08:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* loaded_vmcs points to the VMCS currently used in this vcpu. For a
|
|
|
|
|
* non-nested (L1) guest, it always points to vmcs01. For a nested
|
2019-06-07 19:00:14 +02:00
|
|
|
* guest (L2), it points to a different VMCS.
|
2018-12-03 13:53:08 -08:00
|
|
|
*/
|
|
|
|
|
struct loaded_vmcs vmcs01;
|
|
|
|
|
struct loaded_vmcs *loaded_vmcs;
|
2019-01-25 07:41:05 -08:00
|
|
|
|
2018-12-03 13:53:08 -08:00
|
|
|
struct msr_autoload {
|
|
|
|
|
struct vmx_msrs guest;
|
|
|
|
|
struct vmx_msrs host;
|
|
|
|
|
} msr_autoload;
|
|
|
|
|
|
2019-11-07 21:14:39 -08:00
|
|
|
struct msr_autostore {
|
|
|
|
|
struct vmx_msrs guest;
|
|
|
|
|
} msr_autostore;
|
|
|
|
|
|
2018-12-03 13:53:08 -08:00
|
|
|
struct {
|
|
|
|
|
int vm86_active;
|
|
|
|
|
ulong save_rflags;
|
|
|
|
|
struct kvm_segment segs[8];
|
|
|
|
|
} rmode;
|
|
|
|
|
struct {
|
|
|
|
|
u32 bitmask; /* 4 bits per segment (1 bit per field) */
|
|
|
|
|
struct kvm_save_segment {
|
|
|
|
|
u16 selector;
|
|
|
|
|
unsigned long base;
|
|
|
|
|
u32 limit;
|
|
|
|
|
u32 ar;
|
|
|
|
|
} seg[8];
|
|
|
|
|
} segment_cache;
|
|
|
|
|
int vpid;
|
|
|
|
|
|
|
|
|
|
/* Support for a guest hypervisor (nested VMX) */
|
|
|
|
|
struct nested_vmx nested;
|
|
|
|
|
|
|
|
|
|
/* Dynamic PLE window. */
|
2019-09-06 10:17:21 +08:00
|
|
|
unsigned int ple_window;
|
2018-12-03 13:53:08 -08:00
|
|
|
bool ple_window_dirty;
|
|
|
|
|
|
|
|
|
|
/* Support for PML */
|
2024-12-19 17:10:33 -05:00
|
|
|
#define PML_LOG_NR_ENTRIES 512
|
|
|
|
|
/* PML is written backwards: this is the first entry written by the CPU */
|
|
|
|
|
#define PML_HEAD_INDEX (PML_LOG_NR_ENTRIES-1)
|
|
|
|
|
|
2018-12-03 13:53:08 -08:00
|
|
|
struct page *pml_pg;
|
|
|
|
|
|
|
|
|
|
/* apic deadline value in host tsc */
|
|
|
|
|
u64 hv_deadline_tsc;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Only bits masked by msr_ia32_feature_control_valid_bits can be set in
|
x86/msr-index: Clean up bit defines for IA32_FEATURE_CONTROL MSR
As pointed out by Boris, the defines for bits in IA32_FEATURE_CONTROL
are quite a mouthful, especially the VMX bits which must differentiate
between enabling VMX inside and outside SMX (TXT) operation. Rename the
MSR and its bit defines to abbreviate FEATURE_CONTROL as FEAT_CTL to
make them a little friendlier on the eyes.
Arguably, the MSR itself should keep the full IA32_FEATURE_CONTROL name
to match Intel's SDM, but a future patch will add a dedicated Kconfig,
file and functions for the MSR. Using the full name for those assets is
rather unwieldy, so bite the bullet and use IA32_FEAT_CTL so that its
nomenclature is consistent throughout the kernel.
Opportunistically, fix a few other annoyances with the defines:
- Relocate the bit defines so that they immediately follow the MSR
define, e.g. aren't mistaken as belonging to MISC_FEATURE_CONTROL.
- Add whitespace around the block of feature control defines to make
it clear they're all related.
- Use BIT() instead of manually encoding the bit shift.
- Use "VMX" instead of "VMXON" to match the SDM.
- Append "_ENABLED" to the LMCE (Local Machine Check Exception) bit to
be consistent with the kernel's verbiage used for all other feature
control bits. Note, the SDM refers to the LMCE bit as LMCE_ON,
likely to differentiate it from IA32_MCG_EXT_CTL.LMCE_EN. Ignore
the (literal) one-off usage of _ON, the SDM is simply "wrong".
Signed-off-by: Sean Christopherson <sean.j.christopherson@intel.com>
Signed-off-by: Borislav Petkov <bp@suse.de>
Link: https://lkml.kernel.org/r/20191221044513.21680-2-sean.j.christopherson@intel.com
2019-12-20 20:44:55 -08:00
|
|
|
* msr_ia32_feature_control. FEAT_CTL_LOCKED is always included
|
2018-12-03 13:53:08 -08:00
|
|
|
* in msr_ia32_feature_control_valid_bits.
|
|
|
|
|
*/
|
|
|
|
|
u64 msr_ia32_feature_control;
|
|
|
|
|
u64 msr_ia32_feature_control_valid_bits;
|
2021-04-12 16:21:40 +12:00
|
|
|
/* SGX Launch Control public key hash */
|
|
|
|
|
u64 msr_ia32_sgxlepubkeyhash[4];
|
2022-05-19 20:35:15 -07:00
|
|
|
u64 msr_ia32_mcu_opt_ctrl;
|
|
|
|
|
bool disable_fb_clear;
|
2021-04-12 16:21:40 +12:00
|
|
|
|
2018-10-24 16:05:12 +08:00
|
|
|
struct pt_desc pt_desc;
|
2021-02-02 09:36:08 -05:00
|
|
|
struct lbr_desc lbr_desc;
|
2020-09-25 16:34:20 +02:00
|
|
|
|
2024-01-22 15:53:18 -08:00
|
|
|
/* ve_info must be page aligned. */
|
|
|
|
|
struct vmx_ve_information *ve_info;
|
2018-12-03 13:53:08 -08:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct kvm_vmx {
|
|
|
|
|
struct kvm kvm;
|
|
|
|
|
|
|
|
|
|
unsigned int tss_addr;
|
|
|
|
|
bool ept_identity_pagetable_done;
|
|
|
|
|
gpa_t ept_identity_map_addr;
|
2022-04-19 23:45:10 +08:00
|
|
|
/* Posted Interrupt Descriptor (PID) table for IPI virtualization */
|
|
|
|
|
u64 *pid_table;
|
2018-12-03 13:53:08 -08:00
|
|
|
};
|
|
|
|
|
|
2025-03-14 14:06:48 -04:00
|
|
|
static __always_inline struct vcpu_vt *to_vt(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
return &(container_of(vcpu, struct vcpu_vmx, vcpu)->vt);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline struct kvm_vcpu *vt_to_vcpu(struct vcpu_vt *vt)
|
|
|
|
|
{
|
|
|
|
|
return &(container_of(vt, struct vcpu_vmx, vt)->vcpu);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline union vmx_exit_reason vmx_get_exit_reason(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
return to_vt(vcpu)->exit_reason;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline unsigned long vmx_get_exit_qual(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
struct vcpu_vt *vt = to_vt(vcpu);
|
|
|
|
|
|
|
|
|
|
if (!kvm_register_test_and_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_1) &&
|
|
|
|
|
!WARN_ON_ONCE(is_td_vcpu(vcpu)))
|
|
|
|
|
vt->exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
|
|
|
|
|
|
|
|
|
|
return vt->exit_qualification;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static __always_inline u32 vmx_get_intr_info(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
struct vcpu_vt *vt = to_vt(vcpu);
|
|
|
|
|
|
|
|
|
|
if (!kvm_register_test_and_mark_available(vcpu, VCPU_EXREG_EXIT_INFO_2) &&
|
|
|
|
|
!WARN_ON_ONCE(is_td_vcpu(vcpu)))
|
|
|
|
|
vt->exit_intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
|
|
|
|
|
|
|
|
|
|
return vt->exit_intr_info;
|
|
|
|
|
}
|
|
|
|
|
|
2025-04-29 08:37:19 -07:00
|
|
|
void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu);
|
2018-12-03 13:53:16 -08:00
|
|
|
int allocate_vpid(void);
|
|
|
|
|
void free_vpid(int vpid);
|
|
|
|
|
void vmx_set_constant_host_state(struct vcpu_vmx *vmx);
|
|
|
|
|
void vmx_prepare_switch_to_guest(struct kvm_vcpu *vcpu);
|
2022-02-24 19:19:16 +00:00
|
|
|
void vmx_set_host_fs_gs(struct vmcs_host_state *host, u16 fs_sel, u16 gs_sel,
|
|
|
|
|
unsigned long fs_base, unsigned long gs_base);
|
2018-12-03 13:53:16 -08:00
|
|
|
int vmx_get_cpl(struct kvm_vcpu *vcpu);
|
KVM: x86: Bypass register cache when querying CPL from kvm_sched_out()
When querying guest CPL to determine if a vCPU was preempted while in
kernel mode, bypass the register cache, i.e. always read SS.AR_BYTES from
the VMCS on Intel CPUs. If the kernel is running with full preemption
enabled, using the register cache in the preemption path can result in
stale and/or uninitialized data being cached in the segment cache.
In particular the following scenario is currently possible:
- vCPU is just created, and the vCPU thread is preempted before
SS.AR_BYTES is written in vmx_vcpu_reset().
- When scheduling out the vCPU task, kvm_arch_vcpu_in_kernel() =>
vmx_get_cpl() reads and caches '0' for SS.AR_BYTES.
- vmx_vcpu_reset() => seg_setup() configures SS.AR_BYTES, but doesn't
invoke vmx_segment_cache_clear() to invalidate the cache.
As a result, KVM retains a stale value in the cache, which can be read,
e.g. via KVM_GET_SREGS. Usually this is not a problem because the VMX
segment cache is reset on each VM-Exit, but if the userspace VMM (e.g KVM
selftests) reads and writes system registers just after the vCPU was
created, _without_ modifying SS.AR_BYTES, userspace will write back the
stale '0' value and ultimately will trigger a VM-Entry failure due to
incorrect SS segment type.
Note, the VM-Enter failure can also be avoided by moving the call to
vmx_segment_cache_clear() until after the vmx_vcpu_reset() initializes all
segments. However, while that change is correct and desirable (and will
come along shortly), it does not address the underlying problem that
accessing KVM's register caches from !task context is generally unsafe.
In addition to fixing the immediate bug, bypassing the cache for this
particular case will allow hardening KVM register caching log to assert
that the caches are accessed only when KVM _knows_ it is safe to do so.
Fixes: de63ad4cf497 ("KVM: X86: implement the logic for spinlock optimization")
Reported-by: Maxim Levitsky <mlevitsk@redhat.com>
Closes: https://lore.kernel.org/all/20240716022014.240960-3-mlevitsk@redhat.com
Reviewed-by: Maxim Levitsky <mlevitsk@redhat.com>
Link: https://lore.kernel.org/r/20241009175002.1118178-2-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>
2024-10-09 10:49:59 -07:00
|
|
|
int vmx_get_cpl_no_cache(struct kvm_vcpu *vcpu);
|
2021-09-13 17:09:54 +03:00
|
|
|
bool vmx_emulation_required(struct kvm_vcpu *vcpu);
|
2018-12-03 13:53:16 -08:00
|
|
|
unsigned long vmx_get_rflags(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vmx_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
|
|
|
|
|
u32 vmx_get_interrupt_shadow(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vmx_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask);
|
2020-10-01 14:29:53 +03:00
|
|
|
int vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer);
|
2018-12-03 13:53:16 -08:00
|
|
|
void vmx_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
|
2020-10-06 18:44:15 -07:00
|
|
|
void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4);
|
2018-12-03 13:53:16 -08:00
|
|
|
void set_cr4_guest_host_mask(struct vcpu_vmx *vmx);
|
|
|
|
|
void ept_save_pdptrs(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
|
2021-07-13 09:33:07 -07:00
|
|
|
void __vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg);
|
KVM: x86: Get active PCID only when writing a CR3 value
Retrieve the active PCID only when writing a guest CR3 value, i.e. don't
get the PCID when using EPT or NPT. The PCID is especially problematic
for EPT as the bits have different meaning, and so the PCID and must be
manually stripped, which is annoying and unnecessary. And on VMX,
getting the active PCID also involves reading the guest's CR3 and
CR4.PCIDE, i.e. may add pointless VMREADs.
Opportunistically rename the pgd/pgd_level params to root_hpa and
root_level to better reflect their new roles. Keep the function names,
as "load the guest PGD" is still accurate/correct.
Last, and probably least, pass root_hpa as a hpa_t/u64 instead of an
unsigned long. The EPTP holds a 64-bit value, even in 32-bit mode, so
in theory EPT could support HIGHMEM for 32-bit KVM. Never mind that
doing so would require changing the MMU page allocators and reworking
the MMU to use kmap().
Signed-off-by: Sean Christopherson <sean.j.christopherson@intel.com>
Signed-off-by: Sean Christopherson <seanjc@google.com>
Message-Id: <20210305183123.3978098-2-seanjc@google.com>
Reviewed-by: Vitaly Kuznetsov <vkuznets@redhat.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-03-05 10:31:13 -08:00
|
|
|
u64 construct_eptp(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level);
|
2020-09-23 11:44:48 -07:00
|
|
|
|
2021-06-22 10:22:44 -07:00
|
|
|
bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu);
|
2021-01-14 22:27:54 -05:00
|
|
|
void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu);
|
2020-04-22 19:25:44 -07:00
|
|
|
bool vmx_nmi_blocked(struct kvm_vcpu *vcpu);
|
2024-06-07 10:26:06 -07:00
|
|
|
bool __vmx_interrupt_blocked(struct kvm_vcpu *vcpu);
|
2020-04-22 19:25:44 -07:00
|
|
|
bool vmx_interrupt_blocked(struct kvm_vcpu *vcpu);
|
2018-12-03 13:53:16 -08:00
|
|
|
bool vmx_get_nmi_mask(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vmx_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked);
|
|
|
|
|
void vmx_set_virtual_apic_mode(struct kvm_vcpu *vcpu);
|
2020-09-23 11:04:06 -07:00
|
|
|
struct vmx_uret_msr *vmx_find_uret_msr(struct vcpu_vmx *vmx, u32 msr);
|
2020-09-25 16:34:18 +02:00
|
|
|
void pt_update_intercept_for_msr(struct kvm_vcpu *vcpu);
|
2019-05-20 12:27:47 +08:00
|
|
|
void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp);
|
2022-06-14 23:16:13 +02:00
|
|
|
void vmx_spec_ctrl_restore_host(struct vcpu_vmx *vmx, unsigned int flags);
|
2022-06-14 23:16:12 +02:00
|
|
|
unsigned int __vmx_vcpu_run_flags(struct vcpu_vmx *vmx);
|
|
|
|
|
bool __vmx_vcpu_run(struct vcpu_vmx *vmx, unsigned long *regs,
|
|
|
|
|
unsigned int flags);
|
2020-09-23 11:03:57 -07:00
|
|
|
int vmx_find_loadstore_msr_slot(struct vmx_msrs *m, u32 msr);
|
2020-08-20 16:05:45 -07:00
|
|
|
void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu);
|
KVM: VMX: Invert the inlining of MSR interception helpers
Invert the inline declarations of the MSR interception helpers between
the wrapper, vmx_set_intercept_for_msr(), and the core implementations,
vmx_{dis,en}able_intercept_for_msr(). Letting the compiler _not_
inline the implementation reduces KVM's code footprint by ~3k bytes.
Back when the helpers were added in commit 904e14fb7cb9 ("KVM: VMX: make
MSR bitmaps per-VCPU"), both the wrapper and the implementations were
__always_inline because the end code distilled down to a few conditionals
and a bit operation. Today, the implementations involve a variety of
checks and bit ops in order to support userspace MSR filtering.
Furthermore, the vast majority of calls to manipulate MSR interception
are not performance sensitive, e.g. vCPU creation and x2APIC toggling.
On the other hand, the one path that is performance sensitive, dynamic
LBR passthrough, uses the wrappers, i.e. is largely untouched by
inverting the inlining.
In short, forcing the low level MSR interception code to be inlined no
longer makes sense.
No functional change intended.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Message-Id: <20210423221912.3857243-1-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-04-23 15:19:12 -07:00
|
|
|
|
2025-06-12 01:19:46 -07:00
|
|
|
void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type, bool set);
|
|
|
|
|
|
|
|
|
|
static inline void vmx_disable_intercept_for_msr(struct kvm_vcpu *vcpu,
|
|
|
|
|
u32 msr, int type)
|
|
|
|
|
{
|
|
|
|
|
vmx_set_intercept_for_msr(vcpu, msr, type, false);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu,
|
|
|
|
|
u32 msr, int type)
|
|
|
|
|
{
|
|
|
|
|
vmx_set_intercept_for_msr(vcpu, msr, type, true);
|
|
|
|
|
}
|
KVM: VMX: Invert the inlining of MSR interception helpers
Invert the inline declarations of the MSR interception helpers between
the wrapper, vmx_set_intercept_for_msr(), and the core implementations,
vmx_{dis,en}able_intercept_for_msr(). Letting the compiler _not_
inline the implementation reduces KVM's code footprint by ~3k bytes.
Back when the helpers were added in commit 904e14fb7cb9 ("KVM: VMX: make
MSR bitmaps per-VCPU"), both the wrapper and the implementations were
__always_inline because the end code distilled down to a few conditionals
and a bit operation. Today, the implementations involve a variety of
checks and bit ops in order to support userspace MSR filtering.
Furthermore, the vast majority of calls to manipulate MSR interception
are not performance sensitive, e.g. vCPU creation and x2APIC toggling.
On the other hand, the one path that is performance sensitive, dynamic
LBR passthrough, uses the wrappers, i.e. is largely untouched by
inverting the inlining.
In short, forcing the low level MSR interception code to be inlined no
longer makes sense.
No functional change intended.
Signed-off-by: Sean Christopherson <seanjc@google.com>
Message-Id: <20210423221912.3857243-1-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2021-04-23 15:19:12 -07:00
|
|
|
|
2021-05-26 19:44:13 +01:00
|
|
|
u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu);
|
|
|
|
|
u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu);
|
|
|
|
|
|
2023-09-13 20:42:20 +08:00
|
|
|
gva_t vmx_get_untagged_addr(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags);
|
|
|
|
|
|
2021-02-12 16:50:12 -08:00
|
|
|
void vmx_update_cpu_dirty_logging(struct kvm_vcpu *vcpu);
|
2018-12-03 13:53:16 -08:00
|
|
|
|
2025-06-10 16:20:08 -07:00
|
|
|
u64 vmx_get_supported_debugctl(struct kvm_vcpu *vcpu, bool host_initiated);
|
|
|
|
|
bool vmx_is_valid_debugctl(struct kvm_vcpu *vcpu, u64 data, bool host_initiated);
|
|
|
|
|
|
2025-06-26 09:14:20 -07:00
|
|
|
#define VMX_HOST_OWNED_DEBUGCTL_BITS (DEBUGCTLMSR_FREEZE_IN_SMM)
|
|
|
|
|
|
2025-06-10 16:20:09 -07:00
|
|
|
static inline void vmx_guest_debugctl_write(struct kvm_vcpu *vcpu, u64 val)
|
|
|
|
|
{
|
2025-06-26 09:14:20 -07:00
|
|
|
WARN_ON_ONCE(val & VMX_HOST_OWNED_DEBUGCTL_BITS);
|
KVM: VMX: Preserve host's DEBUGCTLMSR_FREEZE_IN_SMM while running the guest
Set/clear DEBUGCTLMSR_FREEZE_IN_SMM in GUEST_IA32_DEBUGCTL based on the
host's pre-VM-Enter value, i.e. preserve the host's FREEZE_IN_SMM setting
while running the guest. When running with the "default treatment of SMIs"
in effect (the only mode KVM supports), SMIs do not generate a VM-Exit that
is visible to host (non-SMM) software, and instead transitions directly
from VMX non-root to SMM. And critically, DEBUGCTL isn't context switched
by hardware on SMI or RSM, i.e. SMM will run with whatever value was
resident in hardware at the time of the SMI.
Failure to preserve FREEZE_IN_SMM results in the PMU unexpectedly counting
events while the CPU is executing in SMM, which can pollute profiling and
potentially leak information into the guest.
Check for changes in FREEZE_IN_SMM prior to every entry into KVM's inner
run loop, as the bit can be toggled in IRQ context via IPI callback (SMP
function call), by way of /sys/devices/cpu/freeze_on_smi.
Add a field in kvm_x86_ops to communicate which DEBUGCTL bits need to be
preserved, as FREEZE_IN_SMM is only supported and defined for Intel CPUs,
i.e. explicitly checking FREEZE_IN_SMM in common x86 is at best weird, and
at worst could lead to undesirable behavior in the future if AMD CPUs ever
happened to pick up a collision with the bit.
Exempt TDX vCPUs, i.e. protected guests, from the check, as the TDX Module
owns and controls GUEST_IA32_DEBUGCTL.
WARN in SVM if KVM_RUN_LOAD_DEBUGCTL is set, mostly to document that the
lack of handling isn't a KVM bug (TDX already WARNs on any run_flag).
Lastly, explicitly reload GUEST_IA32_DEBUGCTL on a VM-Fail that is missed
by KVM but detected by hardware, i.e. in nested_vmx_restore_host_state().
Doing so avoids the need to track host_debugctl on a per-VMCS basis, as
GUEST_IA32_DEBUGCTL is unconditionally written by prepare_vmcs02() and
load_vmcs12_host_state(). For the VM-Fail case, even though KVM won't
have actually entered the guest, vcpu_enter_guest() will have run with
vmcs02 active and thus could result in vmcs01 being run with a stale value.
Cc: stable@vger.kernel.org
Signed-off-by: Maxim Levitsky <mlevitsk@redhat.com>
Co-developed-by: Sean Christopherson <seanjc@google.com>
Link: https://lore.kernel.org/r/20250610232010.162191-9-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>
2025-06-10 16:20:10 -07:00
|
|
|
|
2025-06-26 09:14:20 -07:00
|
|
|
val |= vcpu->arch.host_debugctl & VMX_HOST_OWNED_DEBUGCTL_BITS;
|
2025-06-10 16:20:09 -07:00
|
|
|
vmcs_write64(GUEST_IA32_DEBUGCTL, val);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline u64 vmx_guest_debugctl_read(void)
|
|
|
|
|
{
|
2025-06-26 09:14:20 -07:00
|
|
|
return vmcs_read64(GUEST_IA32_DEBUGCTL) & ~VMX_HOST_OWNED_DEBUGCTL_BITS;
|
KVM: VMX: Preserve host's DEBUGCTLMSR_FREEZE_IN_SMM while running the guest
Set/clear DEBUGCTLMSR_FREEZE_IN_SMM in GUEST_IA32_DEBUGCTL based on the
host's pre-VM-Enter value, i.e. preserve the host's FREEZE_IN_SMM setting
while running the guest. When running with the "default treatment of SMIs"
in effect (the only mode KVM supports), SMIs do not generate a VM-Exit that
is visible to host (non-SMM) software, and instead transitions directly
from VMX non-root to SMM. And critically, DEBUGCTL isn't context switched
by hardware on SMI or RSM, i.e. SMM will run with whatever value was
resident in hardware at the time of the SMI.
Failure to preserve FREEZE_IN_SMM results in the PMU unexpectedly counting
events while the CPU is executing in SMM, which can pollute profiling and
potentially leak information into the guest.
Check for changes in FREEZE_IN_SMM prior to every entry into KVM's inner
run loop, as the bit can be toggled in IRQ context via IPI callback (SMP
function call), by way of /sys/devices/cpu/freeze_on_smi.
Add a field in kvm_x86_ops to communicate which DEBUGCTL bits need to be
preserved, as FREEZE_IN_SMM is only supported and defined for Intel CPUs,
i.e. explicitly checking FREEZE_IN_SMM in common x86 is at best weird, and
at worst could lead to undesirable behavior in the future if AMD CPUs ever
happened to pick up a collision with the bit.
Exempt TDX vCPUs, i.e. protected guests, from the check, as the TDX Module
owns and controls GUEST_IA32_DEBUGCTL.
WARN in SVM if KVM_RUN_LOAD_DEBUGCTL is set, mostly to document that the
lack of handling isn't a KVM bug (TDX already WARNs on any run_flag).
Lastly, explicitly reload GUEST_IA32_DEBUGCTL on a VM-Fail that is missed
by KVM but detected by hardware, i.e. in nested_vmx_restore_host_state().
Doing so avoids the need to track host_debugctl on a per-VMCS basis, as
GUEST_IA32_DEBUGCTL is unconditionally written by prepare_vmcs02() and
load_vmcs12_host_state(). For the VM-Fail case, even though KVM won't
have actually entered the guest, vcpu_enter_guest() will have run with
vmcs02 active and thus could result in vmcs01 being run with a stale value.
Cc: stable@vger.kernel.org
Signed-off-by: Maxim Levitsky <mlevitsk@redhat.com>
Co-developed-by: Sean Christopherson <seanjc@google.com>
Link: https://lore.kernel.org/r/20250610232010.162191-9-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>
2025-06-10 16:20:10 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void vmx_reload_guest_debugctl(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
u64 val = vmcs_read64(GUEST_IA32_DEBUGCTL);
|
|
|
|
|
|
2025-06-26 09:14:20 -07:00
|
|
|
if (!((val ^ vcpu->arch.host_debugctl) & VMX_HOST_OWNED_DEBUGCTL_BITS))
|
KVM: VMX: Preserve host's DEBUGCTLMSR_FREEZE_IN_SMM while running the guest
Set/clear DEBUGCTLMSR_FREEZE_IN_SMM in GUEST_IA32_DEBUGCTL based on the
host's pre-VM-Enter value, i.e. preserve the host's FREEZE_IN_SMM setting
while running the guest. When running with the "default treatment of SMIs"
in effect (the only mode KVM supports), SMIs do not generate a VM-Exit that
is visible to host (non-SMM) software, and instead transitions directly
from VMX non-root to SMM. And critically, DEBUGCTL isn't context switched
by hardware on SMI or RSM, i.e. SMM will run with whatever value was
resident in hardware at the time of the SMI.
Failure to preserve FREEZE_IN_SMM results in the PMU unexpectedly counting
events while the CPU is executing in SMM, which can pollute profiling and
potentially leak information into the guest.
Check for changes in FREEZE_IN_SMM prior to every entry into KVM's inner
run loop, as the bit can be toggled in IRQ context via IPI callback (SMP
function call), by way of /sys/devices/cpu/freeze_on_smi.
Add a field in kvm_x86_ops to communicate which DEBUGCTL bits need to be
preserved, as FREEZE_IN_SMM is only supported and defined for Intel CPUs,
i.e. explicitly checking FREEZE_IN_SMM in common x86 is at best weird, and
at worst could lead to undesirable behavior in the future if AMD CPUs ever
happened to pick up a collision with the bit.
Exempt TDX vCPUs, i.e. protected guests, from the check, as the TDX Module
owns and controls GUEST_IA32_DEBUGCTL.
WARN in SVM if KVM_RUN_LOAD_DEBUGCTL is set, mostly to document that the
lack of handling isn't a KVM bug (TDX already WARNs on any run_flag).
Lastly, explicitly reload GUEST_IA32_DEBUGCTL on a VM-Fail that is missed
by KVM but detected by hardware, i.e. in nested_vmx_restore_host_state().
Doing so avoids the need to track host_debugctl on a per-VMCS basis, as
GUEST_IA32_DEBUGCTL is unconditionally written by prepare_vmcs02() and
load_vmcs12_host_state(). For the VM-Fail case, even though KVM won't
have actually entered the guest, vcpu_enter_guest() will have run with
vmcs02 active and thus could result in vmcs01 being run with a stale value.
Cc: stable@vger.kernel.org
Signed-off-by: Maxim Levitsky <mlevitsk@redhat.com>
Co-developed-by: Sean Christopherson <seanjc@google.com>
Link: https://lore.kernel.org/r/20250610232010.162191-9-seanjc@google.com
Signed-off-by: Sean Christopherson <seanjc@google.com>
2025-06-10 16:20:10 -07:00
|
|
|
return;
|
|
|
|
|
|
2025-06-26 09:14:20 -07:00
|
|
|
vmx_guest_debugctl_write(vcpu, val & ~VMX_HOST_OWNED_DEBUGCTL_BITS);
|
2025-06-10 16:20:09 -07:00
|
|
|
}
|
|
|
|
|
|
2021-11-09 01:30:46 +00:00
|
|
|
/*
|
|
|
|
|
* Note, early Intel manuals have the write-low and read-high bitmap offsets
|
|
|
|
|
* the wrong way round. The bitmaps control MSRs 0x00000000-0x00001fff and
|
|
|
|
|
* 0xc0000000-0xc0001fff. The former (low) uses bytes 0-0x3ff for reads and
|
|
|
|
|
* 0x800-0xbff for writes. The latter (high) uses 0x400-0x7ff for reads and
|
|
|
|
|
* 0xc00-0xfff for writes. MSRs not covered by either of the ranges always
|
|
|
|
|
* VM-Exit.
|
|
|
|
|
*/
|
|
|
|
|
#define __BUILD_VMX_MSR_BITMAP_HELPER(rtype, action, bitop, access, base) \
|
|
|
|
|
static inline rtype vmx_##action##_msr_bitmap_##access(unsigned long *bitmap, \
|
|
|
|
|
u32 msr) \
|
|
|
|
|
{ \
|
|
|
|
|
int f = sizeof(unsigned long); \
|
|
|
|
|
\
|
|
|
|
|
if (msr <= 0x1fff) \
|
|
|
|
|
return bitop##_bit(msr, bitmap + base / f); \
|
|
|
|
|
else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) \
|
|
|
|
|
return bitop##_bit(msr & 0x1fff, bitmap + (base + 0x400) / f); \
|
|
|
|
|
return (rtype)true; \
|
2021-11-09 01:30:45 +00:00
|
|
|
}
|
2021-11-09 01:30:46 +00:00
|
|
|
#define BUILD_VMX_MSR_BITMAP_HELPERS(ret_type, action, bitop) \
|
|
|
|
|
__BUILD_VMX_MSR_BITMAP_HELPER(ret_type, action, bitop, read, 0x0) \
|
|
|
|
|
__BUILD_VMX_MSR_BITMAP_HELPER(ret_type, action, bitop, write, 0x800)
|
2021-11-09 01:30:45 +00:00
|
|
|
|
2021-11-09 01:30:46 +00:00
|
|
|
BUILD_VMX_MSR_BITMAP_HELPERS(bool, test, test)
|
|
|
|
|
BUILD_VMX_MSR_BITMAP_HELPERS(void, clear, __clear)
|
|
|
|
|
BUILD_VMX_MSR_BITMAP_HELPERS(void, set, __set)
|
2021-11-09 01:30:45 +00:00
|
|
|
|
2018-12-03 13:53:07 -08:00
|
|
|
static inline u8 vmx_get_rvi(void)
|
|
|
|
|
{
|
|
|
|
|
return vmcs_read16(GUEST_INTR_STATUS) & 0xff;
|
|
|
|
|
}
|
|
|
|
|
|
2022-08-30 15:37:27 +02:00
|
|
|
#define __KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS \
|
|
|
|
|
(VM_ENTRY_LOAD_DEBUG_CONTROLS)
|
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
|
#define KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS \
|
|
|
|
|
(__KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS | \
|
|
|
|
|
VM_ENTRY_IA32E_MODE)
|
|
|
|
|
#else
|
|
|
|
|
#define KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS \
|
|
|
|
|
__KVM_REQUIRED_VMX_VM_ENTRY_CONTROLS
|
|
|
|
|
#endif
|
|
|
|
|
#define KVM_OPTIONAL_VMX_VM_ENTRY_CONTROLS \
|
|
|
|
|
(VM_ENTRY_LOAD_IA32_PERF_GLOBAL_CTRL | \
|
|
|
|
|
VM_ENTRY_LOAD_IA32_PAT | \
|
|
|
|
|
VM_ENTRY_LOAD_IA32_EFER | \
|
|
|
|
|
VM_ENTRY_LOAD_BNDCFGS | \
|
|
|
|
|
VM_ENTRY_PT_CONCEAL_PIP | \
|
|
|
|
|
VM_ENTRY_LOAD_IA32_RTIT_CTL)
|
|
|
|
|
|
|
|
|
|
#define __KVM_REQUIRED_VMX_VM_EXIT_CONTROLS \
|
|
|
|
|
(VM_EXIT_SAVE_DEBUG_CONTROLS | \
|
|
|
|
|
VM_EXIT_ACK_INTR_ON_EXIT)
|
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
|
#define KVM_REQUIRED_VMX_VM_EXIT_CONTROLS \
|
|
|
|
|
(__KVM_REQUIRED_VMX_VM_EXIT_CONTROLS | \
|
|
|
|
|
VM_EXIT_HOST_ADDR_SPACE_SIZE)
|
|
|
|
|
#else
|
|
|
|
|
#define KVM_REQUIRED_VMX_VM_EXIT_CONTROLS \
|
|
|
|
|
__KVM_REQUIRED_VMX_VM_EXIT_CONTROLS
|
|
|
|
|
#endif
|
|
|
|
|
#define KVM_OPTIONAL_VMX_VM_EXIT_CONTROLS \
|
|
|
|
|
(VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL | \
|
2022-08-30 15:37:29 +02:00
|
|
|
VM_EXIT_SAVE_IA32_PAT | \
|
2022-08-30 15:37:27 +02:00
|
|
|
VM_EXIT_LOAD_IA32_PAT | \
|
2022-08-30 15:37:29 +02:00
|
|
|
VM_EXIT_SAVE_IA32_EFER | \
|
|
|
|
|
VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | \
|
2022-08-30 15:37:27 +02:00
|
|
|
VM_EXIT_LOAD_IA32_EFER | \
|
|
|
|
|
VM_EXIT_CLEAR_BNDCFGS | \
|
|
|
|
|
VM_EXIT_PT_CONCEAL_PIP | \
|
|
|
|
|
VM_EXIT_CLEAR_IA32_RTIT_CTL)
|
|
|
|
|
|
|
|
|
|
#define KVM_REQUIRED_VMX_PIN_BASED_VM_EXEC_CONTROL \
|
|
|
|
|
(PIN_BASED_EXT_INTR_MASK | \
|
|
|
|
|
PIN_BASED_NMI_EXITING)
|
|
|
|
|
#define KVM_OPTIONAL_VMX_PIN_BASED_VM_EXEC_CONTROL \
|
|
|
|
|
(PIN_BASED_VIRTUAL_NMIS | \
|
|
|
|
|
PIN_BASED_POSTED_INTR | \
|
|
|
|
|
PIN_BASED_VMX_PREEMPTION_TIMER)
|
|
|
|
|
|
|
|
|
|
#define __KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL \
|
|
|
|
|
(CPU_BASED_HLT_EXITING | \
|
|
|
|
|
CPU_BASED_CR3_LOAD_EXITING | \
|
|
|
|
|
CPU_BASED_CR3_STORE_EXITING | \
|
|
|
|
|
CPU_BASED_UNCOND_IO_EXITING | \
|
|
|
|
|
CPU_BASED_MOV_DR_EXITING | \
|
|
|
|
|
CPU_BASED_USE_TSC_OFFSETTING | \
|
|
|
|
|
CPU_BASED_MWAIT_EXITING | \
|
|
|
|
|
CPU_BASED_MONITOR_EXITING | \
|
|
|
|
|
CPU_BASED_INVLPG_EXITING | \
|
|
|
|
|
CPU_BASED_RDPMC_EXITING | \
|
|
|
|
|
CPU_BASED_INTR_WINDOW_EXITING)
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_X86_64
|
|
|
|
|
#define KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL \
|
|
|
|
|
(__KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL | \
|
|
|
|
|
CPU_BASED_CR8_LOAD_EXITING | \
|
|
|
|
|
CPU_BASED_CR8_STORE_EXITING)
|
|
|
|
|
#else
|
|
|
|
|
#define KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL \
|
|
|
|
|
__KVM_REQUIRED_VMX_CPU_BASED_VM_EXEC_CONTROL
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#define KVM_OPTIONAL_VMX_CPU_BASED_VM_EXEC_CONTROL \
|
2022-08-30 15:37:30 +02:00
|
|
|
(CPU_BASED_RDTSC_EXITING | \
|
|
|
|
|
CPU_BASED_TPR_SHADOW | \
|
|
|
|
|
CPU_BASED_USE_IO_BITMAPS | \
|
|
|
|
|
CPU_BASED_MONITOR_TRAP_FLAG | \
|
2022-08-30 15:37:27 +02:00
|
|
|
CPU_BASED_USE_MSR_BITMAPS | \
|
|
|
|
|
CPU_BASED_NMI_WINDOW_EXITING | \
|
2022-08-30 15:37:30 +02:00
|
|
|
CPU_BASED_PAUSE_EXITING | \
|
2022-08-30 15:37:27 +02:00
|
|
|
CPU_BASED_ACTIVATE_SECONDARY_CONTROLS | \
|
|
|
|
|
CPU_BASED_ACTIVATE_TERTIARY_CONTROLS)
|
|
|
|
|
|
|
|
|
|
#define KVM_REQUIRED_VMX_SECONDARY_VM_EXEC_CONTROL 0
|
|
|
|
|
#define KVM_OPTIONAL_VMX_SECONDARY_VM_EXEC_CONTROL \
|
|
|
|
|
(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES | \
|
|
|
|
|
SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE | \
|
|
|
|
|
SECONDARY_EXEC_WBINVD_EXITING | \
|
|
|
|
|
SECONDARY_EXEC_ENABLE_VPID | \
|
|
|
|
|
SECONDARY_EXEC_ENABLE_EPT | \
|
|
|
|
|
SECONDARY_EXEC_UNRESTRICTED_GUEST | \
|
|
|
|
|
SECONDARY_EXEC_PAUSE_LOOP_EXITING | \
|
|
|
|
|
SECONDARY_EXEC_DESC | \
|
|
|
|
|
SECONDARY_EXEC_ENABLE_RDTSCP | \
|
|
|
|
|
SECONDARY_EXEC_ENABLE_INVPCID | \
|
|
|
|
|
SECONDARY_EXEC_APIC_REGISTER_VIRT | \
|
|
|
|
|
SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | \
|
|
|
|
|
SECONDARY_EXEC_SHADOW_VMCS | \
|
2023-08-15 13:36:43 -07:00
|
|
|
SECONDARY_EXEC_ENABLE_XSAVES | \
|
2022-08-30 15:37:27 +02:00
|
|
|
SECONDARY_EXEC_RDSEED_EXITING | \
|
|
|
|
|
SECONDARY_EXEC_RDRAND_EXITING | \
|
|
|
|
|
SECONDARY_EXEC_ENABLE_PML | \
|
|
|
|
|
SECONDARY_EXEC_TSC_SCALING | \
|
|
|
|
|
SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | \
|
|
|
|
|
SECONDARY_EXEC_PT_USE_GPA | \
|
|
|
|
|
SECONDARY_EXEC_PT_CONCEAL_VMX | \
|
|
|
|
|
SECONDARY_EXEC_ENABLE_VMFUNC | \
|
|
|
|
|
SECONDARY_EXEC_BUS_LOCK_DETECTION | \
|
|
|
|
|
SECONDARY_EXEC_NOTIFY_VM_EXITING | \
|
2024-01-22 15:53:18 -08:00
|
|
|
SECONDARY_EXEC_ENCLS_EXITING | \
|
|
|
|
|
SECONDARY_EXEC_EPT_VIOLATION_VE)
|
2022-08-30 15:37:27 +02:00
|
|
|
|
|
|
|
|
#define KVM_REQUIRED_VMX_TERTIARY_VM_EXEC_CONTROL 0
|
|
|
|
|
#define KVM_OPTIONAL_VMX_TERTIARY_VM_EXEC_CONTROL \
|
|
|
|
|
(TERTIARY_EXEC_IPI_VIRT)
|
|
|
|
|
|
|
|
|
|
#define BUILD_CONTROLS_SHADOW(lname, uname, bits) \
|
|
|
|
|
static inline void lname##_controls_set(struct vcpu_vmx *vmx, u##bits val) \
|
|
|
|
|
{ \
|
|
|
|
|
if (vmx->loaded_vmcs->controls_shadow.lname != val) { \
|
|
|
|
|
vmcs_write##bits(uname, val); \
|
|
|
|
|
vmx->loaded_vmcs->controls_shadow.lname = val; \
|
|
|
|
|
} \
|
|
|
|
|
} \
|
|
|
|
|
static inline u##bits __##lname##_controls_get(struct loaded_vmcs *vmcs) \
|
|
|
|
|
{ \
|
|
|
|
|
return vmcs->controls_shadow.lname; \
|
|
|
|
|
} \
|
|
|
|
|
static inline u##bits lname##_controls_get(struct vcpu_vmx *vmx) \
|
|
|
|
|
{ \
|
|
|
|
|
return __##lname##_controls_get(vmx->loaded_vmcs); \
|
|
|
|
|
} \
|
|
|
|
|
static __always_inline void lname##_controls_setbit(struct vcpu_vmx *vmx, u##bits val) \
|
|
|
|
|
{ \
|
|
|
|
|
BUILD_BUG_ON(!(val & (KVM_REQUIRED_VMX_##uname | KVM_OPTIONAL_VMX_##uname))); \
|
|
|
|
|
lname##_controls_set(vmx, lname##_controls_get(vmx) | val); \
|
|
|
|
|
} \
|
|
|
|
|
static __always_inline void lname##_controls_clearbit(struct vcpu_vmx *vmx, u##bits val) \
|
|
|
|
|
{ \
|
|
|
|
|
BUILD_BUG_ON(!(val & (KVM_REQUIRED_VMX_##uname | KVM_OPTIONAL_VMX_##uname))); \
|
|
|
|
|
lname##_controls_set(vmx, lname##_controls_get(vmx) & ~val); \
|
2018-12-03 13:53:07 -08:00
|
|
|
}
|
2022-04-19 23:33:18 +08:00
|
|
|
BUILD_CONTROLS_SHADOW(vm_entry, VM_ENTRY_CONTROLS, 32)
|
|
|
|
|
BUILD_CONTROLS_SHADOW(vm_exit, VM_EXIT_CONTROLS, 32)
|
|
|
|
|
BUILD_CONTROLS_SHADOW(pin, PIN_BASED_VM_EXEC_CONTROL, 32)
|
|
|
|
|
BUILD_CONTROLS_SHADOW(exec, CPU_BASED_VM_EXEC_CONTROL, 32)
|
|
|
|
|
BUILD_CONTROLS_SHADOW(secondary_exec, SECONDARY_VM_EXEC_CONTROL, 32)
|
2022-04-19 23:34:00 +08:00
|
|
|
BUILD_CONTROLS_SHADOW(tertiary_exec, TERTIARY_VM_EXEC_CONTROL, 64)
|
2018-12-03 13:53:07 -08:00
|
|
|
|
2021-11-26 07:00:15 -05:00
|
|
|
/*
|
|
|
|
|
* VMX_REGS_LAZY_LOAD_SET - The set of registers that will be updated in the
|
|
|
|
|
* cache on demand. Other registers not listed here are synced to
|
|
|
|
|
* the cache immediately after VM-Exit.
|
|
|
|
|
*/
|
|
|
|
|
#define VMX_REGS_LAZY_LOAD_SET ((1 << VCPU_REGS_RIP) | \
|
|
|
|
|
(1 << VCPU_REGS_RSP) | \
|
|
|
|
|
(1 << VCPU_EXREG_RFLAGS) | \
|
|
|
|
|
(1 << VCPU_EXREG_PDPTR) | \
|
|
|
|
|
(1 << VCPU_EXREG_SEGMENTS) | \
|
|
|
|
|
(1 << VCPU_EXREG_CR0) | \
|
|
|
|
|
(1 << VCPU_EXREG_CR3) | \
|
|
|
|
|
(1 << VCPU_EXREG_CR4) | \
|
|
|
|
|
(1 << VCPU_EXREG_EXIT_INFO_1) | \
|
|
|
|
|
(1 << VCPU_EXREG_EXIT_INFO_2))
|
KVM: nVMX: Reset register cache (available and dirty masks) on VMCS switch
Reset the per-vCPU available and dirty register masks when switching
between vmcs01 and vmcs02, as the masks track state relative to the
current VMCS. The stale masks don't cause problems in the current code
base because the registers are either unconditionally written on nested
transitions or, in the case of segment registers, have an additional
tracker that is manually reset.
Note, by dropping (previously implicitly, now explicitly) the dirty mask
when switching the active VMCS, KVM is technically losing writes to the
associated fields. But, the only regs that can be dirtied (RIP, RSP and
PDPTRs) are unconditionally written on nested transitions, e.g. explicit
writeback is a waste of cycles, and a WARN_ON would be rather pointless.
Signed-off-by: Sean Christopherson <sean.j.christopherson@intel.com>
Message-Id: <20200415203454.8296-3-sean.j.christopherson@intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2020-04-15 13:34:51 -07:00
|
|
|
|
2023-03-22 02:37:31 +01:00
|
|
|
static inline unsigned long vmx_l1_guest_owned_cr0_bits(void)
|
|
|
|
|
{
|
|
|
|
|
unsigned long bits = KVM_POSSIBLE_CR0_GUEST_BITS;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* CR0.WP needs to be intercepted when KVM is shadowing legacy paging
|
|
|
|
|
* in order to construct shadow PTEs with the correct protections.
|
|
|
|
|
* Note! CR0.WP technically can be passed through to the guest if
|
|
|
|
|
* paging is disabled, but checking CR0.PG would generate a cyclical
|
|
|
|
|
* dependency of sorts due to forcing the caller to ensure CR0 holds
|
|
|
|
|
* the correct value prior to determining which CR0 bits can be owned
|
|
|
|
|
* by L1. Keep it simple and limit the optimization to EPT.
|
|
|
|
|
*/
|
|
|
|
|
if (!enable_ept)
|
|
|
|
|
bits &= ~X86_CR0_WP;
|
|
|
|
|
return bits;
|
|
|
|
|
}
|
|
|
|
|
|
2022-12-13 06:09:09 +00:00
|
|
|
static __always_inline struct kvm_vmx *to_kvm_vmx(struct kvm *kvm)
|
2018-12-03 13:53:08 -08:00
|
|
|
{
|
|
|
|
|
return container_of(kvm, struct kvm_vmx, kvm);
|
|
|
|
|
}
|
|
|
|
|
|
2022-12-13 06:09:09 +00:00
|
|
|
static __always_inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
|
2018-12-03 13:53:08 -08:00
|
|
|
{
|
|
|
|
|
return container_of(vcpu, struct vcpu_vmx, vcpu);
|
|
|
|
|
}
|
|
|
|
|
|
2022-07-27 23:34:24 +00:00
|
|
|
void intel_pmu_cross_mapped_check(struct kvm_pmu *pmu);
|
2022-07-27 23:34:23 +00:00
|
|
|
int intel_pmu_create_guest_lbr_event(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vmx_passthrough_lbr_msrs(struct kvm_vcpu *vcpu);
|
|
|
|
|
|
2019-02-11 11:02:52 -08:00
|
|
|
struct vmcs *alloc_vmcs_cpu(bool shadow, int cpu, gfp_t flags);
|
2018-12-03 13:53:07 -08:00
|
|
|
void free_vmcs(struct vmcs *vmcs);
|
|
|
|
|
int alloc_loaded_vmcs(struct loaded_vmcs *loaded_vmcs);
|
|
|
|
|
void free_loaded_vmcs(struct loaded_vmcs *loaded_vmcs);
|
|
|
|
|
void loaded_vmcs_clear(struct loaded_vmcs *loaded_vmcs);
|
|
|
|
|
|
|
|
|
|
static inline struct vmcs *alloc_vmcs(bool shadow)
|
|
|
|
|
{
|
2019-02-11 11:02:52 -08:00
|
|
|
return alloc_vmcs_cpu(shadow, raw_smp_processor_id(),
|
|
|
|
|
GFP_KERNEL_ACCOUNT);
|
2018-12-03 13:53:07 -08:00
|
|
|
}
|
|
|
|
|
|
2019-07-16 14:55:50 +08:00
|
|
|
static inline bool vmx_has_waitpkg(struct vcpu_vmx *vmx)
|
|
|
|
|
{
|
2021-08-10 10:19:49 -07:00
|
|
|
return secondary_exec_controls_get(vmx) &
|
2019-07-16 14:55:50 +08:00
|
|
|
SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE;
|
|
|
|
|
}
|
|
|
|
|
|
2020-07-10 17:48:08 +02:00
|
|
|
static inline bool vmx_need_pf_intercept(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
2020-09-03 16:11:22 +02:00
|
|
|
if (!enable_ept)
|
|
|
|
|
return true;
|
|
|
|
|
|
2024-03-19 11:11:11 +08:00
|
|
|
return allow_smaller_maxphyaddr &&
|
2024-04-23 15:15:21 -07:00
|
|
|
cpuid_maxphyaddr(vcpu) < kvm_host.maxphyaddr;
|
2020-07-10 17:48:08 +02:00
|
|
|
}
|
|
|
|
|
|
2020-09-21 08:10:25 +00:00
|
|
|
static inline bool is_unrestricted_guest(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
return enable_unrestricted_guest && (!is_guest_mode(vcpu) ||
|
|
|
|
|
(secondary_exec_controls_get(to_vmx(vcpu)) &
|
|
|
|
|
SECONDARY_EXEC_UNRESTRICTED_GUEST));
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-23 11:44:48 -07:00
|
|
|
bool __vmx_guest_state_valid(struct kvm_vcpu *vcpu);
|
|
|
|
|
static inline bool vmx_guest_state_valid(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
return is_unrestricted_guest(vcpu) || __vmx_guest_state_valid(vcpu);
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-18 12:08:40 +00:00
|
|
|
void dump_vmcs(struct kvm_vcpu *vcpu);
|
2019-04-15 15:16:17 +02:00
|
|
|
|
2021-11-09 17:44:25 +00:00
|
|
|
static inline int vmx_get_instr_info_reg2(u32 vmx_instr_info)
|
|
|
|
|
{
|
|
|
|
|
return (vmx_instr_info >> 28) & 0xf;
|
|
|
|
|
}
|
|
|
|
|
|
2022-04-19 23:45:10 +08:00
|
|
|
static inline bool vmx_can_use_ipiv(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
return lapic_in_kernel(vcpu) && enable_ipiv;
|
|
|
|
|
}
|
|
|
|
|
|
2024-07-25 13:52:31 -04:00
|
|
|
static inline void vmx_segment_cache_clear(struct vcpu_vmx *vmx)
|
|
|
|
|
{
|
|
|
|
|
vmx->segment_cache.bitmask = 0;
|
|
|
|
|
}
|
|
|
|
|
|
KVM: VMX: Refactor VMX module init/exit functions
Add vt_init() and vt_exit() as the new module init/exit functions and
refactor existing vmx_init()/vmx_exit() as helper to make room for TDX
specific initialization and teardown.
To support TDX, KVM will need to enable TDX during KVM module loading
time. Enabling TDX requires enabling hardware virtualization first so
that all online CPUs (and the new CPU going online) are in post-VMXON
state.
Currently, the vmx_init() flow is:
1) hv_init_evmcs(),
2) kvm_x86_vendor_init(),
3) Other VMX specific initialization,
4) kvm_init()
The kvm_x86_vendor_init() invokes kvm_x86_init_ops::hardware_setup() to
do VMX specific hardware setup and calls kvm_update_ops() to initialize
kvm_x86_ops to VMX's version.
TDX will have its own version for most of kvm_x86_ops callbacks. It
would be nice if kvm_x86_init_ops::hardware_setup() could also be used
for TDX, but in practice it cannot. The reason is, as mentioned above,
TDX initialization requires hardware virtualization having been enabled,
which must happen after kvm_update_ops(), but hardware_setup() is done
before that.
Also, TDX is based on VMX, and it makes sense to only initialize TDX
after VMX has been initialized. If VMX fails to initialize, TDX is
likely broken anyway.
So the new flow of KVM module init function will be:
1) Current VMX initialization code in vmx_init() before kvm_init(),
2) TDX initialization,
3) kvm_init()
Split vmx_init() into two parts based on above 1) and 3) so that TDX
initialization can fit in between. Make part 1) as the new helper
vmx_init(). Introduce vt_init() as the new module init function which
calls vmx_init() and kvm_init(). TDX initialization will be added
later.
Do the same thing for vmx_exit()/vt_exit().
Signed-off-by: Kai Huang <kai.huang@intel.com>
Message-ID: <3f23f24098bdcf42e213798893ffff7cdc7103be.1731664295.git.kai.huang@intel.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2025-01-22 05:40:25 -05:00
|
|
|
int vmx_init(void);
|
|
|
|
|
void vmx_exit(void);
|
|
|
|
|
|
2018-12-03 13:53:08 -08:00
|
|
|
#endif /* __KVM_X86_VMX_H */
|