2019-06-03 07:44:50 +02:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0-only */
|
2015-11-24 16:51:12 +01:00
|
|
|
/*
|
|
|
|
|
* Copyright (C) 2015, 2016 ARM Ltd.
|
|
|
|
|
*/
|
|
|
|
|
#ifndef __KVM_ARM_VGIC_NEW_H__
|
|
|
|
|
#define __KVM_ARM_VGIC_NEW_H__
|
|
|
|
|
|
2015-12-01 15:02:35 +01:00
|
|
|
#include <linux/irqchip/arm-gic-common.h>
|
2023-01-27 07:54:48 +08:00
|
|
|
#include <asm/kvm_mmu.h>
|
2015-12-01 15:02:35 +01:00
|
|
|
|
2016-04-26 11:06:47 +01:00
|
|
|
#define PRODUCT_ID_KVM 0x4b /* ASCII code K */
|
|
|
|
|
#define IMPLEMENTER_ARM 0x43b
|
|
|
|
|
|
2015-12-21 16:36:04 +01:00
|
|
|
#define VGIC_ADDR_UNDEF (-1)
|
|
|
|
|
#define IS_VGIC_ADDR_UNDEF(_x) ((_x) == VGIC_ADDR_UNDEF)
|
|
|
|
|
|
2016-01-27 14:54:30 +00:00
|
|
|
#define INTERRUPT_ID_BITS_SPIS 10
|
2016-07-15 12:43:34 +01:00
|
|
|
#define INTERRUPT_ID_BITS_ITS 16
|
2024-04-22 20:01:44 +00:00
|
|
|
#define VGIC_LPI_MAX_INTID ((1 << INTERRUPT_ID_BITS_ITS) - 1)
|
2015-12-01 14:34:02 +00:00
|
|
|
#define VGIC_PRI_BITS 5
|
|
|
|
|
|
2015-11-26 17:19:25 +00:00
|
|
|
#define vgic_irq_is_sgi(intid) ((intid) < VGIC_NR_SGIS)
|
|
|
|
|
|
2017-01-26 19:50:47 +05:30
|
|
|
#define VGIC_AFFINITY_0_SHIFT 0
|
|
|
|
|
#define VGIC_AFFINITY_0_MASK (0xffUL << VGIC_AFFINITY_0_SHIFT)
|
|
|
|
|
#define VGIC_AFFINITY_1_SHIFT 8
|
|
|
|
|
#define VGIC_AFFINITY_1_MASK (0xffUL << VGIC_AFFINITY_1_SHIFT)
|
|
|
|
|
#define VGIC_AFFINITY_2_SHIFT 16
|
|
|
|
|
#define VGIC_AFFINITY_2_MASK (0xffUL << VGIC_AFFINITY_2_SHIFT)
|
|
|
|
|
#define VGIC_AFFINITY_3_SHIFT 24
|
|
|
|
|
#define VGIC_AFFINITY_3_MASK (0xffUL << VGIC_AFFINITY_3_SHIFT)
|
|
|
|
|
|
|
|
|
|
#define VGIC_AFFINITY_LEVEL(reg, level) \
|
|
|
|
|
((((reg) & VGIC_AFFINITY_## level ##_MASK) \
|
|
|
|
|
>> VGIC_AFFINITY_## level ##_SHIFT) << MPIDR_LEVEL_SHIFT(level))
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* The Userspace encodes the affinity differently from the MPIDR,
|
|
|
|
|
* Below macro converts vgic userspace format to MPIDR reg format.
|
|
|
|
|
*/
|
|
|
|
|
#define VGIC_TO_MPIDR(val) (VGIC_AFFINITY_LEVEL(val, 0) | \
|
|
|
|
|
VGIC_AFFINITY_LEVEL(val, 1) | \
|
|
|
|
|
VGIC_AFFINITY_LEVEL(val, 2) | \
|
|
|
|
|
VGIC_AFFINITY_LEVEL(val, 3))
|
|
|
|
|
|
2017-01-26 19:50:51 +05:30
|
|
|
/*
|
2020-04-14 18:48:35 +02:00
|
|
|
* As per Documentation/virt/kvm/devices/arm-vgic-v3.rst,
|
2017-01-26 19:50:51 +05:30
|
|
|
* below macros are defined for CPUREG encoding.
|
|
|
|
|
*/
|
|
|
|
|
#define KVM_REG_ARM_VGIC_SYSREG_OP0_MASK 0x000000000000c000
|
|
|
|
|
#define KVM_REG_ARM_VGIC_SYSREG_OP0_SHIFT 14
|
|
|
|
|
#define KVM_REG_ARM_VGIC_SYSREG_OP1_MASK 0x0000000000003800
|
|
|
|
|
#define KVM_REG_ARM_VGIC_SYSREG_OP1_SHIFT 11
|
|
|
|
|
#define KVM_REG_ARM_VGIC_SYSREG_CRN_MASK 0x0000000000000780
|
|
|
|
|
#define KVM_REG_ARM_VGIC_SYSREG_CRN_SHIFT 7
|
|
|
|
|
#define KVM_REG_ARM_VGIC_SYSREG_CRM_MASK 0x0000000000000078
|
|
|
|
|
#define KVM_REG_ARM_VGIC_SYSREG_CRM_SHIFT 3
|
|
|
|
|
#define KVM_REG_ARM_VGIC_SYSREG_OP2_MASK 0x0000000000000007
|
|
|
|
|
#define KVM_REG_ARM_VGIC_SYSREG_OP2_SHIFT 0
|
|
|
|
|
|
|
|
|
|
#define KVM_DEV_ARM_VGIC_SYSREG_MASK (KVM_REG_ARM_VGIC_SYSREG_OP0_MASK | \
|
|
|
|
|
KVM_REG_ARM_VGIC_SYSREG_OP1_MASK | \
|
|
|
|
|
KVM_REG_ARM_VGIC_SYSREG_CRN_MASK | \
|
|
|
|
|
KVM_REG_ARM_VGIC_SYSREG_CRM_MASK | \
|
|
|
|
|
KVM_REG_ARM_VGIC_SYSREG_OP2_MASK)
|
|
|
|
|
|
2025-07-14 13:26:26 +01:00
|
|
|
#define KVM_ICC_SRE_EL2 (ICC_SRE_EL2_ENABLE | ICC_SRE_EL2_SRE | \
|
|
|
|
|
ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB)
|
2025-07-14 13:26:27 +01:00
|
|
|
#define KVM_ICH_VTR_EL2_RES0 (ICH_VTR_EL2_DVIM | \
|
|
|
|
|
ICH_VTR_EL2_A3V | \
|
|
|
|
|
ICH_VTR_EL2_IDbits)
|
|
|
|
|
#define KVM_ICH_VTR_EL2_RES1 ICH_VTR_EL2_nV4
|
|
|
|
|
|
|
|
|
|
static inline u64 kvm_get_guest_vtr_el2(void)
|
|
|
|
|
{
|
|
|
|
|
u64 vtr;
|
|
|
|
|
|
|
|
|
|
vtr = kvm_vgic_global_state.ich_vtr_el2;
|
|
|
|
|
vtr &= ~KVM_ICH_VTR_EL2_RES0;
|
|
|
|
|
vtr |= KVM_ICH_VTR_EL2_RES1;
|
|
|
|
|
|
|
|
|
|
return vtr;
|
|
|
|
|
}
|
2025-07-14 13:26:26 +01:00
|
|
|
|
2017-01-09 16:19:41 +01:00
|
|
|
/*
|
2020-04-14 18:48:35 +02:00
|
|
|
* As per Documentation/virt/kvm/devices/arm-vgic-its.rst,
|
2017-01-09 16:19:41 +01:00
|
|
|
* below macros are defined for ITS table entry encoding.
|
|
|
|
|
*/
|
|
|
|
|
#define KVM_ITS_CTE_VALID_SHIFT 63
|
|
|
|
|
#define KVM_ITS_CTE_VALID_MASK BIT_ULL(63)
|
|
|
|
|
#define KVM_ITS_CTE_RDBASE_SHIFT 16
|
|
|
|
|
#define KVM_ITS_CTE_ICID_MASK GENMASK_ULL(15, 0)
|
2017-05-03 17:38:01 +02:00
|
|
|
#define KVM_ITS_ITE_NEXT_SHIFT 48
|
|
|
|
|
#define KVM_ITS_ITE_PINTID_SHIFT 16
|
|
|
|
|
#define KVM_ITS_ITE_PINTID_MASK GENMASK_ULL(47, 16)
|
|
|
|
|
#define KVM_ITS_ITE_ICID_MASK GENMASK_ULL(15, 0)
|
2017-01-09 16:27:07 +01:00
|
|
|
#define KVM_ITS_DTE_VALID_SHIFT 63
|
|
|
|
|
#define KVM_ITS_DTE_VALID_MASK BIT_ULL(63)
|
|
|
|
|
#define KVM_ITS_DTE_NEXT_SHIFT 49
|
|
|
|
|
#define KVM_ITS_DTE_NEXT_MASK GENMASK_ULL(62, 49)
|
|
|
|
|
#define KVM_ITS_DTE_ITTADDR_SHIFT 5
|
|
|
|
|
#define KVM_ITS_DTE_ITTADDR_MASK GENMASK_ULL(48, 5)
|
|
|
|
|
#define KVM_ITS_DTE_SIZE_MASK GENMASK_ULL(4, 0)
|
|
|
|
|
#define KVM_ITS_L1E_VALID_MASK BIT_ULL(63)
|
|
|
|
|
/* we only support 64 kB translation table page size */
|
|
|
|
|
#define KVM_ITS_L1E_ADDR_MASK GENMASK_ULL(51, 16)
|
2017-01-09 16:19:41 +01:00
|
|
|
|
2018-05-22 09:55:17 +02:00
|
|
|
#define KVM_VGIC_V3_RDIST_INDEX_MASK GENMASK_ULL(11, 0)
|
|
|
|
|
#define KVM_VGIC_V3_RDIST_FLAGS_MASK GENMASK_ULL(15, 12)
|
|
|
|
|
#define KVM_VGIC_V3_RDIST_FLAGS_SHIFT 12
|
|
|
|
|
#define KVM_VGIC_V3_RDIST_BASE_MASK GENMASK_ULL(51, 16)
|
|
|
|
|
#define KVM_VGIC_V3_RDIST_COUNT_MASK GENMASK_ULL(63, 52)
|
|
|
|
|
#define KVM_VGIC_V3_RDIST_COUNT_SHIFT 52
|
|
|
|
|
|
2018-08-03 21:57:03 +08:00
|
|
|
#ifdef CONFIG_DEBUG_SPINLOCK
|
|
|
|
|
#define DEBUG_SPINLOCK_BUG_ON(p) BUG_ON(p)
|
|
|
|
|
#else
|
|
|
|
|
#define DEBUG_SPINLOCK_BUG_ON(p)
|
|
|
|
|
#endif
|
|
|
|
|
|
2022-04-05 19:23:27 +01:00
|
|
|
static inline u32 vgic_get_implementation_rev(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
return vcpu->kvm->arch.vgic.implementation_rev;
|
|
|
|
|
}
|
|
|
|
|
|
2018-03-06 09:21:06 +00:00
|
|
|
/* Requires the irq_lock to be held by the caller. */
|
2017-01-23 14:07:18 +01:00
|
|
|
static inline bool irq_is_pending(struct vgic_irq *irq)
|
|
|
|
|
{
|
|
|
|
|
if (irq->config == VGIC_CONFIG_EDGE)
|
|
|
|
|
return irq->pending_latch;
|
|
|
|
|
else
|
|
|
|
|
return irq->pending_latch || irq->line_level;
|
|
|
|
|
}
|
|
|
|
|
|
KVM: arm/arm64: vgic: Support level-triggered mapped interrupts
Level-triggered mapped IRQs are special because we only observe rising
edges as input to the VGIC, and we don't set the EOI flag and therefore
are not told when the level goes down, so that we can re-queue a new
interrupt when the level goes up.
One way to solve this problem is to side-step the logic of the VGIC and
special case the validation in the injection path, but it has the
unfortunate drawback of having to peak into the physical GIC state
whenever we want to know if the interrupt is pending on the virtual
distributor.
Instead, we can maintain the current semantics of a level triggered
interrupt by sort of treating it as an edge-triggered interrupt,
following from the fact that we only observe an asserting edge. This
requires us to be a bit careful when populating the LRs and when folding
the state back in though:
* We lower the line level when populating the LR, so that when
subsequently observing an asserting edge, the VGIC will do the right
thing.
* If the guest never acked the interrupt while running (for example if
it had masked interrupts at the CPU level while running), we have
to preserve the pending state of the LR and move it back to the
line_level field of the struct irq when folding LR state.
If the guest never acked the interrupt while running, but changed the
device state and lowered the line (again with interrupts masked) then
we need to observe this change in the line_level.
Both of the above situations are solved by sampling the physical line
and set the line level when folding the LR back.
* Finally, if the guest never acked the interrupt while running and
sampling the line reveals that the device state has changed and the
line has been lowered, we must clear the physical active state, since
we will otherwise never be told when the interrupt becomes asserted
again.
This has the added benefit of making the timer optimization patches
(https://lists.cs.columbia.edu/pipermail/kvmarm/2017-July/026343.html) a
bit simpler, because the timer code doesn't have to clear the active
state on the sync anymore. It also potentially improves the performance
of the timer implementation because the GIC knows the state or the LR
and only needs to clear the
active state when the pending bit in the LR is still set, where the
timer has to always clear it when returning from running the guest with
an injected timer interrupt.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2017-08-29 10:40:44 +02:00
|
|
|
static inline bool vgic_irq_is_mapped_level(struct vgic_irq *irq)
|
|
|
|
|
{
|
|
|
|
|
return irq->config == VGIC_CONFIG_LEVEL && irq->hw;
|
|
|
|
|
}
|
|
|
|
|
|
KVM: arm/arm64: vgic: Fix source vcpu issues for GICv2 SGI
Now that we make sure we don't inject multiple instances of the
same GICv2 SGI at the same time, we've made another bug more
obvious:
If we exit with an active SGI, we completely lose track of which
vcpu it came from. On the next entry, we restore it with 0 as a
source, and if that wasn't the right one, too bad. While this
doesn't seem to trouble GIC-400, the architectural model gets
offended and doesn't deactivate the interrupt on EOI.
Another connected issue is that we will happilly make pending
an interrupt from another vcpu, overriding the above zero with
something that is just as inconsistent. Don't do that.
The final issue is that we signal a maintenance interrupt when
no pending interrupts are present in the LR. Assuming we've fixed
the two issues above, we end-up in a situation where we keep
exiting as soon as we've reached the active state, and not be
able to inject the following pending.
The fix comes in 3 parts:
- GICv2 SGIs have their source vcpu saved if they are active on
exit, and restored on entry
- Multi-SGIs cannot go via the Pending+Active state, as this would
corrupt the source field
- Multi-SGIs are converted to using MI on EOI instead of NPIE
Fixes: 16ca6a607d84bef0 ("KVM: arm/arm64: vgic: Don't populate multiple LRs with the same vintid")
Reported-by: Mark Rutland <mark.rutland@arm.com>
Tested-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Christoffer Dall <christoffer.dall@arm.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-04-18 10:39:04 +01:00
|
|
|
static inline int vgic_irq_get_lr_count(struct vgic_irq *irq)
|
|
|
|
|
{
|
|
|
|
|
/* Account for the active state as an interrupt */
|
|
|
|
|
if (vgic_irq_is_sgi(irq->intid) && irq->source)
|
|
|
|
|
return hweight8(irq->source) + irq->active;
|
|
|
|
|
|
|
|
|
|
return irq_is_pending(irq) || irq->active;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline bool vgic_irq_is_multi_sgi(struct vgic_irq *irq)
|
|
|
|
|
{
|
|
|
|
|
return vgic_irq_get_lr_count(irq) > 1;
|
|
|
|
|
}
|
|
|
|
|
|
2023-01-27 07:54:48 +08:00
|
|
|
static inline int vgic_write_guest_lock(struct kvm *kvm, gpa_t gpa,
|
|
|
|
|
const void *data, unsigned long len)
|
|
|
|
|
{
|
|
|
|
|
struct vgic_dist *dist = &kvm->arch.vgic;
|
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
|
|
dist->table_write_in_progress = true;
|
|
|
|
|
ret = kvm_write_guest_lock(kvm, gpa, data, len);
|
|
|
|
|
dist->table_write_in_progress = false;
|
|
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-21 22:05:22 +01:00
|
|
|
/*
|
|
|
|
|
* This struct provides an intermediate representation of the fields contained
|
|
|
|
|
* in the GICH_VMCR and ICH_VMCR registers, such that code exporting the GIC
|
|
|
|
|
* state to userspace can generate either GICv2 or GICv3 CPU interface
|
|
|
|
|
* registers regardless of the hardware backed GIC used.
|
|
|
|
|
*/
|
2015-12-03 11:47:37 +00:00
|
|
|
struct vgic_vmcr {
|
2017-05-20 14:12:34 +02:00
|
|
|
u32 grpen0;
|
|
|
|
|
u32 grpen1;
|
|
|
|
|
|
|
|
|
|
u32 ackctl;
|
|
|
|
|
u32 fiqen;
|
|
|
|
|
u32 cbpr;
|
|
|
|
|
u32 eoim;
|
|
|
|
|
|
2015-12-03 11:47:37 +00:00
|
|
|
u32 abpr;
|
|
|
|
|
u32 bpr;
|
2017-03-21 22:05:22 +01:00
|
|
|
u32 pmr; /* Priority mask field in the GICC_PMR and
|
|
|
|
|
* ICC_PMR_EL1 priority field format */
|
2015-12-03 11:47:37 +00:00
|
|
|
};
|
|
|
|
|
|
2017-01-26 19:50:47 +05:30
|
|
|
struct vgic_reg_attr {
|
|
|
|
|
struct kvm_vcpu *vcpu;
|
|
|
|
|
gpa_t addr;
|
|
|
|
|
};
|
|
|
|
|
|
2025-02-20 14:42:46 -08:00
|
|
|
struct its_device {
|
|
|
|
|
struct list_head dev_list;
|
|
|
|
|
|
|
|
|
|
/* the head for the list of ITTEs */
|
|
|
|
|
struct list_head itt_head;
|
|
|
|
|
u32 num_eventid_bits;
|
|
|
|
|
gpa_t itt_addr;
|
|
|
|
|
u32 device_id;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
#define COLLECTION_NOT_MAPPED ((u32)~0)
|
|
|
|
|
|
|
|
|
|
struct its_collection {
|
|
|
|
|
struct list_head coll_list;
|
|
|
|
|
|
|
|
|
|
u32 collection_id;
|
|
|
|
|
u32 target_addr;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
#define its_is_collection_mapped(coll) ((coll) && \
|
|
|
|
|
((coll)->target_addr != COLLECTION_NOT_MAPPED))
|
|
|
|
|
|
|
|
|
|
struct its_ite {
|
|
|
|
|
struct list_head ite_list;
|
|
|
|
|
|
|
|
|
|
struct vgic_irq *irq;
|
|
|
|
|
struct its_collection *collection;
|
|
|
|
|
u32 event_id;
|
|
|
|
|
};
|
|
|
|
|
|
2017-01-26 19:50:47 +05:30
|
|
|
int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
|
|
|
|
|
struct vgic_reg_attr *reg_attr);
|
|
|
|
|
int vgic_v2_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
|
|
|
|
|
struct vgic_reg_attr *reg_attr);
|
|
|
|
|
const struct vgic_register_region *
|
|
|
|
|
vgic_get_mmio_region(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev,
|
|
|
|
|
gpa_t addr, int len);
|
KVM: arm64: vgic: Make vgic_get_irq() more robust
vgic_get_irq() has an awkward signature, as it takes both a kvm
*and* a vcpu, where the vcpu is allowed to be NULL if the INTID
being looked up is a global interrupt (SPI or LPI).
This leads to potentially problematic situations where the INTID
passed is a private interrupt, but that there is no vcpu.
In order to make things less ambiguous, let have *two* helpers
instead:
- vgic_get_irq(struct kvm *kvm, u32 intid), which is only concerned
with *global* interrupts, as indicated by the lack of vcpu.
- vgic_get_vcpu_irq(struct kvm_vcpu *vcpu, u32 intid), which can
return *any* interrupt class, but must have of course a non-NULL
vcpu.
Most of the code nicely falls under one or the other situations,
except for a couple of cases (close to the UABI or in the debug code)
where we have to distinguish between the two cases.
Signed-off-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20241117165757.247686-3-maz@kernel.org
Signed-off-by: Oliver Upton <oliver.upton@linux.dev>
2024-11-17 16:57:55 +00:00
|
|
|
struct vgic_irq *vgic_get_irq(struct kvm *kvm, u32 intid);
|
|
|
|
|
struct vgic_irq *vgic_get_vcpu_irq(struct kvm_vcpu *vcpu, u32 intid);
|
2016-07-15 12:43:27 +01:00
|
|
|
void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq);
|
KVM: arm/arm64: vgic: Support level-triggered mapped interrupts
Level-triggered mapped IRQs are special because we only observe rising
edges as input to the VGIC, and we don't set the EOI flag and therefore
are not told when the level goes down, so that we can re-queue a new
interrupt when the level goes up.
One way to solve this problem is to side-step the logic of the VGIC and
special case the validation in the injection path, but it has the
unfortunate drawback of having to peak into the physical GIC state
whenever we want to know if the interrupt is pending on the virtual
distributor.
Instead, we can maintain the current semantics of a level triggered
interrupt by sort of treating it as an edge-triggered interrupt,
following from the fact that we only observe an asserting edge. This
requires us to be a bit careful when populating the LRs and when folding
the state back in though:
* We lower the line level when populating the LR, so that when
subsequently observing an asserting edge, the VGIC will do the right
thing.
* If the guest never acked the interrupt while running (for example if
it had masked interrupts at the CPU level while running), we have
to preserve the pending state of the LR and move it back to the
line_level field of the struct irq when folding LR state.
If the guest never acked the interrupt while running, but changed the
device state and lowered the line (again with interrupts masked) then
we need to observe this change in the line_level.
Both of the above situations are solved by sampling the physical line
and set the line level when folding the LR back.
* Finally, if the guest never acked the interrupt while running and
sampling the line reveals that the device state has changed and the
line has been lowered, we must clear the physical active state, since
we will otherwise never be told when the interrupt becomes asserted
again.
This has the added benefit of making the timer optimization patches
(https://lists.cs.columbia.edu/pipermail/kvmarm/2017-July/026343.html) a
bit simpler, because the timer code doesn't have to clear the active
state on the sync anymore. It also potentially improves the performance
of the timer implementation because the GIC knows the state or the LR
and only needs to clear the
active state when the pending bit in the LR is still set, where the
timer has to always clear it when returning from running the guest with
an injected timer interrupt.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2017-08-29 10:40:44 +02:00
|
|
|
bool vgic_get_phys_line_level(struct vgic_irq *irq);
|
KVM: arm/arm64: Support VGIC dist pend/active changes for mapped IRQs
For mapped IRQs (with the HW bit set in the LR) we have to follow some
rules of the architecture. One of these rules is that VM must not be
allowed to deactivate a virtual interrupt with the HW bit set unless the
physical interrupt is also active.
This works fine when injecting mapped interrupts, because we leave it up
to the injector to either set EOImode==1 or manually set the active
state of the physical interrupt.
However, the guest can set virtual interrupt to be pending or active by
writing to the virtual distributor, which could lead to deactivating a
virtual interrupt with the HW bit set without the physical interrupt
being active.
We could set the physical interrupt to active whenever we are about to
enter the VM with a HW interrupt either pending or active, but that
would be really slow, especially on GICv2. So we take the long way
around and do the hard work when needed, which is expected to be
extremely rare.
When the VM sets the pending state for a HW interrupt on the virtual
distributor we set the active state on the physical distributor, because
the virtual interrupt can become active and then the guest can
deactivate it.
When the VM clears the pending state we also clear it on the physical
side, because the injector might otherwise raise the interrupt. We also
clear the physical active state when the virtual interrupt is not
active, since otherwise a SPEND/CPEND sequence from the guest would
prevent signaling of future interrupts.
Changing the state of mapped interrupts from userspace is not supported,
and it's expected that userspace unmaps devices from VFIO before
attempting to set the interrupt state, because the interrupt state is
driven by hardware.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2017-09-01 16:25:12 +02:00
|
|
|
void vgic_irq_set_phys_pending(struct vgic_irq *irq, bool pending);
|
KVM: arm/arm64: vgic: Support level-triggered mapped interrupts
Level-triggered mapped IRQs are special because we only observe rising
edges as input to the VGIC, and we don't set the EOI flag and therefore
are not told when the level goes down, so that we can re-queue a new
interrupt when the level goes up.
One way to solve this problem is to side-step the logic of the VGIC and
special case the validation in the injection path, but it has the
unfortunate drawback of having to peak into the physical GIC state
whenever we want to know if the interrupt is pending on the virtual
distributor.
Instead, we can maintain the current semantics of a level triggered
interrupt by sort of treating it as an edge-triggered interrupt,
following from the fact that we only observe an asserting edge. This
requires us to be a bit careful when populating the LRs and when folding
the state back in though:
* We lower the line level when populating the LR, so that when
subsequently observing an asserting edge, the VGIC will do the right
thing.
* If the guest never acked the interrupt while running (for example if
it had masked interrupts at the CPU level while running), we have
to preserve the pending state of the LR and move it back to the
line_level field of the struct irq when folding LR state.
If the guest never acked the interrupt while running, but changed the
device state and lowered the line (again with interrupts masked) then
we need to observe this change in the line_level.
Both of the above situations are solved by sampling the physical line
and set the line level when folding the LR back.
* Finally, if the guest never acked the interrupt while running and
sampling the line reveals that the device state has changed and the
line has been lowered, we must clear the physical active state, since
we will otherwise never be told when the interrupt becomes asserted
again.
This has the added benefit of making the timer optimization patches
(https://lists.cs.columbia.edu/pipermail/kvmarm/2017-July/026343.html) a
bit simpler, because the timer code doesn't have to clear the active
state on the sync anymore. It also potentially improves the performance
of the timer implementation because the GIC knows the state or the LR
and only needs to clear the
active state when the pending bit in the LR is still set, where the
timer has to always clear it when returning from running the guest with
an injected timer interrupt.
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Reviewed-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
2017-08-29 10:40:44 +02:00
|
|
|
void vgic_irq_set_phys_active(struct vgic_irq *irq, bool active);
|
2016-10-16 22:19:11 +02:00
|
|
|
bool vgic_queue_irq_unlock(struct kvm *kvm, struct vgic_irq *irq,
|
2024-07-23 12:12:04 +02:00
|
|
|
unsigned long flags) __releases(&irq->irq_lock);
|
2016-04-26 11:06:47 +01:00
|
|
|
void vgic_kick_vcpus(struct kvm *kvm);
|
2021-08-19 19:03:05 +01:00
|
|
|
void vgic_irq_handle_resampling(struct vgic_irq *irq,
|
|
|
|
|
bool lr_deactivated, bool lr_pending);
|
2015-11-24 16:51:12 +01:00
|
|
|
|
2021-10-04 18:19:11 -07:00
|
|
|
int vgic_check_iorange(struct kvm *kvm, phys_addr_t ioaddr,
|
|
|
|
|
phys_addr_t addr, phys_addr_t alignment,
|
|
|
|
|
phys_addr_t size);
|
|
|
|
|
|
2015-11-26 17:19:25 +00:00
|
|
|
void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vgic_v2_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr);
|
|
|
|
|
void vgic_v2_clear_lr(struct kvm_vcpu *vcpu, int lr);
|
|
|
|
|
void vgic_v2_set_underflow(struct kvm_vcpu *vcpu);
|
2015-12-21 17:34:52 +01:00
|
|
|
int vgic_v2_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr);
|
2016-04-25 01:11:37 +02:00
|
|
|
int vgic_v2_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
|
|
|
|
|
int offset, u32 *val);
|
2015-12-03 11:48:42 +00:00
|
|
|
int vgic_v2_cpuif_uaccess(struct kvm_vcpu *vcpu, bool is_write,
|
|
|
|
|
int offset, u32 *val);
|
2015-12-03 11:47:37 +00:00
|
|
|
void vgic_v2_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
|
|
|
|
|
void vgic_v2_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
|
2015-12-21 18:09:38 +01:00
|
|
|
void vgic_v2_enable(struct kvm_vcpu *vcpu);
|
2015-12-01 15:02:35 +01:00
|
|
|
int vgic_v2_probe(const struct gic_kvm_info *info);
|
2015-12-21 15:04:42 +01:00
|
|
|
int vgic_v2_map_resources(struct kvm *kvm);
|
2016-04-26 21:32:49 +01:00
|
|
|
int vgic_register_dist_iodev(struct kvm *kvm, gpa_t dist_base_address,
|
|
|
|
|
enum vgic_type);
|
2015-11-26 17:19:25 +00:00
|
|
|
|
2017-03-18 13:56:56 +01:00
|
|
|
void vgic_v2_init_lrs(void);
|
2016-03-24 11:21:04 +01:00
|
|
|
void vgic_v2_load(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vgic_v2_put(struct kvm_vcpu *vcpu);
|
2017-03-18 13:56:56 +01:00
|
|
|
|
2016-12-22 20:39:10 +01:00
|
|
|
void vgic_v2_save_state(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vgic_v2_restore_state(struct kvm_vcpu *vcpu);
|
|
|
|
|
|
2024-02-21 05:42:51 +00:00
|
|
|
static inline bool vgic_try_get_irq_kref(struct vgic_irq *irq)
|
2016-07-17 11:27:23 +01:00
|
|
|
{
|
2024-02-21 05:42:51 +00:00
|
|
|
if (!irq)
|
|
|
|
|
return false;
|
|
|
|
|
|
2016-07-17 11:27:23 +01:00
|
|
|
if (irq->intid < VGIC_MIN_LPI)
|
2024-02-21 05:42:51 +00:00
|
|
|
return true;
|
|
|
|
|
|
|
|
|
|
return kref_get_unless_zero(&irq->refcount);
|
|
|
|
|
}
|
2016-07-17 11:27:23 +01:00
|
|
|
|
2024-02-21 05:42:51 +00:00
|
|
|
static inline void vgic_get_irq_kref(struct vgic_irq *irq)
|
|
|
|
|
{
|
|
|
|
|
WARN_ON_ONCE(!vgic_try_get_irq_kref(irq));
|
2016-07-17 11:27:23 +01:00
|
|
|
}
|
|
|
|
|
|
2015-11-30 13:09:53 +00:00
|
|
|
void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vgic_v3_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr);
|
|
|
|
|
void vgic_v3_clear_lr(struct kvm_vcpu *vcpu, int lr);
|
|
|
|
|
void vgic_v3_set_underflow(struct kvm_vcpu *vcpu);
|
2015-12-03 11:47:37 +00:00
|
|
|
void vgic_v3_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
|
|
|
|
|
void vgic_v3_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
|
2015-12-21 18:09:38 +01:00
|
|
|
void vgic_v3_enable(struct kvm_vcpu *vcpu);
|
2015-12-01 15:02:35 +01:00
|
|
|
int vgic_v3_probe(const struct gic_kvm_info *info);
|
2015-12-21 15:04:42 +01:00
|
|
|
int vgic_v3_map_resources(struct kvm *kvm);
|
2017-05-04 11:19:52 +02:00
|
|
|
int vgic_v3_lpi_sync_pending_status(struct kvm *kvm, struct vgic_irq *irq);
|
2017-01-09 16:28:27 +01:00
|
|
|
int vgic_v3_save_pending_tables(struct kvm *kvm);
|
2018-05-22 09:55:17 +02:00
|
|
|
int vgic_v3_set_redist_base(struct kvm *kvm, u32 index, u64 addr, u32 count);
|
2017-05-08 12:30:24 +02:00
|
|
|
int vgic_register_redist_iodev(struct kvm_vcpu *vcpu);
|
2023-12-07 15:11:59 +00:00
|
|
|
void vgic_unregister_redist_iodev(struct kvm_vcpu *vcpu);
|
2017-05-08 12:23:51 +02:00
|
|
|
bool vgic_v3_check_base(struct kvm *kvm);
|
2016-09-12 15:49:18 +01:00
|
|
|
|
2016-03-24 11:21:04 +01:00
|
|
|
void vgic_v3_load(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vgic_v3_put(struct kvm_vcpu *vcpu);
|
|
|
|
|
|
2016-07-15 12:43:30 +01:00
|
|
|
bool vgic_has_its(struct kvm *kvm);
|
2016-07-15 12:43:38 +01:00
|
|
|
int kvm_vgic_register_its_device(void);
|
2016-07-15 12:43:34 +01:00
|
|
|
void vgic_enable_lpis(struct kvm_vcpu *vcpu);
|
2019-04-02 06:36:23 +01:00
|
|
|
void vgic_flush_pending_lpis(struct kvm_vcpu *vcpu);
|
2016-07-15 12:43:37 +01:00
|
|
|
int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi);
|
2017-01-26 19:50:47 +05:30
|
|
|
int vgic_v3_has_attr_regs(struct kvm_device *dev, struct kvm_device_attr *attr);
|
|
|
|
|
int vgic_v3_dist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
|
|
|
|
|
int offset, u32 *val);
|
|
|
|
|
int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
|
|
|
|
|
int offset, u32 *val);
|
2022-07-04 08:07:44 +01:00
|
|
|
int vgic_v3_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu,
|
|
|
|
|
struct kvm_device_attr *attr, bool is_write);
|
2022-07-03 14:57:29 +01:00
|
|
|
int vgic_v3_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
|
2025-07-18 12:11:53 +01:00
|
|
|
const struct sys_reg_desc *vgic_v3_get_sysreg_table(unsigned int *sz);
|
2017-01-26 19:50:52 +05:30
|
|
|
int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write,
|
2022-07-05 10:16:44 +01:00
|
|
|
u32 intid, u32 *val);
|
2016-07-15 12:43:23 +01:00
|
|
|
int kvm_register_vgic_device(unsigned long type);
|
2017-01-26 19:50:50 +05:30
|
|
|
void vgic_set_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
|
|
|
|
|
void vgic_get_vmcr(struct kvm_vcpu *vcpu, struct vgic_vmcr *vmcr);
|
2015-12-21 18:09:38 +01:00
|
|
|
int vgic_lazy_init(struct kvm *kvm);
|
|
|
|
|
int vgic_init(struct kvm *kvm);
|
2015-11-30 14:01:58 +01:00
|
|
|
|
2018-05-29 18:22:04 +02:00
|
|
|
void vgic_debug_init(struct kvm *kvm);
|
|
|
|
|
void vgic_debug_destroy(struct kvm *kvm);
|
2017-01-17 23:09:13 +01:00
|
|
|
|
2025-06-27 10:09:02 +00:00
|
|
|
int vgic_v5_probe(const struct gic_kvm_info *info);
|
|
|
|
|
|
2017-09-01 11:41:52 +02:00
|
|
|
static inline int vgic_v3_max_apr_idx(struct kvm_vcpu *vcpu)
|
|
|
|
|
{
|
|
|
|
|
struct vgic_cpu *cpu_if = &vcpu->arch.vgic_cpu;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* num_pri_bits are initialized with HW supported values.
|
|
|
|
|
* We can rely safely on num_pri_bits even if VM has not
|
|
|
|
|
* restored ICC_CTLR_EL1 before restoring APnR registers.
|
|
|
|
|
*/
|
|
|
|
|
switch (cpu_if->num_pri_bits) {
|
|
|
|
|
case 7: return 3;
|
|
|
|
|
case 6: return 1;
|
|
|
|
|
default: return 0;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2018-05-22 09:55:09 +02:00
|
|
|
static inline bool
|
|
|
|
|
vgic_v3_redist_region_full(struct vgic_redist_region *region)
|
|
|
|
|
{
|
|
|
|
|
if (!region->count)
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
return (region->free_index >= region->count);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct vgic_redist_region *vgic_v3_rdist_free_slot(struct list_head *rdregs);
|
|
|
|
|
|
2018-05-22 09:55:11 +02:00
|
|
|
static inline size_t
|
|
|
|
|
vgic_v3_rd_region_size(struct kvm *kvm, struct vgic_redist_region *rdreg)
|
|
|
|
|
{
|
|
|
|
|
if (!rdreg->count)
|
|
|
|
|
return atomic_read(&kvm->online_vcpus) * KVM_VGIC_V3_REDIST_SIZE;
|
|
|
|
|
else
|
|
|
|
|
return rdreg->count * KVM_VGIC_V3_REDIST_SIZE;
|
|
|
|
|
}
|
2018-05-22 09:55:17 +02:00
|
|
|
|
|
|
|
|
struct vgic_redist_region *vgic_v3_rdist_region_from_index(struct kvm *kvm,
|
|
|
|
|
u32 index);
|
2024-06-05 18:56:37 +01:00
|
|
|
void vgic_v3_free_redist_region(struct kvm *kvm, struct vgic_redist_region *rdreg);
|
2018-05-22 09:55:17 +02:00
|
|
|
|
2018-05-22 09:55:11 +02:00
|
|
|
bool vgic_v3_rdist_overlap(struct kvm *kvm, gpa_t base, size_t size);
|
|
|
|
|
|
2018-05-22 09:55:12 +02:00
|
|
|
static inline bool vgic_dist_overlap(struct kvm *kvm, gpa_t base, size_t size)
|
|
|
|
|
{
|
|
|
|
|
struct vgic_dist *d = &kvm->arch.vgic;
|
|
|
|
|
|
|
|
|
|
return (base + size > d->vgic_dist_base) &&
|
|
|
|
|
(base < d->vgic_dist_base + KVM_VGIC_V3_DIST_SIZE);
|
|
|
|
|
}
|
|
|
|
|
|
2022-04-05 19:23:25 +01:00
|
|
|
bool vgic_lpis_enabled(struct kvm_vcpu *vcpu);
|
2017-10-27 15:28:35 +01:00
|
|
|
int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,
|
|
|
|
|
u32 devid, u32 eventid, struct vgic_irq **irq);
|
|
|
|
|
struct vgic_its *vgic_msi_to_its(struct kvm *kvm, struct kvm_msi *msi);
|
2019-03-18 10:29:30 +00:00
|
|
|
int vgic_its_inject_cached_translation(struct kvm *kvm, struct kvm_msi *msi);
|
2024-04-22 20:01:46 +00:00
|
|
|
void vgic_its_invalidate_all_caches(struct kvm *kvm);
|
2017-10-27 15:28:35 +01:00
|
|
|
|
2022-04-05 19:23:26 +01:00
|
|
|
/* GICv4.1 MMIO interface */
|
|
|
|
|
int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq);
|
|
|
|
|
int vgic_its_invall(struct kvm_vcpu *vcpu);
|
|
|
|
|
|
2025-07-23 23:28:03 -07:00
|
|
|
bool system_supports_direct_sgis(void);
|
2017-10-27 15:28:37 +01:00
|
|
|
bool vgic_supports_direct_msis(struct kvm *kvm);
|
2025-07-23 23:28:00 -07:00
|
|
|
bool vgic_supports_direct_sgis(struct kvm *kvm);
|
|
|
|
|
|
|
|
|
|
static inline bool vgic_supports_direct_irqs(struct kvm *kvm)
|
|
|
|
|
{
|
2025-07-23 23:28:03 -07:00
|
|
|
/*
|
|
|
|
|
* Deliberately conflate vLPI and vSGI support on GICv4.1 hardware,
|
|
|
|
|
* indirectly allowing userspace to control whether or not vPEs are
|
|
|
|
|
* allocated for the VM.
|
|
|
|
|
*/
|
|
|
|
|
if (system_supports_direct_sgis())
|
|
|
|
|
return vgic_supports_direct_sgis(kvm);
|
|
|
|
|
|
|
|
|
|
return vgic_supports_direct_msis(kvm);
|
2025-07-23 23:28:00 -07:00
|
|
|
}
|
|
|
|
|
|
2017-10-27 15:28:38 +01:00
|
|
|
int vgic_v4_init(struct kvm *kvm);
|
|
|
|
|
void vgic_v4_teardown(struct kvm *kvm);
|
2020-03-04 20:33:26 +00:00
|
|
|
void vgic_v4_configure_vsgis(struct kvm *kvm);
|
2021-03-22 14:01:55 +08:00
|
|
|
void vgic_v4_get_vlpi_state(struct vgic_irq *irq, bool *val);
|
2023-01-19 11:07:59 +00:00
|
|
|
int vgic_v4_request_vpe_irq(struct kvm_vcpu *vcpu, int irq);
|
2017-10-27 15:28:37 +01:00
|
|
|
|
2024-08-27 16:25:07 +01:00
|
|
|
void vcpu_set_ich_hcr(struct kvm_vcpu *vcpu);
|
|
|
|
|
|
2024-08-20 11:03:38 +01:00
|
|
|
static inline bool kvm_has_gicv3(struct kvm *kvm)
|
|
|
|
|
{
|
2024-08-27 16:25:11 +01:00
|
|
|
return kvm_has_feat(kvm, ID_AA64PFR0_EL1, GIC, IMP);
|
2024-08-20 11:03:38 +01:00
|
|
|
}
|
|
|
|
|
|
2025-02-25 17:29:22 +00:00
|
|
|
void vgic_v3_sync_nested(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vgic_v3_load_nested(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vgic_v3_put_nested(struct kvm_vcpu *vcpu);
|
2025-02-25 17:29:24 +00:00
|
|
|
void vgic_v3_handle_nested_maint_irq(struct kvm_vcpu *vcpu);
|
|
|
|
|
void vgic_v3_nested_update_mi(struct kvm_vcpu *vcpu);
|
2025-02-25 17:29:22 +00:00
|
|
|
|
2025-06-27 10:09:02 +00:00
|
|
|
static inline bool vgic_is_v3_compat(struct kvm *kvm)
|
|
|
|
|
{
|
|
|
|
|
return cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF) &&
|
|
|
|
|
kvm_vgic_global_state.has_gcie_v3_compat;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline bool vgic_is_v3(struct kvm *kvm)
|
|
|
|
|
{
|
|
|
|
|
return kvm_vgic_global_state.type == VGIC_V3 || vgic_is_v3_compat(kvm);
|
|
|
|
|
}
|
|
|
|
|
|
2025-02-20 14:42:46 -08:00
|
|
|
int vgic_its_debug_init(struct kvm_device *dev);
|
|
|
|
|
void vgic_its_debug_destroy(struct kvm_device *dev);
|
|
|
|
|
|
2015-11-24 16:51:12 +01:00
|
|
|
#endif
|