linux/arch/arm64/kernel/proton-pack.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Handle detection, reporting and mitigation of Spectre v1, v2, v3a and v4, as
 * detailed at:
 *
 *   https://developer.arm.com/support/arm-security-updates/speculative-processor-vulnerability
 *
 * This code was originally written hastily under an awful lot of stress and so
 * aspects of it are somewhat hacky. Unfortunately, changing anything in here
 * instantly makes me feel ill. Thanks, Jann. Thann.
 *
 * Copyright (C) 2018 ARM Ltd, All Rights Reserved.
 * Copyright (C) 2020 Google LLC
 *
 * "If there's something strange in your neighbourhood, who you gonna call?"
 *
 * Authors: Will Deacon <will@kernel.org> and Marc Zyngier <maz@kernel.org>
 */

#include <linux/arm-smccc.h>
#include <linux/bpf.h>
#include <linux/cpu.h>
#include <linux/device.h>
#include <linux/nospec.h>
#include <linux/prctl.h>
#include <linux/sched/task_stack.h>

#include <asm/debug-monitors.h>
#include <asm/insn.h>
#include <asm/spectre.h>
#include <asm/traps.h>
#include <asm/vectors.h>
#include <asm/virt.h>

/*
 * We try to ensure that the mitigation state can never change as the result of
 * onlining a late CPU.
 */
static void update_mitigation_state(enum mitigation_state *oldp,
				    enum mitigation_state new)
{
	enum mitigation_state state;

	do {
		state = READ_ONCE(*oldp);
		if (new <= state)
			break;

		/* Userspace almost certainly can't deal with this. */
		if (WARN_ON(system_capabilities_finalized()))
			break;
	} while (cmpxchg_relaxed(oldp, state, new) != state);
}

/*
 * Spectre v1.
 *
 * The kernel can't protect userspace for this one: it's each person for
 * themselves. Advertise what we're doing and be done with it.
 */
ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr,
			    char *buf)
{
	return sprintf(buf, "Mitigation: __user pointer sanitization\n");
}

/*
 * Spectre v2.
 *
 * This one sucks. A CPU is either:
 *
 * - Mitigated in hardware and advertised by ID_AA64PFR0_EL1.CSV2.
 * - Mitigated in hardware and listed in our "safe list".
 * - Mitigated in software by firmware.
 * - Mitigated in software by a CPU-specific dance in the kernel and a
 *   firmware call at EL2.
 * - Vulnerable.
 *
 * It's not unlikely for different CPUs in a big.LITTLE system to fall into
 * different camps.
 */
static enum mitigation_state spectre_v2_state;

static bool __read_mostly __nospectre_v2;
static int __init parse_spectre_v2_param(char *str)
{
	__nospectre_v2 = true;
	return 0;
}
early_param("nospectre_v2", parse_spectre_v2_param);

static bool spectre_v2_mitigations_off(void)
{
	bool ret = __nospectre_v2 || cpu_mitigations_off();

	if (ret)
		pr_info_once("spectre-v2 mitigation disabled by command line option\n");

	return ret;
}

static const char *get_bhb_affected_string(enum mitigation_state bhb_state)
{
	switch (bhb_state) {
	case SPECTRE_UNAFFECTED:
		return "";
	default:
	case SPECTRE_VULNERABLE:
		return ", but not BHB";
	case SPECTRE_MITIGATED:
		return ", BHB";
	}
}

static bool _unprivileged_ebpf_enabled(void)
{
#ifdef CONFIG_BPF_SYSCALL
	return !sysctl_unprivileged_bpf_disabled;
#else
	return false;
#endif
}

ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr,
			    char *buf)
{
	enum mitigation_state bhb_state = arm64_get_spectre_bhb_state();
	const char *bhb_str = get_bhb_affected_string(bhb_state);
	const char *v2_str = "Branch predictor hardening";

	switch (spectre_v2_state) {
	case SPECTRE_UNAFFECTED:
		if (bhb_state == SPECTRE_UNAFFECTED)
			return sprintf(buf, "Not affected\n");

		/*
		 * Platforms affected by Spectre-BHB can't report
		 * "Not affected" for Spectre-v2.
		 */
		v2_str = "CSV2";
		fallthrough;
	case SPECTRE_MITIGATED:
		if (bhb_state == SPECTRE_MITIGATED && _unprivileged_ebpf_enabled())
			return sprintf(buf, "Vulnerable: Unprivileged eBPF enabled\n");

		return sprintf(buf, "Mitigation: %s%s\n", v2_str, bhb_str);
	case SPECTRE_VULNERABLE:
		fallthrough;
	default:
		return sprintf(buf, "Vulnerable\n");
	}
}

static enum mitigation_state spectre_v2_get_cpu_hw_mitigation_state(void)
{
	u64 pfr0;
	static const struct midr_range spectre_v2_safe_list[] = {
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
		MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
		MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER),
		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
		{ /* sentinel */ }
	};

	/* If the CPU has CSV2 set, we're safe */
	pfr0 = read_cpuid(ID_AA64PFR0_EL1);
	if (cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL1_CSV2_SHIFT))
		return SPECTRE_UNAFFECTED;

	/* Alternatively, we have a list of unaffected CPUs */
	if (is_midr_in_range_list(spectre_v2_safe_list))
		return SPECTRE_UNAFFECTED;

	return SPECTRE_VULNERABLE;
}

static enum mitigation_state spectre_v2_get_cpu_fw_mitigation_state(void)
{
	int ret;
	struct arm_smccc_res res;

	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
			     ARM_SMCCC_ARCH_WORKAROUND_1, &res);

	ret = res.a0;
	switch (ret) {
	case SMCCC_RET_SUCCESS:
		return SPECTRE_MITIGATED;
	case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
		return SPECTRE_UNAFFECTED;
	default:
		fallthrough;
	case SMCCC_RET_NOT_SUPPORTED:
		return SPECTRE_VULNERABLE;
	}
}

bool has_spectre_v2(const struct arm64_cpu_capabilities *entry, int scope)
{
	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());

	if (spectre_v2_get_cpu_hw_mitigation_state() == SPECTRE_UNAFFECTED)
		return false;

	if (spectre_v2_get_cpu_fw_mitigation_state() == SPECTRE_UNAFFECTED)
		return false;

	return true;
}

enum mitigation_state arm64_get_spectre_v2_state(void)
{
	return spectre_v2_state;
}

DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);

static void install_bp_hardening_cb(bp_hardening_cb_t fn)
{
	__this_cpu_write(bp_hardening_data.fn, fn);

	/*
	 * Vinz Clortho takes the hyp_vecs start/end "keys" at
	 * the door when we're a guest. Skip the hyp-vectors work.
	 */
	if (!is_hyp_mode_available())
		return;

	__this_cpu_write(bp_hardening_data.slot, HYP_VECTOR_SPECTRE_DIRECT);
}

/* Called during entry so must be noinstr */
static noinstr void call_smc_arch_workaround_1(void)
{
	arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
}

/* Called during entry so must be noinstr */
static noinstr void call_hvc_arch_workaround_1(void)
{
	arm_smccc_1_1_hvc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
}

/* Called during entry so must be noinstr */
static noinstr void qcom_link_stack_sanitisation(void)
{
	u64 tmp;

	asm volatile("mov	%0, x30		\n"
		     ".rept	16		\n"
		     "bl	. + 4		\n"
		     ".endr			\n"
		     "mov	x30, %0		\n"
		     : "=&r" (tmp));
}

static bp_hardening_cb_t spectre_v2_get_sw_mitigation_cb(void)
{
	u32 midr = read_cpuid_id();
	if (((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR) &&
	    ((midr & MIDR_CPU_MODEL_MASK) != MIDR_QCOM_FALKOR_V1))
		return NULL;

	return qcom_link_stack_sanitisation;
}

static enum mitigation_state spectre_v2_enable_fw_mitigation(void)
{
	bp_hardening_cb_t cb;
	enum mitigation_state state;

	state = spectre_v2_get_cpu_fw_mitigation_state();
	if (state != SPECTRE_MITIGATED)
		return state;

	if (spectre_v2_mitigations_off())
		return SPECTRE_VULNERABLE;

	switch (arm_smccc_1_1_get_conduit()) {
	case SMCCC_CONDUIT_HVC:
		cb = call_hvc_arch_workaround_1;
		break;

	case SMCCC_CONDUIT_SMC:
		cb = call_smc_arch_workaround_1;
		break;

	default:
		return SPECTRE_VULNERABLE;
	}

	/*
	 * Prefer a CPU-specific workaround if it exists. Note that we
	 * still rely on firmware for the mitigation at EL2.
	 */
	cb = spectre_v2_get_sw_mitigation_cb() ?: cb;
	install_bp_hardening_cb(cb);
	return SPECTRE_MITIGATED;
}

void spectre_v2_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
{
	enum mitigation_state state;

	WARN_ON(preemptible());

	state = spectre_v2_get_cpu_hw_mitigation_state();
	if (state == SPECTRE_VULNERABLE)
		state = spectre_v2_enable_fw_mitigation();

	update_mitigation_state(&spectre_v2_state, state);
}

/*
 * Spectre-v3a.
 *
 * Phew, there's not an awful lot to do here! We just instruct EL2 to use
 * an indirect trampoline for the hyp vectors so that guests can't read
 * VBAR_EL2 to defeat randomisation of the hypervisor VA layout.
 */
bool has_spectre_v3a(const struct arm64_cpu_capabilities *entry, int scope)
{
	static const struct midr_range spectre_v3a_unsafe_list[] = {
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
		{},
	};

	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());
	return is_midr_in_range_list(spectre_v3a_unsafe_list);
}

void spectre_v3a_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
{
	struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);

	if (this_cpu_has_cap(ARM64_SPECTRE_V3A))
		data->slot += HYP_VECTOR_INDIRECT;
}

/*
 * Spectre v4.
 *
 * If you thought Spectre v2 was nasty, wait until you see this mess. A CPU is
 * either:
 *
 * - Mitigated in hardware and listed in our "safe list".
 * - Mitigated in hardware via PSTATE.SSBS.
 * - Mitigated in software by firmware (sometimes referred to as SSBD).
 *
 * Wait, that doesn't sound so bad, does it? Keep reading...
 *
 * A major source of headaches is that the software mitigation is enabled both
 * on a per-task basis, but can also be forced on for the kernel, necessitating
 * both context-switch *and* entry/exit hooks. To make it even worse, some CPUs
 * allow EL0 to toggle SSBS directly, which can end up with the prctl() state
 * being stale when re-entering the kernel. The usual big.LITTLE caveats apply,
 * so you can have systems that have both firmware and SSBS mitigations. This
 * means we actually have to reject late onlining of CPUs with mitigations if
 * all of the currently onlined CPUs are safelisted, as the mitigation tends to
 * be opt-in for userspace. Yes, really, the cure is worse than the disease.
 *
 * The only good part is that if the firmware mitigation is present, then it is
 * present for all CPUs, meaning we don't have to worry about late onlining of a
 * vulnerable CPU if one of the boot CPUs is using the firmware mitigation.
 *
 * Give me a VAX-11/780 any day of the week...
 */
static enum mitigation_state spectre_v4_state;

/* This is the per-cpu state tracking whether we need to talk to firmware */
DEFINE_PER_CPU_READ_MOSTLY(u64, arm64_ssbd_callback_required);

enum spectre_v4_policy {
	SPECTRE_V4_POLICY_MITIGATION_DYNAMIC,
	SPECTRE_V4_POLICY_MITIGATION_ENABLED,
	SPECTRE_V4_POLICY_MITIGATION_DISABLED,
};

static enum spectre_v4_policy __read_mostly __spectre_v4_policy;

static const struct spectre_v4_param {
	const char		*str;
	enum spectre_v4_policy	policy;
} spectre_v4_params[] = {
	{ "force-on",	SPECTRE_V4_POLICY_MITIGATION_ENABLED, },
	{ "force-off",	SPECTRE_V4_POLICY_MITIGATION_DISABLED, },
	{ "kernel",	SPECTRE_V4_POLICY_MITIGATION_DYNAMIC, },
};
static int __init parse_spectre_v4_param(char *str)
{
	int i;

	if (!str || !str[0])
		return -EINVAL;

	for (i = 0; i < ARRAY_SIZE(spectre_v4_params); i++) {
		const struct spectre_v4_param *param = &spectre_v4_params[i];

		if (strncmp(str, param->str, strlen(param->str)))
			continue;

		__spectre_v4_policy = param->policy;
		return 0;
	}

	return -EINVAL;
}
early_param("ssbd", parse_spectre_v4_param);

/*
 * Because this was all written in a rush by people working in different silos,
 * we've ended up with multiple command line options to control the same thing.
 * Wrap these up in some helpers, which prefer disabling the mitigation if faced
 * with contradictory parameters. The mitigation is always either "off",
 * "dynamic" or "on".
 */
static bool spectre_v4_mitigations_off(void)
{
	bool ret = cpu_mitigations_off() ||
		   __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DISABLED;

	if (ret)
		pr_info_once("spectre-v4 mitigation disabled by command-line option\n");

	return ret;
}

/* Do we need to toggle the mitigation state on entry to/exit from the kernel? */
static bool spectre_v4_mitigations_dynamic(void)
{
	return !spectre_v4_mitigations_off() &&
	       __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_DYNAMIC;
}

static bool spectre_v4_mitigations_on(void)
{
	return !spectre_v4_mitigations_off() &&
	       __spectre_v4_policy == SPECTRE_V4_POLICY_MITIGATION_ENABLED;
}

ssize_t cpu_show_spec_store_bypass(struct device *dev,
				   struct device_attribute *attr, char *buf)
{
	switch (spectre_v4_state) {
	case SPECTRE_UNAFFECTED:
		return sprintf(buf, "Not affected\n");
	case SPECTRE_MITIGATED:
		return sprintf(buf, "Mitigation: Speculative Store Bypass disabled via prctl\n");
	case SPECTRE_VULNERABLE:
		fallthrough;
	default:
		return sprintf(buf, "Vulnerable\n");
	}
}

enum mitigation_state arm64_get_spectre_v4_state(void)
{
	return spectre_v4_state;
}

static enum mitigation_state spectre_v4_get_cpu_hw_mitigation_state(void)
{
	static const struct midr_range spectre_v4_safe_list[] = {
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
		MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
		{ /* sentinel */ },
	};

	if (is_midr_in_range_list(spectre_v4_safe_list))
		return SPECTRE_UNAFFECTED;

	/* CPU features are detected first */
	if (this_cpu_has_cap(ARM64_SSBS))
		return SPECTRE_MITIGATED;

	return SPECTRE_VULNERABLE;
}

static enum mitigation_state spectre_v4_get_cpu_fw_mitigation_state(void)
{
	int ret;
	struct arm_smccc_res res;

	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
			     ARM_SMCCC_ARCH_WORKAROUND_2, &res);

	ret = res.a0;
	switch (ret) {
	case SMCCC_RET_SUCCESS:
		return SPECTRE_MITIGATED;
	case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
		fallthrough;
	case SMCCC_RET_NOT_REQUIRED:
		return SPECTRE_UNAFFECTED;
	default:
		fallthrough;
	case SMCCC_RET_NOT_SUPPORTED:
		return SPECTRE_VULNERABLE;
	}
}

bool has_spectre_v4(const struct arm64_cpu_capabilities *cap, int scope)
{
	enum mitigation_state state;

	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());

	state = spectre_v4_get_cpu_hw_mitigation_state();
	if (state == SPECTRE_VULNERABLE)
		state = spectre_v4_get_cpu_fw_mitigation_state();

	return state != SPECTRE_UNAFFECTED;
}

bool try_emulate_el1_ssbs(struct pt_regs *regs, u32 instr)
{
	const u32 instr_mask = ~(1U << PSTATE_Imm_shift);
	const u32 instr_val = 0xd500401f | PSTATE_SSBS;

	if ((instr & instr_mask) != instr_val)
		return false;

	if (instr & BIT(PSTATE_Imm_shift))
		regs->pstate |= PSR_SSBS_BIT;
	else
		regs->pstate &= ~PSR_SSBS_BIT;

	arm64_skip_faulting_instruction(regs, 4);
	return true;
}

static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
{
	enum mitigation_state state;

	/*
	 * If the system is mitigated but this CPU doesn't have SSBS, then
	 * we must be on the safelist and there's nothing more to do.
	 */
	state = spectre_v4_get_cpu_hw_mitigation_state();
	if (state != SPECTRE_MITIGATED || !this_cpu_has_cap(ARM64_SSBS))
		return state;

	if (spectre_v4_mitigations_off()) {
		sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
		set_pstate_ssbs(1);
		return SPECTRE_VULNERABLE;
	}

	/* SCTLR_EL1.DSSBS was initialised to 0 during boot */
	set_pstate_ssbs(0);

	/*
	 * SSBS is self-synchronizing and is intended to affect subsequent
	 * speculative instructions, but some CPUs can speculate with a stale
	 * value of SSBS.
	 *
	 * Mitigate this with an unconditional speculation barrier, as CPUs
	 * could mis-speculate branches and bypass a conditional barrier.
	 */
	if (IS_ENABLED(CONFIG_ARM64_ERRATUM_3194386))
		spec_bar();

	return SPECTRE_MITIGATED;
}

/*
 * Patch a branch over the Spectre-v4 mitigation code with a NOP so that
 * we fallthrough and check whether firmware needs to be called on this CPU.
 */
void __init spectre_v4_patch_fw_mitigation_enable(struct alt_instr *alt,
						  __le32 *origptr,
						  __le32 *updptr, int nr_inst)
{
	BUG_ON(nr_inst != 1); /* Branch -> NOP */

	if (spectre_v4_mitigations_off())
		return;

	if (cpus_have_cap(ARM64_SSBS))
		return;

	if (spectre_v4_mitigations_dynamic())
		*updptr = cpu_to_le32(aarch64_insn_gen_nop());
}

/*
 * Patch a NOP in the Spectre-v4 mitigation code with an SMC/HVC instruction
 * to call into firmware to adjust the mitigation state.
 */
void __init smccc_patch_fw_mitigation_conduit(struct alt_instr *alt,
					       __le32 *origptr,
					       __le32 *updptr, int nr_inst)
{
	u32 insn;

	BUG_ON(nr_inst != 1); /* NOP -> HVC/SMC */

	switch (arm_smccc_1_1_get_conduit()) {
	case SMCCC_CONDUIT_HVC:
		insn = aarch64_insn_get_hvc_value();
		break;
	case SMCCC_CONDUIT_SMC:
		insn = aarch64_insn_get_smc_value();
		break;
	default:
		return;
	}

	*updptr = cpu_to_le32(insn);
}

static enum mitigation_state spectre_v4_enable_fw_mitigation(void)
{
	enum mitigation_state state;

	state = spectre_v4_get_cpu_fw_mitigation_state();
	if (state != SPECTRE_MITIGATED)
		return state;

	if (spectre_v4_mitigations_off()) {
		arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, false, NULL);
		return SPECTRE_VULNERABLE;
	}

	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_WORKAROUND_2, true, NULL);

	if (spectre_v4_mitigations_dynamic())
		__this_cpu_write(arm64_ssbd_callback_required, 1);

	return SPECTRE_MITIGATED;
}

void spectre_v4_enable_mitigation(const struct arm64_cpu_capabilities *__unused)
{
	enum mitigation_state state;

	WARN_ON(preemptible());

	state = spectre_v4_enable_hw_mitigation();
	if (state == SPECTRE_VULNERABLE)
		state = spectre_v4_enable_fw_mitigation();

	update_mitigation_state(&spectre_v4_state, state);
}

static void __update_pstate_ssbs(struct pt_regs *regs, bool state)
{
	u64 bit = compat_user_mode(regs) ? PSR_AA32_SSBS_BIT : PSR_SSBS_BIT;

	if (state)
		regs->pstate |= bit;
	else
		regs->pstate &= ~bit;
}

void spectre_v4_enable_task_mitigation(struct task_struct *tsk)
{
	struct pt_regs *regs = task_pt_regs(tsk);
	bool ssbs = false, kthread = tsk->flags & PF_KTHREAD;

	if (spectre_v4_mitigations_off())
		ssbs = true;
	else if (spectre_v4_mitigations_dynamic() && !kthread)
		ssbs = !test_tsk_thread_flag(tsk, TIF_SSBD);

	__update_pstate_ssbs(regs, ssbs);
}

/*
 * The Spectre-v4 mitigation can be controlled via a prctl() from userspace.
 * This is interesting because the "speculation disabled" behaviour can be
 * configured so that it is preserved across exec(), which means that the
 * prctl() may be necessary even when PSTATE.SSBS can be toggled directly
 * from userspace.
 */
static void ssbd_prctl_enable_mitigation(struct task_struct *task)
{
	task_clear_spec_ssb_noexec(task);
	task_set_spec_ssb_disable(task);
	set_tsk_thread_flag(task, TIF_SSBD);
}

static void ssbd_prctl_disable_mitigation(struct task_struct *task)
{
	task_clear_spec_ssb_noexec(task);
	task_clear_spec_ssb_disable(task);
	clear_tsk_thread_flag(task, TIF_SSBD);
}

static int ssbd_prctl_set(struct task_struct *task, unsigned long ctrl)
{
	switch (ctrl) {
	case PR_SPEC_ENABLE:
		/* Enable speculation: disable mitigation */
		/*
		 * Force disabled speculation prevents it from being
		 * re-enabled.
		 */
		if (task_spec_ssb_force_disable(task))
			return -EPERM;

		/*
		 * If the mitigation is forced on, then speculation is forced
		 * off and we again prevent it from being re-enabled.
		 */
		if (spectre_v4_mitigations_on())
			return -EPERM;

		ssbd_prctl_disable_mitigation(task);
		break;
	case PR_SPEC_FORCE_DISABLE:
		/* Force disable speculation: force enable mitigation */
		/*
		 * If the mitigation is forced off, then speculation is forced
		 * on and we prevent it from being disabled.
		 */
		if (spectre_v4_mitigations_off())
			return -EPERM;

		task_set_spec_ssb_force_disable(task);
		fallthrough;
	case PR_SPEC_DISABLE:
		/* Disable speculation: enable mitigation */
		/* Same as PR_SPEC_FORCE_DISABLE */
		if (spectre_v4_mitigations_off())
			return -EPERM;

		ssbd_prctl_enable_mitigation(task);
		break;
	case PR_SPEC_DISABLE_NOEXEC:
		/* Disable speculation until execve(): enable mitigation */
		/*
		 * If the mitigation state is forced one way or the other, then
		 * we must fail now before we try to toggle it on execve().
		 */
		if (task_spec_ssb_force_disable(task) ||
		    spectre_v4_mitigations_off() ||
		    spectre_v4_mitigations_on()) {
			return -EPERM;
		}

		ssbd_prctl_enable_mitigation(task);
		task_set_spec_ssb_noexec(task);
		break;
	default:
		return -ERANGE;
	}

	spectre_v4_enable_task_mitigation(task);
	return 0;
}

int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
			     unsigned long ctrl)
{
	switch (which) {
	case PR_SPEC_STORE_BYPASS:
		return ssbd_prctl_set(task, ctrl);
	default:
		return -ENODEV;
	}
}

static int ssbd_prctl_get(struct task_struct *task)
{
	switch (spectre_v4_state) {
	case SPECTRE_UNAFFECTED:
		return PR_SPEC_NOT_AFFECTED;
	case SPECTRE_MITIGATED:
		if (spectre_v4_mitigations_on())
			return PR_SPEC_NOT_AFFECTED;

		if (spectre_v4_mitigations_dynamic())
			break;

		/* Mitigations are disabled, so we're vulnerable. */
		fallthrough;
	case SPECTRE_VULNERABLE:
		fallthrough;
	default:
		return PR_SPEC_ENABLE;
	}

	/* Check the mitigation state for this task */
	if (task_spec_ssb_force_disable(task))
		return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;

	if (task_spec_ssb_noexec(task))
		return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;

	if (task_spec_ssb_disable(task))
		return PR_SPEC_PRCTL | PR_SPEC_DISABLE;

	return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
}

int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
{
	switch (which) {
	case PR_SPEC_STORE_BYPASS:
		return ssbd_prctl_get(task);
	default:
		return -ENODEV;
	}
}

/*
 * Spectre BHB.
 *
 * A CPU is either:
 * - Mitigated by a branchy loop a CPU specific number of times, and listed
 *   in our "loop mitigated list".
 * - Mitigated in software by the firmware Spectre v2 call.
 * - Has the ClearBHB instruction to perform the mitigation.
 * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
 *   software mitigation in the vectors is needed.
 * - Has CSV2.3, so is unaffected.
 */
static enum mitigation_state spectre_bhb_state;

enum mitigation_state arm64_get_spectre_bhb_state(void)
{
	return spectre_bhb_state;
}

enum bhb_mitigation_bits {
	BHB_LOOP,
	BHB_FW,
	BHB_HW,
	BHB_INSN,
};
static unsigned long system_bhb_mitigations;

/*
 * This must be called with SCOPE_LOCAL_CPU for each type of CPU, before any
 * SCOPE_SYSTEM call will give the right answer.
 */
static bool is_spectre_bhb_safe(int scope)
{
	static const struct midr_range spectre_bhb_safe_list[] = {
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A510),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A520),
		MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER),
		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
		{},
	};
	static bool all_safe = true;

	if (scope != SCOPE_LOCAL_CPU)
		return all_safe;

	if (is_midr_in_range_list(spectre_bhb_safe_list))
		return true;

	all_safe = false;

	return false;
}

static u8 spectre_bhb_loop_affected(void)
{
	u8 k = 0;

	static const struct midr_range spectre_bhb_k132_list[] = {
		MIDR_ALL_VERSIONS(MIDR_CORTEX_X3),
		MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2),
		{},
	};
	static const struct midr_range spectre_bhb_k38_list[] = {
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A715),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A720),
		{},
	};
	static const struct midr_range spectre_bhb_k32_list[] = {
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A78),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A78AE),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_X1),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A710),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_X2),
		MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2),
		MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1),
		{},
	};
	static const struct midr_range spectre_bhb_k24_list[] = {
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A76),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A76AE),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A77),
		MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1),
		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_GOLD),
		MIDR_ALL_VERSIONS(MIDR_HISI_HIP09),
		{},
	};
	static const struct midr_range spectre_bhb_k11_list[] = {
		MIDR_ALL_VERSIONS(MIDR_AMPERE1),
		{},
	};
	static const struct midr_range spectre_bhb_k8_list[] = {
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
		MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
		{},
	};

	if (is_midr_in_range_list(spectre_bhb_k132_list))
		k = 132;
	else if (is_midr_in_range_list(spectre_bhb_k38_list))
		k = 38;
	else if (is_midr_in_range_list(spectre_bhb_k32_list))
		k = 32;
	else if (is_midr_in_range_list(spectre_bhb_k24_list))
		k = 24;
	else if (is_midr_in_range_list(spectre_bhb_k11_list))
		k = 11;
	else if (is_midr_in_range_list(spectre_bhb_k8_list))
		k =  8;

	return k;
}

static enum mitigation_state spectre_bhb_get_cpu_fw_mitigation_state(void)
{
	int ret;
	struct arm_smccc_res res;

	arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID,
			     ARM_SMCCC_ARCH_WORKAROUND_3, &res);

	ret = res.a0;
	switch (ret) {
	case SMCCC_RET_SUCCESS:
		return SPECTRE_MITIGATED;
	case SMCCC_ARCH_WORKAROUND_RET_UNAFFECTED:
		return SPECTRE_UNAFFECTED;
	default:
		fallthrough;
	case SMCCC_RET_NOT_SUPPORTED:
		return SPECTRE_VULNERABLE;
	}
}

static bool has_spectre_bhb_fw_mitigation(void)
{
	enum mitigation_state fw_state;
	bool has_smccc = arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_NONE;

	fw_state = spectre_bhb_get_cpu_fw_mitigation_state();
	return has_smccc && fw_state == SPECTRE_MITIGATED;
}

static bool supports_ecbhb(int scope)
{
	u64 mmfr1;

	if (scope == SCOPE_LOCAL_CPU)
		mmfr1 = read_sysreg_s(SYS_ID_AA64MMFR1_EL1);
	else
		mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);

	return cpuid_feature_extract_unsigned_field(mmfr1,
						    ID_AA64MMFR1_EL1_ECBHB_SHIFT);
}

static u8 max_bhb_k;

bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry,
			     int scope)
{
	WARN_ON(scope != SCOPE_LOCAL_CPU || preemptible());

	if (supports_csv2p3(scope))
		return false;

	if (is_spectre_bhb_safe(scope))
		return false;

	/*
	 * At this point the core isn't known to be "safe" so we're going to
	 * assume it's vulnerable. We still need to update `max_bhb_k` though,
	 * but only if we aren't mitigating with clearbhb though.
	 */
	if (scope == SCOPE_LOCAL_CPU && !supports_clearbhb(SCOPE_LOCAL_CPU))
		max_bhb_k = max(max_bhb_k, spectre_bhb_loop_affected());

	return true;
}

u8 get_spectre_bhb_loop_value(void)
{
	return max_bhb_k;
}

static void this_cpu_set_vectors(enum arm64_bp_harden_el1_vectors slot)
{
	const char *v = arm64_get_bp_hardening_vector(slot);

	__this_cpu_write(this_cpu_vector, v);

	/*
	 * When KPTI is in use, the vectors are switched when exiting to
	 * user-space.
	 */
	if (cpus_have_cap(ARM64_UNMAP_KERNEL_AT_EL0))
		return;

	write_sysreg(v, vbar_el1);
	isb();
}

bool __read_mostly __nospectre_bhb;
static int __init parse_spectre_bhb_param(char *str)
{
	__nospectre_bhb = true;
	return 0;
}
early_param("nospectre_bhb", parse_spectre_bhb_param);

void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
{
	bp_hardening_cb_t cpu_cb;
	enum mitigation_state state = SPECTRE_VULNERABLE;
	struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);

	if (!is_spectre_bhb_affected(entry, SCOPE_LOCAL_CPU))
		return;

	if (arm64_get_spectre_v2_state() == SPECTRE_VULNERABLE) {
		/* No point mitigating Spectre-BHB alone. */
	} else if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY)) {
		pr_info_once("spectre-bhb mitigation disabled by compile time option\n");
	} else if (cpu_mitigations_off() || __nospectre_bhb) {
		pr_info_once("spectre-bhb mitigation disabled by command line option\n");
	} else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
		state = SPECTRE_MITIGATED;
		set_bit(BHB_HW, &system_bhb_mitigations);
	} else if (supports_clearbhb(SCOPE_LOCAL_CPU)) {
		/*
		 * Ensure KVM uses the indirect vector which will have ClearBHB
		 * added.
		 */
		if (!data->slot)
			data->slot = HYP_VECTOR_INDIRECT;

		this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN);
		state = SPECTRE_MITIGATED;
		set_bit(BHB_INSN, &system_bhb_mitigations);
	} else if (spectre_bhb_loop_affected()) {
		/*
		 * Ensure KVM uses the indirect vector which will have the
		 * branchy-loop added. A57/A72-r0 will already have selected
		 * the spectre-indirect vector, which is sufficient for BHB
		 * too.
		 */
		if (!data->slot)
			data->slot = HYP_VECTOR_INDIRECT;

		this_cpu_set_vectors(EL1_VECTOR_BHB_LOOP);
		state = SPECTRE_MITIGATED;
		set_bit(BHB_LOOP, &system_bhb_mitigations);
	} else if (has_spectre_bhb_fw_mitigation()) {
		/*
		 * Ensure KVM uses one of the spectre bp_hardening
		 * vectors. The indirect vector doesn't include the EL3
		 * call, so needs upgrading to
		 * HYP_VECTOR_SPECTRE_INDIRECT.
		 */
		if (!data->slot || data->slot == HYP_VECTOR_INDIRECT)
			data->slot += 1;

		this_cpu_set_vectors(EL1_VECTOR_BHB_FW);

		/*
		 * The WA3 call in the vectors supersedes the WA1 call
		 * made during context-switch. Uninstall any firmware
		 * bp_hardening callback.
		 */
		cpu_cb = spectre_v2_get_sw_mitigation_cb();
		if (__this_cpu_read(bp_hardening_data.fn) != cpu_cb)
			__this_cpu_write(bp_hardening_data.fn, NULL);

		state = SPECTRE_MITIGATED;
		set_bit(BHB_FW, &system_bhb_mitigations);
	}

	update_mitigation_state(&spectre_bhb_state, state);
}

bool is_spectre_bhb_fw_mitigated(void)
{
	return test_bit(BHB_FW, &system_bhb_mitigations);
}

/* Patched to NOP when enabled */
void noinstr spectre_bhb_patch_loop_mitigation_enable(struct alt_instr *alt,
						     __le32 *origptr,
						      __le32 *updptr, int nr_inst)
{
	BUG_ON(nr_inst != 1);

	if (test_bit(BHB_LOOP, &system_bhb_mitigations))
		*updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
}

/* Patched to NOP when enabled */
void noinstr spectre_bhb_patch_fw_mitigation_enabled(struct alt_instr *alt,
						   __le32 *origptr,
						   __le32 *updptr, int nr_inst)
{
	BUG_ON(nr_inst != 1);

	if (test_bit(BHB_FW, &system_bhb_mitigations))
		*updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
}

/* Patched to correct the immediate */
void noinstr spectre_bhb_patch_loop_iter(struct alt_instr *alt,
				   __le32 *origptr, __le32 *updptr, int nr_inst)
{
	u8 rd;
	u32 insn;

	BUG_ON(nr_inst != 1); /* MOV -> MOV */

	if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY))
		return;

	insn = le32_to_cpu(*origptr);
	rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);
	insn = aarch64_insn_gen_movewide(rd, max_bhb_k, 0,
					 AARCH64_INSN_VARIANT_64BIT,
					 AARCH64_INSN_MOVEWIDE_ZERO);
	*updptr++ = cpu_to_le32(insn);
}

/* Patched to mov WA3 when supported */
void noinstr spectre_bhb_patch_wa3(struct alt_instr *alt,
				   __le32 *origptr, __le32 *updptr, int nr_inst)
{
	u8 rd;
	u32 insn;

	BUG_ON(nr_inst != 1); /* MOV -> MOV */

	if (!IS_ENABLED(CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY) ||
	    !test_bit(BHB_FW, &system_bhb_mitigations))
		return;

	insn = le32_to_cpu(*origptr);
	rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, insn);

	insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_ORR,
						  AARCH64_INSN_VARIANT_32BIT,
						  AARCH64_INSN_REG_ZR, rd,
						  ARM_SMCCC_ARCH_WORKAROUND_3);
	if (WARN_ON_ONCE(insn == AARCH64_BREAK_FAULT))
		return;

	*updptr++ = cpu_to_le32(insn);
}

/* Patched to NOP when not supported */
void __init spectre_bhb_patch_clearbhb(struct alt_instr *alt,
				   __le32 *origptr, __le32 *updptr, int nr_inst)
{
	BUG_ON(nr_inst != 2);

	if (test_bit(BHB_INSN, &system_bhb_mitigations))
		return;

	*updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
	*updptr++ = cpu_to_le32(aarch64_insn_gen_nop());
}

#ifdef CONFIG_BPF_SYSCALL
#define EBPF_WARN "Unprivileged eBPF is enabled, data leaks possible via Spectre v2 BHB attacks!\n"
void unpriv_ebpf_notify(int new_state)
{
	if (spectre_v2_state == SPECTRE_VULNERABLE ||
	    spectre_bhb_state != SPECTRE_MITIGATED)
		return;

	if (!new_state)
		pr_err("WARNING: %s", EBPF_WARN);
}
#endif