2019-05-29 07:18:02 -07:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2016-10-28 15:04:47 -07:00
|
|
|
/*
|
|
|
|
* Resource Director Technology(RDT)
|
|
|
|
* - Cache Allocation code.
|
|
|
|
*
|
|
|
|
* Copyright (C) 2016 Intel Corporation
|
|
|
|
*
|
|
|
|
* Authors:
|
|
|
|
* Fenghua Yu <fenghua.yu@intel.com>
|
|
|
|
* Tony Luck <tony.luck@intel.com>
|
|
|
|
*
|
|
|
|
* More information about RDT be found in the Intel (R) x86 Architecture
|
|
|
|
* Software Developer Manual June 2016, volume 3, section 17.17.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
|
|
|
|
|
2018-12-10 13:21:54 -08:00
|
|
|
#include <linux/cpu.h>
|
x86/resctrl: Queue mon_event_read() instead of sending an IPI
Intel is blessed with an abundance of monitors, one per RMID, that can be
read from any CPU in the domain. MPAMs monitors reside in the MMIO MSC,
the number implemented is up to the manufacturer. This means when there are
fewer monitors than needed, they need to be allocated and freed.
MPAM's CSU monitors are used to back the 'llc_occupancy' monitor file. The
CSU counter is allowed to return 'not ready' for a small number of
micro-seconds after programming. To allow one CSU hardware monitor to be
used for multiple control or monitor groups, the CPU accessing the
monitor needs to be able to block when configuring and reading the
counter.
Worse, the domain may be broken up into slices, and the MMIO accesses
for each slice may need performing from different CPUs.
These two details mean MPAMs monitor code needs to be able to sleep, and
IPI another CPU in the domain to read from a resource that has been sliced.
mon_event_read() already invokes mon_event_count() via IPI, which means
this isn't possible. On systems using nohz-full, some CPUs need to be
interrupted to run kernel work as they otherwise stay in user-space
running realtime workloads. Interrupting these CPUs should be avoided,
and scheduling work on them may never complete.
Change mon_event_read() to pick a housekeeping CPU, (one that is not using
nohz_full) and schedule mon_event_count() and wait. If all the CPUs
in a domain are using nohz-full, then an IPI is used as the fallback.
This function is only used in response to a user-space filesystem request
(not the timing sensitive overflow code).
This allows MPAM to hide the slice behaviour from resctrl, and to keep
the monitor-allocation in monitor.c. When the IPI fallback is used on
machines where MPAM needs to make an access on multiple CPUs, the counter
read will always fail.
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Shaopeng Tan <tan.shaopeng@fujitsu.com>
Reviewed-by: Peter Newman <peternewman@google.com>
Reviewed-by: Reinette Chatre <reinette.chatre@intel.com>
Reviewed-by: Babu Moger <babu.moger@amd.com>
Tested-by: Shaopeng Tan <tan.shaopeng@fujitsu.com>
Tested-by: Peter Newman <peternewman@google.com>
Tested-by: Babu Moger <babu.moger@amd.com>
Tested-by: Carl Worth <carl@os.amperecomputing.com> # arm64
Link: https://lore.kernel.org/r/20240213184438.16675-14-james.morse@arm.com
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
2024-02-13 18:44:27 +00:00
|
|
|
|
2018-11-21 20:28:25 +00:00
|
|
|
#include "internal.h"
|
2016-10-28 15:04:47 -07:00
|
|
|
|
2024-06-28 14:56:04 -07:00
|
|
|
int resctrl_arch_update_one(struct rdt_resource *r, struct rdt_ctrl_domain *d,
|
2022-09-02 15:48:19 +00:00
|
|
|
u32 closid, enum resctrl_conf_type t, u32 cfg_val)
|
|
|
|
{
|
2024-06-28 14:56:04 -07:00
|
|
|
struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(d);
|
2022-09-02 15:48:19 +00:00
|
|
|
struct rdt_hw_resource *hw_res = resctrl_to_arch_res(r);
|
2025-03-11 18:37:14 +00:00
|
|
|
u32 idx = resctrl_get_config_index(closid, t);
|
2022-09-02 15:48:19 +00:00
|
|
|
struct msr_param msr_param;
|
|
|
|
|
2024-06-28 14:56:02 -07:00
|
|
|
if (!cpumask_test_cpu(smp_processor_id(), &d->hdr.cpu_mask))
|
2022-09-02 15:48:19 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
hw_dom->ctrl_val[idx] = cfg_val;
|
|
|
|
|
|
|
|
msr_param.res = r;
|
2024-03-08 13:38:45 -08:00
|
|
|
msr_param.dom = d;
|
2022-09-02 15:48:19 +00:00
|
|
|
msr_param.low = idx;
|
|
|
|
msr_param.high = idx + 1;
|
2024-03-08 13:38:46 -08:00
|
|
|
hw_res->msr_update(&msr_param);
|
2022-09-02 15:48:19 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2021-07-28 17:06:28 +00:00
|
|
|
int resctrl_arch_update_domains(struct rdt_resource *r, u32 closid)
|
2016-10-28 15:04:47 -07:00
|
|
|
{
|
2021-07-28 17:06:26 +00:00
|
|
|
struct resctrl_staged_config *cfg;
|
2024-06-28 14:56:04 -07:00
|
|
|
struct rdt_hw_ctrl_domain *hw_dom;
|
2016-10-28 15:04:47 -07:00
|
|
|
struct msr_param msr_param;
|
2024-06-28 14:56:04 -07:00
|
|
|
struct rdt_ctrl_domain *d;
|
2021-07-28 17:06:27 +00:00
|
|
|
enum resctrl_conf_type t;
|
2021-07-28 17:06:32 +00:00
|
|
|
u32 idx;
|
2016-10-28 15:04:47 -07:00
|
|
|
|
x86/resctrl: Separate arch and fs resctrl locks
resctrl has one mutex that is taken by the architecture-specific code, and the
filesystem parts. The two interact via cpuhp, where the architecture code
updates the domain list. Filesystem handlers that walk the domains list should
not run concurrently with the cpuhp callback modifying the list.
Exposing a lock from the filesystem code means the interface is not cleanly
defined, and creates the possibility of cross-architecture lock ordering
headaches. The interaction only exists so that certain filesystem paths are
serialised against CPU hotplug. The CPU hotplug code already has a mechanism to
do this using cpus_read_lock().
MPAM's monitors have an overflow interrupt, so it needs to be possible to walk
the domains list in irq context. RCU is ideal for this, but some paths need to
be able to sleep to allocate memory.
Because resctrl_{on,off}line_cpu() take the rdtgroup_mutex as part of a cpuhp
callback, cpus_read_lock() must always be taken first.
rdtgroup_schemata_write() already does this.
Most of the filesystem code's domain list walkers are currently protected by
the rdtgroup_mutex taken in rdtgroup_kn_lock_live(). The exceptions are
rdt_bit_usage_show() and the mon_config helpers which take the lock directly.
Make the domain list protected by RCU. An architecture-specific lock prevents
concurrent writers. rdt_bit_usage_show() could walk the domain list using RCU,
but to keep all the filesystem operations the same, this is changed to call
cpus_read_lock(). The mon_config helpers send multiple IPIs, take the
cpus_read_lock() in these cases.
The other filesystem list walkers need to be able to sleep. Add
cpus_read_lock() to rdtgroup_kn_lock_live() so that the cpuhp callbacks can't
be invoked when file system operations are occurring.
Add lockdep_assert_cpus_held() in the cases where the rdtgroup_kn_lock_live()
call isn't obvious.
Resctrl's domain online/offline calls now need to take the rdtgroup_mutex
themselves.
[ bp: Fold in a build fix: https://lore.kernel.org/r/87zfvwieli.ffs@tglx ]
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Shaopeng Tan <tan.shaopeng@fujitsu.com>
Reviewed-by: Reinette Chatre <reinette.chatre@intel.com>
Reviewed-by: Babu Moger <babu.moger@amd.com>
Tested-by: Shaopeng Tan <tan.shaopeng@fujitsu.com>
Tested-by: Peter Newman <peternewman@google.com>
Tested-by: Babu Moger <babu.moger@amd.com>
Tested-by: Carl Worth <carl@os.amperecomputing.com> # arm64
Link: https://lore.kernel.org/r/20240213184438.16675-25-james.morse@arm.com
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
2024-02-13 18:44:38 +00:00
|
|
|
/* Walking r->domains, ensure it can't race with cpuhp */
|
|
|
|
lockdep_assert_cpus_held();
|
|
|
|
|
2024-06-28 14:56:03 -07:00
|
|
|
list_for_each_entry(d, &r->ctrl_domains, hdr.list) {
|
2024-06-28 14:56:04 -07:00
|
|
|
hw_dom = resctrl_to_arch_ctrl_dom(d);
|
2024-03-08 13:38:45 -08:00
|
|
|
msr_param.res = NULL;
|
2021-07-28 17:06:27 +00:00
|
|
|
for (t = 0; t < CDP_NUM_TYPES; t++) {
|
|
|
|
cfg = &hw_dom->d_resctrl.staged_config[t];
|
|
|
|
if (!cfg->have_new_ctrl)
|
|
|
|
continue;
|
|
|
|
|
2025-03-11 18:37:14 +00:00
|
|
|
idx = resctrl_get_config_index(closid, t);
|
2024-03-08 13:38:45 -08:00
|
|
|
if (cfg->new_ctrl == hw_dom->ctrl_val[idx])
|
2021-07-28 17:06:32 +00:00
|
|
|
continue;
|
2024-03-08 13:38:45 -08:00
|
|
|
hw_dom->ctrl_val[idx] = cfg->new_ctrl;
|
2021-07-28 17:06:33 +00:00
|
|
|
|
2021-07-28 17:06:36 +00:00
|
|
|
if (!msr_param.res) {
|
|
|
|
msr_param.low = idx;
|
|
|
|
msr_param.high = msr_param.low + 1;
|
|
|
|
msr_param.res = r;
|
2024-03-08 13:38:45 -08:00
|
|
|
msr_param.dom = d;
|
2021-07-28 17:06:36 +00:00
|
|
|
} else {
|
|
|
|
msr_param.low = min(msr_param.low, idx);
|
|
|
|
msr_param.high = max(msr_param.high, idx + 1);
|
|
|
|
}
|
2021-07-28 17:06:27 +00:00
|
|
|
}
|
2024-03-08 13:38:45 -08:00
|
|
|
if (msr_param.res)
|
2024-06-28 14:56:02 -07:00
|
|
|
smp_call_function_any(&d->hdr.cpu_mask, rdt_ctrl_update, &msr_param, 1);
|
2016-10-28 15:04:47 -07:00
|
|
|
}
|
2018-04-20 15:36:19 -07:00
|
|
|
|
2016-10-28 15:04:47 -07:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2024-06-28 14:56:04 -07:00
|
|
|
u32 resctrl_arch_get_config(struct rdt_resource *r, struct rdt_ctrl_domain *d,
|
2021-08-11 16:38:31 +00:00
|
|
|
u32 closid, enum resctrl_conf_type type)
|
2021-07-28 17:06:29 +00:00
|
|
|
{
|
2024-06-28 14:56:04 -07:00
|
|
|
struct rdt_hw_ctrl_domain *hw_dom = resctrl_to_arch_ctrl_dom(d);
|
2025-03-11 18:37:14 +00:00
|
|
|
u32 idx = resctrl_get_config_index(closid, type);
|
2021-07-28 17:06:29 +00:00
|
|
|
|
2022-09-02 15:48:17 +00:00
|
|
|
return hw_dom->ctrl_val[idx];
|
2021-07-28 17:06:29 +00:00
|
|
|
}
|