openEuler/openEuler_kernel_rk3588
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
openEuler_kernel_rk3588/drivers/platform/mpam/mpam_devices.c
ardu 24cacf092e init
2026-01-29 22:25:33 +08:00

2423 lines
60 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2022 Arm Ltd.
#define pr_fmt(fmt) "mpam: " fmt
#include <linux/acpi.h>
#include <linux/atomic.h>
#include <linux/arm_mpam.h>
#include <linux/bitfield.h>
#include <linux/bitmap.h>
#include <linux/cacheinfo.h>
#include <linux/cpu.h>
#include <linux/cpumask.h>
#include <linux/device.h>
#include <linux/errno.h>
#include <linux/gfp.h>
#include <linux/interrupt.h>
#include <linux/irq.h>
#include <linux/irqdesc.h>
#include <linux/list.h>
#include <linux/lockdep.h>
#include <linux/mutex.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include <linux/platform_device.h>
#include <linux/printk.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include <linux/workqueue.h>
#include <acpi/pcc.h>
#include <asm/mpam.h>
#include "mpam_internal.h"
/*
* mpam_list_lock protects the SRCU lists when writing. Once the
* mpam_enabled key is enabled these lists are read-only,
* unless the error interrupt disables the driver.
*/
static DEFINE_MUTEX(mpam_list_lock);
static LIST_HEAD(mpam_all_msc);
struct srcu_struct mpam_srcu;
/* MPAM isn't available until all the MSC have been probed. */
static u32 mpam_num_msc;
static int mpam_cpuhp_state;
static DEFINE_MUTEX(mpam_cpuhp_state_lock);
/*
* The smallest common values for any CPU or MSC in the system.
* Generating traffic outside this range will result in screaming interrupts.
*/
u16 mpam_partid_max;
u8 mpam_pmg_max;
static bool partid_max_init, partid_max_published;
static DEFINE_SPINLOCK(partid_max_lock);
/*
* mpam is enabled once all devices have been probed from CPU online callbacks,
* scheduled via this work_struct. If access to an MSC depends on a CPU that
* was not brought online at boot, this can happen surprisingly late.
*/
static DECLARE_WORK(mpam_enable_work, &mpam_enable);
/*
* All mpam error interrupts indicate a software bug. On receipt, disable the
* driver.
*/
static DECLARE_WORK(mpam_broken_work, &mpam_disable);
/*
* An MSC is a container for resources, each identified by their RIS index.
* Components are a group of RIS that control the same thing.
* Classes are the set components of the same type.
*
* e.g. The set of RIS that make up the L2 are a component. These are sometimes
* termed slices. They should be configured as if they were one MSC.
*
* e.g. The SoC probably has more than one L2, each attached to a distinct set
* of CPUs. All the L2 components are grouped as a class.
*
* When creating an MSC, struct mpam_msc is added to the all mpam_all_msc list,
* then linked via struct mpam_ris to a component and a class.
* The same MSC may exist under different class->component paths, but the RIS
* index will be unique.
*/
LIST_HEAD(mpam_classes);
static u32 __mpam_read_reg(struct mpam_msc *msc, u16 reg)
{
WARN_ON_ONCE(reg + sizeof(u32) > msc->mapped_hwpage_sz);
WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility));
return readl_relaxed(msc->mapped_hwpage + reg);
}
static void __mpam_write_reg(struct mpam_msc *msc, u16 reg, u32 val)
{
WARN_ON_ONCE(reg + sizeof(u32) > msc->mapped_hwpage_sz);
WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility));
writel_relaxed(val, msc->mapped_hwpage + reg);
}
#define mpam_read_partsel_reg(msc, reg) \
({ \
u32 ____ret; \
\
lockdep_assert_held_once(&msc->part_sel_lock); \
____ret = __mpam_read_reg(msc, MPAMF_##reg); \
\
____ret; \
})
#define mpam_write_partsel_reg(msc, reg, val) \
({ \
lockdep_assert_held_once(&msc->part_sel_lock); \
__mpam_write_reg(msc, MPAMCFG_##reg, val); \
})
#define mpam_read_monsel_reg(msc, reg) \
({ \
u32 ____ret; \
\
lockdep_assert_held_once(&msc->mon_sel_lock); \
____ret = __mpam_read_reg(msc, MSMON_##reg); \
\
____ret; \
})
#define mpam_write_monsel_reg(msc, reg, val) \
({ \
lockdep_assert_held_once(&msc->mon_sel_lock); \
__mpam_write_reg(msc, MSMON_##reg, val); \
})
static u64 mpam_msc_read_idr(struct mpam_msc *msc)
{
u64 idr_high = 0, idr_low;
lockdep_assert_held(&msc->part_sel_lock);
idr_low = mpam_read_partsel_reg(msc, IDR);
if (FIELD_GET(MPAMF_IDR_HAS_EXT, idr_low))
idr_high = mpam_read_partsel_reg(msc, IDR + 4);
return (idr_high << 32) | idr_low;
}
static void mpam_msc_zero_esr(struct mpam_msc *msc)
{
writel_relaxed(0, msc->mapped_hwpage + MPAMF_ESR);
if (msc->has_extd_esr)
writel_relaxed(0, msc->mapped_hwpage + MPAMF_ESR + 4);
}
static u64 mpam_msc_read_esr(struct mpam_msc *msc)
{
u64 esr_high = 0, esr_low;
esr_low = readl_relaxed(msc->mapped_hwpage + MPAMF_ESR);
if (msc->has_extd_esr)
esr_high = readl_relaxed(msc->mapped_hwpage + MPAMF_ESR + 4);
return (esr_high << 32) | esr_low;
}
static void __mpam_part_sel(u8 ris_idx, u16 partid, struct mpam_msc *msc)
{
u32 partsel;
lockdep_assert_held(&msc->part_sel_lock);
partsel = FIELD_PREP(MPAMCFG_PART_SEL_RIS, ris_idx) |
FIELD_PREP(MPAMCFG_PART_SEL_PARTID_SEL, partid);
mpam_write_partsel_reg(msc, PART_SEL, partsel);
}
int mpam_register_requestor(u16 partid_max, u8 pmg_max)
{
int err = 0;
spin_lock(&partid_max_lock);
if (!partid_max_init) {
mpam_partid_max = partid_max;
mpam_pmg_max = pmg_max;
partid_max_init = true;
} else if (!partid_max_published) {
mpam_partid_max = min(mpam_partid_max, partid_max);
mpam_pmg_max = min(mpam_pmg_max, pmg_max);
} else {
/* New requestors can't lower the values */
if ((partid_max < mpam_partid_max) || (pmg_max < mpam_pmg_max))
err = -EBUSY;
}
spin_unlock(&partid_max_lock);
return err;
}
EXPORT_SYMBOL(mpam_register_requestor);
static struct mpam_component *
mpam_component_alloc(struct mpam_class *class, int id, gfp_t gfp)
{
struct mpam_component *comp;
lockdep_assert_held(&mpam_list_lock);
comp = kzalloc(sizeof(*comp), gfp);
if (!comp)
return ERR_PTR(-ENOMEM);
comp->comp_id = id;
INIT_LIST_HEAD_RCU(&comp->ris);
/* affinity is updated when ris are added */
INIT_LIST_HEAD_RCU(&comp->class_list);
comp->class = class;
list_add_rcu(&comp->class_list, &class->components);
return comp;
}
static struct mpam_component *
mpam_component_get(struct mpam_class *class, int id, bool alloc, gfp_t gfp)
{
struct mpam_component *comp;
lockdep_assert_held(&mpam_list_lock);
list_for_each_entry(comp, &class->components, class_list) {
if (comp->comp_id == id)
return comp;
}
if (!alloc)
return ERR_PTR(-ENOENT);
return mpam_component_alloc(class, id, gfp);
}
static struct mpam_class *
mpam_class_alloc(u8 level_idx, enum mpam_class_types type, gfp_t gfp)
{
struct mpam_class *class;
lockdep_assert_held(&mpam_list_lock);
class = kzalloc(sizeof(*class), gfp);
if (!class)
return ERR_PTR(-ENOMEM);
INIT_LIST_HEAD_RCU(&class->components);
/* affinity is updated when ris are added */
class->level = level_idx;
class->type = type;
INIT_LIST_HEAD_RCU(&class->classes_list);
ida_init(&class->ida_csu_mon);
ida_init(&class->ida_mbwu_mon);
list_add_rcu(&class->classes_list, &mpam_classes);
return class;
}
static struct mpam_class *
mpam_class_get(u8 level_idx, enum mpam_class_types type, bool alloc, gfp_t gfp)
{
bool found = false;
struct mpam_class *class;
lockdep_assert_held(&mpam_list_lock);
list_for_each_entry(class, &mpam_classes, classes_list) {
if (class->type == type && class->level == level_idx) {
found = true;
break;
}
}
if (found)
return class;
if (!alloc)
return ERR_PTR(-ENOENT);
return mpam_class_alloc(level_idx, type, gfp);
}
static void mpam_class_destroy(struct mpam_class *class)
{
lockdep_assert_held(&mpam_list_lock);
list_del_rcu(&class->classes_list);
synchronize_srcu(&mpam_srcu);
kfree(class);
}
static void mpam_comp_destroy(struct mpam_component *comp)
{
struct mpam_class *class = comp->class;
lockdep_assert_held(&mpam_list_lock);
list_del_rcu(&comp->class_list);
synchronize_srcu(&mpam_srcu);
kfree(comp);
if (list_empty(&class->components))
mpam_class_destroy(class);
}
/* synchronise_srcu() before freeing ris */
static void mpam_ris_destroy(struct mpam_msc_ris *ris)
{
struct mpam_component *comp = ris->comp;
struct mpam_class *class = comp->class;
struct mpam_msc *msc = ris->msc;
lockdep_assert_held(&mpam_list_lock);
lockdep_assert_preemption_enabled();
clear_bit(ris->ris_idx, msc->ris_idxs);
list_del_rcu(&ris->comp_list);
list_del_rcu(&ris->msc_list);
cpumask_andnot(&comp->affinity, &comp->affinity, &ris->affinity);
cpumask_andnot(&class->affinity, &class->affinity, &ris->affinity);
if (list_empty(&comp->ris))
mpam_comp_destroy(comp);
}
/*
* There are two ways of reaching a struct mpam_msc_ris. Via the
* class->component->ris, or via the msc.
* When destroying the msc, the other side needs unlinking and cleaning up too.
* synchronise_srcu() before freeing msc.
*/
static void mpam_msc_destroy(struct mpam_msc *msc)
{
struct mpam_msc_ris *ris, *tmp;
lockdep_assert_held(&mpam_list_lock);
lockdep_assert_preemption_enabled();
list_for_each_entry_safe(ris, tmp, &msc->ris, msc_list)
mpam_ris_destroy(ris);
}
/*
* The cacheinfo structures are only populated when CPUs are online.
* This helper walks the device tree to include offline CPUs too.
*/
static int get_cpumask_from_cache_id(u32 cache_id, u32 cache_level,
cpumask_t *affinity)
{
int cpu, err;
u32 iter_level;
int iter_cache_id;
struct device_node *iter;
if (!acpi_disabled)
return acpi_pptt_get_cpumask_from_cache_id(cache_id, affinity);
for_each_possible_cpu(cpu) {
iter = of_get_cpu_node(cpu, NULL);
if (!iter) {
pr_err("Failed to find cpu%d device node\n", cpu);
return -ENOENT;
}
while ((iter = of_find_next_cache_node(iter))) {
err = of_property_read_u32(iter, "cache-level",
&iter_level);
if (err || (iter_level != cache_level)) {
of_node_put(iter);
continue;
}
/*
* get_cpu_cacheinfo_id() isn't ready until sometime
* during device_initcall(). Use cache_of_get_id().
*/
iter_cache_id = cache_of_get_id(iter);
if (cache_id == ~0UL) {
of_node_put(iter);
continue;
}
if (iter_cache_id == cache_id)
cpumask_set_cpu(cpu, affinity);
of_node_put(iter);
}
}
return 0;
}
/*
* cpumask_of_node() only knows about online CPUs. This can't tell us whether
* a class is represented on all possible CPUs.
*/
static void get_cpumask_from_node_id(u32 node_id, cpumask_t *affinity)
{
int cpu;
for_each_possible_cpu(cpu) {
if (node_id == cpu_to_node(cpu))
cpumask_set_cpu(cpu, affinity);
}
}
static int get_cpumask_from_cache(struct device_node *cache,
cpumask_t *affinity)
{
int err;
u32 cache_level;
int cache_id;
err = of_property_read_u32(cache, "cache-level", &cache_level);
if (err) {
pr_err("Failed to read cache-level from cache node\n");
return -ENOENT;
}
cache_id = cache_of_get_id(cache);
if (cache_id == ~0UL) {
pr_err("Failed to calculate cache-id from cache node\n");
return -ENOENT;
}
return get_cpumask_from_cache_id(cache_id, cache_level, affinity);
}
static int mpam_ris_get_affinity(struct mpam_msc *msc, cpumask_t *affinity,
enum mpam_class_types type,
struct mpam_class *class,
struct mpam_component *comp)
{
int err;
switch (type) {
case MPAM_CLASS_CACHE:
err = get_cpumask_from_cache_id(comp->comp_id, class->level,
affinity);
if (err)
return err;
if (cpumask_empty(affinity))
pr_warn_once("%s no CPUs associated with cache node",
dev_name(&msc->pdev->dev));
break;
case MPAM_CLASS_MEMORY:
get_cpumask_from_node_id(comp->comp_id, affinity);
if (cpumask_empty(affinity))
pr_warn_once("%s no CPUs associated with memory node",
dev_name(&msc->pdev->dev));
break;
case MPAM_CLASS_UNKNOWN:
return 0;
}
cpumask_and(affinity, affinity, &msc->accessibility);
return 0;
}
static int mpam_ris_create_locked(struct mpam_msc *msc, u8 ris_idx,
enum mpam_class_types type, u8 class_id,
int component_id, gfp_t gfp)
{
int err;
struct mpam_msc_ris *ris;
struct mpam_class *class;
struct mpam_component *comp;
lockdep_assert_held(&mpam_list_lock);
if (test_and_set_bit(ris_idx, msc->ris_idxs))
return -EBUSY;
ris = devm_kzalloc(&msc->pdev->dev, sizeof(*ris), gfp);
if (!ris)
return -ENOMEM;
class = mpam_class_get(class_id, type, true, gfp);
if (IS_ERR(class))
return PTR_ERR(class);
comp = mpam_component_get(class, component_id, true, gfp);
if (IS_ERR(comp)) {
if (list_empty(&class->components))
mpam_class_destroy(class);
return PTR_ERR(comp);
}
err = mpam_ris_get_affinity(msc, &ris->affinity, type, class, comp);
if (err) {
if (list_empty(&class->components))
mpam_class_destroy(class);
return err;
}
ris->ris_idx = ris_idx;
INIT_LIST_HEAD_RCU(&ris->comp_list);
INIT_LIST_HEAD_RCU(&ris->msc_list);
ris->msc = msc;
ris->comp = comp;
cpumask_or(&comp->affinity, &comp->affinity, &ris->affinity);
cpumask_or(&class->affinity, &class->affinity, &ris->affinity);
list_add_rcu(&ris->comp_list, &comp->ris);
list_add_rcu(&ris->msc_list, &msc->ris);
return 0;
}
int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx,
enum mpam_class_types type, u8 class_id, int component_id)
{
int err;
mutex_lock(&mpam_list_lock);
err = mpam_ris_create_locked(msc, ris_idx, type, class_id,
component_id, GFP_KERNEL);
mutex_unlock(&mpam_list_lock);
return err;
}
static struct mpam_msc_ris *mpam_get_or_create_ris(struct mpam_msc *msc,
u8 ris_idx)
{
int err;
struct mpam_msc_ris *ris, *found = ERR_PTR(-ENOENT);
lockdep_assert_held(&mpam_list_lock);
if (!test_bit(ris_idx, msc->ris_idxs)) {
err = mpam_ris_create_locked(msc, ris_idx, MPAM_CLASS_UNKNOWN,
0, 0, GFP_ATOMIC);
if (err)
return ERR_PTR(err);
}
list_for_each_entry(ris, &msc->ris, msc_list) {
if (ris->ris_idx == ris_idx) {
found = ris;
break;
}
}
return found;
}
static void mpam_ris_hw_probe(struct mpam_msc_ris *ris)
{
int err;
struct mpam_msc *msc = ris->msc;
struct mpam_props *props = &ris->props;
struct mpam_class *class = ris->comp->class;
lockdep_assert_held(&msc->lock);
lockdep_assert_held(&msc->part_sel_lock);
/* Cache Capacity Partitioning */
if (FIELD_GET(MPAMF_IDR_HAS_CCAP_PART, ris->idr)) {
u32 ccap_features = mpam_read_partsel_reg(msc, CCAP_IDR);
props->cmax_wd = FIELD_GET(MPAMF_CCAP_IDR_CMAX_WD, ccap_features);
if (props->cmax_wd)
mpam_set_feature(mpam_feat_ccap_part, props);
}
/* Cache Portion partitioning */
if (FIELD_GET(MPAMF_IDR_HAS_CPOR_PART, ris->idr)) {
u32 cpor_features = mpam_read_partsel_reg(msc, CPOR_IDR);
props->cpbm_wd = FIELD_GET(MPAMF_CPOR_IDR_CPBM_WD, cpor_features);
if (props->cpbm_wd)
mpam_set_feature(mpam_feat_cpor_part, props);
}
/* Memory bandwidth partitioning */
if (FIELD_GET(MPAMF_IDR_HAS_MBW_PART, ris->idr)) {
u32 mbw_features = mpam_read_partsel_reg(msc, MBW_IDR);
/* portion bitmap resolution */
props->mbw_pbm_bits = FIELD_GET(MPAMF_MBW_IDR_BWPBM_WD, mbw_features);
if (props->mbw_pbm_bits &&
FIELD_GET(MPAMF_MBW_IDR_HAS_PBM, mbw_features))
mpam_set_feature(mpam_feat_mbw_part, props);
props->bwa_wd = FIELD_GET(MPAMF_MBW_IDR_BWA_WD, mbw_features);
if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MAX, mbw_features))
mpam_set_feature(mpam_feat_mbw_max, props);
if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MIN, mbw_features))
mpam_set_feature(mpam_feat_mbw_min, props);
if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_PROP, mbw_features))
mpam_set_feature(mpam_feat_mbw_prop, props);
}
/* Priority partitioning */
if (FIELD_GET(MPAMF_IDR_HAS_PRI_PART, ris->idr)) {
u32 pri_features = mpam_read_partsel_reg(msc, PRI_IDR);
props->intpri_wd = FIELD_GET(MPAMF_PRI_IDR_INTPRI_WD, pri_features);
if (props->intpri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_INTPRI, pri_features)) {
mpam_set_feature(mpam_feat_intpri_part, props);
if (FIELD_GET(MPAMF_PRI_IDR_INTPRI_0_IS_LOW, pri_features))
mpam_set_feature(mpam_feat_intpri_part_0_low, props);
}
props->dspri_wd = FIELD_GET(MPAMF_PRI_IDR_DSPRI_WD, pri_features);
if (props->dspri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_DSPRI, pri_features)) {
mpam_set_feature(mpam_feat_dspri_part, props);
if (FIELD_GET(MPAMF_PRI_IDR_DSPRI_0_IS_LOW, pri_features))
mpam_set_feature(mpam_feat_dspri_part_0_low, props);
}
}
/* Performance Monitoring */
if (FIELD_GET(MPAMF_IDR_HAS_MSMON, ris->idr)) {
u32 msmon_features = mpam_read_partsel_reg(msc, MSMON_IDR);
if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_CSU, msmon_features)) {
u32 csumonidr, discard;
/*
* If the firmware max-nrdy-us property is missing, the
* CSU counters can't be used. Should we wait forever?
*/
err = device_property_read_u32(&msc->pdev->dev,
"arm,not-ready-us",
&discard);
csumonidr = mpam_read_partsel_reg(msc, CSUMON_IDR);
props->num_csu_mon = FIELD_GET(MPAMF_CSUMON_IDR_NUM_MON, csumonidr);
if (props->num_csu_mon && !err)
mpam_set_feature(mpam_feat_msmon_csu, props);
else if (props->num_csu_mon)
pr_err_once("Counters are not usable because not-ready timeout was not provided by firmware.");
}
if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_MBWU, msmon_features)) {
bool has_long;
u32 mbwumonidr = mpam_read_partsel_reg(msc, MBWUMON_IDR);
props->num_mbwu_mon = FIELD_GET(MPAMF_MBWUMON_IDR_NUM_MON, mbwumonidr);
if (props->num_mbwu_mon)
mpam_set_feature(mpam_feat_msmon_mbwu, props);
if (FIELD_GET(MPAMF_MBWUMON_IDR_HAS_RWBW, mbwumonidr))
mpam_set_feature(mpam_feat_msmon_mbwu_rwbw, props);
/*
* Treat long counter and its extension, lwd as mutually
* exclusive feature bits. Though these are dependent
* fields at the implementation level, there would never
* be a need for mpam_feat_msmon_mbwu_44counter (long
* counter) and mpam_feat_msmon_mbwu_63counter (lwd)
* bits to be set together.
*
* mpam_feat_msmon_mbwu isn't treated as an exclusive
* bit as this feature bit would be used as the "front
* facing feature bit" for any checks related to mbwu
* monitors.
*/
has_long = FIELD_GET(MPAMF_MBWUMON_IDR_HAS_LONG, mbwumonidr);
if (props->num_mbwu_mon && has_long) {
if (FIELD_GET(MPAMF_MBWUMON_IDR_LWD, mbwumonidr))
mpam_set_feature(mpam_feat_msmon_mbwu_63counter, props);
else
mpam_set_feature(mpam_feat_msmon_mbwu_44counter, props);
}
}
}
/*
* RIS with PARTID narrowing don't have enough storage for one
* configuration per PARTID. If these are in a class we could use,
* reduce the supported partid_max to match the numer of intpartid.
* If the class is unknown, just ignore it.
*/
if (FIELD_GET(MPAMF_IDR_HAS_PARTID_NRW, ris->idr) &&
class->type != MPAM_CLASS_UNKNOWN) {
u32 nrwidr = mpam_read_partsel_reg(msc, PARTID_NRW_IDR);
u16 partid_max = FIELD_GET(MPAMF_PARTID_NRW_IDR_INTPARTID_MAX, nrwidr);
mpam_set_feature(mpam_feat_partid_nrw, props);
msc->partid_max = min(msc->partid_max, partid_max);
}
}
static int mpam_msc_hw_probe(struct mpam_msc *msc)
{
u64 idr;
u16 partid_max;
u8 ris_idx, pmg_max;
struct mpam_msc_ris *ris;
lockdep_assert_held(&msc->lock);
spin_lock(&msc->part_sel_lock);
idr = mpam_read_partsel_reg(msc, AIDR);
if ((idr & MPAMF_AIDR_ARCH_MAJOR_REV) != MPAM_ARCHITECTURE_V1) {
pr_err_once("%s does not match MPAM architecture v1.0\n",
dev_name(&msc->pdev->dev));
spin_unlock(&msc->part_sel_lock);
return -EIO;
}
idr = mpam_msc_read_idr(msc);
spin_unlock(&msc->part_sel_lock);
msc->ris_max = FIELD_GET(MPAMF_IDR_RIS_MAX, idr);
/* Use these values so partid/pmg always starts with a valid value */
msc->partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr);
msc->pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr);
for (ris_idx = 0; ris_idx <= msc->ris_max; ris_idx++) {
spin_lock(&msc->part_sel_lock);
__mpam_part_sel(ris_idx, 0, msc);
idr = mpam_msc_read_idr(msc);
spin_unlock(&msc->part_sel_lock);
partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr);
pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr);
msc->partid_max = min(msc->partid_max, partid_max);
msc->pmg_max = min(msc->pmg_max, pmg_max);
msc->has_extd_esr = FIELD_GET(MPAMF_IDR_HAS_EXT_ESR, idr);
ris = mpam_get_or_create_ris(msc, ris_idx);
if (IS_ERR(ris)) {
return PTR_ERR(ris);
}
ris->idr = idr;
spin_lock(&msc->part_sel_lock);
__mpam_part_sel(ris_idx, 0, msc);
mpam_ris_hw_probe(ris);
spin_unlock(&msc->part_sel_lock);
}
spin_lock(&partid_max_lock);
mpam_partid_max = min(mpam_partid_max, msc->partid_max);
mpam_pmg_max = min(mpam_pmg_max, msc->pmg_max);
spin_unlock(&partid_max_lock);
msc->probed = true;
return 0;
}
struct mon_read
{
struct mpam_msc_ris *ris;
struct mon_cfg *ctx;
enum mpam_device_features type;
u64 *val;
int err;
};
static bool mpam_ris_has_mbwu_long_counter(struct mpam_msc_ris *ris)
{
return (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, &ris->props) ||
mpam_has_feature(mpam_feat_msmon_mbwu_44counter, &ris->props));
}
static u64 mpam_msc_read_mbwu_l(struct mpam_msc *msc)
{
int retry = 3;
u32 mbwu_l_low;
u64 mbwu_l_high1, mbwu_l_high2;
lockdep_assert_held_once(&msc->mon_sel_lock);
WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz);
WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility));
mbwu_l_high2 = readl_relaxed(msc->mapped_hwpage + MSMON_MBWU_L + 4);
do {
mbwu_l_high1 = mbwu_l_high2;
mbwu_l_low = readl_relaxed(msc->mapped_hwpage + MSMON_MBWU_L);
mbwu_l_high2 = readl_relaxed(msc->mapped_hwpage + MSMON_MBWU_L + 4);
retry--;
} while (mbwu_l_high1 != mbwu_l_high2 && retry > 0);
if (mbwu_l_high2 == mbwu_l_high1)
return (mbwu_l_high1 << 32) | mbwu_l_low;
return MSMON___NRDY_L;
}
static void mpam_msc_zero_mbwu_l(struct mpam_msc *msc)
{
lockdep_assert_held_once(&msc->mon_sel_lock);
WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz);
WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility));
writel_relaxed(0, msc->mapped_hwpage + MSMON_MBWU_L);
writel_relaxed(0, msc->mapped_hwpage + MSMON_MBWU_L + 4);
}
static void gen_msmon_ctl_flt_vals(struct mon_read *m, u32 *ctl_val,
u32 *flt_val)
{
struct mon_cfg *ctx = m->ctx;
switch (m->type) {
case mpam_feat_msmon_csu:
*ctl_val = MSMON_CFG_MBWU_CTL_TYPE_CSU;
break;
case mpam_feat_msmon_mbwu:
*ctl_val = MSMON_CFG_MBWU_CTL_TYPE_MBWU;
break;
default:
return;
}
/*
* For CSU counters its implementation-defined what happens when not
* filtering by partid.
*/
*ctl_val |= MSMON_CFG_x_CTL_MATCH_PARTID;
*flt_val = FIELD_PREP(MSMON_CFG_MBWU_FLT_PARTID, ctx->partid);
*flt_val |= FIELD_PREP(MSMON_CFG_MBWU_FLT_RWBW, ctx->opts);
if (m->ctx->match_pmg) {
*ctl_val |= MSMON_CFG_x_CTL_MATCH_PMG;
*flt_val |= FIELD_PREP(MSMON_CFG_MBWU_FLT_PMG, ctx->pmg);
}
if (mpam_has_feature(mpam_feat_msmon_mbwu_rwbw, &m->ris->props))
*flt_val |= FIELD_PREP(MSMON_CFG_MBWU_FLT_RWBW, ctx->opts);
}
static void read_msmon_ctl_flt_vals(struct mon_read *m, u32 *ctl_val,
u32 *flt_val)
{
struct mpam_msc *msc = m->ris->msc;
switch (m->type) {
case mpam_feat_msmon_csu:
*ctl_val = mpam_read_monsel_reg(msc, CFG_CSU_CTL);
*flt_val = mpam_read_monsel_reg(msc, CFG_CSU_FLT);
break;
case mpam_feat_msmon_mbwu:
*ctl_val = mpam_read_monsel_reg(msc, CFG_MBWU_CTL);
*flt_val = mpam_read_monsel_reg(msc, CFG_MBWU_FLT);
break;
default:
return;
}
}
static void write_msmon_ctl_flt_vals(struct mon_read *m, u32 ctl_val,
u32 flt_val)
{
struct mpam_msc *msc = m->ris->msc;
struct msmon_mbwu_state *mbwu_state;
/*
* Write the ctl_val with the enable bit cleared, reset the counter,
* then enable counter.
*/
switch (m->type) {
case mpam_feat_msmon_csu:
mpam_write_monsel_reg(msc, CFG_CSU_FLT, flt_val);
mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val);
mpam_write_monsel_reg(msc, CSU, 0);
mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val|MSMON_CFG_x_CTL_EN);
break;
case mpam_feat_msmon_mbwu:
mpam_write_monsel_reg(msc, CFG_MBWU_FLT, flt_val);
mpam_write_monsel_reg(msc, CFG_MBWU_CTL, ctl_val);
if (mpam_ris_has_mbwu_long_counter(m->ris))
mpam_msc_zero_mbwu_l(m->ris->msc);
else
mpam_write_monsel_reg(msc, MBWU, 0);
mpam_write_monsel_reg(msc, CFG_MBWU_CTL, ctl_val|MSMON_CFG_x_CTL_EN);
mbwu_state = &m->ris->mbwu_state[m->ctx->mon];
if (mbwu_state)
mbwu_state->prev_val = 0;
break;
default:
return;
}
}
static u64 mpam_msmon_overflow_val(struct mpam_msc_ris *ris)
{
/* TODO: implement scaling counters */
if (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, &ris->props))
return GENMASK_ULL(62, 0);
else if (mpam_has_feature(mpam_feat_msmon_mbwu_44counter, &ris->props))
return GENMASK_ULL(43, 0);
else
return GENMASK_ULL(30, 0);
}
static void __ris_msmon_read(void *arg)
{
bool nrdy = false;
unsigned long flags;
bool config_mismatch;
struct mon_read *m = arg;
u64 now, overflow_val = 0;
struct mon_cfg *ctx = m->ctx;
bool reset_on_next_read = false;
struct mpam_msc_ris *ris = m->ris;
struct mpam_msc *msc = m->ris->msc;
struct msmon_mbwu_state *mbwu_state;
u32 mon_sel, ctl_val, flt_val, cur_ctl, cur_flt;
lockdep_assert_held(&msc->lock);
spin_lock_irqsave(&msc->mon_sel_lock, flags);
mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, ctx->mon) |
FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx);
mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel);
if (m->type == mpam_feat_msmon_mbwu) {
mbwu_state = &ris->mbwu_state[ctx->mon];
if (mbwu_state) {
reset_on_next_read = mbwu_state->reset_on_next_read;
mbwu_state->reset_on_next_read = false;
}
}
/*
* Read the existing configuration to avoid re-writing the same values.
* This saves waiting for 'nrdy' on subsequent reads.
*/
read_msmon_ctl_flt_vals(m, &cur_ctl, &cur_flt);
gen_msmon_ctl_flt_vals(m, &ctl_val, &flt_val);
config_mismatch = cur_flt != flt_val ||
cur_ctl != (ctl_val | MSMON_CFG_x_CTL_EN);
if (config_mismatch || reset_on_next_read)
write_msmon_ctl_flt_vals(m, ctl_val, flt_val);
switch (m->type) {
case mpam_feat_msmon_csu:
now = mpam_read_monsel_reg(msc, CSU);
nrdy = now & MSMON___NRDY;
now = FIELD_GET(MSMON___VALUE, now);
break;
case mpam_feat_msmon_mbwu:
/*
* If long or lwd counters are supported, use them, else revert
* to the 32 bit counter.
*/
if (mpam_ris_has_mbwu_long_counter(ris)) {
now = mpam_msc_read_mbwu_l(msc);
nrdy = now & MSMON___NRDY_L;
if (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, &ris->props))
now = FIELD_GET(MSMON___LWD_VALUE, now);
else
now = FIELD_GET(MSMON___L_VALUE, now);
} else {
now = mpam_read_monsel_reg(msc, MBWU);
nrdy = now & MSMON___NRDY;
now = FIELD_GET(MSMON___VALUE, now);
}
if (nrdy)
break;
if (!mbwu_state)
break;
/* Add any pre-overflow value to the mbwu_state->val */
if (mbwu_state->prev_val > now)
overflow_val = mpam_msmon_overflow_val(ris) - mbwu_state->prev_val;
mbwu_state->prev_val = now;
mbwu_state->correction += overflow_val;
/* Include bandwidth consumed before the last hardware reset */
now += mbwu_state->correction;
break;
default:
return;
}
spin_unlock_irqrestore(&msc->mon_sel_lock, flags);
if (nrdy) {
m->err = -EBUSY;
return;
}
*(m->val) += now;
}
static int _msmon_read(struct mpam_component *comp, struct mon_read *arg)
{
int err, idx;
struct mpam_msc *msc;
struct mpam_msc_ris *ris;
idx = srcu_read_lock(&mpam_srcu);
list_for_each_entry_rcu(ris, &comp->ris, comp_list) {
arg->ris = ris;
msc = ris->msc;
mutex_lock(&msc->lock);
err = smp_call_function_any(&msc->accessibility,
__ris_msmon_read, arg, true);
mutex_unlock(&msc->lock);
if (!err && arg->err)
err = arg->err;
if (err)
break;
}
srcu_read_unlock(&mpam_srcu, idx);
return err;
}
int mpam_msmon_read(struct mpam_component *comp, struct mon_cfg *ctx,
enum mpam_device_features type, u64 *val)
{
int err;
struct mon_read arg;
u64 wait_jiffies = 0;
struct mpam_props *cprops = &comp->class->props;
might_sleep();
if (!mpam_is_enabled())
return -EIO;
if (!mpam_has_feature(type, cprops))
return -EOPNOTSUPP;
memset(&arg, 0, sizeof(arg));
arg.ctx = ctx;
arg.type = type;
arg.val = val;
*val = 0;
err = _msmon_read(comp, &arg);
if (err == -EBUSY)
wait_jiffies = usecs_to_jiffies(comp->class->nrdy_usec);
while (wait_jiffies)
wait_jiffies = schedule_timeout_uninterruptible(wait_jiffies);
if (err == -EBUSY) {
memset(&arg, 0, sizeof(arg));
arg.ctx = ctx;
arg.type = type;
arg.val = val;
*val = 0;
err = _msmon_read(comp, &arg);
}
return err;
}
void mpam_msmon_reset_all_mbwu(struct mpam_component *comp)
{
int idx, i;
unsigned long flags;
struct mpam_msc *msc;
struct mpam_msc_ris *ris;
if (!mpam_is_enabled())
return;
idx = srcu_read_lock(&mpam_srcu);
list_for_each_entry_rcu(ris, &comp->ris, comp_list) {
if (!mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props))
continue;
msc = ris->msc;
spin_lock_irqsave(&msc->mon_sel_lock, flags);
for(i = 0; i < ris->props.num_mbwu_mon; i++) {
ris->mbwu_state[i].correction = 0;
ris->mbwu_state[i].reset_on_next_read = true;
}
spin_unlock_irqrestore(&msc->mon_sel_lock, flags);
}
srcu_read_unlock(&mpam_srcu, idx);
}
void mpam_msmon_reset_mbwu(struct mpam_component *comp, struct mon_cfg *ctx)
{
int idx;
unsigned long flags;
struct mpam_msc *msc;
struct mpam_msc_ris *ris;
if (!mpam_is_enabled())
return;
idx = srcu_read_lock(&mpam_srcu);
list_for_each_entry_rcu(ris, &comp->ris, comp_list) {
if (!mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props))
continue;
msc = ris->msc;
spin_lock_irqsave(&msc->mon_sel_lock, flags);
ris->mbwu_state[ctx->mon].correction = 0;
ris->mbwu_state[ctx->mon].reset_on_next_read = true;
spin_unlock_irqrestore(&msc->mon_sel_lock, flags);
}
srcu_read_unlock(&mpam_srcu, idx);
}
static void mpam_reset_msc_bitmap(struct mpam_msc *msc, u16 reg, u16 wd)
{
u32 num_words, msb;
u32 bm = ~0;
int i;
lockdep_assert_held(&msc->part_sel_lock);
/*
* Write all ~0 to all but the last 32bit-word, which may
* have fewer bits...
*/
num_words = DIV_ROUND_UP(wd, 32);
for (i = 0; i < num_words - 1; i++, reg += sizeof(bm))
__mpam_write_reg(msc, reg, bm);
/*
* ....and then the last (maybe) partial 32bit word. When wd is a
* multiple of 32, msb should be 31 to write a full 32bit word.
*/
msb = (wd - 1) % 32;
bm = GENMASK(msb , 0);
if (bm)
__mpam_write_reg(msc, reg, bm);
}
static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid,
struct mpam_config *cfg)
{
u32 pri_val = 0;
u16 cmax = MPAMCFG_CMAX_CMAX;
struct mpam_msc *msc = ris->msc;
u16 bwa_fract = MPAMCFG_MBW_MAX_MAX;
struct mpam_props *rprops = &ris->props;
u16 dspri = GENMASK(rprops->dspri_wd, 0);
u16 intpri = GENMASK(rprops->intpri_wd, 0);
spin_lock(&msc->part_sel_lock);
__mpam_part_sel(ris->ris_idx, partid, msc);
if(mpam_has_feature(mpam_feat_partid_nrw, rprops))
mpam_write_partsel_reg(msc, INTPARTID,
(MPAMCFG_PART_SEL_INTERNAL | partid));
if (mpam_has_feature(mpam_feat_cpor_part, rprops)) {
if (mpam_has_feature(mpam_feat_cpor_part, cfg))
mpam_write_partsel_reg(msc, CPBM, cfg->cpbm);
else
mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM,
rprops->cpbm_wd);
}
if (mpam_has_feature(mpam_feat_mbw_part, rprops)) {
if (mpam_has_feature(mpam_feat_mbw_part, cfg))
mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm);
else
mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM,
rprops->mbw_pbm_bits);
}
if (mpam_has_feature(mpam_feat_mbw_min, rprops))
mpam_write_partsel_reg(msc, MBW_MIN, 0);
if (mpam_has_feature(mpam_feat_mbw_max, rprops)) {
if (mpam_has_feature(mpam_feat_mbw_max, cfg))
mpam_write_partsel_reg(msc, MBW_MAX, cfg->mbw_max | MPAMCFG_MBW_MAX_HARDLIM);
else
mpam_write_partsel_reg(msc, MBW_MAX, bwa_fract);
}
if (mpam_has_feature(mpam_feat_mbw_prop, rprops))
mpam_write_partsel_reg(msc, MBW_PROP, bwa_fract);
if (mpam_has_feature(mpam_feat_ccap_part, rprops))
mpam_write_partsel_reg(msc, CMAX, cmax);
if (mpam_has_feature(mpam_feat_intpri_part, rprops) ||
mpam_has_feature(mpam_feat_dspri_part, rprops)) {
/* aces high? */
if (!mpam_has_feature(mpam_feat_intpri_part_0_low, rprops))
intpri = 0;
if (!mpam_has_feature(mpam_feat_dspri_part_0_low, rprops))
dspri = 0;
if (mpam_has_feature(mpam_feat_intpri_part, rprops))
pri_val |= FIELD_PREP(MPAMCFG_PRI_INTPRI, intpri);
if (mpam_has_feature(mpam_feat_dspri_part, rprops))
pri_val |= FIELD_PREP(MPAMCFG_PRI_DSPRI, dspri);
mpam_write_partsel_reg(msc, PRI, pri_val);
}
spin_unlock(&msc->part_sel_lock);
}
struct reprogram_ris {
struct mpam_msc_ris *ris;
struct mpam_config *cfg;
};
/* Call with MSC lock held */
static int mpam_reprogram_ris(void *_arg)
{
u16 partid, partid_max;
struct reprogram_ris *arg = _arg;
struct mpam_msc_ris *ris = arg->ris;
struct mpam_config *cfg = arg->cfg;
if (ris->in_reset_state)
return 0;
spin_lock(&partid_max_lock);
partid_max = mpam_partid_max;
spin_unlock(&partid_max_lock);
for (partid = 0; partid < partid_max; partid++)
mpam_reprogram_ris_partid(ris, partid, cfg);
return 0;
}
static int mpam_restore_mbwu_state(void *_ris)
{
int i;
struct mon_read mwbu_arg;
struct mpam_msc_ris *ris = _ris;
for (i = 0; i < ris->props.num_mbwu_mon; i++) {
if (ris->mbwu_state[i].enabled) {
mwbu_arg.ris = ris;
mwbu_arg.ctx = &ris->mbwu_state[i].cfg;
mwbu_arg.type = mpam_feat_msmon_mbwu;
__ris_msmon_read(&mwbu_arg);
}
}
return 0;
}
static int mpam_save_mbwu_state(void *arg)
{
int i;
u64 val;
struct mon_cfg *cfg;
unsigned long flags;
u32 cur_flt, cur_ctl, mon_sel;
struct mpam_msc_ris *ris = arg;
struct mpam_msc *msc = ris->msc;
struct msmon_mbwu_state *mbwu_state;
for (i = 0; i < ris->props.num_mbwu_mon; i++) {
mbwu_state = &ris->mbwu_state[i];
cfg = &mbwu_state->cfg;
spin_lock_irqsave(&msc->mon_sel_lock, flags);
mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, i) |
FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx);
mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel);
cur_flt = mpam_read_monsel_reg(msc, CFG_MBWU_FLT);
cur_ctl = mpam_read_monsel_reg(msc, CFG_MBWU_CTL);
mpam_write_monsel_reg(msc, CFG_MBWU_CTL, 0);
if (mpam_ris_has_mbwu_long_counter(ris)) {
val = mpam_msc_read_mbwu_l(msc);
mpam_msc_zero_mbwu_l(msc);
} else {
val = mpam_read_monsel_reg(msc, MBWU);
mpam_write_monsel_reg(msc, MBWU, 0);
}
cfg->mon = i;
cfg->pmg = FIELD_GET(MSMON_CFG_MBWU_FLT_PMG, cur_flt);
cfg->match_pmg = FIELD_GET(MSMON_CFG_x_CTL_MATCH_PMG, cur_ctl);
cfg->partid = FIELD_GET(MSMON_CFG_MBWU_FLT_PARTID, cur_flt);
mbwu_state->correction += val;
mbwu_state->enabled = FIELD_GET(MSMON_CFG_x_CTL_EN, cur_ctl);
spin_unlock_irqrestore(&msc->mon_sel_lock, flags);
}
return 0;
}
/*
* Called via smp_call_on_cpu() to prevent migration, while still being
* pre-emptible.
*/
static int mpam_reset_ris(void *arg)
{
struct mpam_msc_ris *ris = arg;
struct reprogram_ris reprogram_arg;
struct mpam_config empty_cfg = { 0 };
if (ris->in_reset_state)
return 0;
reprogram_arg.ris = ris;
reprogram_arg.cfg = &empty_cfg;
mpam_reprogram_ris(&reprogram_arg);
return 0;
}
/*
* Get the preferred CPU for this MSC. If it is accessible from this CPU,
* this CPU is preferred. This can be preempted/migrated, it will only result
* in more work.
*/
static int mpam_get_msc_preferred_cpu(struct mpam_msc *msc)
{
int cpu = raw_smp_processor_id();
if (cpumask_test_cpu(cpu, &msc->accessibility))
return cpu;
return cpumask_first_and(&msc->accessibility, cpu_online_mask);
}
static int mpam_touch_msc(struct mpam_msc *msc, int (*fn)(void *a), void *arg)
{
lockdep_assert_irqs_enabled();
lockdep_assert_cpus_held();
lockdep_assert_held(&msc->lock);
return smp_call_on_cpu(mpam_get_msc_preferred_cpu(msc), fn, arg, true);
}
static void mpam_reset_msc(struct mpam_msc *msc, bool online)
{
int idx;
struct mpam_msc_ris *ris;
lockdep_assert_held(&msc->lock);
idx = srcu_read_lock(&mpam_srcu);
list_for_each_entry_rcu(ris, &msc->ris, msc_list) {
mpam_touch_msc(msc, &mpam_reset_ris, ris);
/*
* Set in_reset_state when coming online. The reset state
* for non-zero partid may be lost while the CPUs are offline.
*/
ris->in_reset_state = online;
if (mpam_is_enabled() && !online)
mpam_touch_msc(msc, &mpam_save_mbwu_state, ris);
}
srcu_read_unlock(&mpam_srcu, idx);
}
static void mpam_reprogram_msc(struct mpam_msc *msc)
{
int idx;
u16 partid;
bool reset;
struct mpam_config *cfg;
struct mpam_msc_ris *ris;
lockdep_assert_held(&msc->lock);
idx = srcu_read_lock(&mpam_srcu);
list_for_each_entry_rcu(ris, &msc->ris, msc_list) {
if (!mpam_is_enabled() && !ris->in_reset_state) {
mpam_touch_msc(msc, &mpam_reset_ris, ris);
ris->in_reset_state = true;
continue;
}
reset = true;
for (partid = 0; partid < mpam_partid_max; partid++) {
cfg = &ris->comp->cfg[partid];
if (cfg->features)
reset = false;
mpam_reprogram_ris_partid(ris, partid, cfg);
}
ris->in_reset_state = reset;
if (mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props))
mpam_touch_msc(msc, &mpam_restore_mbwu_state, ris);
}
srcu_read_unlock(&mpam_srcu, idx);
}
static void _enable_percpu_irq(void *_irq)
{
int *irq = _irq;
enable_percpu_irq(*irq, IRQ_TYPE_NONE);
}
static int mpam_cpu_online(unsigned int cpu)
{
int idx;
struct mpam_msc *msc;
idx = srcu_read_lock(&mpam_srcu);
list_for_each_entry_rcu(msc, &mpam_all_msc, glbl_list) {
if (!cpumask_test_cpu(cpu, &msc->accessibility))
continue;
mutex_lock(&msc->lock);
if (msc->reenable_error_ppi)
_enable_percpu_irq(&msc->reenable_error_ppi);
if (atomic_fetch_inc(&msc->online_refs) == 0)
mpam_reprogram_msc(msc);
mutex_unlock(&msc->lock);
}
srcu_read_unlock(&mpam_srcu, idx);
if (mpam_is_enabled())
mpam_resctrl_online_cpu(cpu);
return 0;
}
/* Before mpam is enabled, try to probe new MSC */
static int mpam_discovery_cpu_online(unsigned int cpu)
{
int err = 0;
struct mpam_msc *msc;
bool new_device_probed = false;
if (mpam_is_enabled())
return 0;
mutex_lock(&mpam_list_lock);
list_for_each_entry(msc, &mpam_all_msc, glbl_list) {
if (!cpumask_test_cpu(cpu, &msc->accessibility))
continue;
mutex_lock(&msc->lock);
if (!msc->probed)
err = mpam_msc_hw_probe(msc);
mutex_unlock(&msc->lock);
if (!err)
new_device_probed = true;
else
break; // mpam_broken
}
mutex_unlock(&mpam_list_lock);
if (new_device_probed && !err)
schedule_work(&mpam_enable_work);
if (err < 0)
return err;
return mpam_cpu_online(cpu);
}
static int mpam_cpu_offline(unsigned int cpu)
{
int idx;
struct mpam_msc *msc;
idx = srcu_read_lock(&mpam_srcu);
list_for_each_entry_rcu(msc, &mpam_all_msc, glbl_list) {
if (!cpumask_test_cpu(cpu, &msc->accessibility))
continue;
mutex_lock(&msc->lock);
if (msc->reenable_error_ppi)
disable_percpu_irq(msc->reenable_error_ppi);
if (atomic_dec_and_test(&msc->online_refs))
mpam_reset_msc(msc, false);
mutex_unlock(&msc->lock);
}
srcu_read_unlock(&mpam_srcu, idx);
if (mpam_is_enabled())
mpam_resctrl_offline_cpu(cpu);
return 0;
}
static void mpam_register_cpuhp_callbacks(int (*online)(unsigned int online))
{
mutex_lock(&mpam_cpuhp_state_lock);
mpam_cpuhp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mpam:online",
online, mpam_cpu_offline);
if (mpam_cpuhp_state <= 0) {
pr_err("Failed to register cpuhp callbacks");
mpam_cpuhp_state = 0;
}
mutex_unlock(&mpam_cpuhp_state_lock);
}
static int __setup_ppi(struct mpam_msc *msc)
{
int cpu;
msc->error_dev_id = alloc_percpu_gfp(struct mpam_msc *, GFP_KERNEL);
if (!msc->error_dev_id)
return -ENOMEM;
for_each_cpu(cpu, &msc->accessibility) {
struct mpam_msc *empty = *per_cpu_ptr(msc->error_dev_id, cpu);
if (empty != NULL) {
pr_err_once("%s shares PPI with %s!\n", dev_name(&msc->pdev->dev),
dev_name(&empty->pdev->dev));
return -EBUSY;
}
*per_cpu_ptr(msc->error_dev_id, cpu) = msc;
}
return 0;
}
static int mpam_msc_setup_error_irq(struct mpam_msc *msc)
{
int irq;
irq = platform_get_irq_byname_optional(msc->pdev, "error");
if (irq <= 0)
return 0;
/* Allocate and initialise the percpu device pointer for PPI */
if (irq_is_percpu(irq))
return __setup_ppi(msc);
/* sanity check: shared interrupts can be routed anywhere? */
if (!cpumask_equal(&msc->accessibility, cpu_possible_mask)) {
pr_err_once("msc:%u is a private resource with a shared error interrupt",
msc->id);
return -EINVAL;
}
return 0;
}
static int mpam_dt_count_msc(void)
{
int count = 0;
struct device_node *np;
for_each_compatible_node(np, NULL, "arm,mpam-msc")
count++;
return count;
}
static int mpam_dt_parse_resource(struct mpam_msc *msc, struct device_node *np,
u32 ris_idx)
{
int err = 0;
u32 level = 0;
unsigned long cache_id;
struct device_node *cache;
do {
if (of_device_is_compatible(np, "arm,mpam-cache")) {
cache = of_parse_phandle(np, "arm,mpam-device", 0);
if (!cache) {
pr_err("Failed to read phandle\n");
break;
}
} else if (of_device_is_compatible(np->parent, "cache")) {
cache = np->parent;
} else {
/* For now, only caches are supported */
cache = NULL;
break;
}
err = of_property_read_u32(cache, "cache-level", &level);
if (err) {
pr_err("Failed to read cache-level\n");
break;
}
cache_id = cache_of_get_id(cache);
if (cache_id == ~0UL) {
err = -ENOENT;
break;
}
err = mpam_ris_create(msc, ris_idx, MPAM_CLASS_CACHE, level,
cache_id);
} while (0);
of_node_put(cache);
return err;
}
static int mpam_dt_parse_resources(struct mpam_msc *msc, void *ignored)
{
int err, num_ris = 0;
const u32 *ris_idx_p;
struct device_node *iter, *np;
np = msc->pdev->dev.of_node;
for_each_child_of_node(np, iter) {
ris_idx_p = of_get_property(iter, "reg", NULL);
if (ris_idx_p) {
num_ris++;
err = mpam_dt_parse_resource(msc, iter, *ris_idx_p);
if (err) {
of_node_put(iter);
return err;
}
}
}
if (!num_ris)
mpam_dt_parse_resource(msc, np, 0);
return err;
}
static int get_msc_affinity(struct mpam_msc *msc)
{
struct device_node *parent;
u32 affinity_id;
int err;
if (!acpi_disabled) {
err = device_property_read_u32(&msc->pdev->dev, "cpu_affinity",
&affinity_id);
if (err) {
cpumask_copy(&msc->accessibility, cpu_possible_mask);
err = 0;
} else {
err = acpi_pptt_get_cpus_from_container(affinity_id,
&msc->accessibility);
}
return err;
}
/* This depends on the path to of_node */
parent = of_get_parent(msc->pdev->dev.of_node);
if (parent == of_root) {
cpumask_copy(&msc->accessibility, cpu_possible_mask);
err = 0;
} else {
if (of_device_is_compatible(parent, "cache")) {
err = get_cpumask_from_cache(parent,
&msc->accessibility);
} else {
err = -EINVAL;
pr_err("Cannot determine accessibility of MSC: %s\n",
dev_name(&msc->pdev->dev));
}
}
of_node_put(parent);
return err;
}
static int fw_num_msc;
static void mpam_pcc_rx_callback(struct mbox_client *cl, void *msg)
{
/* TODO: wake up tasks blocked on this MSC's PCC channel */
}
static int mpam_msc_drv_probe(struct platform_device *pdev)
{
int err;
pgprot_t prot;
void * __iomem io;
struct mpam_msc *msc;
struct resource *msc_res;
void *plat_data = pdev->dev.platform_data;
mutex_lock(&mpam_list_lock);
do {
msc = devm_kzalloc(&pdev->dev, sizeof(*msc), GFP_KERNEL);
if (!msc) {
err = -ENOMEM;
break;
}
INIT_LIST_HEAD_RCU(&msc->glbl_list);
msc->pdev = pdev;
err = device_property_read_u32(&pdev->dev, "arm,not-ready-us",
&msc->nrdy_usec);
if (err) {
/* This will prevent CSU monitors being usable */
msc->nrdy_usec = 0;
}
err = get_msc_affinity(msc);
if (err)
break;
if (cpumask_empty(&msc->accessibility)) {
pr_err_once("msc:%u is not accessible from any CPU!",
msc->id);
err = -EINVAL;
break;
}
mutex_init(&msc->lock);
msc->id = mpam_num_msc++;
INIT_LIST_HEAD_RCU(&msc->ris);
spin_lock_init(&msc->part_sel_lock);
spin_lock_init(&msc->mon_sel_lock);
err = mpam_msc_setup_error_irq(msc);
if (err) {
devm_kfree(&pdev->dev, msc);
msc = ERR_PTR(err);
break;
}
if (device_property_read_u32(&pdev->dev, "pcc-channel",
&msc->pcc_subspace_id))
msc->iface = MPAM_IFACE_MMIO;
else
msc->iface = MPAM_IFACE_PCC;
if (msc->iface == MPAM_IFACE_MMIO) {
io = devm_platform_get_and_ioremap_resource(pdev, 0,
&msc_res);
if (IS_ERR(io)) {
pr_err("Failed to map MSC base address\n");
devm_kfree(&pdev->dev, msc);
err = PTR_ERR(io);
break;
}
msc->mapped_hwpage_sz = msc_res->end - msc_res->start;
msc->mapped_hwpage = io;
} else if (msc->iface == MPAM_IFACE_PCC) {
msc->pcc_cl.dev = &pdev->dev;
msc->pcc_cl.rx_callback = mpam_pcc_rx_callback;
msc->pcc_cl.tx_block = false;
msc->pcc_cl.tx_tout = 1000; /* 1s */
msc->pcc_cl.knows_txdone = false;
msc->pcc_chan = pcc_mbox_request_channel(&msc->pcc_cl,
msc->pcc_subspace_id);
if (IS_ERR(msc->pcc_chan)) {
pr_err("Failed to request MSC PCC channel\n");
devm_kfree(&pdev->dev, msc);
err = PTR_ERR(msc->pcc_chan);
break;
}
prot = __acpi_get_mem_attribute(msc->pcc_chan->shmem_base_addr);
io = ioremap_prot(msc->pcc_chan->shmem_base_addr,
msc->pcc_chan->shmem_size, pgprot_val(prot));
if (IS_ERR(io)) {
pr_err("Failed to map MSC base address\n");
pcc_mbox_free_channel(msc->pcc_chan);
devm_kfree(&pdev->dev, msc);
err = PTR_ERR(io);
break;
}
/* TODO: issue a read to update the registers */
msc->mapped_hwpage_sz = msc->pcc_chan->shmem_size;
msc->mapped_hwpage = io + sizeof(struct acpi_pcct_shared_memory);
}
list_add_rcu(&msc->glbl_list, &mpam_all_msc);
platform_set_drvdata(pdev, msc);
} while (0);
mutex_unlock(&mpam_list_lock);
if (!err) {
/* Create RIS entries described by firmware */
if (!acpi_disabled)
err = acpi_mpam_parse_resources(msc, plat_data);
else
err = mpam_dt_parse_resources(msc, plat_data);
}
if (!err && fw_num_msc == mpam_num_msc)
mpam_register_cpuhp_callbacks(&mpam_discovery_cpu_online);
return err;
}
/*
* If a resource doesn't match class feature/configuration, do the right thing.
* For 'num' properties we can just take the minimum.
* For properties where the mismatched unused bits would make a difference, we
* nobble the class feature, as we can't configure all the resources.
* e.g. The L3 cache is composed of two resources with 13 and 17 portion
* bitmaps respectively.
*/
static void
__resource_props_mismatch(struct mpam_msc_ris *ris, struct mpam_class *class)
{
struct mpam_props *cprops = &class->props;
struct mpam_props *rprops = &ris->props;
lockdep_assert_held(&mpam_list_lock); /* we modify class */
/* Clear missing features */
cprops->features &= rprops->features;
/* Clear incompatible features */
if (cprops->cpbm_wd != rprops->cpbm_wd)
mpam_clear_feature(mpam_feat_cpor_part, &cprops->features);
if (cprops->mbw_pbm_bits != rprops->mbw_pbm_bits)
mpam_clear_feature(mpam_feat_mbw_part, &cprops->features);
/* bwa_wd is a count of bits, fewer bits means less precision */
if (cprops->bwa_wd != rprops->bwa_wd)
cprops->bwa_wd = min(cprops->bwa_wd, rprops->bwa_wd);
/* For num properties, take the minimum */
if (cprops->num_csu_mon != rprops->num_csu_mon)
cprops->num_csu_mon = min(cprops->num_csu_mon, rprops->num_csu_mon);
if (cprops->num_mbwu_mon != rprops->num_mbwu_mon)
cprops->num_mbwu_mon = min(cprops->num_mbwu_mon, rprops->num_mbwu_mon);
if (cprops->intpri_wd != rprops->intpri_wd)
cprops->intpri_wd = min(cprops->intpri_wd, rprops->intpri_wd);
if (cprops->dspri_wd != rprops->dspri_wd)
cprops->dspri_wd = min(cprops->dspri_wd, rprops->dspri_wd);
/* {int,ds}pri may not have differing 0-low behaviour */
if (mpam_has_feature(mpam_feat_intpri_part_0_low, cprops) !=
mpam_has_feature(mpam_feat_intpri_part_0_low, rprops))
mpam_clear_feature(mpam_feat_intpri_part, &cprops->features);
if (mpam_has_feature(mpam_feat_dspri_part_0_low, cprops) !=
mpam_has_feature(mpam_feat_dspri_part_0_low, rprops))
mpam_clear_feature(mpam_feat_dspri_part, &cprops->features);
}
/*
* Copy the first component's first resources's properties and features to the
* class. __resource_props_mismatch() will remove conflicts.
* It is not possible to have a class with no components, or a component with
* no resources.
*/
static void mpam_enable_init_class_features(struct mpam_class *class)
{
struct mpam_msc_ris *ris;
struct mpam_component *comp;
comp = list_first_entry_or_null(&class->components,
struct mpam_component, class_list);
if (WARN_ON(!comp))
return;
ris = list_first_entry_or_null(&comp->ris,
struct mpam_msc_ris, comp_list);
if (WARN_ON(!ris))
return;
class->props = ris->props;
}
/* Merge all the common resource features into class. */
static void mpam_enable_merge_features(void)
{
struct mpam_msc_ris *ris;
struct mpam_class *class;
struct mpam_component *comp;
lockdep_assert_held(&mpam_list_lock);
list_for_each_entry(class, &mpam_classes, classes_list) {
mpam_enable_init_class_features(class);
list_for_each_entry(comp, &class->components, class_list) {
list_for_each_entry(ris, &comp->ris, comp_list) {
__resource_props_mismatch(ris, class);
class->nrdy_usec = max(class->nrdy_usec,
ris->msc->nrdy_usec);
}
}
}
}
static char *mpam_errcode_names[16] = {
[0] = "No error",
[1] = "PARTID_SEL_Range",
[2] = "Req_PARTID_Range",
[3] = "MSMONCFG_ID_RANGE",
[4] = "Req_PMG_Range",
[5] = "Monitor_Range",
[6] = "intPARTID_Range",
[7] = "Unexpected_INTERNAL",
[8] = "Undefined_RIS_PART_SEL",
[9] = "RIS_No_Control",
[10] = "Undefined_RIS_MON_SEL",
[11] = "RIS_No_Monitor",
[12 ... 15] = "Reserved"
};
static int mpam_enable_msc_ecr(void *_msc)
{
struct mpam_msc *msc = _msc;
writel_relaxed(1, msc->mapped_hwpage + MPAMF_ECR);
return 0;
}
static int mpam_disable_msc_ecr(void *_msc)
{
struct mpam_msc *msc = _msc;
writel_relaxed(0, msc->mapped_hwpage + MPAMF_ECR);
return 0;
}
static irqreturn_t __mpam_irq_handler(int irq, struct mpam_msc *msc)
{
u64 reg;
u16 partid;
u8 errcode, pmg, ris;
if (WARN_ON_ONCE(!msc) ||
WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
&msc->accessibility)))
return IRQ_NONE;
reg = mpam_msc_read_esr(msc);
errcode = FIELD_GET(MPAMF_ESR_ERRCODE, reg);
if (!errcode)
return IRQ_NONE;
/* Clear level triggered irq */
mpam_msc_zero_esr(msc);
partid = FIELD_GET(MPAMF_ESR_PARTID_OR_MON, reg);
pmg = FIELD_GET(MPAMF_ESR_PMG, reg);
ris = FIELD_GET(MPAMF_ESR_PMG, reg);
pr_err("error irq from msc:%u '%s', partid:%u, pmg: %u, ris: %u\n",
msc->id, mpam_errcode_names[errcode], partid, pmg, ris);
if (irq_is_percpu(irq)) {
mpam_disable_msc_ecr(msc);
schedule_work(&mpam_broken_work);
return IRQ_HANDLED;
}
return IRQ_WAKE_THREAD;
}
static irqreturn_t mpam_ppi_handler(int irq, void *dev_id)
{
struct mpam_msc *msc = *(struct mpam_msc **)dev_id;
return __mpam_irq_handler(irq, msc);
}
static irqreturn_t mpam_spi_handler(int irq, void *dev_id)
{
struct mpam_msc *msc = dev_id;
return __mpam_irq_handler(irq, msc);
}
static irqreturn_t mpam_disable_thread(int irq, void *dev_id);
static int mpam_register_irqs(void)
{
int err, irq;
struct mpam_msc *msc;
lockdep_assert_cpus_held();
lockdep_assert_held(&mpam_list_lock);
list_for_each_entry(msc, &mpam_all_msc, glbl_list) {
irq = platform_get_irq_byname_optional(msc->pdev, "error");
if (irq <= 0)
continue;
/* The MPAM spec says the interrupt can be SPI, PPI or LPI */
/* We anticipate sharing the interrupt with other MSCs */
if (irq_is_percpu(irq)) {
err = request_percpu_irq(irq, &mpam_ppi_handler,
"mpam:msc:error",
msc->error_dev_id);
if (err)
return err;
mutex_lock(&msc->lock);
msc->reenable_error_ppi = irq;
smp_call_function_many(&msc->accessibility,
&_enable_percpu_irq, &irq,
true);
mutex_unlock(&msc->lock);
} else {
err = devm_request_threaded_irq(&msc->pdev->dev, irq,
&mpam_spi_handler,
&mpam_disable_thread,
IRQF_SHARED,
"mpam:msc:error", msc);
if (err)
return err;
}
mutex_lock(&msc->lock);
msc->error_irq_requested = true;
mpam_touch_msc(msc, mpam_enable_msc_ecr, msc);
msc->error_irq_hw_enabled = true;
mutex_unlock(&msc->lock);
}
return 0;
}
static void mpam_unregister_irqs(void)
{
int irq;
struct mpam_msc *msc;
cpus_read_lock();
/* take the lock as free_irq() can sleep */
mutex_lock(&mpam_list_lock);
list_for_each_entry(msc, &mpam_all_msc, glbl_list) {
irq = platform_get_irq_byname_optional(msc->pdev, "error");
if (irq <= 0)
continue;
mutex_lock(&msc->lock);
if (msc->error_irq_hw_enabled) {
mpam_touch_msc(msc, mpam_disable_msc_ecr, msc);
msc->error_irq_hw_enabled = false;
}
if (msc->error_irq_requested) {
if (irq_is_percpu(irq)) {
msc->reenable_error_ppi = 0;
free_percpu_irq(irq, msc->error_dev_id);
} else {
devm_free_irq(&msc->pdev->dev, irq, msc);
}
msc->error_irq_requested = false;
}
mutex_unlock(&msc->lock);
}
mutex_unlock(&mpam_list_lock);
cpus_read_unlock();
}
static void __destroy_component_cfg(struct mpam_component *comp)
{
unsigned long flags;
struct mpam_msc_ris *ris;
struct msmon_mbwu_state *mbwu_state;
kfree(comp->cfg);
list_for_each_entry(ris, &comp->ris, comp_list) {
mutex_lock(&ris->msc->lock);
spin_lock_irqsave(&ris->msc->mon_sel_lock, flags);
mbwu_state = ris->mbwu_state;
ris->mbwu_state = NULL;
spin_unlock_irqrestore(&ris->msc->mon_sel_lock, flags);
mutex_unlock(&ris->msc->lock);
kfree(mbwu_state);
}
}
static int __allocate_component_cfg(struct mpam_component *comp)
{
unsigned long flags;
struct mpam_msc_ris *ris;
struct msmon_mbwu_state *mbwu_state;
if (comp->cfg)
return 0;
comp->cfg = kcalloc(mpam_partid_max, sizeof(*comp->cfg), GFP_KERNEL);
if (!comp->cfg)
return -ENOMEM;
list_for_each_entry(ris, &comp->ris, comp_list) {
if (!ris->props.num_mbwu_mon)
continue;
mbwu_state = kcalloc(ris->props.num_mbwu_mon,
sizeof(*ris->mbwu_state), GFP_KERNEL);
if (!mbwu_state) {
__destroy_component_cfg(comp);
return -ENOMEM;
}
mutex_lock(&ris->msc->lock);
spin_lock_irqsave(&ris->msc->mon_sel_lock, flags);
ris->mbwu_state = mbwu_state;
spin_unlock_irqrestore(&ris->msc->mon_sel_lock, flags);
mutex_unlock(&ris->msc->lock);
}
return 0;
}
static int mpam_allocate_config(void)
{
int err = 0;
struct mpam_class *class;
struct mpam_component *comp;
lockdep_assert_held(&mpam_list_lock);
list_for_each_entry(class, &mpam_classes, classes_list) {
list_for_each_entry(comp, &class->components, class_list) {
err = __allocate_component_cfg(comp);
if (err)
return err;
}
}
return 0;
}
static void mpam_enable_once(void)
{
int err;
/*
* If all the MSC have been probed, enabling the IRQs happens next.
* That involves cross-calling to a CPU that can reach the MSC, and
* the locks must be taken in this order:
*/
cpus_read_lock();
mutex_lock(&mpam_list_lock);
do {
mpam_enable_merge_features();
err = mpam_allocate_config();
if (err) {
pr_err("Failed to allocate configuration arrays.\n");
break;
}
err = mpam_register_irqs();
if (err) {
pr_warn("Failed to register irqs: %d\n", err);
break;
}
} while (0);
mutex_unlock(&mpam_list_lock);
cpus_read_unlock();
if (!err) {
err = mpam_resctrl_setup();
if (err)
pr_err("Failed to initialise resctrl: %d\n", err);
}
if (err) {
schedule_work(&mpam_broken_work);
return;
}
mutex_lock(&mpam_cpuhp_state_lock);
cpuhp_remove_state(mpam_cpuhp_state);
mpam_cpuhp_state = 0;
mutex_unlock(&mpam_cpuhp_state_lock);
/*
* Once the cpuhp callbacks have been changed, mpam_partid_max can no
* longer change.
*/
spin_lock(&partid_max_lock);
partid_max_published = true;
spin_unlock(&partid_max_lock);
static_branch_enable(&mpam_enabled);
mpam_register_cpuhp_callbacks(mpam_cpu_online);
pr_info("MPAM enabled with %u partid and %u pmg\n",
READ_ONCE(mpam_partid_max) + 1, mpam_pmg_max + 1);
}
void mpam_reset_class(struct mpam_class *class)
{
int idx;
struct mpam_msc_ris *ris;
struct mpam_component *comp;
idx = srcu_read_lock(&mpam_srcu);
list_for_each_entry_rcu(comp, &class->components, class_list) {
memset(comp->cfg, 0, (mpam_partid_max * sizeof(*comp->cfg)));
list_for_each_entry_rcu(ris, &comp->ris, comp_list) {
mutex_lock(&ris->msc->lock);
mpam_touch_msc(ris->msc, mpam_reset_ris, ris);
mutex_unlock(&ris->msc->lock);
ris->in_reset_state = true;
}
}
srcu_read_unlock(&mpam_srcu, idx);
}
/*
* Called in response to an error IRQ.
* All of MPAMs errors indicate a software bug, restore any modified
* controls to their reset values.
*/
static irqreturn_t mpam_disable_thread(int irq, void *dev_id)
{
int idx;
struct mpam_class *class;
mutex_lock(&mpam_cpuhp_state_lock);
if (mpam_cpuhp_state) {
cpuhp_remove_state(mpam_cpuhp_state);
mpam_cpuhp_state = 0;
}
mutex_unlock(&mpam_cpuhp_state_lock);
mpam_resctrl_exit();
static_branch_disable(&mpam_enabled);
mpam_unregister_irqs();
idx = srcu_read_lock(&mpam_srcu);
list_for_each_entry_rcu(class, &mpam_classes, classes_list)
mpam_reset_class(class);
srcu_read_unlock(&mpam_srcu, idx);
return IRQ_HANDLED;
}
void mpam_disable(struct work_struct *ignored)
{
mpam_disable_thread(0, NULL);
}
/*
* Enable mpam once all devices have been probed.
* Scheduled by mpam_discovery_cpu_online() once all devices have been created.
* Also scheduled when new devices are probed when new CPUs come online.
*/
void mpam_enable(struct work_struct *work)
{
static atomic_t once;
struct mpam_msc *msc;
bool all_devices_probed = true;
mutex_lock(&mpam_list_lock);
list_for_each_entry(msc, &mpam_all_msc, glbl_list) {
mutex_lock(&msc->lock);
if (!msc->probed)
all_devices_probed = false;
mutex_unlock(&msc->lock);
if (!all_devices_probed)
break;
}
mutex_unlock(&mpam_list_lock);
if (all_devices_probed && !atomic_fetch_inc(&once))
mpam_enable_once();
}
static int mpam_msc_drv_remove(struct platform_device *pdev)
{
struct mpam_msc *msc = platform_get_drvdata(pdev);
if (!msc)
return 0;
mutex_lock(&mpam_list_lock);
mpam_num_msc--;
platform_set_drvdata(pdev, NULL);
list_del_rcu(&msc->glbl_list);
mpam_msc_destroy(msc);
synchronize_srcu(&mpam_srcu);
mutex_unlock(&mpam_list_lock);
return 0;
}
struct mpam_write_config_arg {
struct mpam_msc_ris *ris;
struct mpam_component *comp;
u16 partid;
};
static int __write_config(void *arg)
{
struct mpam_write_config_arg *c = arg;
mpam_reprogram_ris_partid(c->ris, c->partid, &c->comp->cfg[c->partid]);
return 0;
}
/* TODO: split into write_config/sync_config */
/* TODO: add config_dirty bitmap to drive sync_config */
int mpam_apply_config(struct mpam_component *comp, u16 partid,
struct mpam_config *cfg)
{
struct mpam_write_config_arg arg;
struct mpam_msc_ris *ris;
int idx;
lockdep_assert_cpus_held();
if (!memcmp(&comp->cfg[partid], cfg, sizeof(*cfg)))
return 0;
comp->cfg[partid] = *cfg;
arg.comp = comp;
arg.partid = partid;
idx = srcu_read_lock(&mpam_srcu);
list_for_each_entry_rcu(ris, &comp->ris, comp_list) {
arg.ris = ris;
mutex_lock(&ris->msc->lock);
mpam_touch_msc(ris->msc, __write_config, &arg);
mutex_unlock(&ris->msc->lock);
}
srcu_read_unlock(&mpam_srcu, idx);
return 0;
}
static const struct of_device_id mpam_of_match[] = {
{ .compatible = "arm,mpam-msc", },
{},
};
MODULE_DEVICE_TABLE(of, mpam_of_match);
static struct platform_driver mpam_msc_driver = {
.driver = {
.name = "mpam_msc",
.of_match_table = of_match_ptr(mpam_of_match),
},
.probe = mpam_msc_drv_probe,
.remove = mpam_msc_drv_remove,
};
/*
* MSC that are hidden under caches are not created as platform devices
* as there is no cache driver. Caches are also special-cased in
* get_msc_affinity().
*/
static void mpam_dt_create_foundling_msc(void)
{
int err;
struct device_node *cache;
for_each_compatible_node(cache, NULL, "cache") {
err = of_platform_populate(cache, mpam_of_match, NULL, NULL);
if (err) {
pr_err("Failed to create MSC devices under caches\n");
}
}
}
static int __init mpam_msc_driver_init(void)
{
bool mpam_not_available = false;
if (!mpam_cpus_have_feature())
return -EOPNOTSUPP;
init_srcu_struct(&mpam_srcu);
/*
* If the MPAM CPU interface is not implemented, or reserved by
* firmware, there is no point touching the rest of the hardware.
*/
spin_lock(&partid_max_lock);
if (!partid_max_init || (!mpam_partid_max && !mpam_pmg_max))
mpam_not_available = true;
spin_unlock(&partid_max_lock);
if (mpam_not_available)
return 0;
if (!acpi_disabled)
fw_num_msc = acpi_mpam_count_msc();
else
fw_num_msc = mpam_dt_count_msc();
if (fw_num_msc <= 0) {
pr_err("No MSC devices found in firmware\n");
return -EINVAL;
}
if (acpi_disabled)
mpam_dt_create_foundling_msc();
return platform_driver_register(&mpam_msc_driver);
}
/* Must occur after arm64_mpam_register_cpus() from arch_initcall() */
subsys_initcall(mpam_msc_driver_init);
Reference in New Issue View Git Blame Copy Permalink