2026-01-29 22:25:33 +08:00

577 lines
18 KiB
C

// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2021 Arm Ltd.
#ifndef MPAM_INTERNAL_H
#define MPAM_INTERNAL_H
#include <linux/arm_mpam.h>
#include <linux/atomic.h>
#include <linux/cpumask.h>
#include <linux/io.h>
#include <linux/jump_label.h>
#include <linux/mailbox_client.h>
#include <linux/mutex.h>
#include <linux/resctrl.h>
#include <linux/sizes.h>
#include <linux/srcu.h>
DECLARE_STATIC_KEY_FALSE(mpam_enabled);
/* Value to indicate the allocated monitor is derived from the RMID index. */
#define USE_RMID_IDX (U16_MAX + 1)
/*
* Only these event configuration bits are supported. MPAM can't know if
* data is being written back, these will show up as a write.
*/
#define MPAM_RESTRL_EVT_CONFIG_VALID (READS_TO_LOCAL_MEM | NON_TEMP_WRITE_TO_LOCAL_MEM)
static inline bool mpam_is_enabled(void)
{
return static_branch_likely(&mpam_enabled);
}
struct mpam_msc
{
/* member of mpam_all_msc */
struct list_head glbl_list;
int id;
struct platform_device *pdev;
/* Not modified after mpam_is_enabled() becomes true */
enum mpam_msc_iface iface;
u32 pcc_subspace_id;
struct mbox_client pcc_cl;
struct pcc_mbox_chan *pcc_chan;
u32 nrdy_usec;
cpumask_t accessibility;
bool has_extd_esr;
int reenable_error_ppi;
struct mpam_msc * __percpu *error_dev_id;
atomic_t online_refs;
struct mutex lock;
bool probed;
bool error_irq_requested;
bool error_irq_hw_enabled;
u16 partid_max;
u8 pmg_max;
unsigned long ris_idxs[128 / BITS_PER_LONG];
u32 ris_max;
/* mpam_msc_ris of this component */
struct list_head ris;
/*
* part_sel_lock protects access to the MSC hardware registers that are
* affected by MPAMCFG_PART_SEL. (including the ID registers)
* If needed, take msc->lock first.
*/
spinlock_t part_sel_lock;
spinlock_t mon_sel_lock;
void __iomem * mapped_hwpage;
size_t mapped_hwpage_sz;
};
/*
* When we compact the supported features, we don't care what they are.
* Storing them as a bitmap makes life easy.
*/
typedef u32 mpam_features_t;
/* Bits for mpam_features_t */
enum mpam_device_features {
mpam_feat_ccap_part = 0,
mpam_feat_cpor_part,
mpam_feat_mbw_part,
mpam_feat_mbw_min,
mpam_feat_mbw_max,
mpam_feat_mbw_prop,
mpam_feat_intpri_part,
mpam_feat_intpri_part_0_low,
mpam_feat_dspri_part,
mpam_feat_dspri_part_0_low,
mpam_feat_msmon,
mpam_feat_msmon_csu,
mpam_feat_msmon_csu_capture,
/*
* Having mpam_feat_msmon_mbwu set doesn't mean the regular 31 bit MBWU
* counter would be used. The exact counter used is decided based on the
* status of mpam_feat_msmon_mbwu_l/mpam_feat_msmon_mbwu_lwd as well.
*/
mpam_feat_msmon_mbwu,
mpam_feat_msmon_mbwu_44counter,
mpam_feat_msmon_mbwu_63counter,
mpam_feat_msmon_mbwu_capture,
mpam_feat_msmon_mbwu_rwbw,
mpam_feat_msmon_capt,
mpam_feat_partid_nrw,
MPAM_FEATURE_LAST,
};
#define MPAM_ALL_FEATURES ((1<<MPAM_FEATURE_LAST) - 1)
struct mpam_props
{
mpam_features_t features;
u16 cpbm_wd;
u16 mbw_pbm_bits;
u8 bwa_wd;
u16 cmax_wd;
u16 intpri_wd;
u16 dspri_wd;
u16 num_csu_mon;
u16 num_mbwu_mon;
};
#define mpam_has_feature(_feat, x) ((1<<_feat) & (x)->features)
#define mpam_set_feature(_feat, x) ((x)->features |= (1<<_feat))
static inline void mpam_clear_feature(enum mpam_device_features feat,
mpam_features_t *supported)
{
*supported &= ~(1<<feat);
}
struct mpam_class
{
/* mpam_components in this class */
struct list_head components;
cpumask_t affinity;
struct mpam_props props;
u32 nrdy_usec;
u8 level;
enum mpam_class_types type;
/* member of mpam_classes */
struct list_head classes_list;
struct ida ida_csu_mon;
struct ida ida_mbwu_mon;
};
struct mpam_config {
/* Which configuration values are valid. 0 is used for reset */
mpam_features_t features;
u32 cpbm;
u32 mbw_pbm;
u16 mbw_max;
};
struct mpam_component
{
u32 comp_id;
/* mpam_msc_ris in this component */
struct list_head ris;
cpumask_t affinity;
/*
* Array of configuration values, indexed by partid.
* Read from cpuhp callbacks, hold the cpuhp lock when writing.
*/
struct mpam_config *cfg;
/* member of mpam_class:components */
struct list_head class_list;
/* parent: */
struct mpam_class *class;
};
/* The values for MSMON_CFG_MBWU_FLT.RWBW */
enum mon_filter_options {
COUNT_BOTH = 0,
COUNT_WRITE = 1,
COUNT_READ = 2,
};
struct mon_cfg {
u16 mon;
u8 pmg;
bool match_pmg;
u32 partid;
enum mon_filter_options opts;
};
/*
* Changes to enabled and cfg are protected by the msc->lock.
* Changes to reset_on_next_read, prev_val and correction are protected by the
* msc's mon_sel_lock.
*/
struct msmon_mbwu_state {
bool enabled;
bool reset_on_next_read;
struct mon_cfg cfg;
/* The value last read from the hardware. Used to detect overflow. */
u64 prev_val;
/*
* The value to add to the new reading to account for power management,
* and shifts to trigger the overflow interrupt.
*/
u64 correction;
};
struct mpam_msc_ris {
u8 ris_idx;
u64 idr;
struct mpam_props props;
bool in_reset_state;
cpumask_t affinity;
/* member of mpam_component:ris */
struct list_head comp_list;
/* member of mpam_msc:ris */
struct list_head msc_list;
/* parents: */
struct mpam_msc *msc;
struct mpam_component *comp;
/* msmon mbwu configuration is preserved over reset */
struct msmon_mbwu_state *mbwu_state;
};
struct mpam_resctrl_dom {
struct mpam_component *comp;
struct rdt_domain resctrl_dom;
u32 mbm_local_evt_cfg;
};
struct mpam_resctrl_res {
struct mpam_class *class;
struct rdt_resource resctrl_res;
};
static inline int mpam_alloc_csu_mon(struct mpam_class *class)
{
struct mpam_props *cprops = &class->props;
if (!mpam_has_feature(mpam_feat_msmon_csu, cprops))
return -EOPNOTSUPP;
return ida_alloc_range(&class->ida_csu_mon, 0, cprops->num_csu_mon - 1,
GFP_KERNEL);
}
static inline void mpam_free_csu_mon(struct mpam_class *class, int csu_mon)
{
ida_free(&class->ida_csu_mon, csu_mon);
}
static inline int mpam_alloc_mbwu_mon(struct mpam_class *class)
{
struct mpam_props *cprops = &class->props;
if (!mpam_has_feature(mpam_feat_msmon_mbwu, cprops))
return -EOPNOTSUPP;
return ida_alloc_range(&class->ida_mbwu_mon, 0,
cprops->num_mbwu_mon - 1, GFP_KERNEL);
}
static inline void mpam_free_mbwu_mon(struct mpam_class *class, int mbwu_mon)
{
ida_free(&class->ida_mbwu_mon, mbwu_mon);
}
/* List of all classes */
extern struct list_head mpam_classes;
extern struct srcu_struct mpam_srcu;
/* System wide partid/pmg values */
extern u16 mpam_partid_max;
extern u8 mpam_pmg_max;
/* Scheduled work callback to enable mpam once all MSC have been probed */
void mpam_enable(struct work_struct *work);
void mpam_disable(struct work_struct *work);
void mpam_reset_class(struct mpam_class *class);
int mpam_apply_config(struct mpam_component *comp, u16 partid,
struct mpam_config *cfg);
int mpam_msmon_read(struct mpam_component *comp, struct mon_cfg *ctx,
enum mpam_device_features, u64 *val);
void mpam_msmon_reset_mbwu(struct mpam_component *comp, struct mon_cfg *ctx);
void mpam_msmon_reset_all_mbwu(struct mpam_component *comp);
int mpam_resctrl_online_cpu(unsigned int cpu);
int mpam_resctrl_offline_cpu(unsigned int cpu);
int mpam_resctrl_setup(void);
void mpam_resctrl_exit(void);
/*
* MPAM MSCs have the following register layout. See:
* Arm Architecture Reference Manual Supplement - Memory System Resource
* Partitioning and Monitoring (MPAM), for Armv8-A. DDI 0598A.a
*/
#define MPAM_ARCHITECTURE_V1 0x10
/* Memory mapped control pages: */
/* ID Register offsets in the memory mapped page */
#define MPAMF_IDR 0x0000 /* features id register */
#define MPAMF_MSMON_IDR 0x0080 /* performance monitoring features */
#define MPAMF_IMPL_IDR 0x0028 /* imp-def partitioning */
#define MPAMF_CPOR_IDR 0x0030 /* cache-portion partitioning */
#define MPAMF_CCAP_IDR 0x0038 /* cache-capacity partitioning */
#define MPAMF_MBW_IDR 0x0040 /* mem-bw partitioning */
#define MPAMF_PRI_IDR 0x0048 /* priority partitioning */
#define MPAMF_CSUMON_IDR 0x0088 /* cache-usage monitor */
#define MPAMF_MBWUMON_IDR 0x0090 /* mem-bw usage monitor */
#define MPAMF_PARTID_NRW_IDR 0x0050 /* partid-narrowing */
#define MPAMF_IIDR 0x0018 /* implementer id register */
#define MPAMF_AIDR 0x0020 /* architectural id register */
/* Configuration and Status Register offsets in the memory mapped page */
#define MPAMCFG_PART_SEL 0x0100 /* partid to configure: */
#define MPAMCFG_CPBM 0x1000 /* cache-portion config */
#define MPAMCFG_CMAX 0x0108 /* cache-capacity config */
#define MPAMCFG_MBW_MIN 0x0200 /* min mem-bw config */
#define MPAMCFG_MBW_MAX 0x0208 /* max mem-bw config */
#define MPAMCFG_MBW_WINWD 0x0220 /* mem-bw accounting window config */
#define MPAMCFG_MBW_PBM 0x2000 /* mem-bw portion bitmap config */
#define MPAMCFG_PRI 0x0400 /* priority partitioning config */
#define MPAMCFG_MBW_PROP 0x0500 /* mem-bw stride config */
#define MPAMCFG_INTPARTID 0x0600 /* partid-narrowing config */
#define MSMON_CFG_MON_SEL 0x0800 /* monitor selector */
#define MSMON_CFG_CSU_FLT 0x0810 /* cache-usage monitor filter */
#define MSMON_CFG_CSU_CTL 0x0818 /* cache-usage monitor config */
#define MSMON_CFG_MBWU_FLT 0x0820 /* mem-bw monitor filter */
#define MSMON_CFG_MBWU_CTL 0x0828 /* mem-bw monitor config */
#define MSMON_CSU 0x0840 /* current cache-usage */
#define MSMON_CSU_CAPTURE 0x0848 /* last cache-usage value captured */
#define MSMON_MBWU 0x0860 /* current mem-bw usage value */
#define MSMON_MBWU_CAPTURE 0x0868 /* last mem-bw value captured */
#define MSMON_MBWU_L 0x0880 /* current long mem-bw usage value */
#define MSMON_MBWU_CAPTURE_L 0x0890 /* last long mem-bw value captured */
#define MSMON_CAPT_EVNT 0x0808 /* signal a capture event */
#define MPAMF_ESR 0x00F8 /* error status register */
#define MPAMF_ECR 0x00F0 /* error control register */
/* MPAMF_IDR - MPAM features ID register */
#define MPAMF_IDR_PARTID_MAX GENMASK(15, 0)
#define MPAMF_IDR_PMG_MAX GENMASK(23, 16)
#define MPAMF_IDR_HAS_CCAP_PART BIT(24)
#define MPAMF_IDR_HAS_CPOR_PART BIT(25)
#define MPAMF_IDR_HAS_MBW_PART BIT(26)
#define MPAMF_IDR_HAS_PRI_PART BIT(27)
#define MPAMF_IDR_HAS_EXT BIT(28)
#define MPAMF_IDR_HAS_IMPL_IDR BIT(29)
#define MPAMF_IDR_HAS_MSMON BIT(30)
#define MPAMF_IDR_HAS_PARTID_NRW BIT(31)
#define MPAMF_IDR_HAS_RIS BIT(32)
#define MPAMF_IDR_HAS_EXT_ESR BIT(38)
#define MPAMF_IDR_HAS_ESR BIT(39)
#define MPAMF_IDR_RIS_MAX GENMASK(59, 56)
/* MPAMF_MSMON_IDR - MPAM performance monitoring ID register */
#define MPAMF_MSMON_IDR_MSMON_CSU BIT(16)
#define MPAMF_MSMON_IDR_MSMON_MBWU BIT(17)
#define MPAMF_MSMON_IDR_HAS_LOCAL_CAPT_EVNT BIT(31)
/* MPAMF_CPOR_IDR - MPAM features cache portion partitioning ID register */
#define MPAMF_CPOR_IDR_CPBM_WD GENMASK(15, 0)
/* MPAMF_CCAP_IDR - MPAM features cache capacity partitioning ID register */
#define MPAMF_CCAP_IDR_CMAX_WD GENMASK(5, 0)
/* MPAMF_MBW_IDR - MPAM features memory bandwidth partitioning ID register */
#define MPAMF_MBW_IDR_BWA_WD GENMASK(5, 0)
#define MPAMF_MBW_IDR_HAS_MIN BIT(10)
#define MPAMF_MBW_IDR_HAS_MAX BIT(11)
#define MPAMF_MBW_IDR_HAS_PBM BIT(12)
#define MPAMF_MBW_IDR_HAS_PROP BIT(13)
#define MPAMF_MBW_IDR_WINDWR BIT(14)
#define MPAMF_MBW_IDR_BWPBM_WD GENMASK(28, 16)
/* MPAMF_PRI_IDR - MPAM features priority partitioning ID register */
#define MPAMF_PRI_IDR_HAS_INTPRI BIT(0)
#define MPAMF_PRI_IDR_INTPRI_0_IS_LOW BIT(1)
#define MPAMF_PRI_IDR_INTPRI_WD GENMASK(9, 4)
#define MPAMF_PRI_IDR_HAS_DSPRI BIT(16)
#define MPAMF_PRI_IDR_DSPRI_0_IS_LOW BIT(17)
#define MPAMF_PRI_IDR_DSPRI_WD GENMASK(25, 20)
/* MPAMF_CSUMON_IDR - MPAM cache storage usage monitor ID register */
#define MPAMF_CSUMON_IDR_NUM_MON GENMASK(15, 0)
#define MPAMF_CSUMON_IDR_HAS_CAPTURE BIT(31)
/* MPAMF_MBWUMON_IDR - MPAM memory bandwidth usage monitor ID register */
#define MPAMF_MBWUMON_IDR_NUM_MON GENMASK(15, 0)
#define MPAMF_MBWUMON_IDR_HAS_RWBW BIT(28)
#define MPAMF_MBWUMON_IDR_LWD BIT(29)
#define MPAMF_MBWUMON_IDR_HAS_LONG BIT(30)
#define MPAMF_MBWUMON_IDR_HAS_CAPTURE BIT(31)
/* MPAMF_PARTID_NRW_IDR - MPAM PARTID narrowing ID register */
#define MPAMF_PARTID_NRW_IDR_INTPARTID_MAX GENMASK(15, 0)
/* MPAMF_IIDR - MPAM implementation ID register */
#define MPAMF_IIDR_PRODUCTID GENMASK(31, 20)
#define MPAMF_IIDR_PRODUCTID_SHIFT 20
#define MPAMF_IIDR_VARIANT GENMASK(19, 16)
#define MPAMF_IIDR_VARIANT_SHIFT 16
#define MPAMF_IIDR_REVISON GENMASK(15, 12)
#define MPAMF_IIDR_REVISON_SHIFT 12
#define MPAMF_IIDR_IMPLEMENTER GENMASK(11, 0)
#define MPAMF_IIDR_IMPLEMENTER_SHIFT 0
/* MPAMF_AIDR - MPAM architecture ID register */
#define MPAMF_AIDR_ARCH_MAJOR_REV GENMASK(7, 4)
#define MPAMF_AIDR_ARCH_MINOR_REV GENMASK(3, 0)
/* MPAMCFG_PART_SEL - MPAM partition configuration selection register */
#define MPAMCFG_PART_SEL_PARTID_SEL GENMASK(15, 0)
#define MPAMCFG_PART_SEL_INTERNAL BIT(16)
#define MPAMCFG_PART_SEL_RIS GENMASK(27, 24)
/* MPAMCFG_CMAX - MPAM cache portion bitmap partition configuration register */
#define MPAMCFG_CMAX_CMAX GENMASK(15, 0)
/*
* MPAMCFG_MBW_MIN - MPAM memory minimum bandwidth partitioning configuration
* register
*/
#define MPAMCFG_MBW_MIN_MIN GENMASK(15, 0)
/*
* MPAMCFG_MBW_MAX - MPAM memory maximum bandwidth partitioning configuration
* register
*/
#define MPAMCFG_MBW_MAX_MAX GENMASK(15, 0)
#define MPAMCFG_MBW_MAX_HARDLIM BIT(31)
/*
* MPAMCFG_MBW_WINWD - MPAM memory bandwidth partitioning window width
* register
*/
#define MPAMCFG_MBW_WINWD_US_FRAC GENMASK(7, 0)
#define MPAMCFG_MBW_WINWD_US_INT GENMASK(23, 8)
/* MPAMCFG_PRI - MPAM priority partitioning configuration register */
#define MPAMCFG_PRI_INTPRI GENMASK(15, 0)
#define MPAMCFG_PRI_DSPRI GENMASK(31, 16)
/*
* MPAMCFG_MBW_PROP - Memory bandwidth proportional stride partitioning
* configuration register
*/
#define MPAMCFG_MBW_PROP_STRIDEM1 GENMASK(15, 0)
#define MPAMCFG_MBW_PROP_EN BIT(31)
/*
* MPAMCFG_INTPARTID - MPAM internal partition narrowing configuration register
*/
#define MPAMCFG_INTPARTID_INTPARTID GENMASK(15, 0)
#define MPAMCFG_INTPARTID_INTERNAL BIT(16)
/* MSMON_CFG_MON_SEL - Memory system performance monitor selection register */
#define MSMON_CFG_MON_SEL_MON_SEL GENMASK(7, 0)
#define MSMON_CFG_MON_SEL_RIS GENMASK(27, 24)
/* MPAMF_ESR - MPAM Error Status Register */
#define MPAMF_ESR_PARTID_OR_MON GENMASK(15, 0)
#define MPAMF_ESR_PMG GENMASK(23, 16)
#define MPAMF_ESR_ERRCODE GENMASK(27, 24)
#define MPAMF_ESR_OVRWR BIT(31)
#define MPAMF_ESR_RIS GENMASK(35, 32)
/* MPAMF_ECR - MPAM Error Control Register */
#define MPAMF_ECR_INTEN BIT(0)
/* Error conditions in accessing memory mapped registers */
#define MPAM_ERRCODE_NONE 0
#define MPAM_ERRCODE_PARTID_SEL_RANGE 1
#define MPAM_ERRCODE_REQ_PARTID_RANGE 2
#define MPAM_ERRCODE_MSMONCFG_ID_RANGE 3
#define MPAM_ERRCODE_REQ_PMG_RANGE 4
#define MPAM_ERRCODE_MONITOR_RANGE 5
#define MPAM_ERRCODE_INTPARTID_RANGE 6
#define MPAM_ERRCODE_UNEXPECTED_INTERNAL 7
/*
* MSMON_CFG_CSU_FLT - Memory system performance monitor configure cache storage
* usage monitor filter register
*/
#define MSMON_CFG_CSU_FLT_PARTID GENMASK(15, 0)
#define MSMON_CFG_CSU_FLT_PMG GENMASK(23, 16)
/*
* MSMON_CFG_CSU_CTL - Memory system performance monitor configure cache storage
* usage monitor control register
* MSMON_CFG_MBWU_CTL - Memory system performance monitor configure memory
* bandwidth usage monitor control register
*/
#define MSMON_CFG_x_CTL_TYPE GENMASK(7, 0)
#define MSMON_CFG_x_CTL_MATCH_PARTID BIT(16)
#define MSMON_CFG_x_CTL_MATCH_PMG BIT(17)
#define MSMON_CFG_x_CTL_SCLEN BIT(19)
#define MSMON_CFG_x_CTL_SUBTYPE GENMASK(23, 20)
#define MSMON_CFG_x_CTL_OFLOW_FRZ BIT(24)
#define MSMON_CFG_x_CTL_OFLOW_INTR BIT(25)
#define MSMON_CFG_x_CTL_OFLOW_STATUS BIT(26)
#define MSMON_CFG_x_CTL_CAPT_RESET BIT(27)
#define MSMON_CFG_x_CTL_CAPT_EVNT GENMASK(30, 28)
#define MSMON_CFG_x_CTL_EN BIT(31)
#define MSMON_CFG_MBWU_CTL_TYPE_MBWU 0x42
#define MSMON_CFG_MBWU_CTL_TYPE_CSU 0x43
#define MSMON_CFG_MBWU_CTL_SUBTYPE_NONE 0
#define MSMON_CFG_MBWU_CTL_SUBTYPE_READ 1
#define MSMON_CFG_MBWU_CTL_SUBTYPE_WRITE 2
#define MSMON_CFG_MBWU_CTL_SUBTYPE_BOTH 3
#define MSMON_CFG_MBWU_CTL_SUBTYPE_MAX 3
#define MSMON_CFG_MBWU_CTL_SUBTYPE_MASK 0x3
/*
* MSMON_CFG_MBWU_FLT - Memory system performance monitor configure memory
* bandwidth usage monitor filter register
*/
#define MSMON_CFG_MBWU_FLT_PARTID GENMASK(15, 0)
#define MSMON_CFG_MBWU_FLT_PMG GENMASK(23, 16)
#define MSMON_CFG_MBWU_FLT_RWBW GENMASK(31, 30)
/*
* MSMON_CSU - Memory system performance monitor cache storage usage monitor
* register
* MSMON_CSU_CAPTURE - Memory system performance monitor cache storage usage
* capture register
* MSMON_MBWU - Memory system performance monitor memory bandwidth usage
* monitor register
* MSMON_MBWU_CAPTURE - Memory system performance monitor memory bandwidth usage
* capture register
*/
#define MSMON___VALUE GENMASK(30, 0)
#define MSMON___NRDY BIT(31)
#define MSMON___NRDY_L BIT(63)
#define MSMON___L_VALUE GENMASK(43, 0)
#define MSMON___LWD_VALUE GENMASK(62, 0)
/*
* MSMON_CAPT_EVNT - Memory system performance monitoring capture event
* generation register
*/
#define MSMON_CAPT_EVNT_NOW BIT(0)
#endif /* MPAM_INTERNAL_H */