linux/drivers/perf/marvell_cn10k_ddr_pmu.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver
 *
 * Copyright (C) 2021-2024 Marvell.
 */

#include <linux/init.h>
#include <linux/io.h>
#include <linux/module.h>
#include <linux/of.h>
#include <linux/perf_event.h>
#include <linux/hrtimer.h>
#include <linux/acpi.h>
#include <linux/platform_device.h>

/* Performance Counters Operating Mode Control Registers */
#define CN10K_DDRC_PERF_CNT_OP_MODE_CTRL	0x8020
#define ODY_DDRC_PERF_CNT_OP_MODE_CTRL		0x20020
#define OP_MODE_CTRL_VAL_MANUAL	0x1

/* Performance Counters Start Operation Control Registers */
#define CN10K_DDRC_PERF_CNT_START_OP_CTRL	0x8028
#define ODY_DDRC_PERF_CNT_START_OP_CTRL		0x200A0
#define START_OP_CTRL_VAL_START		0x1ULL
#define START_OP_CTRL_VAL_ACTIVE	0x2

/* Performance Counters End Operation Control Registers */
#define CN10K_DDRC_PERF_CNT_END_OP_CTRL	0x8030
#define ODY_DDRC_PERF_CNT_END_OP_CTRL	0x200E0
#define END_OP_CTRL_VAL_END		0x1ULL

/* Performance Counters End Status Registers */
#define CN10K_DDRC_PERF_CNT_END_STATUS		0x8038
#define ODY_DDRC_PERF_CNT_END_STATUS		0x20120
#define END_STATUS_VAL_END_TIMER_MODE_END	0x1

/* Performance Counters Configuration Registers */
#define CN10K_DDRC_PERF_CFG_BASE		0x8040
#define ODY_DDRC_PERF_CFG_BASE			0x20160

/* 8 Generic event counter + 2 fixed event counters */
#define DDRC_PERF_NUM_GEN_COUNTERS	8
#define DDRC_PERF_NUM_FIX_COUNTERS	2
#define DDRC_PERF_READ_COUNTER_IDX	DDRC_PERF_NUM_GEN_COUNTERS
#define DDRC_PERF_WRITE_COUNTER_IDX	(DDRC_PERF_NUM_GEN_COUNTERS + 1)
#define DDRC_PERF_NUM_COUNTERS		(DDRC_PERF_NUM_GEN_COUNTERS + \
					 DDRC_PERF_NUM_FIX_COUNTERS)

/* Generic event counter registers */
#define DDRC_PERF_CFG(base, n)		((base) + 8 * (n))
#define EVENT_ENABLE			BIT_ULL(63)

/* Two dedicated event counters for DDR reads and writes */
#define EVENT_DDR_READS			101
#define EVENT_DDR_WRITES		100

#define DDRC_PERF_REG(base, n)		((base) + 8 * (n))
/*
 * programmable events IDs in programmable event counters.
 * DO NOT change these event-id numbers, they are used to
 * program event bitmap in h/w.
 */
#define EVENT_DFI_CMD_IS_RETRY			61
#define EVENT_RD_UC_ECC_ERROR			60
#define EVENT_RD_CRC_ERROR			59
#define EVENT_CAPAR_ERROR			58
#define EVENT_WR_CRC_ERROR			57
#define EVENT_DFI_PARITY_POISON			56
#define EVENT_RETRY_FIFO_FULL			46
#define EVENT_DFI_CYCLES			45

#define EVENT_OP_IS_ZQLATCH			55
#define EVENT_OP_IS_ZQSTART			54
#define EVENT_OP_IS_TCR_MRR			53
#define EVENT_OP_IS_DQSOSC_MRR			52
#define EVENT_OP_IS_DQSOSC_MPC			51
#define EVENT_VISIBLE_WIN_LIMIT_REACHED_WR	50
#define EVENT_VISIBLE_WIN_LIMIT_REACHED_RD	49
#define EVENT_BSM_STARVATION			48
#define EVENT_BSM_ALLOC				47
#define EVENT_LPR_REQ_WITH_NOCREDIT		46
#define EVENT_HPR_REQ_WITH_NOCREDIT		45
#define EVENT_OP_IS_ZQCS			44
#define EVENT_OP_IS_ZQCL			43
#define EVENT_OP_IS_LOAD_MODE			42
#define EVENT_OP_IS_SPEC_REF			41
#define EVENT_OP_IS_CRIT_REF			40
#define EVENT_OP_IS_REFRESH			39
#define EVENT_OP_IS_ENTER_MPSM			35
#define EVENT_OP_IS_ENTER_POWERDOWN		31
#define EVENT_OP_IS_ENTER_SELFREF		27
#define EVENT_WAW_HAZARD			26
#define EVENT_RAW_HAZARD			25
#define EVENT_WAR_HAZARD			24
#define EVENT_WRITE_COMBINE			23
#define EVENT_RDWR_TRANSITIONS			22
#define EVENT_PRECHARGE_FOR_OTHER		21
#define EVENT_PRECHARGE_FOR_RDWR		20
#define EVENT_OP_IS_PRECHARGE			19
#define EVENT_OP_IS_MWR				18
#define EVENT_OP_IS_WR				17
#define EVENT_OP_IS_RD				16
#define EVENT_OP_IS_RD_ACTIVATE			15
#define EVENT_OP_IS_RD_OR_WR			14
#define EVENT_OP_IS_ACTIVATE			13
#define EVENT_WR_XACT_WHEN_CRITICAL		12
#define EVENT_LPR_XACT_WHEN_CRITICAL		11
#define EVENT_HPR_XACT_WHEN_CRITICAL		10
#define EVENT_DFI_RD_DATA_CYCLES		9
#define EVENT_DFI_WR_DATA_CYCLES		8
#define EVENT_ACT_BYPASS			7
#define EVENT_READ_BYPASS			6
#define EVENT_HIF_HI_PRI_RD			5
#define EVENT_HIF_RMW				4
#define EVENT_HIF_RD				3
#define EVENT_HIF_WR				2
#define EVENT_HIF_RD_OR_WR			1

/* Event counter value registers */
#define CN10K_DDRC_PERF_CNT_VALUE_BASE	0x8080
#define ODY_DDRC_PERF_CNT_VALUE_BASE	0x201C0

/* Fixed event counter enable/disable register */
#define CN10K_DDRC_PERF_CNT_FREERUN_EN		0x80C0
#define DDRC_PERF_FREERUN_WRITE_EN	0x1
#define DDRC_PERF_FREERUN_READ_EN	0x2

/* Fixed event counter control register */
#define CN10K_DDRC_PERF_CNT_FREERUN_CTRL	0x80C8
#define ODY_DDRC_PERF_CNT_FREERUN_CTRL		0x20240
#define DDRC_FREERUN_WRITE_CNT_CLR	0x1
#define DDRC_FREERUN_READ_CNT_CLR	0x2

/* Fixed event counter clear register, defined only for Odyssey */
#define ODY_DDRC_PERF_CNT_FREERUN_CLR  0x20248

#define DDRC_PERF_CNT_VALUE_OVERFLOW	BIT_ULL(48)
#define DDRC_PERF_CNT_MAX_VALUE		GENMASK_ULL(48, 0)

/* Fixed event counter value register */
#define CN10K_DDRC_PERF_CNT_VALUE_WR_OP		0x80D0
#define CN10K_DDRC_PERF_CNT_VALUE_RD_OP		0x80D8
#define ODY_DDRC_PERF_CNT_VALUE_WR_OP		0x20250
#define ODY_DDRC_PERF_CNT_VALUE_RD_OP		0x20258

struct cn10k_ddr_pmu {
	struct pmu pmu;
	void __iomem *base;
	const struct ddr_pmu_platform_data *p_data;
	const struct ddr_pmu_ops *ops;
	unsigned int cpu;
	struct	device *dev;
	int active_events;
	struct perf_event *events[DDRC_PERF_NUM_COUNTERS];
	struct hrtimer hrtimer;
	struct hlist_node node;
};

struct ddr_pmu_ops {
	void (*enable_read_freerun_counter)(struct cn10k_ddr_pmu *pmu,
					    bool enable);
	void (*enable_write_freerun_counter)(struct cn10k_ddr_pmu *pmu,
					     bool enable);
	void (*clear_read_freerun_counter)(struct cn10k_ddr_pmu *pmu);
	void (*clear_write_freerun_counter)(struct cn10k_ddr_pmu *pmu);
	void (*pmu_overflow_handler)(struct cn10k_ddr_pmu *pmu, int evt_idx);
};

#define to_cn10k_ddr_pmu(p)	container_of(p, struct cn10k_ddr_pmu, pmu)

struct ddr_pmu_platform_data {
	u64 counter_overflow_val;
	u64 counter_max_val;
	u64 cnt_base;
	u64 cfg_base;
	u64 cnt_op_mode_ctrl;
	u64 cnt_start_op_ctrl;
	u64 cnt_end_op_ctrl;
	u64 cnt_end_status;
	u64 cnt_freerun_en;
	u64 cnt_freerun_ctrl;
	u64 cnt_freerun_clr;
	u64 cnt_value_wr_op;
	u64 cnt_value_rd_op;
	bool is_cn10k;
	bool is_ody;
};

static ssize_t cn10k_ddr_pmu_event_show(struct device *dev,
					struct device_attribute *attr,
					char *page)
{
	struct perf_pmu_events_attr *pmu_attr;

	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
	return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);

}

#define CN10K_DDR_PMU_EVENT_ATTR(_name, _id)				     \
	PMU_EVENT_ATTR_ID(_name, cn10k_ddr_pmu_event_show, _id)

static struct attribute *cn10k_ddr_perf_events_attrs[] = {
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_wr_data_access, EVENT_DFI_WR_DATA_CYCLES),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_rd_data_access, EVENT_DFI_RD_DATA_CYCLES),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access,
					EVENT_HPR_XACT_WHEN_CRITICAL),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access,
					EVENT_LPR_XACT_WHEN_CRITICAL),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access,
					EVENT_WR_XACT_WHEN_CRITICAL),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access, EVENT_OP_IS_RD_OR_WR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access, EVENT_OP_IS_RD_ACTIVATE),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr, EVENT_PRECHARGE_FOR_RDWR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other,
					EVENT_PRECHARGE_FOR_OTHER),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown, EVENT_OP_IS_ENTER_POWERDOWN),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hpr_req_with_nocredit,
					EVENT_HPR_REQ_WITH_NOCREDIT),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_lpr_req_with_nocredit,
					EVENT_LPR_REQ_WITH_NOCREDIT),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd,
					EVENT_VISIBLE_WIN_LIMIT_REACHED_RD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr,
					EVENT_VISIBLE_WIN_LIMIT_REACHED_WR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH),
	/* Free run event counters */
	CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES),
	NULL
};

static struct attribute *odyssey_ddr_perf_events_attrs[] = {
	/* Programmable */
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_wr_data_access,
				 EVENT_DFI_WR_DATA_CYCLES),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_rd_data_access,
				 EVENT_DFI_RD_DATA_CYCLES),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access,
				 EVENT_HPR_XACT_WHEN_CRITICAL),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access,
				 EVENT_LPR_XACT_WHEN_CRITICAL),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access,
				 EVENT_WR_XACT_WHEN_CRITICAL),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access,
				 EVENT_OP_IS_RD_OR_WR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access,
				 EVENT_OP_IS_RD_ACTIVATE),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr,
				 EVENT_PRECHARGE_FOR_RDWR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other,
				 EVENT_PRECHARGE_FOR_OTHER),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown,
				 EVENT_OP_IS_ENTER_POWERDOWN),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_cycles, EVENT_DFI_CYCLES),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_retry_fifo_full,
				 EVENT_RETRY_FIFO_FULL),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd,
				 EVENT_VISIBLE_WIN_LIMIT_REACHED_RD),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr,
				 EVENT_VISIBLE_WIN_LIMIT_REACHED_WR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_parity_poison,
				 EVENT_DFI_PARITY_POISON),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_crc_error, EVENT_WR_CRC_ERROR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_capar_error, EVENT_CAPAR_ERROR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_crc_error, EVENT_RD_CRC_ERROR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_uc_ecc_error, EVENT_RD_UC_ECC_ERROR),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_dfi_cmd_is_retry, EVENT_DFI_CMD_IS_RETRY),
	/* Free run event counters */
	CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS),
	CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES),
	NULL
};

static struct attribute_group odyssey_ddr_perf_events_attr_group = {
	.name = "events",
	.attrs = odyssey_ddr_perf_events_attrs,
};

static struct attribute_group cn10k_ddr_perf_events_attr_group = {
	.name = "events",
	.attrs = cn10k_ddr_perf_events_attrs,
};

PMU_FORMAT_ATTR(event, "config:0-8");

static struct attribute *cn10k_ddr_perf_format_attrs[] = {
	&format_attr_event.attr,
	NULL,
};

static struct attribute_group cn10k_ddr_perf_format_attr_group = {
	.name = "format",
	.attrs = cn10k_ddr_perf_format_attrs,
};

static ssize_t cn10k_ddr_perf_cpumask_show(struct device *dev,
					   struct device_attribute *attr,
					   char *buf)
{
	struct cn10k_ddr_pmu *pmu = dev_get_drvdata(dev);

	return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
}

static struct device_attribute cn10k_ddr_perf_cpumask_attr =
	__ATTR(cpumask, 0444, cn10k_ddr_perf_cpumask_show, NULL);

static struct attribute *cn10k_ddr_perf_cpumask_attrs[] = {
	&cn10k_ddr_perf_cpumask_attr.attr,
	NULL,
};

static struct attribute_group cn10k_ddr_perf_cpumask_attr_group = {
	.attrs = cn10k_ddr_perf_cpumask_attrs,
};

static const struct attribute_group *cn10k_attr_groups[] = {
	&cn10k_ddr_perf_events_attr_group,
	&cn10k_ddr_perf_format_attr_group,
	&cn10k_ddr_perf_cpumask_attr_group,
	NULL,
};

static const struct attribute_group *odyssey_attr_groups[] = {
	&odyssey_ddr_perf_events_attr_group,
	&cn10k_ddr_perf_format_attr_group,
	&cn10k_ddr_perf_cpumask_attr_group,
	NULL
};

/* Default poll timeout is 100 sec, which is very sufficient for
 * 48 bit counter incremented max at 5.6 GT/s, which may take many
 * hours to overflow.
 */
static unsigned long cn10k_ddr_pmu_poll_period_sec = 100;
module_param_named(poll_period_sec, cn10k_ddr_pmu_poll_period_sec, ulong, 0644);

static ktime_t cn10k_ddr_pmu_timer_period(void)
{
	return ms_to_ktime((u64)cn10k_ddr_pmu_poll_period_sec * USEC_PER_SEC);
}

static int ddr_perf_get_event_bitmap(int eventid, u64 *event_bitmap,
				     struct cn10k_ddr_pmu *ddr_pmu)
{
	int err = 0;

	switch (eventid) {
	case EVENT_DFI_PARITY_POISON ...EVENT_DFI_CMD_IS_RETRY:
		if (!ddr_pmu->p_data->is_ody) {
			err = -EINVAL;
			break;
		}
		fallthrough;
	case EVENT_HIF_RD_OR_WR ... EVENT_WAW_HAZARD:
	case EVENT_OP_IS_REFRESH ... EVENT_OP_IS_ZQLATCH:
		*event_bitmap = (1ULL << (eventid - 1));
		break;
	case EVENT_OP_IS_ENTER_SELFREF:
	case EVENT_OP_IS_ENTER_POWERDOWN:
	case EVENT_OP_IS_ENTER_MPSM:
		*event_bitmap = (0xFULL << (eventid - 1));
		break;
	default:
		err = -EINVAL;
	}

	if (err)
		pr_err("%s Invalid eventid %d\n", __func__, eventid);
	return err;
}

static int cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu *pmu,
					struct perf_event *event)
{
	u8 config = event->attr.config;
	int i;

	/* DDR read free-run counter index */
	if (config == EVENT_DDR_READS) {
		pmu->events[DDRC_PERF_READ_COUNTER_IDX] = event;
		return DDRC_PERF_READ_COUNTER_IDX;
	}

	/* DDR write free-run counter index */
	if (config == EVENT_DDR_WRITES) {
		pmu->events[DDRC_PERF_WRITE_COUNTER_IDX] = event;
		return DDRC_PERF_WRITE_COUNTER_IDX;
	}

	/* Allocate DDR generic counters */
	for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
		if (pmu->events[i] == NULL) {
			pmu->events[i] = event;
			return i;
		}
	}

	return -ENOENT;
}

static void cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu *pmu, int counter)
{
	pmu->events[counter] = NULL;
}

static int cn10k_ddr_perf_event_init(struct perf_event *event)
{
	struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
	struct hw_perf_event *hwc = &event->hw;

	if (event->attr.type != event->pmu->type)
		return -ENOENT;

	if (is_sampling_event(event)) {
		dev_info(pmu->dev, "Sampling not supported!\n");
		return -EOPNOTSUPP;
	}

	if (event->cpu < 0) {
		dev_warn(pmu->dev, "Can't provide per-task data!\n");
		return -EOPNOTSUPP;
	}

	/*  We must NOT create groups containing mixed PMUs */
	if (event->group_leader->pmu != event->pmu &&
	    !is_software_event(event->group_leader))
		return -EINVAL;

	/* Set ownership of event to one CPU, same event can not be observed
	 * on multiple cpus at same time.
	 */
	event->cpu = pmu->cpu;
	hwc->idx = -1;
	return 0;
}

static void cn10k_ddr_perf_counter_start(struct cn10k_ddr_pmu *ddr_pmu,
					 int counter)
{
	const struct ddr_pmu_platform_data *p_data = ddr_pmu->p_data;
	u64 ctrl_reg = p_data->cnt_start_op_ctrl;

	writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base +
		       DDRC_PERF_REG(ctrl_reg, counter));
}

static void cn10k_ddr_perf_counter_stop(struct cn10k_ddr_pmu *ddr_pmu,
					int counter)
{
	const struct ddr_pmu_platform_data *p_data = ddr_pmu->p_data;
	u64 ctrl_reg = p_data->cnt_end_op_ctrl;

	writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base +
		       DDRC_PERF_REG(ctrl_reg, counter));
}

static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu,
					  int counter, bool enable)
{
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	u64 ctrl_reg = pmu->p_data->cnt_op_mode_ctrl;
	const struct ddr_pmu_ops *ops = pmu->ops;
	bool is_ody = pmu->p_data->is_ody;
	u32 reg;
	u64 val;

	if (counter > DDRC_PERF_NUM_COUNTERS) {
		pr_err("Error: unsupported counter %d\n", counter);
		return;
	}

	if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {
		reg = DDRC_PERF_CFG(p_data->cfg_base, counter);
		val = readq_relaxed(pmu->base + reg);

		if (enable)
			val |= EVENT_ENABLE;
		else
			val &= ~EVENT_ENABLE;

		writeq_relaxed(val, pmu->base + reg);

		if (is_ody) {
			if (enable) {
				/*
				 * Setup the PMU counter to work in
				 * manual mode
				 */
				reg = DDRC_PERF_REG(ctrl_reg, counter);
				writeq_relaxed(OP_MODE_CTRL_VAL_MANUAL,
					       pmu->base + reg);

				cn10k_ddr_perf_counter_start(pmu, counter);
			} else {
				cn10k_ddr_perf_counter_stop(pmu, counter);
			}
		}
	} else {
		if (counter == DDRC_PERF_READ_COUNTER_IDX)
			ops->enable_read_freerun_counter(pmu, enable);
		else
			ops->enable_write_freerun_counter(pmu, enable);
	}
}

static u64 cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu *pmu, int counter)
{
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	u64 val;

	if (counter == DDRC_PERF_READ_COUNTER_IDX)
		return readq_relaxed(pmu->base +
				     p_data->cnt_value_rd_op);

	if (counter == DDRC_PERF_WRITE_COUNTER_IDX)
		return readq_relaxed(pmu->base +
				     p_data->cnt_value_wr_op);

	val = readq_relaxed(pmu->base +
			    DDRC_PERF_REG(p_data->cnt_base, counter));
	return val;
}

static void cn10k_ddr_perf_event_update(struct perf_event *event)
{
	struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	struct hw_perf_event *hwc = &event->hw;
	u64 prev_count, new_count, mask;

	do {
		prev_count = local64_read(&hwc->prev_count);
		new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);
	} while (local64_xchg(&hwc->prev_count, new_count) != prev_count);

	mask = p_data->counter_max_val;

	local64_add((new_count - prev_count) & mask, &event->count);
}

static void cn10k_ddr_perf_event_start(struct perf_event *event, int flags)
{
	struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
	struct hw_perf_event *hwc = &event->hw;
	int counter = hwc->idx;

	local64_set(&hwc->prev_count, 0);

	cn10k_ddr_perf_counter_enable(pmu, counter, true);

	hwc->state = 0;
}

static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags)
{
	struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	const struct ddr_pmu_ops *ops = pmu->ops;
	struct hw_perf_event *hwc = &event->hw;
	u8 config = event->attr.config;
	int counter, ret;
	u32 reg_offset;
	u64 val;

	counter = cn10k_ddr_perf_alloc_counter(pmu, event);
	if (counter < 0)
		return -EAGAIN;

	pmu->active_events++;
	hwc->idx = counter;

	if (pmu->active_events == 1)
		hrtimer_start(&pmu->hrtimer, cn10k_ddr_pmu_timer_period(),
			      HRTIMER_MODE_REL_PINNED);

	if (counter < DDRC_PERF_NUM_GEN_COUNTERS) {
		/* Generic counters, configure event id */
		reg_offset = DDRC_PERF_CFG(p_data->cfg_base, counter);
		ret = ddr_perf_get_event_bitmap(config, &val, pmu);
		if (ret)
			return ret;

		writeq_relaxed(val, pmu->base + reg_offset);
	} else {
		/* fixed event counter, clear counter value */
		if (counter == DDRC_PERF_READ_COUNTER_IDX)
			ops->clear_read_freerun_counter(pmu);
		else
			ops->clear_write_freerun_counter(pmu);
	}

	hwc->state |= PERF_HES_STOPPED;

	if (flags & PERF_EF_START)
		cn10k_ddr_perf_event_start(event, flags);

	return 0;
}

static void cn10k_ddr_perf_event_stop(struct perf_event *event, int flags)
{
	struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
	struct hw_perf_event *hwc = &event->hw;
	int counter = hwc->idx;

	cn10k_ddr_perf_counter_enable(pmu, counter, false);

	if (flags & PERF_EF_UPDATE)
		cn10k_ddr_perf_event_update(event);

	hwc->state |= PERF_HES_STOPPED;
}

static void cn10k_ddr_perf_event_del(struct perf_event *event, int flags)
{
	struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu);
	struct hw_perf_event *hwc = &event->hw;
	int counter = hwc->idx;

	cn10k_ddr_perf_event_stop(event, PERF_EF_UPDATE);

	cn10k_ddr_perf_free_counter(pmu, counter);
	pmu->active_events--;
	hwc->idx = -1;

	/* Cancel timer when no events to capture */
	if (pmu->active_events == 0)
		hrtimer_cancel(&pmu->hrtimer);
}

static void cn10k_ddr_perf_pmu_enable(struct pmu *pmu)
{
	struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);
	const struct ddr_pmu_platform_data *p_data = ddr_pmu->p_data;

	writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base +
		       p_data->cnt_start_op_ctrl);
}

static void cn10k_ddr_perf_pmu_disable(struct pmu *pmu)
{
	struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu);
	const struct ddr_pmu_platform_data *p_data = ddr_pmu->p_data;

	writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base +
		       p_data->cnt_end_op_ctrl);
}

static void cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu *pmu)
{
	struct hw_perf_event *hwc;
	int i;

	for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
		if (pmu->events[i] == NULL)
			continue;

		cn10k_ddr_perf_event_update(pmu->events[i]);
	}

	/* Reset previous count as h/w counter are reset */
	for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
		if (pmu->events[i] == NULL)
			continue;

		hwc = &pmu->events[i]->hw;
		local64_set(&hwc->prev_count, 0);
	}
}

static void ddr_pmu_enable_read_freerun(struct cn10k_ddr_pmu *pmu, bool enable)
{
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	u64 val;

	val = readq_relaxed(pmu->base + p_data->cnt_freerun_en);
	if (enable)
		val |= DDRC_PERF_FREERUN_READ_EN;
	else
		val &= ~DDRC_PERF_FREERUN_READ_EN;

	writeq_relaxed(val, pmu->base + p_data->cnt_freerun_en);
}

static void ddr_pmu_enable_write_freerun(struct cn10k_ddr_pmu *pmu, bool enable)
{
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	u64 val;

	val = readq_relaxed(pmu->base + p_data->cnt_freerun_en);
	if (enable)
		val |= DDRC_PERF_FREERUN_WRITE_EN;
	else
		val &= ~DDRC_PERF_FREERUN_WRITE_EN;

	writeq_relaxed(val, pmu->base + p_data->cnt_freerun_en);
}

static void ddr_pmu_read_clear_freerun(struct cn10k_ddr_pmu *pmu)
{
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	u64 val;

	val = DDRC_FREERUN_READ_CNT_CLR;
	writeq_relaxed(val, pmu->base + p_data->cnt_freerun_ctrl);
}

static void ddr_pmu_write_clear_freerun(struct cn10k_ddr_pmu *pmu)
{
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	u64 val;

	val = DDRC_FREERUN_WRITE_CNT_CLR;
	writeq_relaxed(val, pmu->base + p_data->cnt_freerun_ctrl);
}

static void ddr_pmu_overflow_hander(struct cn10k_ddr_pmu *pmu, int evt_idx)
{
	cn10k_ddr_perf_event_update_all(pmu);
	cn10k_ddr_perf_pmu_disable(&pmu->pmu);
	cn10k_ddr_perf_pmu_enable(&pmu->pmu);
}

static void ddr_pmu_ody_enable_read_freerun(struct cn10k_ddr_pmu *pmu,
					    bool enable)
{
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	u64 val;

	val = readq_relaxed(pmu->base + p_data->cnt_freerun_ctrl);
	if (enable)
		val |= DDRC_PERF_FREERUN_READ_EN;
	else
		val &= ~DDRC_PERF_FREERUN_READ_EN;

	writeq_relaxed(val, pmu->base + p_data->cnt_freerun_ctrl);
}

static void ddr_pmu_ody_enable_write_freerun(struct cn10k_ddr_pmu *pmu,
					     bool enable)
{
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	u64 val;

	val = readq_relaxed(pmu->base + p_data->cnt_freerun_ctrl);
	if (enable)
		val |= DDRC_PERF_FREERUN_WRITE_EN;
	else
		val &= ~DDRC_PERF_FREERUN_WRITE_EN;

	writeq_relaxed(val, pmu->base + p_data->cnt_freerun_ctrl);
}

static void ddr_pmu_ody_read_clear_freerun(struct cn10k_ddr_pmu *pmu)
{
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	u64 val;

	val = DDRC_FREERUN_READ_CNT_CLR;
	writeq_relaxed(val, pmu->base + p_data->cnt_freerun_clr);
}

static void ddr_pmu_ody_write_clear_freerun(struct cn10k_ddr_pmu *pmu)
{
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	u64 val;

	val = DDRC_FREERUN_WRITE_CNT_CLR;
	writeq_relaxed(val, pmu->base + p_data->cnt_freerun_clr);
}

static void ddr_pmu_ody_overflow_hander(struct cn10k_ddr_pmu *pmu, int evt_idx)
{
	/*
	 * On reaching the maximum value of the counter, the counter freezes
	 * there. The particular event is updated and the respective counter
	 * is stopped and started again so that it starts counting from zero
	 */
	cn10k_ddr_perf_event_update(pmu->events[evt_idx]);
	cn10k_ddr_perf_counter_stop(pmu, evt_idx);
	cn10k_ddr_perf_counter_start(pmu, evt_idx);
}

static irqreturn_t cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu *pmu)
{
	const struct ddr_pmu_platform_data *p_data = pmu->p_data;
	const struct ddr_pmu_ops *ops = pmu->ops;
	struct perf_event *event;
	struct hw_perf_event *hwc;
	u64 prev_count, new_count;
	u64 value;
	int i;

	event = pmu->events[DDRC_PERF_READ_COUNTER_IDX];
	if (event) {
		hwc = &event->hw;
		prev_count = local64_read(&hwc->prev_count);
		new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);

		/* Overflow condition is when new count less than
		 * previous count
		 */
		if (new_count < prev_count)
			cn10k_ddr_perf_event_update(event);
	}

	event = pmu->events[DDRC_PERF_WRITE_COUNTER_IDX];
	if (event) {
		hwc = &event->hw;
		prev_count = local64_read(&hwc->prev_count);
		new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx);

		/* Overflow condition is when new count less than
		 * previous count
		 */
		if (new_count < prev_count)
			cn10k_ddr_perf_event_update(event);
	}

	for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) {
		if (pmu->events[i] == NULL)
			continue;

		value = cn10k_ddr_perf_read_counter(pmu, i);
		if (value == p_data->counter_max_val) {
			pr_info("Counter-(%d) reached max value\n", i);
			ops->pmu_overflow_handler(pmu, i);
		}
	}

	return IRQ_HANDLED;
}

static enum hrtimer_restart cn10k_ddr_pmu_timer_handler(struct hrtimer *hrtimer)
{
	struct cn10k_ddr_pmu *pmu = container_of(hrtimer, struct cn10k_ddr_pmu,
						 hrtimer);
	unsigned long flags;

	local_irq_save(flags);
	cn10k_ddr_pmu_overflow_handler(pmu);
	local_irq_restore(flags);

	hrtimer_forward_now(hrtimer, cn10k_ddr_pmu_timer_period());
	return HRTIMER_RESTART;
}

static int cn10k_ddr_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
{
	struct cn10k_ddr_pmu *pmu = hlist_entry_safe(node, struct cn10k_ddr_pmu,
						     node);
	unsigned int target;

	if (cpu != pmu->cpu)
		return 0;

	target = cpumask_any_but(cpu_online_mask, cpu);
	if (target >= nr_cpu_ids)
		return 0;

	perf_pmu_migrate_context(&pmu->pmu, cpu, target);
	pmu->cpu = target;
	return 0;
}

static const struct ddr_pmu_ops ddr_pmu_ops = {
	.enable_read_freerun_counter = ddr_pmu_enable_read_freerun,
	.enable_write_freerun_counter = ddr_pmu_enable_write_freerun,
	.clear_read_freerun_counter = ddr_pmu_read_clear_freerun,
	.clear_write_freerun_counter = ddr_pmu_write_clear_freerun,
	.pmu_overflow_handler = ddr_pmu_overflow_hander,
};

#if defined(CONFIG_ACPI) || defined(CONFIG_OF)
static const struct ddr_pmu_platform_data cn10k_ddr_pmu_pdata = {
	.counter_overflow_val =  BIT_ULL(48),
	.counter_max_val = GENMASK_ULL(48, 0),
	.cnt_base = CN10K_DDRC_PERF_CNT_VALUE_BASE,
	.cfg_base = CN10K_DDRC_PERF_CFG_BASE,
	.cnt_op_mode_ctrl = CN10K_DDRC_PERF_CNT_OP_MODE_CTRL,
	.cnt_start_op_ctrl = CN10K_DDRC_PERF_CNT_START_OP_CTRL,
	.cnt_end_op_ctrl = CN10K_DDRC_PERF_CNT_END_OP_CTRL,
	.cnt_end_status = CN10K_DDRC_PERF_CNT_END_STATUS,
	.cnt_freerun_en = CN10K_DDRC_PERF_CNT_FREERUN_EN,
	.cnt_freerun_ctrl = CN10K_DDRC_PERF_CNT_FREERUN_CTRL,
	.cnt_freerun_clr = 0,
	.cnt_value_wr_op = CN10K_DDRC_PERF_CNT_VALUE_WR_OP,
	.cnt_value_rd_op = CN10K_DDRC_PERF_CNT_VALUE_RD_OP,
	.is_cn10k = TRUE,
};
#endif

static const struct ddr_pmu_ops ddr_pmu_ody_ops = {
	.enable_read_freerun_counter = ddr_pmu_ody_enable_read_freerun,
	.enable_write_freerun_counter = ddr_pmu_ody_enable_write_freerun,
	.clear_read_freerun_counter = ddr_pmu_ody_read_clear_freerun,
	.clear_write_freerun_counter = ddr_pmu_ody_write_clear_freerun,
	.pmu_overflow_handler = ddr_pmu_ody_overflow_hander,
};

#ifdef CONFIG_ACPI
static const struct ddr_pmu_platform_data odyssey_ddr_pmu_pdata = {
	.counter_overflow_val = 0,
	.counter_max_val = GENMASK_ULL(63, 0),
	.cnt_base = ODY_DDRC_PERF_CNT_VALUE_BASE,
	.cfg_base = ODY_DDRC_PERF_CFG_BASE,
	.cnt_op_mode_ctrl = ODY_DDRC_PERF_CNT_OP_MODE_CTRL,
	.cnt_start_op_ctrl = ODY_DDRC_PERF_CNT_START_OP_CTRL,
	.cnt_end_op_ctrl = ODY_DDRC_PERF_CNT_END_OP_CTRL,
	.cnt_end_status = ODY_DDRC_PERF_CNT_END_STATUS,
	.cnt_freerun_en = 0,
	.cnt_freerun_ctrl = ODY_DDRC_PERF_CNT_FREERUN_CTRL,
	.cnt_freerun_clr = ODY_DDRC_PERF_CNT_FREERUN_CLR,
	.cnt_value_wr_op = ODY_DDRC_PERF_CNT_VALUE_WR_OP,
	.cnt_value_rd_op = ODY_DDRC_PERF_CNT_VALUE_RD_OP,
	.is_ody = TRUE,
};
#endif

static int cn10k_ddr_perf_probe(struct platform_device *pdev)
{
	const struct ddr_pmu_platform_data *dev_data;
	struct cn10k_ddr_pmu *ddr_pmu;
	struct resource *res;
	void __iomem *base;
	bool is_cn10k;
	bool is_ody;
	char *name;
	int ret;

	ddr_pmu = devm_kzalloc(&pdev->dev, sizeof(*ddr_pmu), GFP_KERNEL);
	if (!ddr_pmu)
		return -ENOMEM;

	ddr_pmu->dev = &pdev->dev;
	platform_set_drvdata(pdev, ddr_pmu);

	dev_data = device_get_match_data(&pdev->dev);
	if (!dev_data) {
		dev_err(&pdev->dev, "Error: No device match data found\n");
		return -ENODEV;
	}

	base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
	if (IS_ERR(base))
		return PTR_ERR(base);

	ddr_pmu->base = base;

	ddr_pmu->p_data = dev_data;
	is_cn10k = ddr_pmu->p_data->is_cn10k;
	is_ody = ddr_pmu->p_data->is_ody;

	if (is_cn10k) {
		ddr_pmu->ops = &ddr_pmu_ops;
		/* Setup the PMU counter to work in manual mode */
		writeq_relaxed(OP_MODE_CTRL_VAL_MANUAL, ddr_pmu->base +
			       ddr_pmu->p_data->cnt_op_mode_ctrl);

		ddr_pmu->pmu = (struct pmu) {
			.module	      = THIS_MODULE,
			.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
			.task_ctx_nr = perf_invalid_context,
			.attr_groups = cn10k_attr_groups,
			.event_init  = cn10k_ddr_perf_event_init,
			.add	     = cn10k_ddr_perf_event_add,
			.del	     = cn10k_ddr_perf_event_del,
			.start	     = cn10k_ddr_perf_event_start,
			.stop	     = cn10k_ddr_perf_event_stop,
			.read	     = cn10k_ddr_perf_event_update,
			.pmu_enable  = cn10k_ddr_perf_pmu_enable,
			.pmu_disable = cn10k_ddr_perf_pmu_disable,
		};
	}

	if (is_ody) {
		ddr_pmu->ops = &ddr_pmu_ody_ops;

		ddr_pmu->pmu = (struct pmu) {
			.module       = THIS_MODULE,
			.capabilities = PERF_PMU_CAP_NO_EXCLUDE,
			.task_ctx_nr = perf_invalid_context,
			.attr_groups = odyssey_attr_groups,
			.event_init  = cn10k_ddr_perf_event_init,
			.add         = cn10k_ddr_perf_event_add,
			.del         = cn10k_ddr_perf_event_del,
			.start       = cn10k_ddr_perf_event_start,
			.stop        = cn10k_ddr_perf_event_stop,
			.read        = cn10k_ddr_perf_event_update,
		};
	}

	/* Choose this cpu to collect perf data */
	ddr_pmu->cpu = raw_smp_processor_id();

	name = devm_kasprintf(ddr_pmu->dev, GFP_KERNEL, "mrvl_ddr_pmu_%llx",
			      res->start);
	if (!name)
		return -ENOMEM;

	hrtimer_setup(&ddr_pmu->hrtimer, cn10k_ddr_pmu_timer_handler, CLOCK_MONOTONIC,
		      HRTIMER_MODE_REL);

	cpuhp_state_add_instance_nocalls(
				CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
				&ddr_pmu->node);

	ret = perf_pmu_register(&ddr_pmu->pmu, name, -1);
	if (ret)
		goto error;

	pr_info("DDR PMU Driver for ddrc@%llx\n", res->start);
	return 0;
error:
	cpuhp_state_remove_instance_nocalls(
				CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
				&ddr_pmu->node);
	return ret;
}

static void cn10k_ddr_perf_remove(struct platform_device *pdev)
{
	struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev);

	cpuhp_state_remove_instance_nocalls(
				CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
				&ddr_pmu->node);

	perf_pmu_unregister(&ddr_pmu->pmu);
}

#ifdef CONFIG_OF
static const struct of_device_id cn10k_ddr_pmu_of_match[] = {
	{ .compatible = "marvell,cn10k-ddr-pmu", .data = &cn10k_ddr_pmu_pdata },
	{ },
};
MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match);
#endif

#ifdef CONFIG_ACPI
static const struct acpi_device_id cn10k_ddr_pmu_acpi_match[] = {
	{"MRVL000A", (kernel_ulong_t)&cn10k_ddr_pmu_pdata },
	{"MRVL000C", (kernel_ulong_t)&odyssey_ddr_pmu_pdata},
	{},
};
MODULE_DEVICE_TABLE(acpi, cn10k_ddr_pmu_acpi_match);
#endif

static struct platform_driver cn10k_ddr_pmu_driver = {
	.driver	= {
		.name   = "cn10k-ddr-pmu",
		.of_match_table = of_match_ptr(cn10k_ddr_pmu_of_match),
		.acpi_match_table  = ACPI_PTR(cn10k_ddr_pmu_acpi_match),
		.suppress_bind_attrs = true,
	},
	.probe		= cn10k_ddr_perf_probe,
	.remove		= cn10k_ddr_perf_remove,
};

static int __init cn10k_ddr_pmu_init(void)
{
	int ret;

	ret = cpuhp_setup_state_multi(
				CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE,
				"perf/marvell/cn10k/ddr:online", NULL,
				cn10k_ddr_pmu_offline_cpu);
	if (ret)
		return ret;

	ret = platform_driver_register(&cn10k_ddr_pmu_driver);
	if (ret)
		cpuhp_remove_multi_state(
				CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);
	return ret;
}

static void __exit cn10k_ddr_pmu_exit(void)
{
	platform_driver_unregister(&cn10k_ddr_pmu_driver);
	cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE);
}

module_init(cn10k_ddr_pmu_init);
module_exit(cn10k_ddr_pmu_exit);

MODULE_AUTHOR("Bharat Bhushan <bbhushan2@marvell.com>");
MODULE_DESCRIPTION("Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver");
MODULE_LICENSE("GPL v2");