linux/drivers/net/ethernet/marvell/octeontx2/nic/otx2_xsk.c

// SPDX-License-Identifier: GPL-2.0
/* Marvell RVU Ethernet driver
 *
 * Copyright (C) 2024 Marvell.
 *
 */

#include <linux/bpf_trace.h>
#include <linux/stringify.h>
#include <net/xdp_sock_drv.h>
#include <net/xdp.h>

#include "otx2_common.h"
#include "otx2_struct.h"
#include "otx2_xsk.h"

int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool,
			    dma_addr_t *dma, int idx)
{
	struct xdp_buff *xdp;
	int delta;

	xdp = xsk_buff_alloc(pool->xsk_pool);
	if (!xdp)
		return -ENOMEM;

	pool->xdp[pool->xdp_top++] = xdp;
	*dma = OTX2_DATA_ALIGN(xsk_buff_xdp_get_dma(xdp));
	/* Adjust xdp->data for unaligned addresses */
	delta = *dma - xsk_buff_xdp_get_dma(xdp);
	xdp->data += delta;

	return 0;
}

static int otx2_xsk_ctx_disable(struct otx2_nic *pfvf, u16 qidx, int aura_id)
{
	struct nix_cn10k_aq_enq_req *cn10k_rq_aq;
	struct npa_aq_enq_req *aura_aq;
	struct npa_aq_enq_req *pool_aq;
	struct nix_aq_enq_req *rq_aq;

	if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) {
		cn10k_rq_aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
		if (!cn10k_rq_aq)
			return -ENOMEM;
		cn10k_rq_aq->qidx = qidx;
		cn10k_rq_aq->rq.ena = 0;
		cn10k_rq_aq->rq_mask.ena = 1;
		cn10k_rq_aq->ctype = NIX_AQ_CTYPE_RQ;
		cn10k_rq_aq->op = NIX_AQ_INSTOP_WRITE;
	} else {
		rq_aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox);
		if (!rq_aq)
			return -ENOMEM;
		rq_aq->qidx = qidx;
		rq_aq->sq.ena = 0;
		rq_aq->sq_mask.ena = 1;
		rq_aq->ctype = NIX_AQ_CTYPE_RQ;
		rq_aq->op = NIX_AQ_INSTOP_WRITE;
	}

	aura_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox);
	if (!aura_aq)
		goto fail;

	aura_aq->aura_id = aura_id;
	aura_aq->aura.ena = 0;
	aura_aq->aura_mask.ena = 1;
	aura_aq->ctype = NPA_AQ_CTYPE_AURA;
	aura_aq->op = NPA_AQ_INSTOP_WRITE;

	pool_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox);
	if (!pool_aq)
		goto fail;

	pool_aq->aura_id = aura_id;
	pool_aq->pool.ena = 0;
	pool_aq->pool_mask.ena = 1;

	pool_aq->ctype = NPA_AQ_CTYPE_POOL;
	pool_aq->op = NPA_AQ_INSTOP_WRITE;

	return otx2_sync_mbox_msg(&pfvf->mbox);

fail:
	otx2_mbox_reset(&pfvf->mbox.mbox, 0);
	return -ENOMEM;
}

static void otx2_clean_up_rq(struct otx2_nic *pfvf, int qidx)
{
	struct otx2_qset *qset = &pfvf->qset;
	struct otx2_cq_queue *cq;
	struct otx2_pool *pool;
	u64 iova;

	/* If the DOWN flag is set SQs are already freed */
	if (pfvf->flags & OTX2_FLAG_INTF_DOWN)
		return;

	cq = &qset->cq[qidx];
	if (cq)
		otx2_cleanup_rx_cqes(pfvf, cq, qidx);

	pool = &pfvf->qset.pool[qidx];
	iova = otx2_aura_allocptr(pfvf, qidx);
	while (iova) {
		iova -= OTX2_HEAD_ROOM;
		otx2_free_bufs(pfvf, pool, iova, pfvf->rbsize);
		iova = otx2_aura_allocptr(pfvf, qidx);
	}

	mutex_lock(&pfvf->mbox.lock);
	otx2_xsk_ctx_disable(pfvf, qidx, qidx);
	mutex_unlock(&pfvf->mbox.lock);
}

int otx2_xsk_pool_enable(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qidx)
{
	u16 rx_queues = pf->hw.rx_queues;
	u16 tx_queues = pf->hw.tx_queues;
	int err;

	if (qidx >= rx_queues || qidx >= tx_queues)
		return -EINVAL;

	err = xsk_pool_dma_map(pool, pf->dev, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
	if (err)
		return err;

	set_bit(qidx, pf->af_xdp_zc_qidx);
	otx2_clean_up_rq(pf, qidx);
	/* Reconfigure RSS table as 'qidx' cannot be part of RSS now */
	otx2_set_rss_table(pf, DEFAULT_RSS_CONTEXT_GROUP, NULL);
	/* Kick start the NAPI context so that receiving will start */
	return otx2_xsk_wakeup(pf->netdev, qidx, XDP_WAKEUP_RX);
}

int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qidx)
{
	struct net_device *netdev = pf->netdev;
	struct xsk_buff_pool *pool;
	struct otx2_snd_queue *sq;

	pool = xsk_get_pool_from_qid(netdev, qidx);
	if (!pool)
		return -EINVAL;

	sq = &pf->qset.sq[qidx + pf->hw.tx_queues];
	sq->xsk_pool = NULL;
	otx2_clean_up_rq(pf, qidx);
	clear_bit(qidx, pf->af_xdp_zc_qidx);
	xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
	/* Reconfigure RSS table as 'qidx' now need to be part of RSS now */
	otx2_set_rss_table(pf, DEFAULT_RSS_CONTEXT_GROUP, NULL);

	return 0;
}

int otx2_xsk_pool_setup(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qidx)
{
	if (pool)
		return otx2_xsk_pool_enable(pf, pool, qidx);

	return otx2_xsk_pool_disable(pf, qidx);
}

int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
{
	struct otx2_nic *pf = netdev_priv(dev);
	struct otx2_cq_poll *cq_poll = NULL;
	struct otx2_qset *qset = &pf->qset;

	if (pf->flags & OTX2_FLAG_INTF_DOWN)
		return -ENETDOWN;

	if (queue_id >= pf->hw.rx_queues || queue_id >= pf->hw.tx_queues)
		return -EINVAL;

	cq_poll = &qset->napi[queue_id];
	if (!cq_poll)
		return -EINVAL;

	/* Trigger interrupt */
	if (!napi_if_scheduled_mark_missed(&cq_poll->napi)) {
		otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), BIT_ULL(0));
		otx2_write64(pf, NIX_LF_CINTX_INT_W1S(cq_poll->cint_idx), BIT_ULL(0));
	}

	return 0;
}

void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx)
{
	if (test_bit(qidx, pfvf->af_xdp_zc_qidx))
		sq->xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, qidx);
}

static void otx2_xsk_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len,
				   u16 qidx)
{
	struct nix_sqe_hdr_s *sqe_hdr;
	struct otx2_snd_queue *sq;
	int offset;

	sq = &pfvf->qset.sq[qidx];
	memset(sq->sqe_base + 8, 0, sq->sqe_size - 8);

	sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);

	if (!sqe_hdr->total) {
		sqe_hdr->aura = sq->aura_id;
		sqe_hdr->df = 1;
		sqe_hdr->sq = qidx;
		sqe_hdr->pnc = 1;
	}
	sqe_hdr->total = len;
	sqe_hdr->sqe_id = sq->head;

	offset = sizeof(*sqe_hdr);

	otx2_xdp_sqe_add_sg(sq, NULL, iova, len, &offset, OTX2_AF_XDP_FRAME);
	sqe_hdr->sizem1 = (offset / 16) - 1;
	pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
}

void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool,
			  int queue, int budget)
{
	struct xdp_desc *xdp_desc = pool->tx_descs;
	int  i, batch;

	budget = min(budget, otx2_read_free_sqe(pfvf, queue));
	batch = xsk_tx_peek_release_desc_batch(pool, budget);
	if (!batch)
		return;

	for (i = 0; i < batch; i++) {
		dma_addr_t dma_addr;

		dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc[i].addr);
		otx2_xsk_sq_append_pkt(pfvf, dma_addr, xdp_desc[i].len, queue);
	}
}