/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 2021 - 2023, Shanghai Yunsilicon Technology Co., Ltd.
 * All rights reserved.
 */

#ifndef XSC_IB_H
#define XSC_IB_H

#include <linux/kernel.h>
#include <linux/sched.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_smi.h>
#include <rdma/ib_pack.h>
#include "common/xsc_core.h"
#include "common/driver.h"
#include "common/cq.h"
#include "common/qp.h"
#include <linux/types.h>
#include <crypto/hash.h>

#include "xsc_ib_compat.h"

#define xsc_ib_dbg(dev, format, arg...)						\
do {										\
	if (xsc_log_level <= XSC_LOG_LEVEL_DBG)					\
		pr_debug("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name,	\
			__func__, __LINE__, current->pid, ##arg);		\
} while (0)

#define xsc_ib_err(dev, format, arg...)						\
do {										\
	if (xsc_log_level <= XSC_LOG_LEVEL_ERR)					\
		pr_err("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name,	\
			__func__, __LINE__, current->pid, ##arg);		\
} while (0)

#define xsc_ib_warn(dev, format, arg...)					\
do {										\
	if (xsc_log_level <= XSC_LOG_LEVEL_WARN)				\
		pr_warn("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name,	\
			__func__, __LINE__, current->pid, ##arg);		\
} while (0)

struct xsc_ib_ucontext {
	struct ib_ucontext	ibucontext;
	struct list_head	db_page_list;

	/* protect doorbell record alloc/free
	 */
	struct mutex		db_page_mutex;
};

#define field_avail(type, fld, sz) (offsetof(type, fld) +		\
		sizeof(((type *)0)->fld) <= (sz))

static inline struct xsc_ib_ucontext *to_xucontext(struct ib_ucontext *ibucontext)
{
	return container_of(ibucontext, struct xsc_ib_ucontext, ibucontext);
}

struct xsc_ib_pd {
	struct ib_pd		ibpd;
	u32			pdn;
	u32			pa_lkey;
};

/* Use macros here so that don't have to duplicate
 * enum ib_send_flags and enum ib_qp_type for low-level driver
 */

#define XSC_IB_QPT_REG_UMR	IB_QPT_RESERVED1

enum {
	XSC_PAGE_SHIFT_4K	= 12,
	XSC_PAGE_SHIFT_64K	= 16,
	XSC_PAGE_SHIFT_2M	= 21,
	XSC_PAGE_SHIFT_1G	= 30,
};

enum {
	XSC_PAGE_MODE_4K	= 0,
	XSC_PAGE_MODE_64K	= 1,
	XSC_PAGE_MODE_2M	= 2,
	XSC_PAGE_MODE_1G	= 3,
};

struct wr_list {
	u16	opcode;
	u16	next;
};

struct xsc_ib_wq {
	u64		       *wrid;
	u32		       *wr_data;
	struct wr_list	       *w_list;
	unsigned long	       *wqe_head;
	u16		       unsig_count;

	/* serialize post to the work queue
	 */
	spinlock_t		lock;
	int			wqe_cnt;
	int			ds_cnt;
	int			max_post;
	int			max_gs;
	int			offset;
	int			wqe_shift;
	unsigned int		head;
	unsigned int		tail;
	u16			cur_post;
	u16			last_poll;
	void		       *qend;
	void		*hdr_buf;
	u32			hdr_size;
	dma_addr_t	hdr_dma;
	int			mad_queue_depth;
	int			mad_index;
};

enum {
	XSC_QP_USER,
	XSC_QP_KERNEL,
	XSC_QP_EMPTY
};

struct xsc_ib_qp {
	struct ib_qp		ibqp;
	struct xsc_core_qp	xqp;
	struct xsc_buf		buf;

	struct xsc_db		db;
	struct xsc_ib_wq	rq;

	u32			doorbell_qpn;
	u8			sq_signal_bits;
	u8			fm_cache;
	int			sq_max_wqes_per_wr;
	int			sq_spare_wqes;
	struct xsc_ib_wq	sq;

	struct ib_umem	       *umem;
	int			buf_size;

	/* serialize qp state modifications
	 */
	struct mutex		mutex;
	u16			xrcdn;
	u32			flags;
	u8			port;
	u8			alt_port;
	u8			atomic_rd_en;
	u8			resp_depth;
	u8			state;
	int			xsc_type;
	int			wq_sig;
	int			scat_cqe;
	int			max_inline_data;
	int			has_rq;

	int			create_type;
	u32			pa_lkey;
	/* For QP1 */
	struct ib_ud_header	qp1_hdr;
	u32			send_psn;
	struct xsc_qp_context	ctx;
	struct ib_cq		*send_cq;
	struct ib_cq		*recv_cq;
	/* For qp resources */
	spinlock_t		lock;
};

struct xsc_ib_cq_buf {
	struct xsc_buf		buf;
	struct ib_umem		*umem;
	int			cqe_size;
};

enum xsc_ib_qp_flags {
	XSC_IB_QP_BLOCK_MULTICAST_LOOPBACK     = 1 << 0,
	XSC_IB_QP_SIGNATURE_HANDLING           = 1 << 1,
};

struct xsc_shared_mr_info {
	int mr_id;
	struct ib_umem		*umem;
};

struct xsc_ib_cq {
	struct ib_cq		ibcq;
	struct xsc_core_cq	xcq;
	struct xsc_ib_cq_buf	buf;
	struct xsc_db		db;

	/* serialize access to the CQ
	 */
	spinlock_t		lock;

	/* protect resize cq
	 */
	struct mutex		resize_mutex;
	struct xsc_ib_cq_resize *resize_buf;
	struct ib_umem	       *resize_umem;
	int			cqe_size;
};

struct xsc_ib_xrcd {
	struct ib_xrcd		ibxrcd;
	u32			xrcdn;
};

struct xsc_ib_peer_id;

struct xsc_ib_mr {
	struct ib_mr		ibmr;
	struct xsc_core_mr	mmr;
	struct ib_umem	       *umem;
	struct xsc_shared_mr_info	*smr_info;
	struct list_head	list;
	int			order;
	__be64			*pas;
	dma_addr_t		dma;
	int			npages;
	struct completion	done;
	enum ib_wc_status	status;
	struct xsc_ib_peer_id *peer_id;
	atomic_t      invalidated;
	struct completion invalidation_comp;
};

struct xsc_ib_peer_id {
	struct completion comp;
	struct xsc_ib_mr *mr;
};

struct xsc_cache_ent {
	struct list_head	head;
	/* sync access to the cahce entry
	 */
	spinlock_t		lock;

	struct dentry	       *dir;
	char                    name[4];
	u32                     order;
	u32			size;
	u32                     cur;
	u32                     miss;
	u32			limit;

	struct dentry          *fsize;
	struct dentry          *fcur;
	struct dentry          *fmiss;
	struct dentry          *flimit;

	struct xsc_ib_dev     *dev;
	struct work_struct	work;
	struct delayed_work	dwork;
};

struct xsc_mr_cache {
	struct workqueue_struct *wq;
	struct xsc_cache_ent	ent[MAX_MR_CACHE_ENTRIES];
	int			stopped;
	struct dentry		*root;
	unsigned long		last_add;
};

struct xsc_gid {
	u8 data[16];
};

struct xsc_sgid_tbl {
	struct xsc_gid *tbl;
	u32 max;
	u32 count;
};

struct xsc_ib_res {
	struct xsc_sgid_tbl sgid_tbl;
};

struct xsc_ib_resources {
	struct ib_cq	*c0;
	struct ib_xrcd	*x0;
	struct ib_xrcd	*x1;
	struct ib_pd	*p0;
	struct ib_srq	*s0;
};

struct xsc_ib_dev {
	struct ib_device		ib_dev;
	struct uverbs_object_tree_def *driver_trees[6];
	struct net_device	*netdev;
	struct xsc_core_device *xdev;
	XSC_DECLARE_DOORBELL_LOCK(uar_lock);
	struct list_head		eqs_list;
	int				num_ports;
	int				num_comp_vectors;
	/* serialize update of capability mask
	 */
	struct mutex			cap_mask_mutex;
	u8				ib_active;
	/* sync used page count stats
	 */
	spinlock_t			mr_lock;
	struct xsc_ib_res		ib_res;
	struct xsc_ib_resources	devr;
	struct xsc_mr_cache		cache;
	u32				crc_32_table[256];
	int cm_pcp;
	int cm_dscp;
};

union xsc_ib_fw_ver {
	u64 data;
	struct {
		u16	chip_ver_h;
		u16	hotfix_num;
		u16	chip_ver_l;
		u16	feature_flag;
	} s;
};

struct xsc_pa_chunk {
	struct list_head list;
	u64 va;
	dma_addr_t pa;
	size_t length;
};

static inline struct xsc_ib_cq *to_xibcq(struct xsc_core_cq *xcq)
{
	return container_of(xcq, struct xsc_ib_cq, xcq);
}

static inline struct xsc_ib_xrcd *to_mxrcd(struct ib_xrcd *ibxrcd)
{
	return container_of(ibxrcd, struct xsc_ib_xrcd, ibxrcd);
}

static inline struct xsc_ib_dev *to_mdev(struct ib_device *ibdev)
{
	return container_of(ibdev, struct xsc_ib_dev, ib_dev);
}

static inline struct xsc_ib_cq *to_xcq(struct ib_cq *ibcq)
{
	return container_of(ibcq, struct xsc_ib_cq, ibcq);
}

static inline struct xsc_ib_qp *to_xibqp(struct xsc_core_qp *xqp)
{
	return container_of(xqp, struct xsc_ib_qp, xqp);
}

static inline struct xsc_ib_pd *to_mpd(struct ib_pd *ibpd)
{
	return container_of(ibpd, struct xsc_ib_pd, ibpd);
}

static inline struct xsc_ib_qp *to_xqp(struct ib_qp *ibqp)
{
	return container_of(ibqp, struct xsc_ib_qp, ibqp);
}

static inline struct xsc_ib_mr *to_mmr(struct ib_mr *ibmr)
{
	return container_of(ibmr, struct xsc_ib_mr, ibmr);
}

struct xsc_ib_ah {
	struct ib_ah		ibah;
	struct xsc_av		av;
};

static inline struct xsc_ib_ah *to_mah(struct ib_ah *ibah)
{
	return container_of(ibah, struct xsc_ib_ah, ibah);
}

static inline struct xsc_ib_dev *xdev2ibdev(struct xsc_core_device *xdev)
{
	return container_of((void *)xdev, struct xsc_ib_dev, xdev);
}

int xsc_ib_query_port(struct ib_device *ibdev, u32 port,
		      struct ib_port_attr *props);

int xsc_ib_create_qp(struct ib_qp *ibqp,
		     struct ib_qp_init_attr *init_attr,
		     struct ib_udata *udata);
void __xsc_ib_cq_clean(struct xsc_ib_cq *cq, u32 qpn);
void xsc_ib_cq_clean(struct xsc_ib_cq *cq, u32 qpn);

int xsc_ib_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
int xsc_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
		     int attr_mask, struct ib_udata *udata);
int xsc_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
		    struct ib_qp_init_attr *qp_init_attr);

int xsc_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
		     const struct ib_send_wr **bad_wr);
int xsc_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
		     const struct ib_recv_wr **bad_wr);

void *xsc_get_send_wqe(struct xsc_ib_qp *qp, int n);
int xsc_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
int xsc_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
struct ib_mr *xsc_ib_get_dma_mr(struct ib_pd *pd, int acc);
struct ib_mr *xsc_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
				 u64 virt_addr, int access_flags,
				 struct ib_udata *udata);
int xsc_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
void xsc_ib_cont_pages(struct ib_umem *umem, u64 addr, int *count, int *shift,
		       int *ncont, int *order);
void xsc_ib_populate_pas(struct xsc_ib_dev *dev, struct ib_umem *umem,
			 int page_shift, __be64 *pas, int npages, bool need_to_devide);
const struct uverbs_object_tree_def *xsc_ib_get_devx_tree(void);

int xsc_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
		     int sg_nents, unsigned int *sg_offset);
int xsc_wr_reg_mr(struct xsc_ib_dev *dev, const struct ib_send_wr *wr);
int xsc_wr_invalidate_mr(struct xsc_ib_dev *dev, const struct ib_send_wr *wr);
int xsc_find_best_pgsz(struct ib_umem *umem, unsigned long pgsz_bitmap,
		       unsigned long addr, int *npage, int *shift, u64 **pas);

static inline void init_query_mad(struct ib_smp *mad)
{
	mad->base_version  = 1;
	mad->mgmt_class    = IB_MGMT_CLASS_SUBN_LID_ROUTED;
	mad->class_version = 1;
	mad->method	   = IB_MGMT_METHOD_GET;
}

static inline u8 convert_access(int acc)
{
	return (acc & IB_ACCESS_REMOTE_ATOMIC ? XSC_PERM_ATOMIC       : 0) |
	       (acc & IB_ACCESS_REMOTE_WRITE  ? XSC_PERM_REMOTE_WRITE : 0) |
	       (acc & IB_ACCESS_REMOTE_READ   ? XSC_PERM_REMOTE_READ  : 0) |
	       (acc & IB_ACCESS_LOCAL_WRITE   ? XSC_PERM_LOCAL_WRITE  : 0) |
	       XSC_PERM_LOCAL_READ;
}

static inline enum ib_mtu xsc_net_to_ib_mtu(unsigned int mtu)
{
	mtu = mtu - (IB_GRH_BYTES + IB_UDP_BYTES + IB_BTH_BYTES +
		     IB_EXT_XRC_BYTES + IB_EXT_ATOMICETH_BYTES +
		     IB_ICRC_BYTES);

	if (mtu >= ib_mtu_enum_to_int(IB_MTU_4096))
		return IB_MTU_4096;
	else if (mtu >= ib_mtu_enum_to_int(IB_MTU_1024))
		return IB_MTU_1024;
	else
		return 0;
}

/**
 * UDP source port selection must adhere IANA port allocation ranges. Thus
 * we will be using IANA recommendation for Ephemeral port range of:
 * 49152-65535, or in hex: 0xC000-0xFFFF.
 */
#define IB_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000)
#define IB_ROCE_UDP_ENCAP_VALID_PORT_MAX (0xFFFF)
#define IB_GRH_FLOWLABEL_MASK (0x000FFFFF)

/**
 * rdma_flow_label_to_udp_sport - generate a RoCE v2 UDP src port value based
 *                               on the flow_label
 *
 * This function will convert the 20 bit flow_label input to a valid RoCE v2
 * UDP src port 14 bit value. All RoCE V2 drivers should use this same
 * convention.
 */
static inline u16 xsc_flow_label_to_udp_sport(u32 fl)
{
	u32 fl_low = fl & 0x03fff, fl_high = fl & 0xFC000;

	fl_low ^= fl_high >> 14;
	return (u16)(fl_low | IB_ROCE_UDP_ENCAP_VALID_PORT_MIN);
}

#endif /* XSC_IB_H */