.\" Automatically generated by Pandoc 3.1.3
.\"
.\" Define V font for inline verbatim, using C font in formats
.\" that render this, and otherwise B font.
.ie "\f[CB]x\f[]"x" \{\
. ftr V B
. ftr VI BI
. ftr VB B
. ftr VBI BI
.\}
.el \{\
. ftr V CR
. ftr VI CI
. ftr VB CB
. ftr VBI CBI
.\}
.TH "MLX5DV_WR" "3" "2019-02-24" "mlx5" "mlx5 Programmer\[cq]s Manual"
.hy
.SH NAME
.PP
mlx5dv_wr_set_dc_addr - Attach a DC info to the last work request
.PP
mlx5dv_wr_raw_wqe - Build a raw work request
.PP
mlx5dv_wr_memcpy - Build a DMA memcpy work request
.SH SYNOPSIS
.IP
.nf
\f[C]
#include <infiniband/mlx5dv.h>

static inline void mlx5dv_wr_set_dc_addr(struct mlx5dv_qp_ex *mqp,
                                         struct ibv_ah *ah,
                                         uint32_t remote_dctn,
                                         uint64_t remote_dc_key);

static inline void mlx5dv_wr_set_dc_addr_stream(struct mlx5dv_qp_ex *mqp,
                        struct ibv_ah *ah,
                        uint32_t remote_dctn,
                        uint64_t remote_dc_key,
                        uint16_t stream_id);

struct mlx5dv_mr_interleaved {
    uint64_t        addr;
    uint32_t        bytes_count;
    uint32_t        bytes_skip;
    uint32_t        lkey;
};

static inline void mlx5dv_wr_mr_interleaved(struct mlx5dv_qp_ex *mqp,
                        struct mlx5dv_mkey *mkey,
                        uint32_t access_flags, /* use enum ibv_access_flags */
                        uint32_t repeat_count,
                        uint16_t num_interleaved,
                        struct mlx5dv_mr_interleaved *data);

static inline void mlx5dv_wr_mr_list(struct mlx5dv_qp_ex *mqp,
                      struct mlx5dv_mkey *mkey,
                      uint32_t access_flags, /* use enum ibv_access_flags */
                      uint16_t num_sges,
                      struct ibv_sge *sge);

static inline int mlx5dv_wr_raw_wqe(struct mlx5dv_qp_ex *mqp, const void *wqe);

static inline void mlx5dv_wr_memcpy(struct mlx5dv_qp_ex *mqp_ex,
                    uint32_t dest_lkey, uint64_t dest_addr,
                    uint32_t src_lkey, uint64_t src_addr,
                    size_t length)
\f[R]
.fi
.SH DESCRIPTION
.PP
The MLX5DV work request APIs (mlx5dv_wr_*) is an extension for IBV work
request API (ibv_wr_*) with mlx5 specific features for send work
request.
This may be used together with or without ibv_wr_* calls.
.SH USAGE
.PP
To use these APIs a QP must be created using mlx5dv_create_qp() with
\f[I]send_ops_flags\f[R] of struct ibv_qp_init_attr_ex set.
.PP
If the QP does not support all the requested work request types then QP
creation will fail.
.PP
The mlx5dv_qp_ex is extracted from the IBV_QP by ibv_qp_to_qp_ex() and
mlx5dv_qp_ex_from_ibv_qp_ex().
This should be used to apply the mlx5 specific features on the posted
WR.
.PP
A work request creation requires to use the ibv_qp_ex as described in
the man for ibv_wr_post and mlx5dv_qp with its available builders and
setters.
.SS QP Specific builders
.TP
\f[I]RC\f[R] QPs
\f[I]mlx5dv_wr_mr_interleaved()\f[R]
.RS
.PP
registers an interleaved memory layout by using an indirect mkey and
some interleaved data.
The layout of the memory pointed by the mkey after its registration will
be the \f[I]data\f[R] representation for the \f[I]num_interleaved\f[R]
entries.
This single layout representation is repeated by \f[I]repeat_count\f[R].
.PP
The \f[I]data\f[R] as described by struct mlx5dv_mr_interleaved will
hold real data defined by \f[I]bytes_count\f[R] and then a padding of
\f[I]bytes_skip\f[R].
Post a successful registration, RDMA operations can use this
\f[I]mkey\f[R].
The hardware will scatter the data according to the pattern.
The \f[I]mkey\f[R] should be used in a zero-based mode.
The \f[I]addr\f[R] field in its \f[I]ibv_sge\f[R] is an offset in the
total data.
To create this \f[I]mkey\f[R] mlx5dv_create_mkey() should be used.
.PP
Current implementation requires the IBV_SEND_INLINE option to be on in
\f[I]ibv_qp_ex->wr_flags\f[R] field.
To be able to have more than 3 \f[I]num_interleaved\f[R] entries, the QP
should be created with a larger WQE size that may fit it.
This should be done using the \f[I]max_inline_data\f[R] attribute of
\f[I]struct ibv_qp_cap\f[R] upon its creation.
.PP
As one entry will be consumed for strided header, the \f[I]mkey\f[R]
should be created with one more entry than the required
\f[I]num_interleaved\f[R].
.PP
In case \f[I]ibv_qp_ex->wr_flags\f[R] turns on IBV_SEND_SIGNALED, the
reported WC opcode will be MLX5DV_WC_UMR.
Unregister the \f[I]mkey\f[R] to enable another pattern registration
should be done via ibv_post_send with IBV_WR_LOCAL_INV opcode.
.RE
\f[I]mlx5dv_wr_mr_list()\f[R]
.RS
.PP
registers a memory layout based on list of ibv_sge.
The layout of the memory pointed by the \f[I]mkey\f[R] after its
registration will be based on the list of \f[I]sge\f[R] counted by
\f[I]num_sges\f[R].
Post a successful registration RDMA operations can use this
\f[I]mkey\f[R], the hardware will scatter the data according to the
pattern.
The \f[I]mkey\f[R] should be used in a zero-based mode, the
\f[I]addr\f[R] field in its \f[I]ibv_sge\f[R] is an offset in the total
data.
.PP
Current implementation requires the IBV_SEND_INLINE option to be on in
\f[I]ibv_qp_ex->wr_flags\f[R] field.
To be able to have more than 4 \f[I]num_sge\f[R] entries, the QP should
be created with a larger WQE size that may fit it.
This should be done using the \f[I]max_inline_data\f[R] attribute of
\f[I]struct ibv_qp_cap\f[R] upon its creation.
.PP
In case \f[I]ibv_qp_ex->wr_flags\f[R] turns on IBV_SEND_SIGNALED, the
reported WC opcode will be MLX5DV_WC_UMR.
Unregister the \f[I]mkey\f[R] to enable other pattern registration
should be done via ibv_post_send with IBV_WR_LOCAL_INV opcode.
.RE
.TP
\f[I]RC\f[R] or \f[I]DCI\f[R] QPs
\f[I]mlx5dv_wr_memcpy()\f[R]
.RS
.PP
Builds a DMA memcpy work request to copy data of length \f[I]length\f[R]
from \f[I]src_addr\f[R] to \f[I]dest_addr\f[R].
The copy operation will be done using the DMA MMO functionality of the
device to copy data on PCI bus.
.PP
The MLX5DV_QP_EX_WITH_MEMCPY flag in
\f[I]mlx5dv_qp_init_attr.send_ops_flags\f[R] needs to be set during QP
creation.
If the device or QP doesn\[cq]t support it then QP creation will fail.
The maximum memcpy length that is supported by the device is reported in
\f[I]mlx5dv_context->max_wr_memcpy_length\f[R].
A zero value in \f[I]mlx5dv_context->max_wr_memcpy_length\f[R] means the
device doesn\[cq]t support memcpy operations.
.PP
IBV_SEND_FENCE indicator should be used on a following send request
which is dependent on \f[I]dest_addr\f[R] of the memcpy operation.
.PP
In case \f[I]ibv_qp_ex->wr_flags\f[R] turns on IBV_SEND_SIGNALED, the
reported WC opcode will be MLX5DV_WC_MEMCPY.
.RE
.SS Raw WQE builders
.TP
\f[I]mlx5dv_wr_raw_wqe()\f[R]
It is used to build a custom work request (WQE) and post it on a normal
QP.
The caller needs to set all details of the WQE (except the
\[lq]ctrl.wqe_index\[rq] and \[lq]ctrl.signature\[rq] fields, which is
the driver\[cq]s responsibility to set).
The MLX5DV_QP_EX_WITH_RAW_WQE flag in mlx5_qp_attr.send_ops_flags needs
to be set.
.RS
.PP
The wr_flags are ignored as it\[cq]s the caller\[cq]s responsibility to
set flags in WQE.
.PP
No matter what the send opcode is, the work completion opcode for a raw
WQE is IBV_WC_DRIVER2.
.RE
.SS QP Specific setters
.TP
\f[I]DCI\f[R] QPs
\f[I]mlx5dv_wr_set_dc_addr()\f[R] must be called to set the DCI WR
properties.
The destination address of the work is specified by \f[I]ah\f[R], the
remote DCT number is specified by \f[I]remote_dctn\f[R] and the DC key
is specified by \f[I]remote_dc_key\f[R].
This setter is available when the QP transport is DCI and send_ops_flags
in struct ibv_qp_init_attr_ex is set.
The available builders and setters for DCI QP are the same as RC QP.
DCI QP created with MLX5DV_QP_INIT_ATTR_MASK_DCI_STREAMS can call
\f[I]mlx5dv_wr_set_dc_addr_stream()\f[R] to define the
\f[I]stream_id\f[R] of the operation to allow HW to choose one of the
multiple concurrent DCI resources.
Calls to \f[I]mlx5dv_wr_set_dc_addr()\f[R] are equivalent to using
\f[I]stream_id\f[R]=0
.SH EXAMPLE
.IP
.nf
\f[C]
/* create DC QP type and specify the required send opcodes */
attr_ex.qp_type = IBV_QPT_DRIVER;
attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE;

attr_dv.comp_mask |= MLX5DV_QP_INIT_ATTR_MASK_DC;
attr_dv.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCI;

ibv_qp *qp = mlx5dv_create_qp(ctx, attr_ex, attr_dv);
ibv_qp_ex *qpx = ibv_qp_to_qp_ex(qp);
mlx5dv_qp_ex *mqpx = mlx5dv_qp_ex_from_ibv_qp_ex(qpx);

ibv_wr_start(qpx);

/* Use ibv_qp_ex object to set WR generic attributes */
qpx->wr_id = my_wr_id_1;
qpx->wr_flags = IBV_SEND_SIGNALED;
ibv_wr_rdma_write(qpx, rkey, remote_addr_1);
ibv_wr_set_sge(qpx, lkey, local_addr_1, length_1);

/* Use mlx5 DC setter using mlx5dv_qp_ex object */
mlx5dv_wr_set_wr_dc_addr(mqpx, ah, remote_dctn, remote_dc_key);

ret = ibv_wr_complete(qpx);
\f[R]
.fi
.SH SEE ALSO
.PP
\f[B]ibv_post_send\f[R](3), \f[B]ibv_create_qp_ex(3)\f[R],
\f[B]ibv_wr_post(3)\f[R], \f[B]mlx5dv_create_mkey(3)\f[R].
.SH AUTHOR
.PP
Guy Levi <guyle@mellanox.com>
.PP
Mark Zhang <markzhang@nvidia.com>