prov/efa: adjust the memory barrier positions

We need such barrier in two transitions:
1. from wqe write to doorbell ring
2. from doorbell ring to wqe write.

For #2, we currently have the barrier right after the doorbell ring,
which is different from rdma-core implementation that has the doorbell
right in the next wr_start, and according to benchmark such choice
hurts performance on ARM platform. This patch moves the #2 barrier
to the beginning of wr start, before the wqe write, to be consistent
with rdma-core and improves the performance.

Signed-off-by: Shi Jin <[email protected]>

Author: shijin-aws

prov/efa/src/efa_data_path_direct_entry.h

@@ -400,9 +400,16 @@ static inline int efa_data_path_direct_post_send(
 		return err;
 	}
 
+	/* This means we are starting from fresh after a db ring so we need a barrier */
+	if (!sq->num_wqe_pending)
+		mmio_wc_start();
+
 	/* when reaching the sq max_batch, ring the db */
-	if (sq->num_wqe_pending == sq->wq.max_batch)
+	if (sq->num_wqe_pending == sq->wq.max_batch) {
 		efa_data_path_direct_send_wr_ring_db(sq);
+		/* we will prepare more WQE after db ring, so we need a barrier */
+		mmio_wc_start();
+	}
 
 	/* Set work request ID */
 	qp->ibv_qp_ex->wr_id = wr_id;
@@ -481,12 +488,22 @@ static inline int efa_data_path_direct_post_read(
 	/* Validate SGE count for RDMA operations */
 	if (OFI_UNLIKELY(sge_count > EFA_IO_TX_DESC_NUM_RDMA_BUFS)) {
 		EFA_WARN(FI_LOG_EP_DATA, "EFA device doesn't support > %d iov for rdma operations\n", EFA_IO_TX_DESC_NUM_RDMA_BUFS);
+		/* ring db for earlier wqes if there is any */
+		if (sq->num_wqe_pending)
+			efa_data_path_direct_send_wr_ring_db(sq);
 		return EINVAL;
 	}
 
+	/* This means we are starting from fresh after a db ring so we need a barrier */
+	if (!sq->num_wqe_pending)
+		mmio_wc_start();
+
 	/* when reaching the sq max_batch, ring the db */
-	if (sq->num_wqe_pending == sq->wq.max_batch)
+	if (sq->num_wqe_pending == sq->wq.max_batch) {
 		efa_data_path_direct_send_wr_ring_db(sq);
+		/* we will prepare more WQE after db ring, so we need a barrier */
+		mmio_wc_start();
+	}
 
 	/* Set work request ID */
 	qp->ibv_qp_ex->wr_id = wr_id;
@@ -552,6 +569,16 @@ efa_data_path_direct_post_write(
 	struct efa_io_remote_mem_addr *remote_mem = &local_wqe.data.rdma_req.remote_mem;
 	int err;
 
+	/* Validate SGE count for RDMA operations */
+	if (OFI_UNLIKELY(sge_count > EFA_IO_TX_DESC_NUM_RDMA_BUFS)) {
+		EFA_WARN(FI_LOG_EP_DATA, "EFA device doesn't support > %d iov for rdma operations\n", EFA_IO_TX_DESC_NUM_RDMA_BUFS);
+		/* ring db for earlier wqes if there is any */
+		if (sq->num_wqe_pending) {
+			efa_data_path_direct_send_wr_ring_db(sq);
+		}
+		return EINVAL;
+	}
+
 	/* Validate queue space */
 	err = efa_post_send_validate(qp);
 	if (OFI_UNLIKELY(err)) {
@@ -564,12 +591,22 @@ efa_data_path_direct_post_write(
 	/* Validate SGE count for RDMA operations */
 	if (OFI_UNLIKELY(sge_count > EFA_IO_TX_DESC_NUM_RDMA_BUFS)) {
 		EFA_WARN(FI_LOG_EP_DATA, "EFA device doesn't support > %d iov for rdma operations\n", EFA_IO_TX_DESC_NUM_RDMA_BUFS);
+		/* ring db for earlier wqes if there is any */
+		if (sq->num_wqe_pending)
+			efa_data_path_direct_send_wr_ring_db(sq);
 		return EINVAL;
 	}
 
+	/* This means we are starting from fresh after a db ring so we need a barrier */
+	if (!sq->num_wqe_pending)
+		mmio_wc_start();
+
 	/* when reaching the sq max_batch, ring the db */
-	if (sq->num_wqe_pending == sq->wq.max_batch)
+	if (sq->num_wqe_pending == sq->wq.max_batch) {
 		efa_data_path_direct_send_wr_ring_db(sq);
+		/* we will prepare more WQE after db ring, so we need a barrier */
+		mmio_wc_start();
+	}
 
 	/* Set work request ID */
 	qp->ibv_qp_ex->wr_id = wr_id;

prov/efa/src/efa_data_path_direct_internal.h

@@ -585,7 +585,6 @@ efa_data_path_direct_send_wr_ring_db(struct efa_data_path_direct_sq *sq)
 {
 	mmio_flush_writes();
 	efa_sq_ring_doorbell(sq, sq->wq.pc);
-	mmio_wc_start();
 	sq->num_wqe_pending = 0;
 }