Move RDMA control messages to own endpoint

For the long message RDMA protocol, we want to make sure that we
never starve the sender for data to move, which means prioritizing
control messages from the receiver to the sender.  This patch moves
both the communicator setup and recv control messages to a new
endpoint, which is always on device rail 0.  Future patches will
optimize polling of the control message cq in the send path
and setting priority bits on the control cq.

Signed-off-by: Brian Barrett <bbarrett@amazon.com>

include/nccl_ofi_rdma.h

@@ -68,6 +68,8 @@ typedef enum nccl_net_ofi_rdma_req_type {
  * allocate a rdma memory registration handle with `num_rails' rails.
  */
 typedef struct nccl_net_ofi_rdma_mr_handle {
+	struct fid_mr *control_mr;
+
 	int num_rails;
 
 	/* Array of size `num_rails' */
@@ -103,6 +105,8 @@ struct nccl_net_ofi_rdma_req;
 struct nccl_net_ofi_rdma_ep;
 typedef struct nccl_net_ofi_rdma_req nccl_net_ofi_rdma_req_t;
 typedef struct nccl_net_ofi_rdma_ep nccl_net_ofi_rdma_ep_t;
+struct nccl_net_ofi_ep_rail;
+typedef struct nccl_net_ofi_ep_rail nccl_net_ofi_ep_rail_t;
 
 typedef struct {
 	/* Bounce buffer freelist item */
@@ -117,7 +121,7 @@ typedef struct {
 	 * This is useful for re-posting the bounce buffer on the same rail
 	 * when it gets completed.
 	 */
-	int bounce_rail_id;
+	nccl_net_ofi_ep_rail_t *rail;
 	/*
 	 * Back-pointer to associated endpoint
 	 */
@@ -158,10 +162,6 @@ typedef struct {
 typedef struct {
 	/* Pointer to the allocated control buffer from freelist */
 	nccl_net_ofi_rdma_ctrl_fl_item_t *ctrl_fl_item;
-	/* Schedule used to transfer the control buffer. We save the
-	 * pointer to reference it when transferring the buffer over
-	 * network. */
-	nccl_net_ofi_schedule_t *ctrl_schedule;
 	/* Pointer to recv parent request */
 	nccl_net_ofi_rdma_req_t *recv_req;
 } rdma_req_send_ctrl_data_t;
@@ -298,6 +298,8 @@ typedef struct nccl_ofi_rdma_connection_info {
 	   side. The receiver must use this ID when sending messages to sender */
 	uint64_t local_comm_id;
 
+	nccl_ofi_rdma_ep_name_t control_ep_name;
+
 	/* Number of rails */
 	int num_rails;
 
@@ -358,6 +360,8 @@ typedef struct nccl_net_ofi_rdma_send_comm {
 
 	nccl_ofi_msgbuff_t *msgbuff;
 
+	nccl_net_ofi_rdma_send_comm_rail_t control_rail;
+
 	/* Number of rails */
 	int num_rails;
 
@@ -431,6 +435,8 @@ typedef struct nccl_net_ofi_rdma_recv_comm {
 	/* Free list to track control buffers, for sending RDMA control messages */
 	nccl_ofi_freelist_t *ctrl_buff_fl;
 
+	nccl_net_ofi_rdma_recv_comm_rail_t control_rail;
+
 	/* Number of rails */
 	int num_rails;
 
@@ -468,7 +474,9 @@ typedef struct nccl_net_ofi_rdma_listen_comm {
  * Endpoint rail encapsulates data of an endpoint for a
  * specific rail.
  */
-typedef struct nccl_net_ofi_ep_rail {
+struct nccl_net_ofi_ep_rail {
+	int rail_id;
+
 	/* Local libfabric endpoint handle */
 	struct fid_ep *ofi_ep;
 
@@ -493,7 +501,7 @@ typedef struct nccl_net_ofi_ep_rail {
 	size_t max_bounce_posted;
 	/* Mutex for bounce buffer operations */
 	pthread_mutex_t bounce_mutex;
-} nccl_net_ofi_ep_rail_t;
+};
 
 /*
  * @brief	RDMA Endpoint
@@ -517,6 +525,8 @@ struct nccl_net_ofi_rdma_ep {
 	/* ID pool */
 	nccl_ofi_idpool_t *comm_idpool;
 
+	nccl_net_ofi_ep_rail_t control_rail;
+
 	/* Number of rails */
 	int num_rails;