prov/efa: Disable zero copy receive if p2p is not available

Fix the bug of using zcpy_rx for an iface that does not support p2p. Signed-off-by: Jessie Yang <jiaxiyan@amazon.com>
a-szegel · Nov 2, 2024 · c0225bd · c0225bd
1 parent a9796d9
commit c0225bd
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 9 deletions.
diff --git a/prov/efa/src/rdm/efa_rdm_ep_fiops.c b/prov/efa/src/rdm/efa_rdm_ep_fiops.c
@@ -481,11 +481,11 @@ void efa_rdm_ep_set_use_zcpy_rx(struct efa_rdm_ep *ep)
 	}
 
 	/* Zero-copy receive requires P2P support. Disable it if any initialized HMEM iface does not support P2P. */
-	EFA_HMEM_IFACE_FOREACH(iface) {
+	EFA_HMEM_IFACE_FOREACH_NON_SYSTEM(iface) {
 		if (g_efa_hmem_info[iface].initialized &&
-		    !ofi_hmem_p2p_disabled() &&
-		    ep->hmem_p2p_opt != FI_HMEM_P2P_DISABLED &&
-		    !g_efa_hmem_info[iface].p2p_supported_by_device) {
+		    (ofi_hmem_p2p_disabled() ||
+		    ep->hmem_p2p_opt == FI_HMEM_P2P_DISABLED ||
+		    !g_efa_hmem_info[iface].p2p_supported_by_device)) {
 			EFA_INFO(FI_LOG_EP_CTRL,
 			         "%s does not support P2P, zero-copy receive "
 			         "protocol will be disabled\n",

diff --git a/prov/efa/test/efa_unit_test_ep.c b/prov/efa/test/efa_unit_test_ep.c
@@ -948,6 +948,9 @@ static void test_efa_rdm_ep_use_zcpy_rx_impl(struct efa_resource *resource,
 
 	ep = container_of(resource->ep, struct efa_rdm_ep, base_ep.util_ep.ep_fid);
 
+	if (cuda_p2p_supported)
+		ep->hmem_p2p_opt = FI_HMEM_P2P_ENABLED;
+
 	/* Set sufficiently small max_msg_size */
 	assert_int_equal(fi_setopt(&resource->ep->fid, FI_OPT_ENDPOINT, FI_OPT_MAX_MSG_SIZE,
 			&max_msg_size, sizeof max_msg_size), 0);
@@ -1003,9 +1006,9 @@ void test_efa_rdm_ep_user_zcpy_rx_disabled(struct efa_resource **state)
 }
 
 /**
- * @brief Verify zcpy_rx is enabled if CUDA P2P is explictly disabled
+ * @brief Verify zcpy_rx is disabled if CUDA P2P is explictly disabled
  */
-void test_efa_rdm_ep_user_disable_p2p_zcpy_rx_happy(struct efa_resource **state)
+void test_efa_rdm_ep_user_disable_p2p_zcpy_rx_disabled(struct efa_resource **state)
 {
 	struct efa_resource *resource = *state;
 
@@ -1015,7 +1018,7 @@ void test_efa_rdm_ep_user_disable_p2p_zcpy_rx_happy(struct efa_resource **state)
 	resource->hints->mode = FI_MSG_PREFIX;
 	resource->hints->caps = FI_MSG;
 
-	test_efa_rdm_ep_use_zcpy_rx_impl(resource, true, false, true);
+	test_efa_rdm_ep_use_zcpy_rx_impl(resource, true, false, false);
 }
 
 /**

diff --git a/prov/efa/test/efa_unit_tests.c b/prov/efa/test/efa_unit_tests.c
@@ -105,7 +105,7 @@ int main(void)
 		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_rma_without_caps, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_atomic_without_caps, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_user_zcpy_rx_disabled, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
-		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_user_disable_p2p_zcpy_rx_happy, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_user_disable_p2p_zcpy_rx_disabled, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_user_zcpy_rx_unhappy_due_to_sas, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_user_p2p_not_supported_zcpy_rx_happy, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_rdm_ep_user_zcpy_rx_unhappy_due_to_no_mr_local, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),

diff --git a/prov/efa/test/efa_unit_tests.h b/prov/efa/test/efa_unit_tests.h
@@ -120,7 +120,7 @@ void test_efa_rdm_ep_setopt_shared_memory_permitted();
 void test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_good();
 void test_efa_rdm_ep_enable_qp_in_order_aligned_128_bytes_bad();
 void test_efa_rdm_ep_user_zcpy_rx_disabled();
-void test_efa_rdm_ep_user_disable_p2p_zcpy_rx_happy();
+void test_efa_rdm_ep_user_disable_p2p_zcpy_rx_disabled();
 void test_efa_rdm_ep_user_zcpy_rx_unhappy_due_to_sas();
 void test_efa_rdm_ep_user_p2p_not_supported_zcpy_rx_happy();
 void test_efa_rdm_ep_user_zcpy_rx_unhappy_due_to_no_mr_local();