Merge branch 'master' into tswhison/opae.io_sriov

CMakeLists.txt

@@ -793,7 +793,7 @@ endif()
 
 set(OPAE_VERSION_LOCAL   "" CACHE STRING "OPAE local version")
 set(OPAE_VERSION_MAJOR    2 CACHE STRING "OPAE major version" FORCE)
-set(OPAE_VERSION_MINOR    9 CACHE STRING "OPAE minor version" FORCE)
+set(OPAE_VERSION_MINOR   11 CACHE STRING "OPAE minor version" FORCE)
 set(OPAE_VERSION_REVISION 0${OPAE_VERSION_LOCAL} CACHE STRING "OPAE revision version" FORCE)
 set(OPAE_VERSION ${OPAE_VERSION_MAJOR}.${OPAE_VERSION_MINOR}.${OPAE_VERSION_REVISION}
     CACHE STRING "OPAE version" FORCE)

binaries/fpgabist/dma/fpga_dma.cpp

@@ -1,4 +1,4 @@
-// Copyright(c) 2018-2020, Intel Corporation
+// Copyright(c) 2018-2023, Intel Corporation
 //
 // Redistribution  and	use  in source	and  binary  forms,  with  or  without
 // modification, are permitted provided that the following conditions are met:
@@ -438,7 +438,7 @@ static void *dispatcherWorker(void* dma_handle) {
 	uint64_t desc_count = 1;
 	msgdma_sw_desc_t *sw_desc[FPGA_DMA_BLOCK_SIZE+1];
 	msgdma_sw_desc_t *first_sw_desc;
-	msgdma_hw_descp_t *hw_descp;
+	msgdma_hw_descp_t *hw_descp = nullptr;
 	bool is_owned_by_hw;
 	uint8_t block_size = 0;
 	uint8_t format;
@@ -516,7 +516,7 @@ static void *dispatcherWorker(void* dma_handle) {
 
 				// Skip invalid descriptors
 				for(k=1; k<= (FPGA_DMA_BLOCK_SIZE-desc_count); k++) {
-					msgdma_hw_descp_t *unused_hw_descp;
+					msgdma_hw_descp_t *unused_hw_descp = nullptr;
 					while(dma_h->free_desc.empty());
 					dma_h->free_desc.try_pop(unused_hw_descp);
 					dump_hw_desc_log(0, unused_hw_descp->hw_desc, disp_log);
@@ -544,7 +544,7 @@ static void *completionWorker(void* dma_handle) {
 		FPGA_DMA_ERR("Invalid DMA handle\n");
 		return NULL;
 	}
-	msgdma_sw_desc_t *sw_desc;
+	msgdma_sw_desc_t *sw_desc = nullptr;
 
 	debug_print("started completion worker\n");
 	while (1) {
@@ -560,7 +560,7 @@ static void *completionWorker(void* dma_handle) {
 
 			if(sw_desc->last == 1 && (sw_desc->hw_descp->hw_desc_id < (FPGA_DMA_BLOCK_SIZE - 1))) {
 				for(i = (sw_desc->hw_descp->hw_desc_id + 1) ; i < FPGA_DMA_BLOCK_SIZE ; i++) {
-					msgdma_hw_descp_t *unused_hw_descp;
+					msgdma_hw_descp_t *unused_hw_descp = nullptr;
 					dma_h->invalid_desc_queue.try_pop(unused_hw_descp);
 					dma_h->free_desc.push(unused_hw_descp);
 				}

binaries/fpgadiag/opae/diag/fecmode.py

@@ -1,5 +1,5 @@
 #! /usr/bin/env python3
-# Copyright(c) 2020, Intel Corporation
+# Copyright(c) 2020-2023, Intel Corporation
 #
 # Redistribution  and  use  in source  and  binary  forms,  with  or  without
 # modification, are permitted provided that the following conditions are met:
@@ -42,8 +42,8 @@
 CONF_FILE = '/etc/modprobe.d/dfl-fme.conf'
 OPTION_LINE = 'options dfl_n3000_nios fec_mode='
 DRV_MODE = '/sys/module/dfl_n3000_nios/parameters/fec_mode'
-REMOVE_MOD = 'rmmod dfl_n3000_nios'
-PROBE_MOD = 'modprobe dfl_n3000_nios'
+REMOVE_MOD = ['rmmod', 'dfl_n3000_nios']
+PROBE_MOD = ['modprobe', 'dfl_n3000_nios']
 
 
 def get_fpga_sysfs_path(sbdf):
@@ -85,14 +85,11 @@ def do_rsu(sbdf, debug):
         return None
 
     try:
-        cmd = "rsu bmcimg {}".format(sbdf)
+        cmd = ['rsu', 'bmcimg', sbdf]
         if debug:
+            cmd.append('-d')
             print(cmd)
-            cmd += ' -d'
-        rc = subprocess.call(cmd, shell=True)
-        if rc != 0:
-            print("failed to '{}'".format(cmd))
-            return None
+        subprocess.run(cmd, check=True)
     except subprocess.CalledProcessError as e:
         print('failed call')
         return None
@@ -145,10 +142,7 @@ def reload_driver(fec_mode, debug):
     try:
         if debug:
             print(REMOVE_MOD)
-        rc = subprocess.call(REMOVE_MOD, shell=True)
-        if rc != 0:
-            print("failed to '{}'".format(REMOVE_MOD))
-            return rc
+        subprocess.run(REMOVE_MOD, check=True)
     except subprocess.CalledProcessError as e:
         print('failed call')
         return 2
@@ -158,10 +152,7 @@ def reload_driver(fec_mode, debug):
     try:
         if debug:
             print(PROBE_MOD)
-        rc = subprocess.call(PROBE_MOD, shell=True)
-        if rc != 0:
-            print("failed to '{}'".format(PROBE_MOD))
-            return rc
+        subprocess.run(PROBE_MOD, check=True)
     except subprocess.CalledProcessError as e:
         print(e)
         return 2

binaries/fpgadiag/opae/diag/fpgadiag.py

@@ -1,5 +1,5 @@
 #! /usr/bin/env python3
-# Copyright(c) 2017-2020 Intel Corporation
+# Copyright(c) 2017-2023 Intel Corporation
 #
 # Redistribution  and  use  in source  and  binary  forms,  with  or  without
 # modification, are permitted provided that the following conditions are met:
@@ -67,10 +67,9 @@ def main():
 
     cmdline[0] = os.path.join(cwd, cmdline[0])
     cmdline = cmdline + ['-t', args.target] + leftover
-    cmdline = ' '.join(cmdline)
 
     try:
-        subprocess.check_call(cmdline, shell=True)
+        subprocess.run(cmdline, check=True)
     except CalledProcessError as e:
         exit(e.returncode)
 

binaries/fpgadiag/opae/diag/fpgastats.py

@@ -1,5 +1,5 @@
 #! /usr/bin/env python3
-# Copyright(c) 2018-2019, Intel Corporation
+# Copyright(c) 2018-2023, Intel Corporation
 #
 # Redistribution  and  use  in source  and  binary  forms,  with  or  without
 # modification, are permitted provided that the following conditions are met:
@@ -372,15 +372,13 @@ def eth_stats(self):
             print("Ethernet Interface Name:", eth_name[1])
             print("------------------------------")
             try:
-                cmd = "ethtool  {}".format(eth_name[1])
+                cmd = ['ethtool', eth_name[1]]
                 print(cmd)
-                rc = subprocess.call(cmd, shell=True)
-                cmd = "ethtool -S {}".format(eth_name[1])
+                subprocess.run(cmd, check=True)
+
+                cmd = ['ethtool', '-S', eth_name[1]]
                 print(cmd)
-                rc = subprocess.call(cmd, shell=True)
-                if rc != 0:
-                    print("failed to '{}'".format(cmd))
-                    return None
+                subprocess.run(cmd, check=True)
             except subprocess.CalledProcessError as e:
                 print('failed call')
                 return None

binaries/fpgametrics/fpgametrics.c

@@ -254,14 +254,13 @@ int main(int argc, char *argv[])
 	}
 
 	printf("---------------------------------------------------------------------------------------------------\n");
-	printf("metric_num              qualifier_name      group_name             metric_name            metric_units\n");
+	printf("%-11s | %-19s  | %-50s  | %-5s \n", "metric_num", "group_name", "metric_name", "metric_units");
 	printf("---------------------------------------------------------------------------------------------------\n");
 
 	for (i = 0; i < num_metrics; i++) {
 
-		printf("%-3ld  | %-30s  | %-15s  | %-30s  | %-10s \n",
-						metric_info[i].metric_num,
-						metric_info[i].qualifier_name,
+		printf("%-10ld  | %-20s | %-50s  | %-1s \n",
+						metric_info[i].metric_num + 1,
 						metric_info[i].group_name,
 						metric_info[i].metric_name,
 						metric_info[i].metric_units);
@@ -279,36 +278,36 @@ int main(int argc, char *argv[])
 	ON_ERR_GOTO(res, out_close, "get num of metrics value by index");
 
 	printf("\n\n\n");
-	printf("-------------------------------------------------------------------------------------------------\n ");
-	printf("    metric_num              qualifier_name                    metric_name              value     \n ");
-	printf("-------------------------------------------------------------------------------------------------\n ");
+	printf("-------------------------------------------------------------------------------------------------\n");
+	printf("%-11s | %-19s  | %-50s  | %-20s \n", "metric_num", "group_name", "metric_name", "value");
+	printf("-------------------------------------------------------------------------------------------------\n");
 
 	for (i = 0; i < num_metrics; i++) {
 
 		uint64_t num = metric_array[i].metric_num;
 		if (metric_info[num].metric_datatype == FPGA_METRIC_DATATYPE_INT &&
 			metric_array[i].isvalid) {
 
-				printf("%-20ld  | %-30s   | %-25s  | %ld %-20s \n ",
-					metric_info[num].metric_num,
-					metric_info[num].qualifier_name,
+				printf("%-10ld  | %-19s  | %-50s  | %ld %-20s \n",
+					metric_info[num].metric_num +1,
+					metric_info[i].group_name,
 					metric_info[num].metric_name,
 					metric_array[i].value.ivalue,
 					metric_info[num].metric_units);
 		} else if (metric_info[num].metric_datatype == FPGA_METRIC_DATATYPE_DOUBLE &&
 			metric_array[i].isvalid) {
 
-			printf("%-20ld  | %-30s   | %-25s  | %0.2f %-20s \n ",
-				metric_info[num].metric_num,
-				metric_info[num].qualifier_name,
+			printf("%-10ld  | %-19s  | %-50s  | %0.2f %-20s \n",
+				metric_info[num].metric_num + 1,
+				metric_info[i].group_name,
 				metric_info[num].metric_name,
 				metric_array[i].value.dvalue,
 				metric_info[num].metric_units);
 		} else {
 
-			printf("%-20ld  | %-30s   | %-25s  | %s \n ",
-				metric_info[num].metric_num,
-				metric_info[num].qualifier_name,
+			printf("%-10ld  | %-19s  | %-50s  | %s \n",
+				metric_info[num].metric_num + 1,
+				metric_info[i].group_name,
 				metric_info[num].metric_name,
 				"Fails to read metric value");
 		}

binaries/opae.io/main.h

@@ -121,6 +121,9 @@ struct vfio_device {
     close();
   }
 
+  vfio_device(const vfio_device &) = delete;
+  vfio_device &operator=(const vfio_device &) = delete;
+
   static vfio_device* open(const char *pci_addr)
   {
     opae_vfio *v = new opae_vfio();
@@ -243,6 +246,5 @@ struct vfio_device {
   opae_vfio *v_;
   vfio_device(opae_vfio *v)
     : v_(v){}
-  vfio_device(const vfio_device &);
 };
 

include/opae/access.h

@@ -72,6 +72,36 @@ extern "C" {
 fpga_result fpgaOpen(fpga_token token, fpga_handle *handle,
 		     int flags);
 
+/**
+ * Extract the handles of children of a previously fpgaOpen()ed resource.
+ * Only AFUs with feature parameters that name child AFU GUIDs will have
+ * children.
+ *
+ * Child AFU handles may be used to connect to child-specific MMIO regions
+ * and manage interrupts. Children will be closed automatically along with
+ * the parent. fpgaClose() should not be called on returned child handles.
+ *
+ * Child handles may not be passed to fpgaPrepareBuffer(). All shared
+ * memory management must be associated with the parent.
+ *
+ * @param[in]  handle       Handle to previously opened FPGA object
+ * @param[in]  max_children Maximum number of handles that may be returned
+ *                          in the `children` array.
+ * @param[out] children     Pointer to an array of child handles currently
+ *                          open. When NULL or `max_children` is 0, the
+ *                          number of children will still be returned in
+ *                          `num_children`.
+ * @param[out] num_children Number of children belonging to the parent.
+ *                          This number may be higher than `max_children`.
+ * @returns                 FPGA_OK on success.
+ *                          FPGA_INVALID_PARAM if handle does not refer to
+ *                          an acquired resource, or if handle is NULL.
+ *                          FPGA_EXCEPTION if an internal error occurred
+ *                          while accessing the handle.
+ */
+fpga_result fpgaGetChildren(fpga_handle handle, uint32_t max_children,
+			    fpga_handle *children, uint32_t *num_children);
+
 /**
  * Close a previously opened FPGA object
  *

libraries/libopae-c/CMakeLists.txt

@@ -29,6 +29,7 @@ set(SRC
     api-shell.c
     init.c
     props.c
+    multi-port-afu.c
     cfg-file.c
     fpgad-cfg.c
     fpgainfo-cfg.c

libraries/libopae-c/adapter.h

@@ -102,8 +102,22 @@ typedef struct _opae_api_adapter_table {
 
 	fpga_result (*fpgaGetIOAddress)(fpga_handle handle, uint64_t wsid,
 					uint64_t *ioaddr);
+
 	fpga_result (*fpgaBindSVA)(fpga_handle handle, uint32_t *pasid);
 
+	// Internal methods between shell and plugin to pin/unpin an existing
+	// buffer at a specific ioaddr. Used when managing the same address
+	// space on parent and child AFU ports, all opened by the same process.
+	fpga_result (*fpgaPinBuffer)(fpga_handle handle, void *buf_addr,
+				     uint64_t len, uint64_t ioaddr);
+	fpga_result (*fpgaUnpinBuffer)(fpga_handle handle, void *buf_addr,
+				       uint64_t len, uint64_t ioaddr);
+
+	// Internal method to extract details of a workspace.
+	fpga_result (*fpgaGetWSInfo)(fpga_handle handle, uint64_t wsid,
+				     uint64_t *ioaddr,
+				     void **buf_addr, uint64_t *len);
+
 	/*
 	**	fpga_result (*fpgaGetOPAECVersion)(fpga_version *version);
 	**