From 8b508a5507e2eea0194b7db4be908622c804caf8 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Wed, 23 Nov 2022 10:38:48 +0000 Subject: [PATCH 1/8] iio.h: Add API function iio_block_disable_cpu_access() This function is meant to be used when the data is never accessed directly, but passed around as DMABUF objects. In that case, keeping the data in DMA space means that it doesn't have to be synchronized with the CPU cache, which improves performance, and saves two ioctl calls. When CPU access is disabled, all API functions to access a block's data should not be used. Signed-off-by: Paul Cercueil --- block.c | 10 ++++++++++ include/iio/iio-backend.h | 1 + include/iio/iio.h | 11 +++++++++++ 3 files changed, 22 insertions(+) diff --git a/block.c b/block.c index f6b0f9bbc..28a7a0279 100644 --- a/block.c +++ b/block.c @@ -295,3 +295,13 @@ struct iio_buffer * iio_block_get_buffer(const struct iio_block *block) { return block->buffer; } + +int iio_block_disable_cpu_access(struct iio_block *block, bool disable) +{ + const struct iio_backend_ops *ops = block->buffer->dev->ctx->ops; + + if (ops->disable_cpu_access) + return ops->disable_cpu_access(block->pdata, disable); + + return -ENOSYS; +} diff --git a/include/iio/iio-backend.h b/include/iio/iio-backend.h index a97e76605..823da8a9b 100644 --- a/include/iio/iio-backend.h +++ b/include/iio/iio-backend.h @@ -125,6 +125,7 @@ struct iio_backend_ops { int (*dequeue_block)(struct iio_block_pdata *pdata, bool nonblock); int (*get_dmabuf_fd)(struct iio_block_pdata *pdata); + int (*disable_cpu_access)(struct iio_block_pdata *pdata, bool disable); struct iio_event_stream_pdata *(*open_ev)(const struct iio_device *dev); void (*close_ev)(struct iio_event_stream_pdata *pdata); diff --git a/include/iio/iio.h b/include/iio/iio.h index 6a353230e..1cc910437 100644 --- a/include/iio/iio.h +++ b/include/iio/iio.h @@ -1139,6 +1139,17 @@ iio_buffer_create_block(struct iio_buffer *buffer, size_t size); __api void iio_block_destroy(struct iio_block *block); +/** @brief Disable CPU access of a given block + * @param block A pointer to an iio_block structure + * @param disable Whether or not to disable CPU access + * + * NOTE:Disabling CPU access is useful when manipulating DMABUF objects. + * If CPU access is disabled, the block's internal buffer of samples should not + * be accessed. Therefore, the following functions should not be called: + * iio_block_start, iio_block_first, iio_block_end, iio_block_foreach_sample. */ +__api int iio_block_disable_cpu_access(struct iio_block *block, bool disable); + + /** @brief Get the start address of the block * @param block A pointer to an iio_block structure * @return A pointer corresponding to the start address of the block */ From 361949a3092d850a3f2455c0fc8bd49c6e7aafbe Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Wed, 23 Nov 2022 12:12:11 +0000 Subject: [PATCH 2/8] iio.h: Add API function iio_block_get_dmabuf_fd() This API function can be used to retrieve a file descriptor of the underlying DMABUF object. This file descriptor can then be used for instance to attach the block to a different device driver for zero-copy applications. Signed-off-by: Paul Cercueil --- block.c | 17 +++++++++++++++-- include/iio/iio.h | 8 ++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/block.c b/block.c index 28a7a0279..fe19e0608 100644 --- a/block.c +++ b/block.c @@ -21,6 +21,8 @@ struct iio_block { struct iio_task_token *token; size_t bytes_used; + + int dmabuf_fd; }; struct iio_block * @@ -41,13 +43,19 @@ iio_buffer_create_block(struct iio_buffer *buf, size_t size) if (!block) return iio_ptr(-ENOMEM); + block->dmabuf_fd = -EINVAL; + if (ops->create_block) { pdata = ops->create_block(buf->pdata, size, &block->data); ret = iio_err(pdata); - if (!ret) + if (!ret) { block->pdata = pdata; - else if (ret != -ENOSYS) + + if (ops->get_dmabuf_fd) + block->dmabuf_fd = ops->get_dmabuf_fd(pdata); + } else if (ret != -ENOSYS) { goto err_free_block; + } } if (!block->pdata) { @@ -296,6 +304,11 @@ struct iio_buffer * iio_block_get_buffer(const struct iio_block *block) return block->buffer; } +int iio_block_get_dmabuf_fd(const struct iio_block *block) +{ + return block->dmabuf_fd; +} + int iio_block_disable_cpu_access(struct iio_block *block, bool disable) { const struct iio_backend_ops *ops = block->buffer->dev->ctx->ops; diff --git a/include/iio/iio.h b/include/iio/iio.h index 1cc910437..82395aabf 100644 --- a/include/iio/iio.h +++ b/include/iio/iio.h @@ -1139,6 +1139,14 @@ iio_buffer_create_block(struct iio_buffer *buffer, size_t size); __api void iio_block_destroy(struct iio_block *block); +/** @brief Get the file descriptor of the underlying DMABUF object + * @param block A pointer to an iio_block structure + * @return The file descriptor of the underlying DMABUF object. + * If the iio_block is not backed by a DMABUF object, -EINVAL is returned. + * Otherwise, the file descriptor will be valid until the block is destroyed. */ +__api __check_ret int iio_block_get_dmabuf_fd(const struct iio_block *block); + + /** @brief Disable CPU access of a given block * @param block A pointer to an iio_block structure * @param disable Whether or not to disable CPU access From a9c0229d276974385f0b28de515ea3cbebf761fe Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Wed, 30 Nov 2022 11:20:47 +0000 Subject: [PATCH 3/8] local: Update to new DMABUF-based kernel API The previous DMABUF-based API was refused upstream as it had several problems and was very complex. A lot of that complexity was lifted by making the IIO core a DMABUF importer instead of an exporter. This means that the IIO core is no more responsible for creating the DMABUF objects, and to map them in the DMA space. This task is now delegated to the "dma-heap" kernel driver, available since kernel 5.6, which allows userspace programs to create DMABUF objects from the system heap. The DMABUF interface of the IIO core then simply consists in three IOCTLs: one to attach a DMABUF to the interface, one to detach it, and one to request a data transfer. Signed-off-by: Paul Cercueil --- local-dmabuf.c | 136 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 101 insertions(+), 35 deletions(-) diff --git a/local-dmabuf.c b/local-dmabuf.c index f49e83293..7993d4d6f 100644 --- a/local-dmabuf.c +++ b/local-dmabuf.c @@ -6,20 +6,35 @@ * Author: Paul Cercueil */ +#define _GNU_SOURCE #include "local.h" #include +#include #include #include +#include #include #include +#include #include #include +#include #include #include -#define IIO_DMABUF_ALLOC_IOCTL _IOW('i', 0x92, struct iio_dmabuf_req) -#define IIO_DMABUF_ENQUEUE_IOCTL _IOW('i', 0x93, struct iio_dmabuf) +struct iio_dmabuf_heap_data { + uint64_t len; + uint32_t fd; + uint32_t fd_flags; + uint64_t heap_flags; +}; + +#define IIO_DMA_HEAP_ALLOC _IOWR('H', 0x0, struct iio_dmabuf_heap_data) + +#define IIO_DMABUF_ATTACH_IOCTL _IOW('i', 0x92, int) +#define IIO_DMABUF_DETACH_IOCTL _IOW('i', 0x93, int) +#define IIO_DMABUF_ENQUEUE_IOCTL _IOW('i', 0x94, struct iio_dmabuf) #define IIO_DMABUF_SYNC_IOCTL _IOW('b', 0, struct dma_buf_sync) #define IIO_DMABUF_FLAG_CYCLIC (1 << 0) @@ -30,11 +45,6 @@ #define DMA_BUF_SYNC_START (0 << 2) #define DMA_BUF_SYNC_END (1 << 2) -struct iio_dmabuf_req { - uint64_t size; - uint64_t resv; -}; - struct iio_dmabuf { int32_t fd; uint32_t flags; @@ -45,30 +55,54 @@ struct dma_buf_sync { uint64_t flags; }; +static int enable_cpu_access(struct iio_block_pdata *pdata, bool enable) +{ + struct dma_buf_sync dbuf_sync = { 0 }; + int fd = (int)(intptr_t) pdata->pdata; + + dbuf_sync.flags = DMA_BUF_SYNC_RW; + + if (enable) + dbuf_sync.flags |= DMA_BUF_SYNC_START; + else + dbuf_sync.flags |= DMA_BUF_SYNC_END; + + return ioctl_nointr(fd, IIO_DMABUF_SYNC_IOCTL, &dbuf_sync); +} + struct iio_block_pdata * local_create_dmabuf(struct iio_buffer_pdata *pdata, size_t size, void **data) { + struct iio_dmabuf_heap_data req = { + .len = size, + .fd_flags = O_CLOEXEC | O_RDWR, + }; struct iio_block_pdata *priv; - struct iio_dmabuf_req req; - int ret, fd; + int ret, fd, devfd; priv = zalloc(sizeof(*priv)); if (!priv) return iio_ptr(-ENOMEM); - req.size = size; - req.resv = 0; + devfd = open("/dev/dma_heap/system", O_RDONLY | O_CLOEXEC | O_NOFOLLOW); /* Flawfinder: ignore */ + if (devfd < 0) { + ret = -errno; - ret = ioctl_nointr(pdata->fd, IIO_DMABUF_ALLOC_IOCTL, &req); + /* If we're running on an old kernel, return -ENOSYS to mark + * the DMABUF interface as unavailable */ + if (ret == -ENOENT) + ret = -ENOSYS; - /* If we get -ENODEV or -EINVAL errors here, the ioctl is wrong and the - * high-speed DMABUF interface is not supported. */ - if (ret == -ENODEV || ret == -EINVAL || ret == -ENOTTY) - ret = -ENOSYS; - if (ret < 0) goto err_free_priv; + } + + ret = ioctl(devfd, IIO_DMA_HEAP_ALLOC, &req); + if (ret < 0) { + ret = -errno; + goto err_close_devfd; + } - fd = ret; + fd = req.fd; *data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); if (*data == MAP_FAILED) { @@ -83,10 +117,35 @@ local_create_dmabuf(struct iio_buffer_pdata *pdata, size_t size, void **data) priv->dequeued = true; pdata->dmabuf_supported = true; + /* The new block is dequeued by default, so enable CPU access */ + ret = enable_cpu_access(priv, true); + if (ret) + goto err_data_unmap; + + /* Attach DMABUF to the buffer */ + ret = ioctl(pdata->fd, IIO_DMABUF_ATTACH_IOCTL, &fd); + if (ret) { + ret = -errno; + + if (ret == -ENODEV) { + /* If the ioctl is not available, return -ENOSYS to mark + * the DMABUF interface as unavailable */ + ret = -ENOSYS; + } + + goto err_data_unmap; + } + + close(devfd); + return priv; +err_data_unmap: + munmap(priv->data, priv->size); err_close_fd: close(fd); +err_close_devfd: + close(devfd); err_free_priv: free(priv); return iio_ptr(ret); @@ -94,7 +153,11 @@ local_create_dmabuf(struct iio_buffer_pdata *pdata, size_t size, void **data) void local_free_dmabuf(struct iio_block_pdata *pdata) { - int fd = (int)(intptr_t) pdata->pdata; + int ret, fd = (int)(intptr_t) pdata->pdata; + + ret = ioctl(pdata->buf->fd, IIO_DMABUF_DETACH_IOCTL, &fd); + if (ret) + dev_perror(pdata->buf->dev, ret, "Unable to detach DMABUF"); munmap(pdata->data, pdata->size); close(fd); @@ -104,30 +167,34 @@ void local_free_dmabuf(struct iio_block_pdata *pdata) int local_enqueue_dmabuf(struct iio_block_pdata *pdata, size_t bytes_used, bool cyclic) { - struct dma_buf_sync dbuf_sync; struct iio_dmabuf dmabuf; int ret, fd = (int)(intptr_t) pdata->pdata; if (!pdata->dequeued) return -EPERM; + if (bytes_used > pdata->size || bytes_used == 0) + return -EINVAL; + dmabuf.fd = fd; dmabuf.flags = 0; dmabuf.bytes_used = bytes_used; if (cyclic) - dmabuf.flags |= IIO_DMABUF_FLAG_CYCLIC; + dmabuf.flags |= IIO_DMABUF_FLAG_CYCLIC; - dbuf_sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_RW; - - /* Disable CPU access to last block */ - ret = ioctl_nointr(fd, IIO_DMABUF_SYNC_IOCTL, &dbuf_sync); - if (ret) - return ret; + if (!pdata->cpu_access_disabled) { + /* Disable CPU access to last block */ + ret = enable_cpu_access(pdata, false); + if (ret) + return ret; + } ret = ioctl_nointr(pdata->buf->fd, IIO_DMABUF_ENQUEUE_IOCTL, &dmabuf); - if (ret) + if (ret) { + dev_perror(pdata->buf->dev, ret, "Unable to enqueue DMABUF"); return ret; + } pdata->dequeued = false; @@ -137,7 +204,6 @@ int local_enqueue_dmabuf(struct iio_block_pdata *pdata, int local_dequeue_dmabuf(struct iio_block_pdata *pdata, bool nonblock) { struct iio_buffer_pdata *buf_pdata = pdata->buf; - struct dma_buf_sync dbuf_sync; struct timespec start, *time_ptr = NULL; int ret, fd = (int)(intptr_t) pdata->pdata; @@ -153,12 +219,12 @@ int local_dequeue_dmabuf(struct iio_block_pdata *pdata, bool nonblock) if (ret < 0) return ret; - dbuf_sync.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_RW; - - /* Enable CPU access to new block */ - ret = ioctl_nointr(fd, IIO_DMABUF_SYNC_IOCTL, &dbuf_sync); - if (ret < 0) - return ret; + if (!pdata->cpu_access_disabled) { + /* Enable CPU access to new block */ + ret = enable_cpu_access(pdata, true); + if (ret < 0) + return ret; + } pdata->dequeued = true; From c13087ead95c007a18d6db82a2439b68a1d01af7 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Wed, 30 Nov 2022 11:21:32 +0000 Subject: [PATCH 4/8] local: Implement .disable_cpu_access() callback Implement the .disable_cpu_access() callback. This will only work when the DMABUF interface is used. Signed-off-by: Paul Cercueil --- local-dmabuf.c | 15 +++++++++++++++ local.c | 14 ++++++++++++++ local.h | 3 +++ 3 files changed, 32 insertions(+) diff --git a/local-dmabuf.c b/local-dmabuf.c index 7993d4d6f..e30461aa2 100644 --- a/local-dmabuf.c +++ b/local-dmabuf.c @@ -230,3 +230,18 @@ int local_dequeue_dmabuf(struct iio_block_pdata *pdata, bool nonblock) return 0; } + +int local_dmabuf_disable_cpu_access(struct iio_block_pdata *pdata, bool disable) +{ + int ret; + + if (pdata->dequeued) { + ret = enable_cpu_access(pdata, !disable); + if (ret) + return ret; + } + + pdata->cpu_access_disabled = disable; + + return 0; +} diff --git a/local.c b/local.c index 77f8f20b6..55f9d4c5f 100644 --- a/local.c +++ b/local.c @@ -1697,6 +1697,18 @@ static int local_read_event(struct iio_event_stream_pdata *pdata, return 0; } +static int local_disable_cpu_access(struct iio_block_pdata *pdata, bool disable) +{ + if (WITH_LOCAL_DMABUF_API && pdata->buf->dmabuf_supported) { + if (pdata->cpu_access_disabled == disable) + return 0; + + return local_dmabuf_disable_cpu_access(pdata, disable); + } + + return -EINVAL; +} + static const struct iio_backend_ops local_ops = { .scan = local_context_scan, .create = local_create_context, @@ -1722,6 +1734,8 @@ static const struct iio_backend_ops local_ops = { .open_ev = local_open_events_fd, .close_ev = local_close_events_fd, .read_ev = local_read_event, + + .disable_cpu_access = local_disable_cpu_access, }; const struct iio_backend iio_local_backend = { diff --git a/local.h b/local.h index 4d256f210..932152732 100644 --- a/local.h +++ b/local.h @@ -33,6 +33,7 @@ struct iio_block_pdata { size_t size; void *data; bool dequeued; + bool cpu_access_disabled; }; int ioctl_nointr(int fd, unsigned long request, void *data); @@ -48,6 +49,8 @@ int local_enqueue_dmabuf(struct iio_block_pdata *pdata, size_t bytes_used, bool cyclic); int local_dequeue_dmabuf(struct iio_block_pdata *pdata, bool nonblock); +int local_dmabuf_disable_cpu_access(struct iio_block_pdata *pdata, bool disable); + struct iio_block_pdata * local_create_mmap_block(struct iio_buffer_pdata *pdata, size_t size, void **data); From 6e6a1429b4fbed6ed3bb5d05d33bf5776d348804 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Wed, 30 Nov 2022 11:21:58 +0000 Subject: [PATCH 5/8] local: Implement .get_dmabuf_fd() callback Implement the .get_dmabuf_fd() callback. This will only work when the DMABUF interface is used. Signed-off-by: Paul Cercueil --- local-dmabuf.c | 5 +++++ local.c | 9 +++++++++ local.h | 1 + 3 files changed, 15 insertions(+) diff --git a/local-dmabuf.c b/local-dmabuf.c index e30461aa2..a453803d5 100644 --- a/local-dmabuf.c +++ b/local-dmabuf.c @@ -164,6 +164,11 @@ void local_free_dmabuf(struct iio_block_pdata *pdata) free(pdata); } +int local_dmabuf_get_fd(struct iio_block_pdata *pdata) +{ + return (int)(intptr_t) pdata->pdata; +} + int local_enqueue_dmabuf(struct iio_block_pdata *pdata, size_t bytes_used, bool cyclic) { diff --git a/local.c b/local.c index 55f9d4c5f..9322c4b67 100644 --- a/local.c +++ b/local.c @@ -1697,6 +1697,14 @@ static int local_read_event(struct iio_event_stream_pdata *pdata, return 0; } +static int local_get_dmabuf_fd(struct iio_block_pdata *pdata) +{ + if (WITH_LOCAL_DMABUF_API && pdata->buf->dmabuf_supported) + return local_dmabuf_get_fd(pdata); + + return -EINVAL; +} + static int local_disable_cpu_access(struct iio_block_pdata *pdata, bool disable) { if (WITH_LOCAL_DMABUF_API && pdata->buf->dmabuf_supported) { @@ -1735,6 +1743,7 @@ static const struct iio_backend_ops local_ops = { .close_ev = local_close_events_fd, .read_ev = local_read_event, + .get_dmabuf_fd = local_get_dmabuf_fd, .disable_cpu_access = local_disable_cpu_access, }; diff --git a/local.h b/local.h index 932152732..547eb9e42 100644 --- a/local.h +++ b/local.h @@ -49,6 +49,7 @@ int local_enqueue_dmabuf(struct iio_block_pdata *pdata, size_t bytes_used, bool cyclic); int local_dequeue_dmabuf(struct iio_block_pdata *pdata, bool nonblock); +int local_dmabuf_get_fd(struct iio_block_pdata *pdata); int local_dmabuf_disable_cpu_access(struct iio_block_pdata *pdata, bool disable); struct iio_block_pdata * From 81605a3763c00c4c9e26ae963c40de4ed54e6bb6 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 25 Nov 2022 16:21:38 +0000 Subject: [PATCH 6/8] iiod: Support zero-copy to USB Add support for passing sample data as DMABUF objects between IIO devices, and the USB stack. This mechanism has the benefit that the CPU will never access the data to copy it from one hardware buffer to another, and therefore results in a much higher throughput at a much lower CPU usage. For this new mechanism to work, the DMABUF-based IIO kernel API must be available and used by the local backend, and the FunctionFS stack must also support importing DMABUFs. If those conditions are not met, the standard way of transferring the data will be used. Signed-off-by: Paul Cercueil --- iio-config.h.cmakein | 1 + iiod/CMakeLists.txt | 6 +++++ iiod/ops.h | 8 ++++++ iiod/responder.c | 64 +++++++++++++++++++++++++++++++++++++------- iiod/usb-dmabuf.c | 60 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 10 deletions(-) create mode 100644 iiod/usb-dmabuf.c diff --git a/iio-config.h.cmakein b/iio-config.h.cmakein index 3f647eb09..e3ea030a0 100644 --- a/iio-config.h.cmakein +++ b/iio-config.h.cmakein @@ -32,6 +32,7 @@ #cmakedefine01 WITH_IIOD_NETWORK #cmakedefine01 WITH_IIOD_USBD #cmakedefine01 WITH_IIOD_SERIAL +#cmakedefine01 WITH_IIOD_USB_DMABUF #cmakedefine01 WITH_IIOD_V0_COMPAT #cmakedefine01 WITH_LOCAL_CONFIG #cmakedefine01 WITH_LOCAL_DMABUF_API diff --git a/iiod/CMakeLists.txt b/iiod/CMakeLists.txt index b419bbbb7..27e9cfcff 100644 --- a/iiod/CMakeLists.txt +++ b/iiod/CMakeLists.txt @@ -68,6 +68,12 @@ if (WITH_AIO) endif() target_sources(iiod PRIVATE usbd.c) + + + option(WITH_IIOD_USB_DMABUF "Enable DMABUF support on the USB stack" OFF) + if (WITH_IIOD_USB_DMABUF) + target_sources(iiod PRIVATE usb-dmabuf.c) + endif() endif() target_include_directories(iiod PRIVATE ${LIBAIO_INCLUDE_DIR}) diff --git a/iiod/ops.h b/iiod/ops.h index 3eaea7f1c..e4d9681e5 100644 --- a/iiod/ops.h +++ b/iiod/ops.h @@ -47,6 +47,7 @@ struct iio_mutex; struct iio_task; struct iiod_io; struct pollfd; +struct parser_pdata; struct thread_pool; extern struct thread_pool *main_thread_pool; struct DevEntry; @@ -61,6 +62,8 @@ struct block_entry { uint64_t bytes_used; uint16_t idx; bool cyclic; + int dmabuf_fd; + int ep_fd; }; struct buffer_entry { @@ -71,6 +74,7 @@ struct buffer_entry { struct iio_task *enqueue_task, *dequeue_task; uint32_t *words; uint16_t idx; + bool is_tx; SLIST_HEAD(BlockList, block_entry) blocklist; struct iio_mutex *lock; @@ -139,6 +143,10 @@ int start_network_daemon(struct iio_context *ctx, struct thread_pool *pool, const void *xml_zstd, size_t xml_zstd_len, uint16_t port); +int usb_attach_dmabuf(int ep_fd, int fd); +int usb_detach_dmabuf(int ep_fd, int fd); +int usb_transfer_dmabuf(int ep_fd, int fd, uint64_t size); + int binary_parse(struct parser_pdata *pdata); void enable_binary(struct parser_pdata *pdata); diff --git a/iiod/responder.c b/iiod/responder.c index d9d8d8c7c..756ec0cc5 100644 --- a/iiod/responder.c +++ b/iiod/responder.c @@ -325,7 +325,17 @@ static int buffer_dequeue_block(void *priv, void *d) if (ret < 0) goto out_send_response; - if (!iio_buffer_is_tx(buffer->buf)) { + if (!buffer->is_tx) { + if (WITH_IIOD_USB_DMABUF && entry->dmabuf_fd > 0) { + /* We need to send the error code before the data. + * If usb_transfer_dmabuf() fails, we're screwed... */ + iiod_io_send_response_code(entry->io, entry->bytes_used); + + return usb_transfer_dmabuf(buffer->pdata->fd_out, + entry->dmabuf_fd, + entry->bytes_used); + } + data.ptr = iio_block_start(entry->block); data.size = iio_block_end(entry->block) - data.ptr; nb_data++; @@ -364,6 +374,8 @@ static void handle_create_buffer(struct parser_pdata *pdata, goto err_send_response; } + entry->pdata = pdata; + nb_channels = iio_device_get_channels_count(dev); nb_words = (nb_channels + 31) / 32; @@ -434,6 +446,8 @@ static void handle_create_buffer(struct parser_pdata *pdata, entry->words[BIT_WORD(i)] &= ~BIT_MASK(i); } + entry->is_tx = iio_buffer_is_tx(buf); + /* Success, destroy the temporary mask object */ iio_channels_mask_destroy(mask); @@ -628,7 +642,7 @@ static void handle_create_block(struct parser_pdata *pdata, struct iiod_buf data; uint64_t block_size; struct iiod_io *io; - int ret; + int ret, ep_fd; io = iiod_command_create_io(cmd, cmd_data); ret = iio_err(io); @@ -674,6 +688,25 @@ static void handle_create_block(struct parser_pdata *pdata, entry->io = io; entry->idx = cmd->code >> 16; + if (WITH_IIOD_USB_DMABUF && pdata->is_usb) { + entry->dmabuf_fd = iio_block_get_dmabuf_fd(block); + if (entry->dmabuf_fd > 0) { + ep_fd = buf_entry->is_tx ? pdata->fd_in : pdata->fd_out; + entry->ep_fd = ep_fd; + + ret = usb_attach_dmabuf(ep_fd, entry->dmabuf_fd); + if (!ret) { + /* We could attach to functionfs. Disable CPU + * access to the block as we won't need it. */ + iio_block_disable_cpu_access(block, true); + } else { + /* If we can't attach - no problem. The + * data will be transferred the regular way. */ + entry->dmabuf_fd = -ENOSYS; + } + } + } + /* Keep a reference to the iiod_io until the block is freed. */ iiod_io_ref(io); @@ -695,7 +728,7 @@ static void handle_free_block(struct parser_pdata *pdata, struct iio_buffer *buf; struct iio_block *block; struct iiod_io *io; - int ret; + int ret, ep_fd; buf = get_iio_buffer(pdata, cmd, &buf_entry); ret = iio_err(buf); @@ -715,6 +748,9 @@ static void handle_free_block(struct parser_pdata *pdata, if (entry->block != block) continue; + if (WITH_IIOD_USB_DMABUF && entry->dmabuf_fd > 0) + usb_detach_dmabuf(entry->ep_fd, entry->dmabuf_fd); + SLIST_REMOVE(&buf_entry->blocklist, entry, block_entry, entry); free_block_entry(entry); @@ -779,13 +815,21 @@ static void handle_transfer_block(struct parser_pdata *pdata, } /* Read the data into the block if we are dealing with a TX buffer */ - if (iio_buffer_is_tx(buf)) { - readbuf.ptr = iio_block_start(block); - readbuf.size = iio_block_end(block) - readbuf.ptr; - - ret = iiod_command_data_read(cmd_data, &readbuf); - if (ret < 0) - goto out_send_response; + if (entry->is_tx) { + if (WITH_IIOD_USB_DMABUF && block_entry->dmabuf_fd > 0) { + ret = usb_transfer_dmabuf(pdata->fd_in, + block_entry->dmabuf_fd, + bytes_used); + if (ret) + goto out_send_response; + } else { + readbuf.ptr = iio_block_start(block); + readbuf.size = iio_block_end(block) - readbuf.ptr; + + ret = iiod_command_data_read(cmd_data, &readbuf); + if (ret < 0) + goto out_send_response; + } } block_entry->bytes_used = bytes_used; diff --git a/iiod/usb-dmabuf.c b/iiod/usb-dmabuf.c new file mode 100644 index 000000000..36dea651d --- /dev/null +++ b/iiod/usb-dmabuf.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +/* + * libiio - Library for interfacing industrial I/O (IIO) devices + * + * Copyright (C) 2023 Analog Devices, Inc. + * Author: Paul Cercueil + */ + +#include +#include +#include +#include + +#define IIO_FFS_DMABUF_ATTACH _IOW('g', 131, int) +#define IIO_FFS_DMABUF_DETACH _IOW('g', 132, int) +#define IIO_FFS_DMABUF_TRANSFER _IOW('g', 133, struct iio_ffs_dmabuf_transfer) + +struct iio_ffs_dmabuf_transfer { + int fd; + uint32_t flags; + uint64_t length; +}; + +int usb_attach_dmabuf(int ep_fd, int fd) +{ + int ret; + + ret = ioctl(ep_fd, IIO_FFS_DMABUF_ATTACH, &fd); + if (ret == -1) + return -errno; + + return 0; +} + +int usb_detach_dmabuf(int ep_fd, int fd) +{ + int ret; + + ret = ioctl(ep_fd, IIO_FFS_DMABUF_DETACH, &fd); + if (ret == -1) + return -errno; + + return 0; +} + +int usb_transfer_dmabuf(int ep_fd, int fd, uint64_t size) +{ + struct iio_ffs_dmabuf_transfer req; + int ret; + + req.fd = fd; + req.length = size; + req.flags = 0; + + ret = ioctl(ep_fd, IIO_FFS_DMABUF_TRANSFER, &req); + if (ret == -1) + return -errno; + + return 0; +} From e93064edf3928562043a307961f4f121ce0b7689 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 25 Aug 2023 11:18:43 +0200 Subject: [PATCH 7/8] README_BUILD.md: Document WITH_IIOD_USB_DMABUF Add an entry in the CMake options table for the WITH_IIOD_USB_DMABUF toggle. Signed-off-by: Paul Cercueil --- README_BUILD.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README_BUILD.md b/README_BUILD.md index 29e92e103..54fbe0378 100644 --- a/README_BUILD.md +++ b/README_BUILD.md @@ -109,6 +109,7 @@ Cmake Options | Default | Description | `WITH_IIOD_NETWORK` | ON | Add network (TCP/IP) support | `WITH_IIOD_SERIAL` | ON | Add serial (UART) support | `WITH_IIOD_USBD` | ON | Add support for USB through FunctionFS within IIOD | +`WITH_IIOD_USB_DMABUF` | OFF | Enable DMABUF support on the USB stack | `WITH_IIOD_V0_COMPAT` | ON | Add support for Libiio v0.x protocol and clients | `WITH_LIBTINYIIOD` | OFF | Build libtinyiiod | `WITH_AIO` | ON | Build IIOD with async. I/O support | From 557ebb4664923896ef344362d5203139f7a942d5 Mon Sep 17 00:00:00 2001 From: Nuno Sa Date: Wed, 7 Aug 2024 09:33:09 +0200 Subject: [PATCH 8/8] local-dmabuf: only set dmabuf_supported in the end We were setting dmabuf_supported to true before calling IIO_DMABUF_ATTACH_IOCTL that can still fail. Thererefore we would be wrongly using the DMABUF interface for things like enqueue/dequeue blocks. Signed-off-by: Nuno Sa --- local-dmabuf.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/local-dmabuf.c b/local-dmabuf.c index a453803d5..1b5f1414e 100644 --- a/local-dmabuf.c +++ b/local-dmabuf.c @@ -115,7 +115,6 @@ local_create_dmabuf(struct iio_buffer_pdata *pdata, size_t size, void **data) priv->size = size; priv->buf = pdata; priv->dequeued = true; - pdata->dmabuf_supported = true; /* The new block is dequeued by default, so enable CPU access */ ret = enable_cpu_access(priv, true); @@ -136,6 +135,8 @@ local_create_dmabuf(struct iio_buffer_pdata *pdata, size_t size, void **data) goto err_data_unmap; } + pdata->dmabuf_supported = true; + close(devfd); return priv;