rdcp

Signed-off-by: Roi Dayan <roid@mellanox.com>

.gitignore

@@ -0,0 +1,2 @@
+cscope.*
+tags

Makefile

@@ -0,0 +1,19 @@
+CC = gcc
+CFLAGS = -ggdb
+# -Wall
+LDFLAGS = -libverbs -lrdmacm -lpthread
+XSLTPROC = /usr/bin/xsltproc
+
+all: rdcp rdcp.8
+default: all
+
+.PHONY: rdcp.8
+rdcp.8: rdcp.8.xml
+	-test -z "$(XSLTPROC)" || $(XSLTPROC) -o $@ http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl $<
+
+.PHONY: clean
+clean:
+	rm -f rdcp
+	rm -f rdcp.8
+
+rdcp:

list.h

@@ -0,0 +1,123 @@
+#ifndef __LIST_H__
+#define __LIST_H__
+
+/* taken from linux kernel */
+
+#undef offsetof
+#ifdef __compiler_offsetof
+#define offsetof(TYPE,MEMBER) __compiler_offsetof(TYPE,MEMBER)
+#else
+#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER)
+#endif
+
+#ifndef container_of
+#define container_of(ptr, type, member) ({			\
+        const typeof( ((type *)0)->member ) *__mptr = (ptr);	\
+        (type *)( (char *)__mptr - offsetof(type,member) );})
+#endif
+
+struct list_head {
+	struct list_head *next, *prev;
+};
+
+#define LIST_HEAD_INIT(name) { &(name), &(name) }
+
+#define LIST_HEAD(name) \
+	struct list_head name = LIST_HEAD_INIT(name)
+
+static inline void INIT_LIST_HEAD(struct list_head *list)
+{
+	list->next = list;
+	list->prev = list;
+}
+
+#define list_first_entry(ptr, type, member) \
+	list_entry((ptr)->next, type, member)
+
+static inline int list_empty(const struct list_head *head)
+{
+	return head->next == head;
+}
+
+#define list_entry(ptr, type, member) \
+	container_of(ptr, type, member)
+
+#define list_for_each(pos, head) \
+	for (pos = (head)->next; pos != (head); pos = pos->next)
+
+#define list_for_each_prev(pos, head) \
+	for (pos = (head)->prev; pos != (head); pos = pos->prev)
+
+#define list_for_each_entry(pos, head, member)				\
+	for (pos = list_entry((head)->next, typeof(*pos), member);	\
+	     &pos->member != (head);				 	\
+	     pos = list_entry(pos->member.next, typeof(*pos), member))
+
+#define list_for_each_entry_safe(pos, n, head, member)			\
+	for (pos = list_entry((head)->next, typeof(*pos), member),	\
+		n = list_entry(pos->member.next, typeof(*pos), member);	\
+	     &pos->member != (head); 					\
+	     pos = n, n = list_entry(n->member.next, typeof(*n), member))
+
+static inline void __list_add(struct list_head *new,
+			      struct list_head *prev,
+			      struct list_head *next)
+{
+	next->prev = new;
+	new->next = next;
+	new->prev = prev;
+	prev->next = new;
+}
+
+static inline void list_add(struct list_head *new, struct list_head *head)
+{
+	__list_add(new, head, head->next);
+}
+
+static inline void list_add_tail(struct list_head *new, struct list_head *head)
+{
+	__list_add(new, head->prev, head);
+}
+
+static inline void __list_del(struct list_head * prev, struct list_head * next)
+{
+	next->prev = prev;
+	prev->next = next;
+}
+
+static inline void list_del(struct list_head *entry)
+{
+	__list_del(entry->prev, entry->next);
+	entry->next = entry->prev = NULL;
+}
+
+static inline void list_del_init(struct list_head *entry)
+{
+	__list_del(entry->prev, entry->next);
+	INIT_LIST_HEAD(entry);
+}
+
+static inline void __list_splice(const struct list_head *list,
+				 struct list_head *prev,
+				 struct list_head *next)
+{
+	struct list_head *first = list->next;
+	struct list_head *last = list->prev;
+
+	first->prev = prev;
+	prev->next = first;
+
+	last->next = next;
+	next->prev = last;
+}
+
+static inline void list_splice_init(struct list_head *list,
+				    struct list_head *head)
+{
+	if (!list_empty(list)) {
+		__list_splice(list, head, head->next);
+		INIT_LIST_HEAD(list);
+	}
+}
+
+#endif

paper.txt

@@ -0,0 +1,125 @@
+server:
+
+queue = 100
+
+pool 100 (of size 5)
+post_recv 100 pool_item_id
+
+event completion:
+  if completion < 50
+	post_recv 100 - completions
+
+
+
+client:
+
+size 1000 / 5 = 200  = total sends
+
+loop:
+   post_send
+
+
+---------------------
+
+s: rdma listen
+s: connected
+s: create qp
+s: post recv metadata
+s: post recv multi rdma_info
+c: create qp
+c: post recv
+c: connect s
+c: post send metadata
+loop:
+-c: post send addr+key
+-s: rdma read from c addr+key
+-s: post send read ok
+c: post send done (TODO: set last buf flag in last post_send)
+c: disconnect
+
+
+-------------
+send_tasks 100
+recv_tasks 100
+
+s: post recv 100
+c: post recv 100
+c: while have send_tasks: allocate task + read file to buffer + post send task
+s: recv event: rdma read + post send done with task id
+   post recv
+   rdma read complete: write buffer to file, free buffer
+   send complete: ?
+c: recv event: free send_task
+   post recv
+
+
+
+method 1
+
+sqA.buf=1 sqB.buf=2 sqC.buf=3
+
+c: post send sqA sqB sqC (buf + id)
+s: recv rqA.buf=1 rqB.buf=2 rqC.buf=3 -> add rr_task
+s: rdma read 1 2 3 (from rr_task_list)
+s: post send s.q1 sq.2 sq.3 (buf + id)
+c: recv completion rq.1 rq.2 rq.3 (buf + id)
+   send_tasks (id id id)
+   free_task(&send_task[id])
+
+
+method 2
+
+buf 1 2 3
+sq
+rq
+
+c: post send sq.1 sq.2 sq.3
+s: post recc 1 2 3
+c: send completion sq sq sq. free send tasks sq sq sq
+s: rdma read
+s: post send done 1 2 3
+c: post recv 1 2 3. free bufs 1 2 3
+
+
+c: while can_send (note: have send task + have free buffer)
+
+
+
+---
+
+client
+post send
+wait - recv completion
+post send
+wait - recv completion
+post send
+
+
+
+
+------------1-------------
+multiple post_send
+wait
+-------2------------
+event send_comp: free send_task
+event recv_comp: sem_post
+
+
+
+--------------------
+step1
+get_event
+step2
+get_event
+
+
+-----------------
+c: post recv md
+c: post recv 16 tasks
+
+send_tasks
+c: post send 16 tasks
+c: wait. on each wc_recv do post recv
+goto send_tasks
+
+