add co-simulating, testing and docker image building ci

.github/workflows/build_docker_image.yml

@@ -0,0 +1,40 @@
+name: Create and publish a Docker image
+
+on:
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        with:
+          context: .
+          push: True
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/test_rdma.yml

@@ -0,0 +1,32 @@
+name: Test RDMA
+
+on:
+  workflow_dispatch:
+
+jobs:
+  test-rdma:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/datenlord/open-rdma-driver
+      volumes:
+        - my_docker_volume:/volume_mount
+      options: --privileged --user root
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: addnab/docker-run-action@v3
+        with:
+          image: ghcr.io/datenlord/open-rdma-driver
+          options: --privileged -v ${{ github.workspace }}:/workspaces/open-rdma-driver
+          run: |
+            cd /workspaces/open-rdma-driver
+            make model
+            make driver
+            make user
+            NON_INTERACTIVE=1 ./scripts/run_qemu.sh
+      - uses: actions/upload-artifact@v3
+        with:
+          name: test-results
+          path: |
+            .build/qemu.log
+            .build/model.log

.gitignore

@@ -5,5 +5,6 @@
 *.ko
 *.mod
 *.mod.c
+.build
 .cache
-compile_commands.json
+compile_commands.json

.gitmodules

@@ -1,4 +1,8 @@
 [submodule "rdma-core"]
 	path = rdma-core
-	url = git@github.com:datenlord/rdma-core.git
-	branch = add-dtld-driver
+	url = https://github.com/datenlord/rdma-core.git
+	branch = dtld-main
+[submodule "DistroSim"]
+	path = DistroSim
+	url = https://github.com/ruiqurm/DistroSim.git
+	branch = master

Dockerfile

@@ -0,0 +1,106 @@
+FROM ubuntu:22.04
+
+ARG KERNEL_SOURCE_GIT="git://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/jammy"
+ARG KERNEL_SOURCE_BRANCH="Ubuntu-hwe-5.19-5.19.0-50.50"
+ARG UBUNTU_RELEASE_NAME="jammy"
+ARG UBUNTU_ROOTFS_URL="https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64-root.tar.xz"
+ARG ROOTFS_TARBALL_PATH="/vm/origin_disk.img"
+ARG ROOTFS_DIR="/rootfs"
+ARG LINUX_SRC_DIR="/linux-src"
+ARG VM_DIR="/vm"
+ARG XILINX_QEMU_GIT="https://github.com/Xilinx/qemu.git"
+ARG XILINX_QEMU_GIT_BRANCH="master"
+
+# linux-image-kvm is installed to make libguestfs-tools happy, because it need a kernel and an simple rootfs to run VM
+RUN apt-get update && \
+	apt-get install -y wget vim git tmux\
+					# the following line is for qemu build
+					zlib1g-dev libglib2.0-dev libpixman-1-dev libfdt-dev libcap-ng-dev libattr1-dev \
+					# the following lines is for linux kernel build
+					build-essential cmake gcc libudev-dev libnl-3-dev libnl-route-3-dev \
+					ninja-build pkg-config valgrind python3-dev cython3 python3-docutils pandoc \
+					bc fakeroot libncurses5-dev libssl-dev ccache bison flex libelf-dev dwarves \ 
+					rsync libguestfs-tools linux-image-kvm && \
+	rm -rf /var/lib/apt/lists/*
+
+RUN	git clone $XILINX_QEMU_GIT --depth=1 --branch $XILINX_QEMU_GIT_BRANCH /qemu && \
+	cd /qemu && \
+    mkdir qemu_build && \
+    cd qemu_build && \
+    ../configure  --target-list=x86_64-softmmu --enable-virtfs && \
+    make -j && \
+	make install && \
+	rm -rf /qemu
+
+RUN mkdir -p $VM_DIR && \
+	wget $UBUNTU_ROOTFS_URL -O $ROOTFS_TARBALL_PATH && \
+	git clone $KERNEL_SOURCE_GIT --depth=1 --branch $KERNEL_SOURCE_BRANCH ./linux-src
+
+
+RUN cd $LINUX_SRC_DIR && \
+	echo "" > config_patch.config && \
+	echo "CONFIG_INFINIBAND=m" >> config_patch.config && \
+	echo "CONFIG_INFINIBAND_USER_MAD=m" >> config_patch.config && \
+	echo "CONFIG_INFINIBAND_USER_ACCESS=m" >> config_patch.config && \
+	echo "CONFIG_RDMA_RXE=m" >> config_patch.config && \
+	echo "# CONFIG_WERROR is not set" >> config_patch.config && \
+	echo "# CONFIG_RANDOMIZE_BASE is not set" >> config_patch.config && \
+	echo "CONFIG_DEBUG_INFO_DWARF5=y" >> config_patch.config && \
+	echo "CONFIG_GDB_SCRIPTS=y" >> config_patch.config && \
+	echo "# CONFIG_DEBUG_INFO_REDUCED is not set" >> config_patch.config && \
+	make defconfig && \
+	./scripts/kconfig/merge_config.sh .config ./config_patch.config && \
+	make -j20 && \
+	make scripts_gdb
+
+RUN mkdir -p $ROOTFS_DIR && \
+	tar -Jxf $ROOTFS_TARBALL_PATH -C $ROOTFS_DIR && \
+	cd $LINUX_SRC_DIR && \
+	make INSTALL_MOD_PATH=$ROOTFS_DIR modules_install && \
+	echo "root:root" | chpasswd -R $ROOTFS_DIR && \
+	# generate a systemd service config file to mount 9p fs at booting. adding it to fstab seems not working.
+	# after mounting,we also run a script mounted from host to do some init work. 
+	echo "[Unit]" > $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \
+	echo "Description=Mount 9p to access host fs" >> $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \
+	echo "After=network.target" >> $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \
+	echo "[Service]" >> $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \
+	echo "ExecStart=bash -c \"mount -t 9p -o trans=virtio,version=9p2000.L,access=any hostshare /host && /host/workspaces/open-rdma-driver/scripts/for_qemu/boot_init.sh\"" >> $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \
+	echo "[Install]" >> $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \
+	echo "WantedBy=default.target" >> $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \
+	# == finish generating systemd config
+	#
+	# After switch to Xilinx Qemu, it seems that the network service has some problems. It will block boot for a long time, so we need to fix it
+	# The following code should be removed when the network problem has been solved.
+	sed -i '/ExecStart=/s/$/ --timeout 1/' $ROOTFS_DIR/etc/systemd/system/network-online.target.wants/systemd-networkd-wait-online.service && \
+	#
+	# generate a tmp script to run in chroot environment to modify the rootfs for qemu
+	echo "ssh-keygen -A " >> /tmp/vm_init.sh && \
+	echo "apt-get purge --auto-remove -y snapd multipath-tools" >> /tmp/vm_init.sh && \
+	echo 'mkdir -p /run/systemd/resolve/' >> /tmp/vm_init.sh && \
+	echo 'echo "nameserver 8.8.8.8" > /run/systemd/resolve/stub-resolv.conf' >> /tmp/vm_init.sh && \
+	echo "apt-get update && apt-get install -y build-essential gdb libudev-dev libnl-3-dev libnl-route-3-dev tmux " >> /tmp/vm_init.sh && \
+	echo "mkdir -p /host" >> /tmp/vm_init.sh && \
+	#  -- this link below is to make rdma-core happy, since the build system of rdma-core can only build binary that "run inplace"
+	#  -- with this link, the binary in qemu has the save path as it in the devcontainer. so we can build it in devcontainer and 
+	#  -- run it in qemu.
+	echo "ln -sf /host/workspaces /workspaces" >> /tmp/vm_init.sh && \
+	#  -- make run scripts stored in devcontainer more easily.
+	echo "echo \"export PATH=$PATH:/host/workspaces/open-rdma-driver/scripts/for_qemu:/host/workspaces/open-rdma-driver/rdma-core/build/bin\" > /etc/profile.d/set_path.sh" >> /tmp/vm_init.sh && \
+	echo "systemctl enable mount_9p" >> /tmp/vm_init.sh && \
+    # enable auto login
+    echo "echo \"[Service]\" >> /etc/systemd/system/serial-getty@ttyS0.service" >> /tmp/vm_init.sh && \
+    echo "echo \"ExecStart=\" >> /etc/systemd/system/serial-getty@ttyS0.service" >> /tmp/vm_init.sh && \
+    echo "echo \"ExecStart=/sbin/agetty --autologin root -8 --keep-baud 115200,38400,9600 ttyS0 $TERM\" >> /etc/systemd/system/serial-getty@ttyS0.service" >> /tmp/vm_init.sh && \
+	# auto run test script
+    echo "echo \"[Unit]\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \
+    echo "echo \"[Service]\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \
+    echo "echo \"ExecStartPre=/bin/sleep 3\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \
+    echo "echo \"ExecStart=/workspaces/open-rdma-driver/scripts/for_qemu/auto_test.sh\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \
+    echo "echo \"Type=oneshot\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \
+    echo "echo \"[Install]\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \
+    echo "echo \"WantedBy=default.target\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \
+    echo "systemctl enable auto_test.service" >> /tmp/vm_init.sh && \
+    #
+	chroot $ROOTFS_DIR /bin/sh < /tmp/vm_init.sh && \
+	# finish generate and run tmp script.
+	virt-make-fs --label cloudimg-rootfs --format=qcow2 --type=ext4 --size=+1G $ROOTFS_DIR /rootfs.qcow2

Makefile

@@ -1,6 +1,6 @@
 KERNEL_SRC_DIR ?= /linux-src
 
-.PHONY: kernel driver user gdb
+.PHONY: kernel driver user model gdb
 
 kernel:
 	$(MAKE) -C $(KERNEL_SRC_DIR)
@@ -11,6 +11,9 @@ driver:
 user:
 	export CMAKE_EXPORT_COMPILE_COMMANDS=1 && ./rdma-core/build.sh
 
+model:
+	./scripts/build_model.sh
+
 qemu:
 	./scripts/run_qemu.sh
 
@@ -20,4 +23,4 @@ gdb:
 # language server config, e.g., clangd
 lsp:
 	cd $(KERNEL_SRC_DIR) && ./scripts/clang-tools/gen_compile_commands.py
-	cd driver && $(KERNEL_SRC_DIR)/scripts/clang-tools/gen_compile_commands.py -d $(KERNEL_SRC_DIR) ./
+	cd driver && $(KERNEL_SRC_DIR)/scripts/clang-tools/gen_compile_commands.py -d $(KERNEL_SRC_DIR) ./

README.md

@@ -7,22 +7,24 @@ This project recommends to use the devcontainer. The corresponsding devcontainer
 * gdb debug scripts and configs for both vscode based GUI debug and pure cli debug.
 
 
-suppose you will clone this project and it's submodules, now you will have a top folder named `dtld-rdma-driver`, make sure your folder use this name, since when using devcontainer, this folder will be mounted as `/workspaces/dtld-rdma-driver` in the container, and this path is hardcoded in many scripts. So, ***don't change this name***.
+suppose you will clone this project and it's submodules, now you will have a top folder named `open-rdma-driver`, make sure your folder use this name, since when using devcontainer, this folder will be mounted as `/workspaces/open-rdma-driver` in the container, and this path is hardcoded in many scripts. So, ***don't change this name***.
 
 
 #### Dev Container Structure
 after open this git repo in devcontainer, you will see the following folders, before we go on, let's keep one thing in mind:
 
-> our code should only rely on the kernel, but not modify the kernel. So, any change that be made should go into the `/workspaces/dtld-rdma-driver` folder，**ALL THE OTHER FOLDERS WILL BE DROPPED WHEN REBUILDING THE DEV CONTAINER**. for example, you should not save anything in the QEMU VM, since the qemu's disk is not stored in `/workspaces/dtld-rdma-driver`. The dev container and QEMU VM is only useful for testing and debugging, so it may be dropped and recreated very often. All the changes that you want to keep between drop and recreate should go into `/workspaces/dtld-rdma-driver`.
+> our code should only rely on the kernel, but not modify the kernel. So, any change that be made should go into the `/workspaces/open-rdma-driver` folder，**ALL THE OTHER FOLDERS WILL BE DROPPED WHEN REBUILDING THE DEV CONTAINER**. for example, you should not save anything in the QEMU VM, since the qemu's disk is not stored in `/workspaces/open-rdma-driver`. The dev container and QEMU VM is only useful for testing and debugging, so it may be dropped and recreated very often. All the changes that you want to keep between drop and recreate should go into `/workspaces/open-rdma-driver`.
 
 * `/linux-src`: the linux kernel source and prebuild binarys.
 * `/vm`: this is the rootfs disk image for qemu
-* `/workspaces/dtld-rdma-driver/driver`: this is the kernel driver, run in kernel space
+* `/workspaces/open-rdma-driver/driver`: this is the kernel driver, run in kernel space
     * you can run `make driver` to build it
-* `/workspaces/dtld-rdma-driver/rdma-code`: this is a submodule for RDMA user space driver
+* `/workspaces/open-rdma-driver/rdma-code`: this is a submodule for RDMA user space driver
     * you can run `make user` to build the user-space code
-* `/workspaces/dtld-rdma-driver/scripts` this provides many useful scripts.
-    * `/workspaces/dtld-rdma-driver/scripts/for_qemu` this floder will be mounted into the QEMU VM and be added to the `PATH` in the QEMU's guest os, so you can run the scripts under this folder easily in QEMU.
+* `/workspaces/open-rdma-driver/DistroSim`: submodule for co-simulating
+    * you can run `make model` to build the model
+* `/workspaces/open-rdma-driver/scripts` this provides many useful scripts.
+    * `/workspaces/open-rdma-driver/scripts/for_qemu` this floder will be mounted into the QEMU VM and be added to the `PATH` in the QEMU's guest os, so you can run the scripts under this folder easily in QEMU.
 
 
 #### QEMU Guest OS Structure
@@ -34,9 +36,10 @@ in the qemu, there are some pre-mounted folders:
     * we need this trick because the built binary for `rdma-core` must be run *in-place*, i.e., you can't build the binary, copy it to some other path, and then run it. because we build it in devcontainer, and run it in qemu, so we must make the path in qemu looks like where it was built in the dev container. 
 
 * the following path is added to the `PATH` in the qemu's guest OS, so we can run scripts or RDMA user space tools easily.
-    * `/host/workspaces/dtld-rdma-driver/scripts/for_qemu`
-    * `/host/workspaces/dtld-rdma-driver/rdma-core/build/bin`
+    * `/host/workspaces/open-rdma-driver/scripts/for_qemu`
+    * `/host/workspaces/open-rdma-driver/rdma-core/build/bin`
 
+to automatically run tests in `/workspaces/open-rdma-driver/scripts/for_qemu/tests`, execute `/workspaces/open-rdma-driver/scripts/run_qemu.sh` with env `NON_INTERACTIVE` set.
 
 #### How to debug the kernel?
 there is out-of-the-box support for debugging. the gdb config has been set for you, you need to first run `make qemu` and then:

scripts/build_model.sh

@@ -0,0 +1,27 @@
+#!/bin/env bash
+
+set -o errexit
+set -o nounset
+
+GIT_DIR=$(cd $(dirname $0)/..; pwd)
+BUILD_DIR=${GIT_DIR}/.build
+mkdir -p ${BUILD_DIR}
+
+cd ${BUILD_DIR}
+
+if [ ! -d "${BUILD_DIR}/systemc-2.3.3" ]; then
+    wget -q https://www.accellera.org/images/downloads/standards/systemc/systemc-2.3.3.tar.gz && tar xzf systemc-2.3.3.tar.gz
+    cd systemc-2.3.3
+    ./configure --prefix=${BUILD_DIR}/systemc-2.3.3
+    make -j
+    make install
+fi
+
+pushd ${GIT_DIR}/DistroSim
+cat <<EOF > .config.mk
+SYSTEMC=${BUILD_DIR}/systemc-2.3.3
+EOF
+cat .config.mk
+make pcie/versal/xdma-demo
+mv pcie/versal/xdma-demo ${BUILD_DIR}/
+popd

scripts/for_qemu/auto_test.sh

@@ -0,0 +1,20 @@
+#!/bin/env bash
+
+set -o pipefail
+
+if [ -e "/workspaces/open-rdma-driver/.build/non_interactive" ]; then
+    rm -rf /workspaces/open-rdma-driver/.build/non_interactive
+    rm -rf /workspaces/open-rdma-driver/.build/qemu.log
+
+    bash /workspaces/open-rdma-driver/scripts/for_qemu/dtld_insmod.sh
+
+    for file in /workspaces/open-rdma-driver/scripts/for_qemu/tests/*.sh; do
+        if [ -f "$file" ]; then
+            echo "testing $file" >> /workspaces/open-rdma-driver/.build/qemu.log
+            bash "$file" 2>&1 | tee -a /workspaces/open-rdma-driver/.build/qemu.log
+            echo "" >> /workspaces/open-rdma-driver/.build/qemu.log
+        fi
+    done
+
+    poweroff
+fi

scripts/for_qemu/dtld_insmod.sh

@@ -2,6 +2,8 @@
 
 # make -C /host/linux-src INSTALL_MOD_PATH=/ modules_install
 
+GIT_DIR=$(cd $(dirname $0)/../..; pwd)
+
 modprobe ib_core
 modprobe ib_uverbs
-insmod /workspaces/dtld-rdma-driver/driver/dtld_ib.ko
+insmod ${GIT_DIR}/driver/dtld_ib.ko

scripts/for_qemu/tests/device_info.sh

@@ -0,0 +1,13 @@
+#!/bin/env bash
+
+output=$(/workspaces/open-rdma-driver/rdma-core/build/bin/ibv_devinfo)
+
+echo "$output"
+
+if echo "$output" | grep -q "dtld-dev"; then
+    echo "RDMA device dtld-dev found"
+    exit 0
+else
+    echo "RDMA device dtld-dev not found"
+    exit 1
+fi

scripts/for_qemu/tests/list_devices.sh

@@ -0,0 +1,13 @@
+#!/bin/env bash
+
+output=$(/workspaces/open-rdma-driver/rdma-core/build/bin/ibv_devices)
+
+echo "$output"
+
+if echo "$output" | grep -q "dtld-dev"; then
+    echo "RDMA device dtld-dev found"
+    exit 0
+else
+    echo "RDMA device dtld-dev not found"
+    exit 1
+fi

scripts/gdb_init_cmd.txt

@@ -6,4 +6,4 @@ file /linux-src/vmlinux
 # we will pass that argument by cli.
 ## target remote :1234
 
-lx-symbols /linux-src/drivers/infiniband /workspaces/dtld-rdma-driver/driver
+lx-symbols /linux-src/drivers/infiniband /workspaces/open-rdma-driver/driver