From 3343bf294bb0bb6f2976fcd28c50e760fc7495a3 Mon Sep 17 00:00:00 2001 From: EAimTY Date: Sun, 22 Oct 2023 03:05:19 +0000 Subject: [PATCH 1/3] import DistroSim as submodule --- .gitmodules | 9 +++++++-- DistroSim | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) create mode 160000 DistroSim diff --git a/.gitmodules b/.gitmodules index 1bbb113..b1e41e5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,9 @@ [submodule "rdma-core"] path = rdma-core - url = git@github.com:datenlord/rdma-core.git - branch = add-dtld-driver + url = https://github.com/datenlord/rdma-core.git + branch = dtld-main + +[submodule "DistroSim"] + path = DistroSim + url = https://github.com/ruiqurm/DistroSim.git + branch = master diff --git a/DistroSim b/DistroSim new file mode 160000 index 0000000..89e501c --- /dev/null +++ b/DistroSim @@ -0,0 +1 @@ +Subproject commit 89e501c563ba90b48c149259df12f6852b570ef7 From a2a9db57d7d323ad915b8ff51a3fc999ee029f67 Mon Sep 17 00:00:00 2001 From: EAimTY Date: Sun, 22 Oct 2023 10:39:53 +0000 Subject: [PATCH 2/3] connect xdma model into qemu --- .gitignore | 3 ++- Makefile | 7 +++++-- scripts/build_model.sh | 27 +++++++++++++++++++++++++++ scripts/run_qemu.sh | 38 ++++++++++++++++++++++++++------------ 4 files changed, 60 insertions(+), 15 deletions(-) create mode 100755 scripts/build_model.sh diff --git a/.gitignore b/.gitignore index 1691785..a564239 100644 --- a/.gitignore +++ b/.gitignore @@ -5,5 +5,6 @@ *.ko *.mod *.mod.c +.build .cache -compile_commands.json \ No newline at end of file +compile_commands.json diff --git a/Makefile b/Makefile index 82c6d24..8c8479c 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ KERNEL_SRC_DIR ?= /linux-src -.PHONY: kernel driver user gdb +.PHONY: kernel driver user model gdb kernel: $(MAKE) -C $(KERNEL_SRC_DIR) @@ -11,6 +11,9 @@ driver: user: export CMAKE_EXPORT_COMPILE_COMMANDS=1 && ./rdma-core/build.sh +model: + ./scripts/build_model.sh + qemu: ./scripts/run_qemu.sh @@ -20,4 +23,4 @@ gdb: # language server config, e.g., clangd lsp: cd $(KERNEL_SRC_DIR) && ./scripts/clang-tools/gen_compile_commands.py - cd driver && $(KERNEL_SRC_DIR)/scripts/clang-tools/gen_compile_commands.py -d $(KERNEL_SRC_DIR) ./ \ No newline at end of file + cd driver && $(KERNEL_SRC_DIR)/scripts/clang-tools/gen_compile_commands.py -d $(KERNEL_SRC_DIR) ./ diff --git a/scripts/build_model.sh b/scripts/build_model.sh new file mode 100755 index 0000000..180e6fc --- /dev/null +++ b/scripts/build_model.sh @@ -0,0 +1,27 @@ +#!/bin/env bash + +set -o errexit +set -o nounset + +GIT_DIR=$(cd $(dirname $0)/..; pwd) +BUILD_DIR=${GIT_DIR}/.build +mkdir -p ${BUILD_DIR} + +cd ${BUILD_DIR} + +if [ ! -d "${BUILD_DIR}/systemc-2.3.3" ]; then + wget -q https://www.accellera.org/images/downloads/standards/systemc/systemc-2.3.3.tar.gz && tar xzf systemc-2.3.3.tar.gz + cd systemc-2.3.3 + ./configure --prefix=${BUILD_DIR}/systemc-2.3.3 + make -j + make install +fi + +pushd ${GIT_DIR}/DistroSim +cat < .config.mk +SYSTEMC=${BUILD_DIR}/systemc-2.3.3 +EOF +cat .config.mk +make pcie/versal/xdma-demo +mv pcie/versal/xdma-demo ${BUILD_DIR}/ +popd diff --git a/scripts/run_qemu.sh b/scripts/run_qemu.sh index e402629..91ca559 100755 --- a/scripts/run_qemu.sh +++ b/scripts/run_qemu.sh @@ -2,6 +2,13 @@ set -x +GIT_DIR=$(cd $(dirname $0)/..; pwd) +BUILD_DIR=${GIT_DIR}/.build + +PCIE_ROOT_SLOT_NUM="1" +RP_PCIE_SLOT_NUM="0" +RP_CHAN_NUM="0" + :"${BDF:=02:00.0}" :"${KERNEL_IMG:=/linux-src/arch/x86_64/boot/bzImage}" @@ -14,21 +21,28 @@ group_number=$(readlink /sys/bus/pci/devices/0000:$BDF/iommu_group | awk -F'/' ' sh -c "chown $CUR_USER /dev/vfio/$group_number" qemu_args="" -qemu_args="$qemu_args -m 4G -smp 4 -display none -enable-kvm -cpu host " -qemu_args="$qemu_args -kernel $KERNEL_IMG " -qemu_args="$qemu_args -append \"root=/dev/sda rootwait console=tty1 console=ttyS0 intel_iommu=on ip=10.0.2.15::10.0.2.1:255.255.255.0 nokaslr\" " -qemu_args="$qemu_args -drive file=/rootfs.qcow2 " -qemu_args="$qemu_args -fsdev local,security_model=passthrough,id=fsdev0,multidevs=remap,path=/ " -qemu_args="$qemu_args -device virtio-9p-pci,fsdev=fsdev0,mount_tag=hostshare " -qemu_args="$qemu_args -netdev user,id=n1 -device virtio-net-pci,netdev=n1 " -qemu_args="$qemu_args -serial mon:stdio " +qemu_args="$qemu_args -m 4G -smp 4 -display none -enable-kvm -cpu host" +qemu_args="$qemu_args -M q35,accel=kvm,kernel-irqchip=split" +qemu_args="$qemu_args -device intel-iommu,intremap=on,device-iotlb=on" +qemu_args="$qemu_args -kernel $KERNEL_IMG" +qemu_args="$qemu_args -append \"root=/dev/sda rootwait console=tty1 console=ttyS0 intel_iommu=on ip=10.0.2.15::10.0.2.1:255.255.255.0 nokaslr\"" +qemu_args="$qemu_args -drive file=/rootfs.qcow2" +qemu_args="$qemu_args -fsdev local,security_model=passthrough,id=fsdev0,multidevs=remap,path=/" +qemu_args="$qemu_args -device virtio-9p-pci,fsdev=fsdev0,mount_tag=hostshare" +qemu_args="$qemu_args -netdev user,id=n1 -device virtio-net-pci,netdev=n1" +qemu_args="$qemu_args -machine-path ${BUILD_DIR}" +qemu_args="$qemu_args -device ioh3420,id=rootport1,slot=${PCIE_ROOT_SLOT_NUM}" +qemu_args="$qemu_args -device remote-port-pci-adaptor,bus=rootport1,id=rp0" +qemu_args="$qemu_args -device remote-port-pcie-root-port,id=rprootport,slot=${RP_PCIE_SLOT_NUM},rp-adaptor0=rp,rp-chan0=${RP_CHAN_NUM}" +qemu_args="$qemu_args -serial mon:stdio" qemu_args="$qemu_args -s" - - if [ -n "$ATTACH_PCI" ]; then qemu_args="$qemu_args -device vfio-pci,host=$BDF,multifunction=on" -fi +fi +killall -u ${USER} xdma-demo qemu-system-x86_64 &>/dev/null || true -eval qemu-system-x86_64 $qemu_args \ No newline at end of file +LD_LIBRARY_PATH=${BUILD_DIR}/systemc-2.3.3/lib-linux64/ ${BUILD_DIR}/xdma-demo unix:${BUILD_DIR}/qemu-rport-_machine_peripheral_rp0_rp 10000 & disown; +sleep 1 +eval qemu-system-x86_64 $qemu_args From 81d71c262df80bb1383778388cab3575e8dc5763 Mon Sep 17 00:00:00 2001 From: EAimTY Date: Thu, 2 Nov 2023 01:15:00 +0000 Subject: [PATCH 3/3] add testing and docker image building ci --- .github/workflows/build_docker_image.yml | 40 +++++++++ .github/workflows/test_rdma.yml | 32 +++++++ .gitmodules | 1 - Dockerfile | 106 +++++++++++++++++++++++ README.md | 19 ++-- scripts/for_qemu/auto_test.sh | 20 +++++ scripts/for_qemu/dtld_insmod.sh | 4 +- scripts/for_qemu/tests/device_info.sh | 13 +++ scripts/for_qemu/tests/list_devices.sh | 13 +++ scripts/gdb_init_cmd.txt | 2 +- scripts/run_qemu.sh | 21 +++-- 11 files changed, 254 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/build_docker_image.yml create mode 100644 .github/workflows/test_rdma.yml create mode 100644 Dockerfile create mode 100755 scripts/for_qemu/auto_test.sh create mode 100755 scripts/for_qemu/tests/device_info.sh create mode 100755 scripts/for_qemu/tests/list_devices.sh diff --git a/.github/workflows/build_docker_image.yml b/.github/workflows/build_docker_image.yml new file mode 100644 index 0000000..003c39d --- /dev/null +++ b/.github/workflows/build_docker_image.yml @@ -0,0 +1,40 @@ +name: Create and publish a Docker image + +on: + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: True + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/test_rdma.yml b/.github/workflows/test_rdma.yml new file mode 100644 index 0000000..95c4666 --- /dev/null +++ b/.github/workflows/test_rdma.yml @@ -0,0 +1,32 @@ +name: Test RDMA + +on: + workflow_dispatch: + +jobs: + test-rdma: + runs-on: ubuntu-latest + container: + image: ghcr.io/datenlord/open-rdma-driver + volumes: + - my_docker_volume:/volume_mount + options: --privileged --user root + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: addnab/docker-run-action@v3 + with: + image: ghcr.io/datenlord/open-rdma-driver + options: --privileged -v ${{ github.workspace }}:/workspaces/open-rdma-driver + run: | + cd /workspaces/open-rdma-driver + make model + make driver + make user + NON_INTERACTIVE=1 ./scripts/run_qemu.sh + - uses: actions/upload-artifact@v3 + with: + name: test-results + path: | + .build/qemu.log + .build/model.log diff --git a/.gitmodules b/.gitmodules index b1e41e5..f67988e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,7 +2,6 @@ path = rdma-core url = https://github.com/datenlord/rdma-core.git branch = dtld-main - [submodule "DistroSim"] path = DistroSim url = https://github.com/ruiqurm/DistroSim.git diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..44e32bf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,106 @@ +FROM ubuntu:22.04 + +ARG KERNEL_SOURCE_GIT="git://git.launchpad.net/~ubuntu-kernel/ubuntu/+source/linux/+git/jammy" +ARG KERNEL_SOURCE_BRANCH="Ubuntu-hwe-5.19-5.19.0-50.50" +ARG UBUNTU_RELEASE_NAME="jammy" +ARG UBUNTU_ROOTFS_URL="https://cloud-images.ubuntu.com/releases/jammy/release/ubuntu-22.04-server-cloudimg-amd64-root.tar.xz" +ARG ROOTFS_TARBALL_PATH="/vm/origin_disk.img" +ARG ROOTFS_DIR="/rootfs" +ARG LINUX_SRC_DIR="/linux-src" +ARG VM_DIR="/vm" +ARG XILINX_QEMU_GIT="https://github.com/Xilinx/qemu.git" +ARG XILINX_QEMU_GIT_BRANCH="master" + +# linux-image-kvm is installed to make libguestfs-tools happy, because it need a kernel and an simple rootfs to run VM +RUN apt-get update && \ + apt-get install -y wget vim git tmux\ + # the following line is for qemu build + zlib1g-dev libglib2.0-dev libpixman-1-dev libfdt-dev libcap-ng-dev libattr1-dev \ + # the following lines is for linux kernel build + build-essential cmake gcc libudev-dev libnl-3-dev libnl-route-3-dev \ + ninja-build pkg-config valgrind python3-dev cython3 python3-docutils pandoc \ + bc fakeroot libncurses5-dev libssl-dev ccache bison flex libelf-dev dwarves \ + rsync libguestfs-tools linux-image-kvm && \ + rm -rf /var/lib/apt/lists/* + +RUN git clone $XILINX_QEMU_GIT --depth=1 --branch $XILINX_QEMU_GIT_BRANCH /qemu && \ + cd /qemu && \ + mkdir qemu_build && \ + cd qemu_build && \ + ../configure --target-list=x86_64-softmmu --enable-virtfs && \ + make -j && \ + make install && \ + rm -rf /qemu + +RUN mkdir -p $VM_DIR && \ + wget $UBUNTU_ROOTFS_URL -O $ROOTFS_TARBALL_PATH && \ + git clone $KERNEL_SOURCE_GIT --depth=1 --branch $KERNEL_SOURCE_BRANCH ./linux-src + + +RUN cd $LINUX_SRC_DIR && \ + echo "" > config_patch.config && \ + echo "CONFIG_INFINIBAND=m" >> config_patch.config && \ + echo "CONFIG_INFINIBAND_USER_MAD=m" >> config_patch.config && \ + echo "CONFIG_INFINIBAND_USER_ACCESS=m" >> config_patch.config && \ + echo "CONFIG_RDMA_RXE=m" >> config_patch.config && \ + echo "# CONFIG_WERROR is not set" >> config_patch.config && \ + echo "# CONFIG_RANDOMIZE_BASE is not set" >> config_patch.config && \ + echo "CONFIG_DEBUG_INFO_DWARF5=y" >> config_patch.config && \ + echo "CONFIG_GDB_SCRIPTS=y" >> config_patch.config && \ + echo "# CONFIG_DEBUG_INFO_REDUCED is not set" >> config_patch.config && \ + make defconfig && \ + ./scripts/kconfig/merge_config.sh .config ./config_patch.config && \ + make -j20 && \ + make scripts_gdb + +RUN mkdir -p $ROOTFS_DIR && \ + tar -Jxf $ROOTFS_TARBALL_PATH -C $ROOTFS_DIR && \ + cd $LINUX_SRC_DIR && \ + make INSTALL_MOD_PATH=$ROOTFS_DIR modules_install && \ + echo "root:root" | chpasswd -R $ROOTFS_DIR && \ + # generate a systemd service config file to mount 9p fs at booting. adding it to fstab seems not working. + # after mounting,we also run a script mounted from host to do some init work. + echo "[Unit]" > $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \ + echo "Description=Mount 9p to access host fs" >> $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \ + echo "After=network.target" >> $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \ + echo "[Service]" >> $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \ + echo "ExecStart=bash -c \"mount -t 9p -o trans=virtio,version=9p2000.L,access=any hostshare /host && /host/workspaces/open-rdma-driver/scripts/for_qemu/boot_init.sh\"" >> $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \ + echo "[Install]" >> $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \ + echo "WantedBy=default.target" >> $ROOTFS_DIR/etc/systemd/system/mount_9p.service && \ + # == finish generating systemd config + # + # After switch to Xilinx Qemu, it seems that the network service has some problems. It will block boot for a long time, so we need to fix it + # The following code should be removed when the network problem has been solved. + sed -i '/ExecStart=/s/$/ --timeout 1/' $ROOTFS_DIR/etc/systemd/system/network-online.target.wants/systemd-networkd-wait-online.service && \ + # + # generate a tmp script to run in chroot environment to modify the rootfs for qemu + echo "ssh-keygen -A " >> /tmp/vm_init.sh && \ + echo "apt-get purge --auto-remove -y snapd multipath-tools" >> /tmp/vm_init.sh && \ + echo 'mkdir -p /run/systemd/resolve/' >> /tmp/vm_init.sh && \ + echo 'echo "nameserver 8.8.8.8" > /run/systemd/resolve/stub-resolv.conf' >> /tmp/vm_init.sh && \ + echo "apt-get update && apt-get install -y build-essential gdb libudev-dev libnl-3-dev libnl-route-3-dev tmux " >> /tmp/vm_init.sh && \ + echo "mkdir -p /host" >> /tmp/vm_init.sh && \ + # -- this link below is to make rdma-core happy, since the build system of rdma-core can only build binary that "run inplace" + # -- with this link, the binary in qemu has the save path as it in the devcontainer. so we can build it in devcontainer and + # -- run it in qemu. + echo "ln -sf /host/workspaces /workspaces" >> /tmp/vm_init.sh && \ + # -- make run scripts stored in devcontainer more easily. + echo "echo \"export PATH=$PATH:/host/workspaces/open-rdma-driver/scripts/for_qemu:/host/workspaces/open-rdma-driver/rdma-core/build/bin\" > /etc/profile.d/set_path.sh" >> /tmp/vm_init.sh && \ + echo "systemctl enable mount_9p" >> /tmp/vm_init.sh && \ + # enable auto login + echo "echo \"[Service]\" >> /etc/systemd/system/serial-getty@ttyS0.service" >> /tmp/vm_init.sh && \ + echo "echo \"ExecStart=\" >> /etc/systemd/system/serial-getty@ttyS0.service" >> /tmp/vm_init.sh && \ + echo "echo \"ExecStart=/sbin/agetty --autologin root -8 --keep-baud 115200,38400,9600 ttyS0 $TERM\" >> /etc/systemd/system/serial-getty@ttyS0.service" >> /tmp/vm_init.sh && \ + # auto run test script + echo "echo \"[Unit]\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \ + echo "echo \"[Service]\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \ + echo "echo \"ExecStartPre=/bin/sleep 3\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \ + echo "echo \"ExecStart=/workspaces/open-rdma-driver/scripts/for_qemu/auto_test.sh\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \ + echo "echo \"Type=oneshot\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \ + echo "echo \"[Install]\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \ + echo "echo \"WantedBy=default.target\" >> /etc/systemd/system/auto_test.service" >> /tmp/vm_init.sh && \ + echo "systemctl enable auto_test.service" >> /tmp/vm_init.sh && \ + # + chroot $ROOTFS_DIR /bin/sh < /tmp/vm_init.sh && \ + # finish generate and run tmp script. + virt-make-fs --label cloudimg-rootfs --format=qcow2 --type=ext4 --size=+1G $ROOTFS_DIR /rootfs.qcow2 diff --git a/README.md b/README.md index 7869416..57b2b06 100644 --- a/README.md +++ b/README.md @@ -7,22 +7,24 @@ This project recommends to use the devcontainer. The corresponsding devcontainer * gdb debug scripts and configs for both vscode based GUI debug and pure cli debug. -suppose you will clone this project and it's submodules, now you will have a top folder named `dtld-rdma-driver`, make sure your folder use this name, since when using devcontainer, this folder will be mounted as `/workspaces/dtld-rdma-driver` in the container, and this path is hardcoded in many scripts. So, ***don't change this name***. +suppose you will clone this project and it's submodules, now you will have a top folder named `open-rdma-driver`, make sure your folder use this name, since when using devcontainer, this folder will be mounted as `/workspaces/open-rdma-driver` in the container, and this path is hardcoded in many scripts. So, ***don't change this name***. #### Dev Container Structure after open this git repo in devcontainer, you will see the following folders, before we go on, let's keep one thing in mind: -> our code should only rely on the kernel, but not modify the kernel. So, any change that be made should go into the `/workspaces/dtld-rdma-driver` folder，**ALL THE OTHER FOLDERS WILL BE DROPPED WHEN REBUILDING THE DEV CONTAINER**. for example, you should not save anything in the QEMU VM, since the qemu's disk is not stored in `/workspaces/dtld-rdma-driver`. The dev container and QEMU VM is only useful for testing and debugging, so it may be dropped and recreated very often. All the changes that you want to keep between drop and recreate should go into `/workspaces/dtld-rdma-driver`. +> our code should only rely on the kernel, but not modify the kernel. So, any change that be made should go into the `/workspaces/open-rdma-driver` folder，**ALL THE OTHER FOLDERS WILL BE DROPPED WHEN REBUILDING THE DEV CONTAINER**. for example, you should not save anything in the QEMU VM, since the qemu's disk is not stored in `/workspaces/open-rdma-driver`. The dev container and QEMU VM is only useful for testing and debugging, so it may be dropped and recreated very often. All the changes that you want to keep between drop and recreate should go into `/workspaces/open-rdma-driver`. * `/linux-src`: the linux kernel source and prebuild binarys. * `/vm`: this is the rootfs disk image for qemu -* `/workspaces/dtld-rdma-driver/driver`: this is the kernel driver, run in kernel space +* `/workspaces/open-rdma-driver/driver`: this is the kernel driver, run in kernel space * you can run `make driver` to build it -* `/workspaces/dtld-rdma-driver/rdma-code`: this is a submodule for RDMA user space driver +* `/workspaces/open-rdma-driver/rdma-code`: this is a submodule for RDMA user space driver * you can run `make user` to build the user-space code -* `/workspaces/dtld-rdma-driver/scripts` this provides many useful scripts. - * `/workspaces/dtld-rdma-driver/scripts/for_qemu` this floder will be mounted into the QEMU VM and be added to the `PATH` in the QEMU's guest os, so you can run the scripts under this folder easily in QEMU. +* `/workspaces/open-rdma-driver/DistroSim`: submodule for co-simulating + * you can run `make model` to build the model +* `/workspaces/open-rdma-driver/scripts` this provides many useful scripts. + * `/workspaces/open-rdma-driver/scripts/for_qemu` this floder will be mounted into the QEMU VM and be added to the `PATH` in the QEMU's guest os, so you can run the scripts under this folder easily in QEMU. #### QEMU Guest OS Structure @@ -34,9 +36,10 @@ in the qemu, there are some pre-mounted folders: * we need this trick because the built binary for `rdma-core` must be run *in-place*, i.e., you can't build the binary, copy it to some other path, and then run it. because we build it in devcontainer, and run it in qemu, so we must make the path in qemu looks like where it was built in the dev container. * the following path is added to the `PATH` in the qemu's guest OS, so we can run scripts or RDMA user space tools easily. - * `/host/workspaces/dtld-rdma-driver/scripts/for_qemu` - * `/host/workspaces/dtld-rdma-driver/rdma-core/build/bin` + * `/host/workspaces/open-rdma-driver/scripts/for_qemu` + * `/host/workspaces/open-rdma-driver/rdma-core/build/bin` +to automatically run tests in `/workspaces/open-rdma-driver/scripts/for_qemu/tests`, execute `/workspaces/open-rdma-driver/scripts/run_qemu.sh` with env `NON_INTERACTIVE` set. #### How to debug the kernel? there is out-of-the-box support for debugging. the gdb config has been set for you, you need to first run `make qemu` and then: diff --git a/scripts/for_qemu/auto_test.sh b/scripts/for_qemu/auto_test.sh new file mode 100755 index 0000000..8ab7bb8 --- /dev/null +++ b/scripts/for_qemu/auto_test.sh @@ -0,0 +1,20 @@ +#!/bin/env bash + +set -o pipefail + +if [ -e "/workspaces/open-rdma-driver/.build/non_interactive" ]; then + rm -rf /workspaces/open-rdma-driver/.build/non_interactive + rm -rf /workspaces/open-rdma-driver/.build/qemu.log + + bash /workspaces/open-rdma-driver/scripts/for_qemu/dtld_insmod.sh + + for file in /workspaces/open-rdma-driver/scripts/for_qemu/tests/*.sh; do + if [ -f "$file" ]; then + echo "testing $file" >> /workspaces/open-rdma-driver/.build/qemu.log + bash "$file" 2>&1 | tee -a /workspaces/open-rdma-driver/.build/qemu.log + echo "" >> /workspaces/open-rdma-driver/.build/qemu.log + fi + done + + poweroff +fi diff --git a/scripts/for_qemu/dtld_insmod.sh b/scripts/for_qemu/dtld_insmod.sh index 27b947b..652b329 100755 --- a/scripts/for_qemu/dtld_insmod.sh +++ b/scripts/for_qemu/dtld_insmod.sh @@ -2,6 +2,8 @@ # make -C /host/linux-src INSTALL_MOD_PATH=/ modules_install +GIT_DIR=$(cd $(dirname $0)/../..; pwd) + modprobe ib_core modprobe ib_uverbs -insmod /workspaces/dtld-rdma-driver/driver/dtld_ib.ko \ No newline at end of file +insmod ${GIT_DIR}/driver/dtld_ib.ko diff --git a/scripts/for_qemu/tests/device_info.sh b/scripts/for_qemu/tests/device_info.sh new file mode 100755 index 0000000..158e7a6 --- /dev/null +++ b/scripts/for_qemu/tests/device_info.sh @@ -0,0 +1,13 @@ +#!/bin/env bash + +output=$(/workspaces/open-rdma-driver/rdma-core/build/bin/ibv_devinfo) + +echo "$output" + +if echo "$output" | grep -q "dtld-dev"; then + echo "RDMA device dtld-dev found" + exit 0 +else + echo "RDMA device dtld-dev not found" + exit 1 +fi diff --git a/scripts/for_qemu/tests/list_devices.sh b/scripts/for_qemu/tests/list_devices.sh new file mode 100755 index 0000000..4d7f0a4 --- /dev/null +++ b/scripts/for_qemu/tests/list_devices.sh @@ -0,0 +1,13 @@ +#!/bin/env bash + +output=$(/workspaces/open-rdma-driver/rdma-core/build/bin/ibv_devices) + +echo "$output" + +if echo "$output" | grep -q "dtld-dev"; then + echo "RDMA device dtld-dev found" + exit 0 +else + echo "RDMA device dtld-dev not found" + exit 1 +fi diff --git a/scripts/gdb_init_cmd.txt b/scripts/gdb_init_cmd.txt index a826d4a..b4b4714 100644 --- a/scripts/gdb_init_cmd.txt +++ b/scripts/gdb_init_cmd.txt @@ -6,4 +6,4 @@ file /linux-src/vmlinux # we will pass that argument by cli. ## target remote :1234 -lx-symbols /linux-src/drivers/infiniband /workspaces/dtld-rdma-driver/driver +lx-symbols /linux-src/drivers/infiniband /workspaces/open-rdma-driver/driver diff --git a/scripts/run_qemu.sh b/scripts/run_qemu.sh index 91ca559..908a365 100755 --- a/scripts/run_qemu.sh +++ b/scripts/run_qemu.sh @@ -4,6 +4,7 @@ set -x GIT_DIR=$(cd $(dirname $0)/..; pwd) BUILD_DIR=${GIT_DIR}/.build +mkdir -p ${BUILD_DIR} PCIE_ROOT_SLOT_NUM="1" RP_PCIE_SLOT_NUM="0" @@ -30,19 +31,27 @@ qemu_args="$qemu_args -drive file=/rootfs.qcow2" qemu_args="$qemu_args -fsdev local,security_model=passthrough,id=fsdev0,multidevs=remap,path=/" qemu_args="$qemu_args -device virtio-9p-pci,fsdev=fsdev0,mount_tag=hostshare" qemu_args="$qemu_args -netdev user,id=n1 -device virtio-net-pci,netdev=n1" -qemu_args="$qemu_args -machine-path ${BUILD_DIR}" -qemu_args="$qemu_args -device ioh3420,id=rootport1,slot=${PCIE_ROOT_SLOT_NUM}" -qemu_args="$qemu_args -device remote-port-pci-adaptor,bus=rootport1,id=rp0" -qemu_args="$qemu_args -device remote-port-pcie-root-port,id=rprootport,slot=${RP_PCIE_SLOT_NUM},rp-adaptor0=rp,rp-chan0=${RP_CHAN_NUM}" qemu_args="$qemu_args -serial mon:stdio" qemu_args="$qemu_args -s" if [ -n "$ATTACH_PCI" ]; then qemu_args="$qemu_args -device vfio-pci,host=$BDF,multifunction=on" +else + qemu_args="$qemu_args -machine-path ${BUILD_DIR}" + qemu_args="$qemu_args -device ioh3420,id=rootport1,slot=${PCIE_ROOT_SLOT_NUM}" + qemu_args="$qemu_args -device remote-port-pci-adaptor,bus=rootport1,id=rp0" + qemu_args="$qemu_args -device remote-port-pcie-root-port,id=rprootport,slot=${RP_PCIE_SLOT_NUM},rp-adaptor0=rp,rp-chan0=${RP_CHAN_NUM}" +fi + +if [ -n "$NON_INTERACTIVE" ]; then + touch ${BUILD_DIR}/non_interactive fi killall -u ${USER} xdma-demo qemu-system-x86_64 &>/dev/null || true -LD_LIBRARY_PATH=${BUILD_DIR}/systemc-2.3.3/lib-linux64/ ${BUILD_DIR}/xdma-demo unix:${BUILD_DIR}/qemu-rport-_machine_peripheral_rp0_rp 10000 & disown; -sleep 1 +if [ -z "$ATTACH_PCI" ]; then + LD_LIBRARY_PATH=${BUILD_DIR}/systemc-2.3.3/lib-linux64/ ${BUILD_DIR}/xdma-demo unix:${BUILD_DIR}/qemu-rport-_machine_peripheral_rp0_rp 10000 > ${BUILD_DIR}/model.log 2>&1 & disown; + sleep 1 +fi + eval qemu-system-x86_64 $qemu_args