resources: Scripts to build disk with gem5 driver

src/ubuntu-generic-diskimages/build-arm.sh → src/ubuntu-generic-disk-images/build-arm.sh

@@ -29,7 +29,21 @@ if [[ "$ubuntu_version" != "22.04" && "$ubuntu_version" != "24.04" ]]; then
     exit 1
 fi
 
-# make the flash0.sh file
+# Create a dictionary to store the path to the kernel modules for each Ubuntu version
+declare -A kernel_modules_paths
+kernel_modules_paths["22.04"]="modules/u2204"
+kernel_modules_paths["24.04"]="modules/u2404/"
+
+# Get the path to the kernel modules for the specified Ubuntu version
+kernel_modules_path=${kernel_modules_paths[$ubuntu_version]}
+pushd $kernel_modules_path
+./copy_modules.sh
+popd
+
+# Store the image name from the second command line argument or default to "arm-ubuntu"
+image_name="${2:-arm-ubuntu}"
+
+# make the flash0.img file
 cd ./files
 dd if=/dev/zero of=flash0.img bs=1M count=64
 dd if=/usr/share/qemu-efi-aarch64/QEMU_EFI.fd of=flash0.img conv=notrunc
@@ -39,4 +53,4 @@ cd ..
 ./packer init ./packer-scripts/arm-ubuntu.pkr.hcl
 
 # Build the image with the specified Ubuntu version
-./packer build -var "ubuntu_version=${ubuntu_version}" ./packer-scripts/arm-ubuntu.pkr.hcl
+./packer build -var "ubuntu_version=${ubuntu_version}" -var "image_name=${image_name}" ./packer-scripts/arm-ubuntu.pkr.hcl

src/ubuntu-generic-diskimages/build-x86.sh → src/ubuntu-generic-disk-images/build-x86.sh

@@ -27,8 +27,11 @@ if [[ "$ubuntu_version" != "22.04" && "$ubuntu_version" != "24.04" ]]; then
     exit 1
 fi
 
+# Store the image name from the second command line argument or default to "x86-ubuntu"
+image_name="${2:-x86-ubuntu}"
+
 # Install the needed plugins
 ./packer init ./packer-scripts/x86-ubuntu.pkr.hcl
 
 # Build the image with the specified Ubuntu version
-./packer build -var "ubuntu_version=${ubuntu_version}" ./packer-scripts/x86-ubuntu.pkr.hcl
+./packer build -var "ubuntu_version=${ubuntu_version}" -var "image_name=${image_name}" ./packer-scripts/x86-ubuntu.pkr.hcl

src/ubuntu-generic-diskimages/files/arm/gem5_init.sh → src/ubuntu-generic-disk-images/files/arm/gem5_init.sh

@@ -15,6 +15,25 @@ mount -t sysfs /sys /sys
 cmdline=$(cat /proc/cmdline)
 no_systemd=false
 
+# Load gem5_bridge driver
+## Default parameters (ARM64)
+gem5_bridge_baseaddr=0x10010000
+gem5_bridge_rangesize=0x10000
+## Try to read overloads from kernel arguments
+if [[ $cmdline =~ gem5_bridge_baseaddr=([[:alnum:]]+) ]]; then
+    gem5_bridge_baseaddr=${BASH_REMATCH[1]}
+fi
+if [[ $cmdline =~ gem5_bridge_rangesize=([[:alnum:]]+) ]]; then
+    gem5_bridge_rangesize=${BASH_REMATCH[1]}
+fi
+## Insert driver
+modprobe gem5_bridge \
+    gem5_bridge_baseaddr=$gem5_bridge_baseaddr \
+    gem5_bridge_rangesize=$gem5_bridge_rangesize
+
+# see if this modprode fails or not
+# print warning if it fails, gem5-bridge module is not going to work, you will need sudo for running exit events
+
 # gem5-bridge exit signifying that kernel is booted
 # This will cause the simulation to exit. Note that this will
 # cause qemu to fail.

src/ubuntu-generic-disk-images/files/serial-getty@.service-override.conf

@@ -0,0 +1,3 @@
+[Service]
+ExecStart=
+ExecStart=-/sbin/agetty --autologin gem5 --keep-baud 115200,38400,9600 %I $TERM

src/ubuntu-generic-diskimages/http/arm-22-04/user-data → src/ubuntu-generic-disk-images/http/arm-2204/user-data

@@ -70,7 +70,7 @@ autoinstall:
       type: format
       id: format-0
     - device: disk-vda
-      size: 4257218560
+      size: 20909654016
       wipe: superblock
       flag: ''
       number: 2
@@ -92,4 +92,4 @@ autoinstall:
       type: mount
       id: mount-0
   updates: security
-  version: 1
+  version: 1

src/ubuntu-generic-diskimages/http/arm-24-04/user-data → src/ubuntu-generic-disk-images/http/arm-2404/user-data

@@ -62,7 +62,7 @@ autoinstall:
       type: format
       id: format-0
     - device: disk-vda
-      size: 4257218560
+      size: 20909654016
       wipe: superblock
       flag: ''
       number: 2
@@ -84,4 +84,4 @@ autoinstall:
       type: mount
       id: mount-0
   updates: security
-  version: 1
+  version: 1

src/ubuntu-generic-disk-images/modules/u2204/Dockerfile

@@ -0,0 +1,46 @@
+FROM ubuntu:22.04
+
+# Install necessary packages for kernel build
+RUN apt update && apt install -y \
+    build-essential \
+    libncurses-dev \
+    bison \
+    flex \
+    libssl-dev \
+    libelf-dev \
+    bc \
+    wget \
+    git \
+    kmod \
+    apt-src \
+    vim \
+    curl \
+    file
+
+RUN sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list
+RUN apt update
+RUN mkdir /workspace
+RUN cd /workspace && apt source linux-image-unsigned-5.15.0-25-generic
+
+RUN cd /workspace/linux-5.15.0 && \
+    cd scripts && \
+    chmod +x pahole-version.sh && \
+    cd .. && \
+    make defconfig && \
+    make -j 32 && \
+    make INSTALL_MOD_PATH=/workspace/output modules_install
+
+RUN git clone https://github.com/nkrim/gem5.git --depth=1 --filter=blob:none --no-checkout --sparse --single-branch --branch=gem5-bridge && \
+    cd gem5 && \
+    git sparse-checkout add util/m5 && \
+    git sparse-checkout add util/gem5_bridge && \
+    git sparse-checkout add include && \
+    git checkout
+
+RUN cd gem5/util/gem5_bridge && \
+    make KMAKEDIR=/workspace/linux-5.15.0 INSTALL_MOD_PATH=/workspace/output build install
+
+RUN cd /workspace/output/lib/modules/5.15.167 && \
+    rm -rf build source
+
+CMD ["/bin/bash"]

src/ubuntu-generic-disk-images/modules/u2204/copy_modules.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Copyright (c) 2025 The Regents of the University of California.
+# SPDX-License-Identifier: BSD 3-Clause
+
+# Variables
+IMAGE_NAME=ms-u2204-kernel-modules-source-image
+CONTAINER_NAME=ms-u2204-kernel-modules-source-container
+
+# Check if the image exists
+if ! docker image inspect "$IMAGE_NAME" > /dev/null 2>&1; then
+    echo "Docker image $IMAGE_NAME not found. Building the image..."
+
+    # Replace with the actual Docker build command
+    docker build -t $IMAGE_NAME .
+
+    if [ $? -ne 0 ]; then
+        echo "Failed to build the Docker image."
+        exit 1
+    fi
+else
+    echo "Docker image $IMAGE_NAME exists."
+fi
+
+# Check if the container already exists and stop/remove it if necessary
+if docker ps -a --format '{{.Names}}' | grep -q "^$CONTAINER_NAME\$"; then
+    echo "Container $CONTAINER_NAME already exists. Removing it..."
+    docker rm -f $CONTAINER_NAME
+fi
+
+
+# Create and start the container
+echo "Creating and starting container $CONTAINER_NAME..."
+docker create --name $CONTAINER_NAME $IMAGE_NAME
+docker start -a $CONTAINER_NAME
+
+# Ensure the target directory exists
+mkdir -p files
+
+# Copy files from the container to the host
+docker cp $CONTAINER_NAME:/workspace/linux-5.15.0/vmlinux files/
+docker cp $CONTAINER_NAME:/workspace/output/lib/modules/5.15.167 files/
+
+# Stop and remove the container after the copy operation
+echo "Stopping and removing the container..."
+docker stop "$CONTAINER_NAME"
+docker rm "$CONTAINER_NAME"

src/ubuntu-generic-disk-images/modules/u2404/Dockerfile

@@ -0,0 +1,42 @@
+# Start from Ubuntu 24.04 base image
+FROM ubuntu:24.04
+
+# Install necessary packages for kernel and module build
+RUN apt update && apt install -y \
+    build-essential \
+    libncurses-dev \
+    bison \
+    flex \
+    libssl-dev \
+    libelf-dev \
+    bc \
+    wget \
+    git \
+    kmod
+
+
+RUN sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources
+RUN apt update
+RUN mkdir /workspace
+RUN cd /workspace && apt source linux-image-unsigned-6.8.0-47-generic
+
+RUN cd /workspace/linux-6.8.0 && \
+    make defconfig && \
+    make -j 32 && \
+    make INSTALL_MOD_PATH=/workspace/output modules_install
+
+RUN git clone https://github.com/nkrim/gem5.git --depth=1 --filter=blob:none --no-checkout --sparse --single-branch --branch=gem5-bridge && \
+    cd gem5 && \
+    git sparse-checkout add util/m5 && \
+    git sparse-checkout add util/gem5_bridge && \
+    git sparse-checkout add include && \
+    git checkout
+
+RUN cd gem5/util/gem5_bridge && \
+    make KMAKEDIR=/workspace/linux-6.8.0 INSTALL_MOD_PATH=/workspace/output build install
+
+RUN cd /workspace/output/lib/modules/6.8.12 && \
+    rm -rf build
+
+# Use bash as the default shell
+CMD ["/bin/bash"]

src/ubuntu-generic-disk-images/modules/u2404/copy_modules.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# Copyright (c) 2025 The Regents of the University of California.
+# SPDX-License-Identifier: BSD 3-Clause
+
+# Variables
+IMAGE_NAME=ms-u2404-kernel-modules-source-image
+CONTAINER_NAME=ms-u2404-kernel-modules-source-container
+
+# Check if the image exists
+if ! docker image inspect "$IMAGE_NAME" > /dev/null 2>&1; then
+    echo "Docker image $IMAGE_NAME not found. Building the image..."
+
+    # Replace with the actual Docker build command
+    docker build -t $IMAGE_NAME .
+
+    if [ $? -ne 0 ]; then
+        echo "Failed to build the Docker image."
+        exit 1
+    fi
+else
+    echo "Docker image $IMAGE_NAME exists."
+fi
+
+# Check if the container already exists and stop/remove it if necessary
+if docker ps -a --format '{{.Names}}' | grep -q "^$CONTAINER_NAME\$"; then
+    echo "Container $CONTAINER_NAME already exists. Removing it..."
+    docker rm -f $CONTAINER_NAME
+fi
+
+
+# Create and start the container
+echo "Creating and starting container $CONTAINER_NAME..."
+docker create --name $CONTAINER_NAME $IMAGE_NAME
+docker start -a $CONTAINER_NAME
+
+# Ensure the target directory exists
+mkdir -p files
+
+# Copy files from the container to the host
+docker cp $CONTAINER_NAME:/workspace/linux-6.8.0/vmlinux files/
+docker cp $CONTAINER_NAME:/workspace/output/lib/modules/6.8.12 files/
+
+# Stop and remove the container after the copy operation
+echo "Stopping and removing the container..."
+docker stop "$CONTAINER_NAME"
+docker rm "$CONTAINER_NAME"

src/ubuntu-generic-diskimages/packer-scripts/arm-ubuntu.pkr.hcl → src/ubuntu-generic-disk-images/packer-scripts/arm-ubuntu.pkr.hcl

@@ -9,7 +9,6 @@ packer {
 
 variable "image_name" {
   type    = string
-  default = "arm-ubuntu"
 }
 
 variable "ssh_password" {
@@ -36,14 +35,18 @@ locals {
     "22.04" = {
       iso_url       = "https://old-releases.ubuntu.com/releases/jammy/ubuntu-22.04-live-server-arm64.iso"
       iso_checksum  = "sha256:c209ab013280d3cd26a344def60b7b19fbb427de904ea285057d94ca6ac82dd5"
-      output_dir    = "arm-disk-image-22-04"
-      http_directory = "http/arm-22-04"
+      output_dir    = "${var.image_name}-2204"
+      http_directory = "http/arm-2204"
+      kernel_version = "5.15.167"
+      modules_dir = "modules/u2204/files/"
     }
     "24.04" = {
-      iso_url       = "https://cdimage.ubuntu.com/releases/24.04/release/ubuntu-24.04-live-server-arm64.iso"
-      iso_checksum  = "sha256:d2d9986ada3864666e36a57634dfc97d17ad921fa44c56eeaca801e7dab08ad7"
-      output_dir    = "arm-disk-image-24-04"
-      http_directory = "http/arm-24-04"
+      iso_url       = "https://cdimage.ubuntu.com/releases/24.04/release/ubuntu-24.04.1-live-server-arm64.iso"
+      iso_checksum  = "sha256:5ceecb7ef5f976e8ab3fffee7871518c8e9927ec221a3bb548ee1193989e1773"
+      output_dir    = "${var.image_name}-2404"
+      http_directory = "http/arm-2404"
+      kernel_version = "6.8.12"
+      modules_dir = "modules/u2404/files/"
     }
   }
 }
@@ -91,22 +94,22 @@ source "qemu" "initialize" {
                       "<enter>",
                       "<wait>"
                       ]
-  cpus             = "4"
-  disk_size        = "4600"
+  cpus             = "32"
+  disk_size        = "21600"
   format           = "raw"
   headless         = "true"
   http_directory   = local.iso_data[var.ubuntu_version].http_directory
   iso_checksum     = local.iso_data[var.ubuntu_version].iso_checksum
   iso_urls         = [local.iso_data[var.ubuntu_version].iso_url]
-  memory           = "8192"
+  memory           = "65536"
   output_directory = local.iso_data[var.ubuntu_version].output_dir
   qemu_binary      = "/usr/bin/qemu-system-aarch64"
   qemuargs         = local.qemuargs
   shutdown_command = "echo '${var.ssh_password}'|sudo -S shutdown -P now"
   ssh_password     = "${var.ssh_password}"
   ssh_username     = "${var.ssh_username}"
   ssh_wait_timeout = "60m"
-  vm_name          = "${var.image_name}"
+  vm_name          = "disk-image"
   ssh_handshake_attempts = "1000"
 }
 
@@ -115,28 +118,39 @@ build {
 
   provisioner "file" {
     destination = "/home/gem5/"
-    source      = "files/exit.sh"
+    source      = "files/arm/gem5_init.sh"
   }
 
   provisioner "file" {
     destination = "/home/gem5/"
-    source      = "files/arm/gem5_init.sh"
+    source      = "files/arm/after_boot.sh"
   }
 
   provisioner "file" {
     destination = "/home/gem5/"
-    source      = "files/arm/after_boot.sh"
+    source      = "files/serial-getty@.service-override.conf"
   }
 
   provisioner "file" {
     destination = "/home/gem5/"
-    source      = "files/serial-getty@.service"
+    source      = "${local.iso_data[var.ubuntu_version].modules_dir}/${local.iso_data[var.ubuntu_version].kernel_version}"
   }
 
   provisioner "shell" {
     execute_command = "echo '${var.ssh_password}' | {{ .Vars }} sudo -E -S bash '{{ .Path }}'"
-    scripts         = ["scripts/post-installation.sh"]
-    environment_vars = ["ISA=arm64"]
+    scripts         = ["scripts/install-commons.sh", "scripts/update-modules.sh", "scripts/install-gem5-bridge-driver.sh", "scripts/install-user-packages.sh"]
+    environment_vars = ["ISA=arm64", "KERNEL_VERSION=${local.iso_data[var.ubuntu_version].kernel_version}", "DEBIAN_FRONTEND=noninteractive"]
     expect_disconnect = true
   }
+
+  provisioner "shell" {
+    scripts = ["scripts/install-benchmarks.sh"]
+  }
+
+  provisioner "shell" {
+    execute_command = "echo '${var.ssh_password}' | {{ .Vars }} sudo -E -S bash '{{ .Path }}'"
+    scripts         = ["scripts/install-gem5-init.sh", "scripts/disable-network.sh"]
+    expect_disconnect = true
+  }
+
 }