reproduce.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np, pandas as pd\n",
    "import matplotlib.pyplot as plt, colorsys\n",
    "from matplotlib import rcParams as rc\n",
    "import os, subprocess, platform\n",
    "\n",
    "rc[\"legend.markerscale\"] = 2.0\n",
    "rc[\"legend.framealpha\"] = 0\n",
    "rc[\"legend.labelspacing\"] = 0.1\n",
    "rc['figure.figsize'] = (20,10)\n",
    "rc['axes.autolimit_mode'] = 'data'\n",
    "rc['axes.xmargin'] = 0\n",
    "rc['axes.ymargin'] = 0.10\n",
    "rc['axes.titlesize'] = 30\n",
    "rc['axes.labelsize'] = 24\n",
    "rc['font.sans-serif'] = \"Times New Roman\"\n",
    "rc['font.serif'] = \"Times New Roman\"\n",
    "rc['xtick.labelsize'] = 20\n",
    "rc['ytick.labelsize'] = 20\n",
    "rc['axes.grid'] = True\n",
    "rc['grid.linestyle'] = '-'\n",
    "rc['grid.alpha'] = 0.2\n",
    "rc['legend.fontsize'] = 20\n",
    "rc['legend.loc'] = 'upper left'\n",
    "rc[\"figure.autolayout\"] = True\n",
    "rc[\"savefig.dpi\"] = 300\n",
    "rc[\"text.usetex\"] = True\n",
    "rc[\"font.family\"] = \"Times New Roman\"\n",
    "rc.update({\n",
    "  \"text.usetex\": True,\n",
    "  \"font.family\": \"Times New Roman\"\n",
    "})\n",
    "\n",
    "Ngpu_runs = 10 #Set to 100 for more accurate results, much smaller standard deviation.\n",
    "Ncpu_runs = 10 #Set to 100 for more accurate results, much smaller standard deviation.\n",
    "Ncpu_warmup = 5 #Warmup caches and branch predictor.\n",
    "Ngpu_warmup = 1 #No branch prediction on GPU, but if prefetching is disabled, the first run will fetch data.\n",
    "#Change this number if the simulation is taking too long.\n",
    "#Setting this number to -1 will reduce the batch sizes by 1 power of 2\n",
    "OFFSET_BS = 0\n",
    "\n",
    "hostname = platform.node()\n",
    "\n",
    "# Benchmark result filenames\n",
    "path = f'/{os.getcwd()}/output/{hostname}/'\n",
    "buildpath = f'/{os.getcwd()}/build/'\n",
    "fname_base = f'{path}base.csv'\n",
    "fname_one_gpu_v0 = f'{path}one_gpu_v0.csv'\n",
    "fname_one_gpu_v1 = f'{path}one_gpu_v1.csv'\n",
    "fname_multi_gpu_v0 = f'{path}multi_gpu_v0.csv'\n",
    "fname_multi_gpu_v1 = f'{path}multi_gpu_v1.csv'\n",
    "fname_multi_gpu_weak = f'{path}multi_gpu_weak.csv'\n",
    "fname_single_gpu_bs = f'{path}single_gpu_bs.csv'\n",
    "fname_single_gpu_bs = f'{path}single_gpu_bs.csv'\n",
    "fname_base_pipeline = f'{path}base_pipeline.csv'\n",
    "fname_full_pipeline = f'{path}full_pipeline.csv'\n",
    "\n",
    "\n",
    "# Run the command and capture its output\n",
    "if os.system('which nvidia-smi') == 0:\n",
    "    output = subprocess.check_output(['nvidia-smi', '-L'])\n",
    "\n",
    "    # Convert the byte string to a regular string\n",
    "    output_str = output.decode('utf-8')\n",
    "\n",
    "    # Count the number of lines in the output\n",
    "    num_gpus = len(output_str.strip().split('\\n'))\n",
    "else:\n",
    "    num_gpus = 0\n",
    "\n",
    "\n",
    "# Print the number of GPUs found\n",
    "print(f'Found {num_gpus} GPUs on {hostname}')\n",
    "\n",
    "if not os.path.exists(\"output\"):\n",
    "    os.mkdir(\"output\")\n",
    "\n",
    "if not os.path.exists(\"output/\" + hostname):\n",
    "    os.mkdir(\"output/\" + hostname)\n",
    "\n",
    "if not os.path.exists(\"output/\" + hostname + \"/figures\"):\n",
    "    os.mkdir(\"output/\" + hostname + \"/figures\")\n",
    "\n",
    "colors = [\"#1f77b4\", \"#d62728\", \"#9467bd\", \"#8c564b\", \"#e377c2\"]\n",
    "CD = { \"Baseline\" : colors[0], \"GPU_V0\" : colors[1], \"GPU_V1\" : colors[2], \"2 GPU_V0\" :  colors[3] ,\"2 GPU_V1\" : colors[4]}\n",
    "\n",
    "def adjust_brightness(color, amount):\n",
    "    \"\"\"Adjust the brightness of a color by a given amount (-1 to 1).\"\"\"\n",
    "    # Convert the color to the RGB color space\n",
    "    r, g, b = tuple(int(color[i:i+2], 16) for i in (1, 3, 5))\n",
    "    # Convert the color to the HLS color space\n",
    "    h, l, s = colorsys.rgb_to_hls(r/255, g/255, b/255)\n",
    "    # Modify the lightness value\n",
    "    l = max(0, min(1, l + amount))\n",
    "    # Convert the color back to RGB and return it\n",
    "    r, g, b = tuple(round(c * 255) for c in colorsys.hls_to_rgb(h, l, s))\n",
    "    return f\"#{r:02x}{g:02x}{b:02x}\"\n",
    "# Modify the brightness of the colors\n",
    "colors = [adjust_brightness(color, 0.2) for color in colors]\n",
    "\n",
    "CD = { \"Baseline\" : 'r', \"GPU_V0\" : colors[2], \"GPU_V1\" : colors[3], \"2 GPU_V0\" :  colors[4] ,\"2 GPU_V1\" : colors[0], \"Dual\" : f'#7570b3', \"Generate\" : f'#d95f02', \"Projection\" : f'#e7298a', \"Tutte\" : f'#66a61e', \"Opt\" : f'#8931EF' }\n",
    "\n",
    "KName0 = r\"SYCL Kernel 0\"\n",
    "KName1 = r\"SYCL Kernel 1\"\n",
    "\n",
    "def source_and_get_environment(script_path, base_environment=None):\n",
    "    \"\"\"Source script and return the updated environment.\"\"\"\n",
    "    if base_environment is None:\n",
    "        base_environment = os.environ.copy()\n",
    "    command = ['/bin/bash', '-c', f'source {script_path} && env']\n",
    "    proc = subprocess.Popen(command, stdout=subprocess.PIPE, env=base_environment)\n",
    "    output, _ = proc.communicate()\n",
    "    env = dict((line.split(\"=\", 1) for line in output.decode().splitlines() if \"=\" in line))\n",
    "    return env\n",
    "\n",
    "env = source_and_get_environment('/opt/intel/oneapi/setvars.sh')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create files to store results, if they already exist, empty them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reset_file(filename):\n",
    "    # Check if the file already exists\n",
    "    if os.path.isfile(filename):\n",
    "        # If it does, empty it by opening it in write mode with the 'truncate' option\n",
    "        with open(filename, 'w', newline='') as f:\n",
    "            f.truncate()\n",
    "    else:\n",
    "        # If it doesn't, create an empty file by opening it in write mode\n",
    "        with open(filename, 'w', newline='') as f:\n",
    "            pass\n",
    "\n",
    "reset_file(fname_base)\n",
    "reset_file(fname_one_gpu_v0)\n",
    "reset_file(fname_one_gpu_v1)\n",
    "reset_file(fname_multi_gpu_v0)\n",
    "reset_file(fname_multi_gpu_v1)\n",
    "reset_file(fname_multi_gpu_weak)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Validate SYCL kernels against baseline dualisation\n",
    "\n",
    "process = subprocess.Popen(['/bin/bash', '-c', f'{buildpath}validation/sycl/sycl_validation gpu'], env=env).wait()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the batch size experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_file(fname_single_gpu_bs)\n",
    "\n",
    "\n",
    "for i in range(0,21):\n",
    "    if(num_gpus>0):\n",
    "        proc = subprocess.Popen(['/bin/bash', '-c', f'{buildpath}benchmarks/sycl/dualisation gpu {200} {2**(i)} {Ngpu_runs} {Ngpu_warmup} 1 1 {fname_single_gpu_bs}'], env=env); proc.wait()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot the batch size experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_single_gpu_bs = pd.read_csv(fname_single_gpu_bs)\n",
    "import matplotlib.ticker as ticker\n",
    "from matplotlib.ticker import MaxNLocator\n",
    "from mpl_toolkits.axes_grid1.inset_locator import (inset_axes, mark_inset)\n",
    "fig,ax = plt.subplots(figsize=(15,10))\n",
    "def add_line(ax, BS, T, SD, label, color, marker, linestyle):\n",
    "    ax.plot(BS, T, marker=marker, color=color, label=label, linestyle=linestyle)\n",
    "    ax.fill_between(BS, T - SD, T + SD, alpha=0.1, color='k')\n",
    " \n",
    "ax.set_yscale('log')\n",
    "ax.set_xscale('log')\n",
    "ax.set_ylabel(\"Time / Graph [ns]\")\n",
    "ax.set_xlabel(\"Batch Size [1]\")\n",
    "if(num_gpus>0):\n",
    "    add_line(ax, df_single_gpu_bs[\"BS\"].to_numpy(), df_single_gpu_bs[\"T\"].to_numpy(), df_single_gpu_bs[\"TSD\"].to_numpy(), \"Lockstep Parallel Dualization\", CD[\"GPU_V1\"], 'o', ':')\n",
    "ax.legend(loc='best')\n",
    "\n",
    "#Set xticks to powers of 2\n",
    "ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n",
    "ax.xaxis.set_major_formatter(ticker.ScalarFormatter())\n",
    "ax.xaxis.set_minor_formatter(ticker.NullFormatter())\n",
    "ax.xaxis.set_minor_locator(ticker.NullLocator())\n",
    "ax.xaxis.set_ticks([2**i for i in range(0,21)])\n",
    "#Labels should be 2^i\n",
    "ax.xaxis.set_ticklabels([f'$2^{{{i}}}$' for i in range(0,21)])\n",
    "ax.grid(True, which=\"both\", ls=\"--\", alpha=0.2)\n",
    "#remove the ticks from the bottom edge\n",
    "ax.tick_params(axis='x', which='both', bottom=False)\n",
    "\n",
    "#Create inset axes zooming in on the lower right corner\n",
    "axins = ax.inset_axes([0.5, 0.5, 0.4, 0.3])\n",
    "axins.set_yscale('log')\n",
    "axins.set_xscale('log')\n",
    "axins.set_ylabel(\"Time / Graph [ns]\")\n",
    "axins.set_xlabel(\"Batch Size [1]\")\n",
    "#Add line for the last 5 data points\n",
    "if(num_gpus>0):\n",
    "    add_line(axins, df_single_gpu_bs[\"BS\"].to_numpy()[-7:], df_single_gpu_bs[\"T\"].to_numpy()[-7:], df_single_gpu_bs[\"TSD\"].to_numpy()[-7:], \"Lockstep Parallel Dualization\", CD[\"GPU_V1\"], 'o', ':')\n",
    "axins.xaxis.set_major_locator(MaxNLocator(integer=True))\n",
    "axins.xaxis.set_major_formatter(ticker.ScalarFormatter())\n",
    "axins.xaxis.set_minor_formatter(ticker.NullFormatter())\n",
    "axins.xaxis.set_minor_locator(ticker.NullLocator())\n",
    "axins.yaxis.set_major_locator(MaxNLocator(integer=True))\n",
    "axins.yaxis.set_major_formatter(ticker.ScalarFormatter())\n",
    "axins.yaxis.set_minor_formatter(ticker.NullFormatter())\n",
    "axins.xaxis.set_ticks([2**i for i in range(14,21)])\n",
    "#Labels should be 2^i\n",
    "axins.xaxis.set_ticklabels([f'$2^{{{i}}}$' for i in range(14,21)])\n",
    "axins.grid(True, which=\"both\", ls=\"--\", alpha=0.2)\n",
    "#remove the ticks from the bottom edge\n",
    "axins.tick_params(axis='x', which='both', bottom=False)\n",
    "#Mark the zoomed area\n",
    "mark_inset(ax, axins, loc1=3, loc2=1, fc=\"none\", ec=\"0.5\")\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "plt.savefig(f'{path}figures/batch_size_benchmark.pdf', bbox_inches='tight')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Run the benchmarks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reset_file(fname_base)\n",
    "reset_file(fname_one_gpu_v0)\n",
    "reset_file(fname_one_gpu_v1)\n",
    "reset_file(fname_multi_gpu_v0)\n",
    "reset_file(fname_multi_gpu_v1)\n",
    "reset_file(fname_multi_gpu_weak)\n",
    "\n",
    "import subprocess\n",
    "import os\n",
    "\n",
    "def source_and_get_environment(script_path, base_environment=None):\n",
    "    \"\"\"Source script and return the updated environment.\"\"\"\n",
    "    if base_environment is None:\n",
    "        base_environment = os.environ.copy()\n",
    "    command = ['/bin/bash', '-c', f'source {script_path} && env']\n",
    "    proc = subprocess.Popen(command, stdout=subprocess.PIPE, env=base_environment)\n",
    "    output, _ = proc.communicate()\n",
    "    env = dict((line.split(\"=\", 1) for line in output.decode().splitlines() if \"=\" in line))\n",
    "    return env\n",
    "\n",
    "env = source_and_get_environment('/opt/intel/oneapi/setvars.sh')\n",
    "\n",
    "for i in range(20,201,2):\n",
    "    os.system(f'{buildpath}benchmarks/baseline {i} {2**(6+OFFSET_BS)} {Ncpu_runs} {Ncpu_warmup} 0 {fname_base}')\n",
    "    if(num_gpus>0):\n",
    "        proc = subprocess.Popen(['/bin/bash', '-c', f'{buildpath}benchmarks/sycl/dualisation gpu {i} {2**(20+OFFSET_BS)} {Ngpu_runs} {Ngpu_warmup} 0 1 {fname_one_gpu_v0}'], env=env);  proc.wait()\n",
    "        proc = subprocess.Popen(['/bin/bash', '-c', f'{buildpath}benchmarks/sycl/dualisation gpu {i} {2**(20+OFFSET_BS)} {Ngpu_runs} {Ngpu_warmup} 1 1 {fname_one_gpu_v1}'], env=env); proc.wait()\n",
    "        proc = subprocess.Popen(['/bin/bash', '-c', f'{buildpath}benchmarks/sycl/dualisation gpu {i} {2**(20+OFFSET_BS)} {Ngpu_runs} {Ngpu_warmup} 0 {num_gpus} {fname_multi_gpu_v0}'], env=env); proc.wait()\n",
    "        proc = subprocess.Popen(['/bin/bash', '-c', f'{buildpath}benchmarks/sycl/dualisation gpu {i} {2**(20+OFFSET_BS)} {Ngpu_runs} {Ngpu_warmup} 1 {num_gpus} {fname_multi_gpu_v1}'], env=env); proc.wait()\n",
    "        proc = subprocess.Popen(['/bin/bash', '-c', f'{buildpath}benchmarks/sycl/dualisation gpu {i} {num_gpus*2**(20+OFFSET_BS)} {Ngpu_runs} {Ngpu_warmup} 1 {num_gpus} {fname_multi_gpu_weak}'], env=env); proc.wait()\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Baseline Benchmark Plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_base = pd.read_csv(fname_base)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(15,15), nrows=2, sharex=True, dpi=200)\n",
    "ax[0].plot(df_base[\"N\"].to_numpy(), df_base[\"T\"].to_numpy()/1e3, 'o:', color=CD[\"Baseline\"], label=\"Baseline Sequential Dualisation\")\n",
    "ax[0].fill_between(df_base[\"N\"].to_numpy(), (df_base[\"T\"].to_numpy() - df_base[\"TSD\"].to_numpy())/1e3, (df_base[\"T\"].to_numpy() + df_base[\"TSD\"].to_numpy())/1e3, alpha=0.1, color='k')\n",
    "ax[0].set_ylabel(r\"Time / Graph [$\\mu$s]\")\n",
    "ax[0].legend()\n",
    "\n",
    "ax[1].plot(df_base[\"N\"].to_numpy(), df_base[\"T\"].to_numpy() / df_base[\"N\"].to_numpy(), 'o:', color=CD[\"Baseline\"], label=\"Baseline Sequential Dualisation\")\n",
    "ax[1].fill_between(df_base[\"N\"].to_numpy(), (df_base[\"T\"].to_numpy() - df_base[\"TSD\"].to_numpy()*2) / df_base[\"N\"].to_numpy(), (df_base[\"T\"].to_numpy()+df_base[\"TSD\"].to_numpy()*2) / df_base[\"N\"].to_numpy(), color='k', alpha=0.1, label=r\"2$\\sigma$\")\n",
    "ax[1].set_ylabel(r\"Time / Node [ns]\")\n",
    "ax[1].set_xlabel(r\"Cubic Graph Size [# Nodes]\")\n",
    "ax[1].legend()\n",
    "plt.savefig(path + \"figures/baseline.pdf\", bbox_inches='tight')\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot Dual GPU Benchmark of V0 and V1 Kernels (CUDA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if(num_gpus>0):\n",
    "    df0 = pd.read_csv(fname_multi_gpu_v0)\n",
    "    df1 = pd.read_csv(fname_multi_gpu_v1)\n",
    "\n",
    "    fig, ax = plt.subplots(figsize=(15,15), nrows=2, sharex=True, dpi=200)\n",
    "    ax[0].plot(df0[\"N\"].to_numpy(), df1[\"T\"].to_numpy(), 'o:', color=CD[\"2 GPU_V1\"], label=KName1)\n",
    "    ax[0].plot(df0[\"N\"].to_numpy(), df0[\"T\"].to_numpy(), 'o:', color=CD[\"2 GPU_V0\"], label=KName0)\n",
    "    ax[0].fill_between(df0[\"N\"].to_numpy(), (df0[\"T\"].to_numpy() - df0[\"TSD\"].to_numpy()*2), (df0[\"T\"].to_numpy()+df0[\"TSD\"].to_numpy()*2), color='k', alpha=0.1, label=r\"2$\\sigma$\")\n",
    "    ax[0].fill_between(df0[\"N\"].to_numpy(), (df1[\"T\"].to_numpy() - df1[\"TSD\"].to_numpy()*2), (df1[\"T\"].to_numpy()+df1[\"TSD\"].to_numpy()*2), color='k', alpha=0.1)\n",
    "    ylow = ax[0].get_ylim()[0]\n",
    "    yhigh = ax[0].get_ylim()[1]\n",
    "    ax[0].vlines(96, ylow, yhigh, color=CD[\"2 GPU_V0\"], linestyle='--', label=r\"Saturation Kernel 0\")\n",
    "    ax[0].vlines(188, ylow, yhigh, color=CD[\"2 GPU_V1\"], linestyle='--', label=r\"Saturation Kernel 1\")\n",
    "    ax[0].set_ylabel(r\"Time / Graph [ns]\")\n",
    "    ax[0].set_ymargin(0.0)\n",
    "    ax[0].legend(loc=\"upper left\")\n",
    "\n",
    "    ax[1].plot(df0[\"N\"].to_numpy(), df1[\"T\"].to_numpy()*1e3 / df0[\"N\"].to_numpy(), 'o:', color=CD[\"2 GPU_V1\"], label=KName1)\n",
    "    ax[1].plot(df0[\"N\"].to_numpy(), df0[\"T\"].to_numpy()*1e3 / df0[\"N\"].to_numpy(), 'o:', color=CD[\"2 GPU_V0\"], label=KName0)\n",
    "    ax[1].fill_between(df0[\"N\"].to_numpy(), (df0[\"T\"].to_numpy() - df0[\"TSD\"].to_numpy()*2)*1e3 / df0[\"N\"].to_numpy(), (df0[\"T\"].to_numpy()+df0[\"TSD\"].to_numpy()*2)*1e3 / df0[\"N\"].to_numpy(), color='k', alpha=0.1, label=r\"2$\\sigma$\")\n",
    "    ax[1].fill_between(df0[\"N\"].to_numpy(), (df1[\"T\"].to_numpy() - df1[\"TSD\"].to_numpy()*2)*1e3 / df0[\"N\"].to_numpy(), (df1[\"T\"].to_numpy()+df1[\"TSD\"].to_numpy()*2)*1e3 / df0[\"N\"].to_numpy(), color='k', alpha=0.1)\n",
    "    ylow = ax[1].get_ylim()[0]\n",
    "    yhigh = ax[1].get_ylim()[1]\n",
    "    ax[1].vlines(96, ylow, yhigh, color=CD[\"2 GPU_V0\"], linestyle='--', label=r\"Kernel 0 Saturation\")\n",
    "    ax[1].vlines(188, ylow, yhigh, color=CD[\"2 GPU_V1\"], linestyle='--', label=r\"Kernel 1 Saturation\")\n",
    "    ax[1].legend(bbox_to_anchor=(0.5, 0.9))\n",
    "    ax[1].set_ymargin(0.0)\n",
    "    ax[1].set_xlabel(r\"Cubic Graph Size [# Nodes]\")\n",
    "    ax[1].set_ylabel(r\"Time / Node [ps]\")\n",
    "    plt.savefig(path + \"figures/cuda_kernel_benchmark.pdf\", bbox_inches='tight')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot Strong and Weak Scaling of the V1 Kernel (CUDA) from 1 to 2 GPUs "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if(num_gpus>1):\n",
    "    df1 = pd.read_csv(fname_one_gpu_v1)\n",
    "    df3 = pd.read_csv(fname_multi_gpu_v1)\n",
    "    df2 = pd.read_csv(fname_multi_gpu_weak)\n",
    "    def std_div(a,b, a_std, b_std):\n",
    "        return a/b * np.sqrt((a_std/a)**2 + (b_std/b)**2)\n",
    "    fig, ax     = plt.subplots(figsize=(15, 15), nrows=2, sharex=True)\n",
    "    ax[0].plot(df1[\"N\"].to_numpy(), df1[\"T\"].to_numpy(), 'o:',  color=CD[\"GPU_V1\"], label=f\"1 GPU, $B_s = 2^{{{20+OFFSET_BS}}}$\")\n",
    "    ax[0].plot(df3[\"N\"].to_numpy(), df3[\"T\"].to_numpy(), 'o:',  color=CD[\"2 GPU_V1\"], label=f\"2 GPUs $B_s = 2^{{{21+OFFSET_BS}}}$\")\n",
    "    ax[0].plot(df2[\"N\"].to_numpy(), df2[\"T\"].to_numpy(), 'x--',  color=CD[\"2 GPU_V1\"], label=f\"2 GPUs $B_s = 2^{{{20+OFFSET_BS}}}$\")\n",
    "    ax[0].fill_between(df1[\"N\"].to_numpy(), df1[\"T\"].to_numpy() - df1[\"TSD\"].to_numpy()*1, df1[\"T\"].to_numpy() + df1[\"TSD\"].to_numpy()*1, alpha=0.1, color='k', label=r\"1$\\sigma$\")\n",
    "    ax[0].fill_between(df3[\"N\"].to_numpy(), df3[\"T\"].to_numpy() - df3[\"TSD\"].to_numpy()*1, df3[\"T\"].to_numpy() + df3[\"TSD\"].to_numpy()*1, alpha=0.1, color='k')\n",
    "    ax[0].fill_between(df2[\"N\"].to_numpy(), df2[\"T\"].to_numpy() - df2[\"TSD\"].to_numpy()*1, df2[\"T\"].to_numpy() + df2[\"TSD\"].to_numpy()*1, alpha=0.1, color='k')\n",
    "    ax[0].set_ylabel(\"Time / Graph [ns]\")\n",
    "    ax[0].legend(loc='upper left')\n",
    "\n",
    "\n",
    "    #Plot speedup\n",
    "    ax[1].plot(df1[\"N\"].to_numpy(), df1[\"T\"].to_numpy()/df3[\"T\"].to_numpy(), 'o:',  color=CD[\"2 GPU_V1\"], label=f\"2 GPUs $B_s = 2^{{{21+OFFSET_BS}}}$\")\n",
    "    ax[1].plot(df1[\"N\"].to_numpy(), df1[\"T\"].to_numpy()/df2[\"T\"].to_numpy(), 'x--',  color=CD[\"2 GPU_V1\"], label=f\"2 GPUs $B_s = 2^{{{20+OFFSET_BS}}}$\")\n",
    "    std_1 = std_div(df1[\"T\"].to_numpy(), df3[\"T\"].to_numpy(), df1[\"TSD\"].to_numpy(), df3[\"TSD\"].to_numpy())\n",
    "    std_2 = std_div(df1[\"T\"].to_numpy(), df2[\"T\"].to_numpy(), df1[\"TSD\"].to_numpy(), df2[\"TSD\"].to_numpy())\n",
    "    ax[1].fill_between(df1[\"N\"].to_numpy(), df1[\"T\"].to_numpy()/df3[\"T\"].to_numpy() - std_1, df1[\"T\"].to_numpy()/df3[\"T\"].to_numpy() + std_1, alpha=0.1, color='k', label=r\"1$\\sigma$\")\n",
    "    ax[1].fill_between(df1[\"N\"].to_numpy(), df1[\"T\"].to_numpy()/df2[\"T\"].to_numpy() - std_2, df1[\"T\"].to_numpy()/df2[\"T\"].to_numpy() + std_2, alpha=0.1, color='k')\n",
    "    ax[1].hlines(2, 20, 200, linestyles='dashed', color='k', label=r\"Perfect Scaling\")\n",
    "    ax[1].set_ylabel(\"Speedup [1]\")\n",
    "    ax[1].set_xlabel(r\"Number of Nodes [\\#]\")\n",
    "    ax[1].set_ylim(0.95,num_gpus*1.05)\n",
    "    ax[1].legend(loc='lower right', ncol=2)\n",
    "    plt.savefig(path + \"figures/cuda_scaling.pdf\", bbox_inches='tight')\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#Benchmarking full pipeline with sequential dualisation\n",
    "reset_file(fname_base_pipeline)\n",
    "reset_file(fname_full_pipeline)\n",
    "Nbuckygen = 1000000\n",
    "for i in range(72,201,2):\n",
    "    proc = subprocess.Popen(['/bin/bash', '-c', f'{buildpath}benchmarks/sycl/baseline_pipeline gpu {i} {Nbuckygen} {10000} {5} {Ngpu_warmup} 1 1 {fname_base_pipeline}'], env=env).wait()\n",
    "    proc = subprocess.Popen(['/bin/bash', '-c', f'{buildpath}benchmarks/sycl/pipeline gpu {i} {Nbuckygen} {10000} {5} {Ngpu_warmup} 1 1 {fname_full_pipeline}'], env=env).wait()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.ticker as ticker\n",
    "\n",
    "df_base_pipeline = pd.read_csv(fname_base_pipeline)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(20,10), nrows=1, sharex=True, dpi=150)\n",
    "\n",
    "opt = df_base_pipeline[\"T_opt\"].to_numpy()\n",
    "opt_sd = df_base_pipeline[\"TSD_opt\"].to_numpy()\n",
    "tutte = df_base_pipeline[\"T_tutte\"].to_numpy()\n",
    "tutte_sd = df_base_pipeline[\"TSD_tutte\"].to_numpy()\n",
    "project = df_base_pipeline[\"T_project\"].to_numpy()\n",
    "project_sd = df_base_pipeline[\"TSD_project\"].to_numpy()\n",
    "overhead = df_base_pipeline[\"T_overhead\"].to_numpy()\n",
    "overhead_sd = df_base_pipeline[\"TSD_overhead\"].to_numpy()\n",
    "gen = df_base_pipeline[\"T_gen\"].to_numpy()\n",
    "gen_sd = df_base_pipeline[\"TSD_gen\"].to_numpy()\n",
    "dual = df_base_pipeline[\"T_dual\"].to_numpy()\n",
    "dual_sd = df_base_pipeline[\"TSD_dual\"].to_numpy()\n",
    "natoms = df_base_pipeline[\"N\"].to_numpy()\n",
    "\n",
    "\n",
    "parallel = opt + tutte + project\n",
    "parallel_sd = np.sqrt(opt_sd**2 + tutte_sd**2 + project_sd**2)\n",
    "total = parallel + overhead + gen + dual\n",
    "\n",
    "def plot_line_normalized(ax, x, y, y_sd, label, color, marker, linestyle, mfc_bool=True):\n",
    "    if mfc_bool:\n",
    "        ax.plot(x, 1e2* y/total, marker=marker, color=color, label=label, linestyle=linestyle, mfc=color) #Normalized to total time, shown as percentage\n",
    "    else:\n",
    "        ax.plot(x, 1e2* y/total, marker=marker, color=color, label=label, linestyle=linestyle, mfc=\"None\") #Normalized to total time, shown as percentage\n",
    "    ax.fill_between(x, 1e2*(y - y_sd)/total, 1e2*(y + y_sd)/total, alpha=0.1, color='k')\n",
    "\n",
    "\n",
    "plot_line_normalized(ax, natoms, gen, gen_sd, \"Isomer-space graph generation\", CD[\"Generate\"], 'o', ':', False)\n",
    "plot_line_normalized(ax, natoms, dual, dual_sd, \"Baseline Sequential Dualization\", CD[\"Dual\"], 'o', ':', False)\n",
    "plot_line_normalized(ax, natoms, parallel, parallel_sd, \"Lockstep-parallel geometry optimization\", \"k\", '*', ':')\n",
    "plot_line_normalized(ax, natoms, overhead, overhead_sd, \"Overhead\", \"blue\", 'o', ':')\n",
    "\n",
    "ax.set_ylabel(r\"Time / Graph [$\\mu$s]\")\n",
    "ax.legend()\n",
    "ax.set_xlabel(r\"Isomerspace $C_N$ [1]\")\n",
    "ax.set_ylim(0,100)\n",
    "#percentage formatting\n",
    "ax.yaxis.set_major_formatter(ticker.PercentFormatter())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.ticker as ticker\n",
    "\n",
    "df_full_pipeline = pd.read_csv(fname_full_pipeline)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(20,10), nrows=1, sharex=True, dpi=150)\n",
    "\n",
    "opt = df_full_pipeline[\"T_opt\"].to_numpy()\n",
    "opt_sd = df_full_pipeline[\"TSD_opt\"].to_numpy()\n",
    "tutte = df_full_pipeline[\"T_tutte\"].to_numpy()\n",
    "tutte_sd = df_full_pipeline[\"TSD_tutte\"].to_numpy()\n",
    "project = df_full_pipeline[\"T_project\"].to_numpy()\n",
    "project_sd = df_full_pipeline[\"TSD_project\"].to_numpy()\n",
    "overhead = df_full_pipeline[\"T_overhead\"].to_numpy()\n",
    "overhead_sd = df_full_pipeline[\"TSD_overhead\"].to_numpy()\n",
    "gen = df_full_pipeline[\"T_gen\"].to_numpy()\n",
    "gen_sd = df_full_pipeline[\"TSD_gen\"].to_numpy()\n",
    "dual = df_full_pipeline[\"T_dual\"].to_numpy()\n",
    "dual_sd = df_full_pipeline[\"TSD_dual\"].to_numpy()\n",
    "natoms = df_full_pipeline[\"N\"].to_numpy()\n",
    "\n",
    "\n",
    "parallel = opt + tutte + project\n",
    "parallel_sd = np.sqrt(opt_sd**2 + tutte_sd**2 + project_sd**2)\n",
    "total = parallel + overhead + gen + dual\n",
    "\n",
    "def plot_normalized_line(ax, x, y, y_sd, label, color, marker, linestyle, mfc_bool=True):\n",
    "    if mfc_bool:\n",
    "        ax.plot(x, 1e2* y/total, marker=marker, color=color, label=label, linestyle=linestyle, mfc=color) #Normalized to total time, shown as percentage\n",
    "    else:\n",
    "        ax.plot(x, 1e2* y/total, marker=marker, color=color, label=label, linestyle=linestyle, mfc=\"None\") #Normalized to total time, shown as percentage\n",
    "    ax.fill_between(x, 1e2*(y - y_sd)/total, 1e2*(y + y_sd)/total, alpha=0.1, color='k')\n",
    "\n",
    "\n",
    "plot_normalized_line(ax, natoms, gen, gen_sd, \"Isomer-space graph generation\", CD[\"Generate\"], 'o', ':', False)\n",
    "plot_normalized_line(ax, natoms, parallel, parallel_sd, \"Lockstep-parallel geometry optimization\", \"k\", '*', ':')\n",
    "plot_normalized_line(ax, natoms, overhead, overhead_sd, \"Overhead\", \"blue\", 'o', ':')\n",
    "plot_normalized_line(ax, natoms, dual, dual_sd, \"Baseline Sequential Dualization\", CD[\"Dual\"], '*', ':')\n",
    "\n",
    "ax.set_ylabel(r\"Time / Graph [$\\mu$s]\")\n",
    "ax.legend()\n",
    "ax.set_xlabel(r\"Isomerspace $C_N$ [1]\")\n",
    "ax.set_ylim(0,100)\n",
    "#percentage formatting\n",
    "ax.yaxis.set_major_formatter(ticker.PercentFormatter())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.ticker as ticker\n",
    "\n",
    "df_full_pipeline = pd.read_csv(fname_full_pipeline)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(20,10), nrows=1, sharex=True, dpi=150)\n",
    "\n",
    "opt = df_full_pipeline[\"T_opt\"].to_numpy()\n",
    "opt_sd = df_full_pipeline[\"TSD_opt\"].to_numpy()\n",
    "tutte = df_full_pipeline[\"T_tutte\"].to_numpy()\n",
    "tutte_sd = df_full_pipeline[\"TSD_tutte\"].to_numpy()\n",
    "project = df_full_pipeline[\"T_project\"].to_numpy()\n",
    "project_sd = df_full_pipeline[\"TSD_project\"].to_numpy()\n",
    "overhead = df_full_pipeline[\"T_overhead\"].to_numpy()\n",
    "overhead_sd = df_full_pipeline[\"TSD_overhead\"].to_numpy()\n",
    "gen = df_full_pipeline[\"T_gen\"].to_numpy()\n",
    "gen_sd = df_full_pipeline[\"TSD_gen\"].to_numpy()\n",
    "dual = df_full_pipeline[\"T_dual\"].to_numpy()\n",
    "dual_sd = df_full_pipeline[\"TSD_dual\"].to_numpy()\n",
    "natoms = df_full_pipeline[\"N\"].to_numpy()\n",
    "\n",
    "\n",
    "parallel = opt + tutte + project\n",
    "parallel_sd = np.sqrt(opt_sd**2 + tutte_sd**2 + project_sd**2)\n",
    "total = parallel + overhead + gen + dual\n",
    "\n",
    "def plot_absolute_line(ax, x, y, y_sd, label, color, marker, linestyle, mfc_bool=True):\n",
    "    if mfc_bool:\n",
    "        ax.plot(x,  y/1e3, marker=marker, color=color, label=label, linestyle=linestyle, mfc=color) #Normalized to total time, shown as percentage\n",
    "    else:\n",
    "        ax.plot(x,  y/1e3, marker=marker, color=color, label=label, linestyle=linestyle, mfc=\"None\") #Normalized to total time, shown as percentage\n",
    "    ax.fill_between(x, (y - y_sd)/1e3, (y + y_sd)/1e3, alpha=0.1, color='k')\n",
    "\n",
    "\n",
    "plot_absolute_line(ax, natoms, parallel, parallel_sd, \"Lockstep-parallel geometry optimization\", \"k\", '*', ':')\n",
    "plot_absolute_line(ax, natoms, gen, gen_sd, \"Isomer-space graph generation\", CD[\"Generate\"], 'o', ':', False)\n",
    "plot_absolute_line(ax, natoms, overhead, overhead_sd, \"Overhead\", \"blue\", 'o', ':')\n",
    "plot_absolute_line(ax, natoms, dual, dual_sd, \"Lockstep-parallel dualization\", CD[\"Dual\"], '*', ':')\n",
    "\n",
    "ax.set_ylabel(r\"Time / Graph [$\\mu$s]\")\n",
    "ax.legend()\n",
    "ax.set_xlabel(r\"Isomerspace $C_N$ [1]\")\n",
    "ax.set_yscale('log')\n",
    "#percentage formatting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Speedup of the V1 Kernel (SYCL) on 1 GPU compared to sequential CPU dualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Speedup of dualization plot\n",
    "fig, ax = plt.subplots(figsize=(20,10), nrows=1, sharex=True, dpi=150)\n",
    "\n",
    "sequential = df_base_pipeline[\"T_dual\"].to_numpy()\n",
    "sequential_sd = df_base_pipeline[\"TSD_dual\"].to_numpy()\n",
    "parallel = df_full_pipeline[\"T_dual\"].to_numpy()\n",
    "parallel_sd = df_full_pipeline[\"TSD_dual\"].to_numpy()\n",
    "natoms = df_full_pipeline[\"N\"].to_numpy()\n",
    "\n",
    "speedup = sequential / parallel\n",
    "speedup_sd = np.sqrt((sequential_sd/sequential)**2 + (parallel_sd/parallel)**2) * speedup\n",
    "\n",
    "ax.plot(natoms, speedup, 'o:', color=\"r\", label=\"Speedup\")\n",
    "ax.fill_between(natoms, speedup - speedup_sd, speedup + speedup_sd, alpha=0.1, color='k')\n",
    "ax.set_ylabel(r\"Speedup [1]\")\n",
    "ax.legend()\n",
    "ax.set_xlabel(r\"Isomerspace $C_N$ [1]\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}