From 3816b3d3b879122b1cd11386d40d4893a230b720 Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Tue, 11 Feb 2025 14:04:51 +0100
Subject: [PATCH 01/15] Give `end-to-end tests` steps descriptive names

---
 .github/workflows/desktop-e2e.yml | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index 2165308d766f..2e67ee41b6bc 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -154,7 +154,8 @@ jobs:
       matrix:
         os: ${{ fromJSON(needs.prepare-matrices.outputs.linux_matrix) }}
     steps:
-      - uses: actions/download-artifact@v4
+      - name: Download App
+        uses: actions/download-artifact@v4
         if: ${{ needs.build-linux.result == 'success' }}
         with:
           name: linux-build
@@ -167,7 +168,8 @@ jobs:
           git fetch --tags --prune-tags --force
           export TEST_FILTERS="${{ github.event.inputs.tests }}"
           ./test/scripts/ci-runtests.sh ${{ matrix.os }}
-      - uses: actions/upload-artifact@v4
+      - name: Upload test report
+        uses: actions/upload-artifact@v4
         if: '!cancelled()'
         with:
           name: ${{ matrix.os }}_report
@@ -242,7 +244,8 @@ jobs:
       matrix:
         os: ${{ fromJSON(needs.prepare-matrices.outputs.windows_matrix) }}
     steps:
-      - uses: actions/download-artifact@v4
+      - name: Download App
+        uses: actions/download-artifact@v4
         if: ${{ needs.build-windows.result == 'success' }}
         with:
           name: windows-build
@@ -255,7 +258,8 @@ jobs:
           git fetch --tags --prune-tags --force
           export TEST_FILTERS="${{ github.event.inputs.tests }}"
           ./test/scripts/ci-runtests.sh ${{ matrix.os }}
-      - uses: actions/upload-artifact@v4
+      - name: Upload test report
+        uses: actions/upload-artifact@v4
         if: '!cancelled()'
         with:
           name: ${{ matrix.os }}_report
@@ -314,7 +318,8 @@ jobs:
       matrix:
         os: ${{ fromJSON(needs.prepare-matrices.outputs.macos_matrix) }}
     steps:
-      - uses: actions/download-artifact@v4
+      - name: Download App
+        uses: actions/download-artifact@v4
         if: ${{ needs.build-macos.result == 'success' }}
         with:
           name: macos-build
@@ -327,7 +332,8 @@ jobs:
           git fetch --tags --prune-tags --force
           export TEST_FILTERS="${{ github.event.inputs.tests }}"
           ./test/scripts/ci-runtests.sh ${{ matrix.os }}
-      - uses: actions/upload-artifact@v4
+      - name: Upload test report
+        uses: actions/upload-artifact@v4
         if: '!cancelled()'
         with:
           name: ${{ matrix.os }}_report

From 7914b6b993fb26973e44ea2a3348530a74e32b8a Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Wed, 12 Feb 2025 13:57:40 +0100
Subject: [PATCH 02/15] Rename some scripts (and move them to subdirectories)

Rename `test-utils.sh` to `lib.sh` & start to break up `lib.sh`

Start off with the download app package / e2e tests binary in
`download.sh`
---
 test/README.md                                |  14 +-
 test/scripts/build.sh                         |   4 +-
 .../runner-image.sh}                          |  13 +-
 .../test-manager.sh}                          |   2 +-
 .../{build-runner.sh => build/test-runner.sh} |   6 +-
 test/scripts/{ci-runtests.sh => run/ci.sh}    |  26 +++-
 test/scripts/{test-utils.sh => utils/lib.sh}  | 129 ++++++------------
 test/test-by-version.sh                       |   4 +-
 8 files changed, 84 insertions(+), 114 deletions(-)
 rename test/scripts/{build-runner-image.sh => build/runner-image.sh} (76%)
 rename test/scripts/{build-manager.sh => build/test-manager.sh} (92%)
 rename test/scripts/{build-runner.sh => build/test-runner.sh} (86%)
 rename test/scripts/{ci-runtests.sh => run/ci.sh} (67%)
 rename test/scripts/{test-utils.sh => utils/lib.sh} (78%)

diff --git a/test/README.md b/test/README.md
index 112edb65d9bd..66ba136ddf1a 100644
--- a/test/README.md
+++ b/test/README.md
@@ -107,33 +107,33 @@ Next: build the `test-runner`
 
 ### Building the test runner
 
-Building the `test-runner` binary is done with the `build-runner.sh` script.
+Building the `test-runner` binary is done with the `build/test-runner.sh` script.
 Currently, only `x86_64` platforms are supported for Windows/Linux and `ARM64` (Apple Silicon) for macOS.
 
 For example, building `test-runner` for Windows would look like this:
 
 ``` bash
-./scripts/container-run.sh ./scripts/build-runner.sh windows
+./scripts/container-run.sh ./scripts/build/test-runner.sh windows
 ```
 
 #### Linux
 Using `podman` is the recommended way to build the `test-runner`. See the [Linux section under Prerequisities](#prerequisites) for more details.
 
 ``` bash
-./scripts/container-run.sh ./scripts/build-runner.sh linux
+./scripts/container-run.sh ./scripts/build/test-runner.sh linux
 ```
 
 #### macOS
 
 ``` bash
-./scripts/build-runner.sh macos
+./scripts/build/test-runner.sh macos
 ```
 
 #### Windows
 The `test-runner` binary for Windows may be cross-compiled from a Linux host.
 
 ``` bash
-./scripts/container-run.sh ./scripts/build-runner.sh windows
+./scripts/container-run.sh ./scripts/build/test-runner.sh windows
 ```
 
 ### Running the tests
@@ -164,6 +164,6 @@ cargo run --bin test-manager run-tests --vm macos-ventura \
     --app-package-to-upgrade-from 2023.2
 ```
 
-## Note on `scripts/ci-runtests.sh`
+## Note on `scripts/run/ci.sh`
 
-`scripts/ci-runtests.sh` is the script that GitHub actions uses to invokes the `test-manager`, with similar functionality as `test-by-version.sh`. Note that account numbers are read (newline-delimited) from the path specified by the environment variable `ACCOUNT_TOKENS`. Round robin is used to select an account for each VM.
+`scripts/run/ci.sh` is the script that GitHub actions uses to invokes the `test-manager`, with similar functionality as `test-by-version.sh`. Note that account numbers are read (newline-delimited) from the path specified by the environment variable `ACCOUNT_TOKENS`. Round robin is used to select an account for each VM.
diff --git a/test/scripts/build.sh b/test/scripts/build.sh
index 7cfaede311f2..99b7b3494274 100755
--- a/test/scripts/build.sh
+++ b/test/scripts/build.sh
@@ -13,11 +13,11 @@ REPO_ROOT="$SCRIPT_DIR/../.."
 build_linux() {
     mkdir -p "$TEST_FRAMEWORK_ROOT/dist"
     # Build the test manager
-    "$SCRIPT_DIR/build-manager.sh" linux
+    "$SCRIPT_DIR/build/test-manager.sh" linux
     cp "$TEST_FRAMEWORK_ROOT/target/release/test-manager" "$TEST_FRAMEWORK_ROOT/dist/"
 
     # Build the test runner
-    "$SCRIPT_DIR/build-runner.sh" linux
+    "$SCRIPT_DIR/build/test-runner.sh" linux
     cp "$TEST_FRAMEWORK_ROOT/target/x86_64-unknown-linux-gnu/release/test-runner" "$TEST_FRAMEWORK_ROOT/dist/"
     cp "$TEST_FRAMEWORK_ROOT/target/x86_64-unknown-linux-gnu/release/connection-checker" "$TEST_FRAMEWORK_ROOT/dist/"
 
diff --git a/test/scripts/build-runner-image.sh b/test/scripts/build/runner-image.sh
similarity index 76%
rename from test/scripts/build-runner-image.sh
rename to test/scripts/build/runner-image.sh
index 512dc93a4cd8..47c41b2e3871 100755
--- a/test/scripts/build-runner-image.sh
+++ b/test/scripts/build/runner-image.sh
@@ -4,6 +4,9 @@
 
 set -eu
 
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
+
 TEST_RUNNER_IMAGE_SIZE_MB=5000
 
 case $TARGET in
@@ -20,10 +23,8 @@ echo "************************************************************"
 echo "* Preparing test runner image: $TARGET"
 echo "************************************************************"
 
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-mkdir -p "${SCRIPT_DIR}/../testrunner-images/"
-TEST_RUNNER_IMAGE_PATH="${SCRIPT_DIR}/../testrunner-images/${TEST_RUNNER_IMAGE_FILENAME}"
+mkdir -p "${TEST_FRAMEWORK_REPO}/testrunner-images"
+TEST_RUNNER_IMAGE_PATH="${TEST_FRAMEWORK_REPO}/testrunner-images/${TEST_RUNNER_IMAGE_FILENAME}"
 
 case $TARGET in
 
@@ -32,8 +33,8 @@ case $TARGET in
         mformat -F -i "${TEST_RUNNER_IMAGE_PATH}" "::"
         mcopy \
             -i "${TEST_RUNNER_IMAGE_PATH}" \
-            "${SCRIPT_DIR}/../target/$TARGET/release/test-runner.exe" \
-            "${SCRIPT_DIR}/../target/$TARGET/release/connection-checker.exe" \
+            "${TEST_FRAMEWORK_ROOT}/target/$TARGET/release/test-runner.exe" \
+            "${TEST_FRAMEWORK_ROOT}/target/$TARGET/release/connection-checker.exe" \
             "${PACKAGE_DIR}/"*.exe \
             "::"
         mdir -i "${TEST_RUNNER_IMAGE_PATH}"
diff --git a/test/scripts/build-manager.sh b/test/scripts/build/test-manager.sh
similarity index 92%
rename from test/scripts/build-manager.sh
rename to test/scripts/build/test-manager.sh
index 60cf5da98cbc..517bdf310fd4 100755
--- a/test/scripts/build-manager.sh
+++ b/test/scripts/build/test-manager.sh
@@ -4,7 +4,7 @@ set -eu
 
 # Build `test-manager`
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/.."
+TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
 REPO_DIR="$TEST_FRAMEWORK_ROOT/.."
 
 # shellcheck disable=SC1091
diff --git a/test/scripts/build-runner.sh b/test/scripts/build/test-runner.sh
similarity index 86%
rename from test/scripts/build-runner.sh
rename to test/scripts/build/test-runner.sh
index ffdb2b861a26..3e01353ac6a4 100755
--- a/test/scripts/build-runner.sh
+++ b/test/scripts/build/test-runner.sh
@@ -3,7 +3,9 @@
 set -eu
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-REPO_DIR="$SCRIPT_DIR/../.."
+TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
+REPO_DIR="$TEST_FRAMEWORK_ROOT/.."
+
 cd "$SCRIPT_DIR"
 
 # shellcheck disable=SC1091
@@ -35,5 +37,5 @@ cargo build \
 
 # Only build runner image for Windows
 if [[ $TARGET == x86_64-pc-windows-gnu ]]; then
-    TARGET="$TARGET" ./build-runner-image.sh
+    TARGET="$TARGET" ./runner-image.sh
 fi
diff --git a/test/scripts/ci-runtests.sh b/test/scripts/run/ci.sh
similarity index 67%
rename from test/scripts/ci-runtests.sh
rename to test/scripts/run/ci.sh
index 88a89441647b..b55b640e0321 100755
--- a/test/scripts/ci-runtests.sh
+++ b/test/scripts/run/ci.sh
@@ -1,11 +1,17 @@
 #!/usr/bin/env bash
 
+# TODO: Break this down into multiple, smaller scripts and compose them in this file.
+
 set -eu
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
+TEST_DIR="$TEST_FRAMEWORK_ROOT"
+
 cd "$SCRIPT_DIR"
-TEST_DIR="$SCRIPT_DIR/.."
 
+# Parse arguments
+# TODO: Add support for either passing in --account-tokens or reading from env variable.
 if [[ "$#" -lt 1 ]]; then
     echo "usage: $0 TEST_OS" 1>&2
     exit 1
@@ -19,15 +25,17 @@ if [[ "$(uname -s)" == "Darwin" ]]; then
     rustup update
 fi
 
-# shellcheck source=test/scripts/test-utils.sh
-source "test-utils.sh"
+# shellcheck source=test/scripts/utils/lib.sh
+source "../utils/lib.sh"
+# shellcheck source=test/scripts/utils/download.sh
+source "../utils/download.sh" # TODO: Do not source it, call it instead.
 
 echo "**********************************"
 echo "* Version to upgrade from: $LATEST_STABLE_RELEASE"
 echo "* Version to test: $CURRENT_VERSION"
 echo "**********************************"
 
-
+# TODO: Add support for either passing in --account-tokens or reading from env variable.
 if [[ -z "${ACCOUNT_TOKENS+x}" ]]; then
     echo "'ACCOUNT_TOKENS' must be specified" 1>&2
     exit 1
@@ -36,11 +44,15 @@ if ! readarray -t tokens < "${ACCOUNT_TOKENS}"; then
     echo "Specify account numbers in 'ACCOUNT_TOKENS' file" 1>&2
     exit 1
 fi
+
+# TODO: Can we get rid of this? Seemse excessive / leaves a trail
 CI_LOGS_DIR="$TEST_DIR/.ci-logs"
 mkdir -p "$CI_LOGS_DIR"
 echo "$CURRENT_VERSION" > "$CI_LOGS_DIR/last-version.log"
 
 
+# TODO: This should def be it's own step in the GitHub actions workflow
+
 echo "**********************************"
 echo "* Downloading app packages"
 echo "**********************************"
@@ -50,16 +62,22 @@ nice_time download_app_package "$LATEST_STABLE_RELEASE" "$TEST_OS"
 nice_time download_app_package "$CURRENT_VERSION" "$TEST_OS"
 nice_time download_e2e_executable "$CURRENT_VERSION" "$TEST_OS"
 
+# TODO: This should def be it's own step in the GitHub actions workflow
+
 echo "**********************************"
 echo "* Building test manager"
 echo "**********************************"
 
 cargo build -p test-manager
 
+# TODO: This should def be it's own step in the GitHub actions workflow
+
 echo "**********************************"
 echo "* Running tests"
 echo "**********************************"
 
+# TODO: Should we really care about logging in this script?
+
 mkdir -p "$CI_LOGS_DIR/os/"
 export TEST_REPORT="$CI_LOGS_DIR/${TEST_OS}_report"
 rm -f "$TEST_REPORT"
diff --git a/test/scripts/test-utils.sh b/test/scripts/utils/lib.sh
similarity index 78%
rename from test/scripts/test-utils.sh
rename to test/scripts/utils/lib.sh
index be826a6ef053..939289d3bf48 100755
--- a/test/scripts/test-utils.sh
+++ b/test/scripts/utils/lib.sh
@@ -2,33 +2,38 @@
 
 set -eu
 
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
+REPO_ROOT="$TEST_FRAMEWORK_ROOT/.."
+
+export BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases"
+export BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds"
+
 function executable_not_found_in_dist_error {
     1>&2 echo "Executable \"$1\" not found in specified dist dir. Exiting."
     exit 1
 }
 
-# Returns the directory of the test-utils.sh script
+# Returns the directory of the lib.sh script
 function get_test_utls_dir {
-    local script_path="${BASH_SOURCE[0]}"
-    local script_dir
-    if [[ -n "$script_path" ]]; then
-        script_dir="$(cd "$(dirname "$script_path")" >/dev/null && pwd)"
-    else
-        script_dir="$(cd "$(dirname "$0")" >/dev/null && pwd)"
-    fi
-    echo "$script_dir"
+    ## local script_path="${BASH_SOURCE[0]}"
+    ## local script_dir
+    ## if [[ -n "$script_path" ]]; then
+    ##     script_dir="$(cd "$(dirname "$script_path")" >/dev/null && pwd)"
+    ## else
+    ##     script_dir="$(cd "$(dirname "$0")" >/dev/null && pwd)"
+    ## fi
+    ## echo "$script_dir"
+    echo "$SCRIPT_DIR"
 }
 
-export BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases"
-export BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds"
-
 # Infer stable version from GitHub repo
 RELEASES=$(curl -sf https://api.github.com/repos/mullvad/mullvadvpn-app/releases | jq -r '[.[] | select(((.tag_name|(startswith("android") or startswith("ios"))) | not))]')
 LATEST_STABLE_RELEASE=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_name' <<<"$RELEASES")
 
 function get_current_version {
     local app_dir
-    app_dir="$(get_test_utls_dir)/../.."
+    app_dir="$REPO_ROOT"
     if [ -n "${TEST_DIST_DIR+x}" ]; then
         if [ ! -x "${TEST_DIST_DIR%/}/mullvad-version" ]; then
             executable_not_found_in_dist_error mullvad-version
@@ -114,25 +119,22 @@ function is_dev_version {
     return 1
 }
 
-function get_app_filename {
+function get_e2e_filename {
     local version=$1
     local os=$2
     if is_dev_version "$version"; then
         parse_build_version "$version"
-        version="${BUILD_VERSION}${COMMIT_HASH}${TAG:-}"
+        version="${BUILD_VERSION}${COMMIT_HASH}"
     fi
     case $os in
-    debian* | ubuntu*)
-        echo "MullvadVPN-${version}_amd64.deb"
-        ;;
-    fedora*)
-        echo "MullvadVPN-${version}_x86_64.rpm"
+    debian* | ubuntu* | fedora*)
+        echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu"
         ;;
     windows*)
-        echo "MullvadVPN-${version}.exe"
+        echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe"
         ;;
     macos*)
-        echo "MullvadVPN-${version}.pkg"
+        echo "app-e2e-tests-${version}-aarch64-apple-darwin"
         ;;
     *)
         echo "Unsupported target: $os" 1>&2
@@ -141,50 +143,25 @@ function get_app_filename {
     esac
 }
 
-function download_app_package {
-    local version=$1
-    local os=$2
-    local package_repo=""
-
-    if is_dev_version "$version"; then
-        package_repo="${BUILD_DEV_REPOSITORY}"
-    else
-        package_repo="${BUILD_RELEASE_REPOSITORY}"
-    fi
-
-    local filename
-    filename=$(get_app_filename "$version" "$os")
-    local url="${package_repo}/$version/$filename"
-
-    local package_dir
-    package_dir=$(get_package_dir)
-    if [[ ! -f "$package_dir/$filename" ]]; then
-        echo "Downloading build for $version ($os) from $url"
-        if ! curl -sf -o "$package_dir/$filename" "$url"; then
-            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
-            exit 1
-        fi
-    else
-        echo "App package for version $version ($os) already exists at $package_dir/$filename, skipping download"
-    fi
-}
-
-function get_e2e_filename {
+function get_app_filename {
     local version=$1
     local os=$2
     if is_dev_version "$version"; then
         parse_build_version "$version"
-        version="${BUILD_VERSION}${COMMIT_HASH}"
+        version="${BUILD_VERSION}${COMMIT_HASH}${TAG:-}"
     fi
     case $os in
-    debian* | ubuntu* | fedora*)
-        echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu"
+    debian* | ubuntu*)
+        echo "MullvadVPN-${version}_amd64.deb"
+        ;;
+    fedora*)
+        echo "MullvadVPN-${version}_x86_64.rpm"
         ;;
     windows*)
-        echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe"
+        echo "MullvadVPN-${version}.exe"
         ;;
     macos*)
-        echo "app-e2e-tests-${version}-aarch64-apple-darwin"
+        echo "MullvadVPN-${version}.pkg"
         ;;
     *)
         echo "Unsupported target: $os" 1>&2
@@ -193,44 +170,16 @@ function get_e2e_filename {
     esac
 }
 
-function download_e2e_executable {
-    local version=${1:?Error: version not set}
-    local os=${2:?Error: os not set}
-    local package_repo
-
-    if is_dev_version "$version"; then
-        package_repo="${BUILD_DEV_REPOSITORY}"
-    else
-        package_repo="${BUILD_RELEASE_REPOSITORY}"
-    fi
-
-    local filename
-    filename=$(get_e2e_filename "$version" "$os")
-    local url="${package_repo}/$version/additional-files/$filename"
-
-    local package_dir
-    package_dir=$(get_package_dir)
-    if [[ ! -f "$package_dir/$filename" ]]; then
-        echo "Downloading e2e executable for $version ($os) from $url"
-        if ! curl -sf -o "$package_dir/$filename" "$url"; then
-            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
-            exit 1
-        fi
-    else
-        echo "GUI e2e executable for version $version ($os) already exists at $package_dir/$filename, skipping download"
-    fi
-}
-
 function build_test_runner {
     local script_dir
-    script_dir=$(get_test_utls_dir)
+    script_dir="$(get_test_utls_dir)/../"
     local test_os=${1:?Error: test os not set}
     if [[ "${test_os}" =~ "debian"|"ubuntu"|"fedora" ]]; then
-        "$script_dir"/container-run.sh scripts/build-runner.sh linux || exit 1
+        "$script_dir"/container-run.sh scripts/build/test-runner.sh linux || exit 1
     elif [[ "${test_os}" =~ "windows" ]]; then
-        "$script_dir"/container-run.sh scripts/build-runner.sh windows || exit 1
+        "$script_dir"/container-run.sh scripts/build/test-runner.sh windows || exit 1
     elif [[ "${test_os}" =~ "macos" ]]; then
-        "$script_dir"/build-runner.sh macos || exit 1
+        "$script_dir"/build/test-runner.sh macos || exit 1
     fi
 }
 
@@ -279,7 +228,7 @@ function run_tests_for_os {
     local package_dir
     package_dir=$(get_package_dir)
     local test_dir
-    test_dir=$(get_test_utls_dir)/..
+    test_dir=$(get_test_utls_dir)/../..
     read -ra test_filters_arg <<<"${TEST_FILTERS:-}" # Split the string by words into an array
     pushd "$test_dir"
     if [ -n "${TEST_DIST_DIR+x}" ]; then
@@ -321,7 +270,7 @@ function run_tests_for_os {
 # Currently unused, but may be useful in the future
 function build_current_version {
     local app_dir
-    app_dir="$(get_test_utls_dir)/../.."
+    app_dir="$REPO_ROOT"
     local app_filename
     # TODO: TEST_OS must be set to local OS manually, should be set automatically
     app_filename=$(get_app_filename "$CURRENT_VERSION" "${TEST_OS:?Error: TEST_OS not set}")
diff --git a/test/test-by-version.sh b/test/test-by-version.sh
index f409a214cc9d..3b97d45461ff 100755
--- a/test/test-by-version.sh
+++ b/test/test-by-version.sh
@@ -21,8 +21,8 @@ usage() {
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR"
 
-# shellcheck source=test/scripts/test-utils.sh
-source "scripts/test-utils.sh"
+# shellcheck source=test/scripts/utils/lib.sh
+source "scripts/utils/lib.sh"
 
 if [[ ("$*" == "--help") || "$*" == "-h" ]]; then
     usage

From b518c07359d7b71309d56ec692702486b1043257 Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Mon, 17 Feb 2025 16:18:52 +0100
Subject: [PATCH 03/15] Accept output folder in `test-runner.sh`

---
 test/scripts/build/test-runner.sh | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/test/scripts/build/test-runner.sh b/test/scripts/build/test-runner.sh
index 3e01353ac6a4..05d2132679b2 100755
--- a/test/scripts/build/test-runner.sh
+++ b/test/scripts/build/test-runner.sh
@@ -6,7 +6,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
 REPO_DIR="$TEST_FRAMEWORK_ROOT/.."
 
-cd "$SCRIPT_DIR"
+pushd "$SCRIPT_DIR"
 
 # shellcheck disable=SC1091
 source "$REPO_DIR/scripts/utils/log"
@@ -39,3 +39,20 @@ cargo build \
 if [[ $TARGET == x86_64-pc-windows-gnu ]]; then
     TARGET="$TARGET" ./runner-image.sh
 fi
+
+popd
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        # Optionally move binaries to some known location
+        --output)
+            ARTIFACTS_DIR="$TEST_FRAMEWORK_ROOT/target/$TARGET/release"
+            mv -t "$1" "$ARTIFACTS_DIR/test-runner" "$ARTIFACTS_DIR/connection-checker"
+            ;;
+        *)
+            log_error "Unknown parameter: $1"
+            exit 1
+            ;;
+    esac
+    shift
+done

From d5e51c2e283a05c6cd7968ed4e418445161b2b2e Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Wed, 12 Feb 2025 15:52:29 +0100
Subject: [PATCH 04/15] Create composite action for checking out app repo (with
 submodules)

---
 .github/actions/checkout-app/action.yml | 14 ++++++++++++++
 .github/workflows/desktop-e2e.yml       | 20 ++++++--------------
 2 files changed, 20 insertions(+), 14 deletions(-)
 create mode 100644 .github/actions/checkout-app/action.yml

diff --git a/.github/actions/checkout-app/action.yml b/.github/actions/checkout-app/action.yml
new file mode 100644
index 000000000000..3c9290201ff5
--- /dev/null
+++ b/.github/actions/checkout-app/action.yml
@@ -0,0 +1,14 @@
+name: "Checkout app"
+description: "Checkout Mullvad VPN app repository (including submodules)"
+# TODO: Define input for path ?
+runs:
+  using: "composite"
+  steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    - name: Checkout submodules
+      run: |
+        git config --global --add safe.directory '*'
+        git submodule update --init --depth=1 dist-assets/binaries
+        git submodule update --init wireguard-go-rs/libwg/wireguard-go
+      shell: bash
diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index 2e67ee41b6bc..f53073ccddc6 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -120,11 +120,8 @@ jobs:
         run: echo "HOME=/root" >> $GITHUB_ENV
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Checkout submodules
-        run: |
-          git config --global --add safe.directory '*'
-          git submodule update --init --depth=1 dist-assets/binaries
-          git submodule update --init wireguard-go-rs/libwg/wireguard-go
+      - name: Checkout repository
+        uses: ./.github/actions/checkout-app
       - name: Build app
         env:
           USE_MOLD: false
@@ -185,11 +182,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-        with:
-          submodules: true
-      - name: Checkout submodules
-        run: |
-          git submodule update --init wireguard-go-rs/libwg/wireguard-go
+      - name: Checkout repository
+        uses: ./.github/actions/checkout-app
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
         with:
@@ -275,10 +269,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Checkout submodules
-        run: |
-          git config --global --add safe.directory '*'
-          git submodule update --init wireguard-go-rs/libwg/wireguard-go
+      - name: Checkout repository
+        uses: ./.github/actions/checkout-app
       - name: Install Go
         uses: actions/setup-go@v3
         with:

From abd95300bedd3b59051a63a2c0f375e591ca0adf Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Wed, 12 Feb 2025 15:56:07 +0100
Subject: [PATCH 05/15] Remove `USE_MOLD` env variable

Use default value instead (which is false)
---
 .github/workflows/desktop-e2e.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index f53073ccddc6..d3c9dd45801f 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -123,8 +123,6 @@ jobs:
       - name: Checkout repository
         uses: ./.github/actions/checkout-app
       - name: Build app
-        env:
-          USE_MOLD: false
         run: ./build.sh
       - name: Build test executable
         run: ./desktop/packages/mullvad-vpn/scripts/build-test-executable.sh

From 94d3b44cf6a0c9f9b9561323b028ebd8d06e159e Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Wed, 12 Feb 2025 14:39:45 +0100
Subject: [PATCH 06/15] Build test-manager once per e2e-test run

Build & cache test-manager for Linux. This includes not building `test-manager` in `run/ci.sh` (by default)
and building `test-manager` as a completely statically linked binary.
---
 .github/workflows/desktop-e2e.yml |  92 ++++++++++--
 test/scripts/run/ci.sh            |   8 --
 test/scripts/utils/download.sh    | 227 ++++++++++++++++++++++++++++++
 test/scripts/utils/lib.sh         |   7 +-
 4 files changed, 317 insertions(+), 17 deletions(-)
 create mode 100755 test/scripts/utils/download.sh

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index d3c9dd45801f..36ab128fe8a4 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -106,8 +106,9 @@ jobs:
           echo "inner_container_image=$(cat ./building/linux-container-image.txt)" >> $GITHUB_ENV
     outputs:
       container_image: ${{ env.inner_container_image }}
-  build-linux:
-    name: Build Linux
+
+  build-linux-app:
+    name: Build Linux App
     needs: prepare-linux
     runs-on: ubuntu-latest
     container:
@@ -126,7 +127,8 @@ jobs:
         run: ./build.sh
       - name: Build test executable
         run: ./desktop/packages/mullvad-vpn/scripts/build-test-executable.sh
-      - uses: actions/upload-artifact@v4
+      - name: Upload app
+        uses: actions/upload-artifact@v4
         if: '!cancelled()'
         with:
           name: linux-build
@@ -135,9 +137,59 @@ jobs:
             ./dist/*.deb
             ./dist/app-e2e-*
 
+  build-mullvad-version-linux:
+    name: Build mullvad-version
+    needs: prepare-linux
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ needs.prepare-linux.outputs.container_image }}
+    continue-on-error: true
+    steps:
+      # Fix for HOME path overridden by GH runners when building in containers, see:
+      # https://github.com/actions/runner/issues/863
+      - name: Fix HOME path
+        run: echo "HOME=/root" >> $GITHUB_ENV
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Build mullvad-version
+        run: |
+          cargo build --package mullvad-version --release
+          # TODO: Remove
+          mkdir bin
+          mv -t ./bin/ "$CARGO_TARGET_DIR/release/mullvad-version"
+        shell: bash
+      - name: Upload mullvad-version
+        uses: actions/upload-artifact@v4
+        with:
+          name: mullvad-version-linux
+          path: ./bin/mullvad-version
+          if-no-files-found: error
+      - name: Clean up Cargo artifacts
+        run: |
+          cargo clean
+
+  build-test-manager-linux:
+    name: Build Test Manager
+    needs: prepare-linux
+    runs-on: [self-hosted, desktop-test, Linux]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Build test-manager
+        run: ./test/scripts/container-run.sh cargo build --package test-manager --release
+      - uses: actions/upload-artifact@v4
+        if: '!cancelled()'
+        with:
+          name: linux-test-manager-build
+          path: |
+            ./test/target/release/test-manager
+      - name: Clean up Cargo artifacts
+        run: |
+          cargo clean
+
   e2e-test-linux:
     name: Linux end-to-end tests
-    needs: [prepare-matrices, build-linux]
+    needs: [prepare-matrices, build-linux-app, build-mullvad-version-linux, build-test-manager-linux]
     if: |
       !cancelled() &&
       needs.prepare-matrices.outputs.linux_matrix != '[]' &&
@@ -149,20 +201,44 @@ jobs:
       matrix:
         os: ${{ fromJSON(needs.prepare-matrices.outputs.linux_matrix) }}
     steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Create binaries directory & add to PATH
+        shell: bash -ieo pipefail {0}
+        run: |
+          # TODO: Put all binaries in a known folder: test-runner, connection-checker, mullvad-version
+          mkdir "${{ github.workspace }}/bin"
+          echo "${{ github.workspace }}/bin/" >> "$GITHUB_PATH"
+      - name: Download Test Manager
+        uses: actions/download-artifact@v4
+        if: ${{ needs.build-test-manager-linux.result == 'success' }}
+        with:
+          name: linux-test-manager-build
+          path: ${{ github.workspace }}/bin
+      - name: Download mullvad-version
+        uses: actions/download-artifact@v4
+        if: ${{ needs.build-test-manager-linux.result == 'success' }}
+        with:
+          name: mullvad-version-linux
+          path: ${{ github.workspace }}/bin
+      - name: Check binaries
+        run: ls -la ${{ github.workspace }}/bin
+        shell: bash
       - name: Download App
         uses: actions/download-artifact@v4
-        if: ${{ needs.build-linux.result == 'success' }}
+        if: ${{ needs.build-linux-app.result == 'success' }}
         with:
           name: linux-build
           path: ~/.cache/mullvad-test/packages
-      - name: Checkout repository
-        uses: actions/checkout@v4
       - name: Run end-to-end tests
         shell: bash -ieo pipefail {0}
         run: |
+          # TODO: Explain this. A directory with all the binaries required to run test-manager
+          export TEST_DIST_DIR="${{ github.workspace }}/bin/"
           git fetch --tags --prune-tags --force
           export TEST_FILTERS="${{ github.event.inputs.tests }}"
-          ./test/scripts/ci-runtests.sh ${{ matrix.os }}
+          # TEST_DIST_DIR
+          ./test/scripts/run/ci.sh ${{ matrix.os }}
       - name: Upload test report
         uses: actions/upload-artifact@v4
         if: '!cancelled()'
diff --git a/test/scripts/run/ci.sh b/test/scripts/run/ci.sh
index b55b640e0321..b1aff3a6e493 100755
--- a/test/scripts/run/ci.sh
+++ b/test/scripts/run/ci.sh
@@ -64,14 +64,6 @@ nice_time download_e2e_executable "$CURRENT_VERSION" "$TEST_OS"
 
 # TODO: This should def be it's own step in the GitHub actions workflow
 
-echo "**********************************"
-echo "* Building test manager"
-echo "**********************************"
-
-cargo build -p test-manager
-
-# TODO: This should def be it's own step in the GitHub actions workflow
-
 echo "**********************************"
 echo "* Running tests"
 echo "**********************************"
diff --git a/test/scripts/utils/download.sh b/test/scripts/utils/download.sh
new file mode 100755
index 000000000000..37549aea928d
--- /dev/null
+++ b/test/scripts/utils/download.sh
@@ -0,0 +1,227 @@
+#!/usr/bin/env bash
+
+set -eu
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
+REPO_ROOT="$TEST_FRAMEWORK_ROOT/.."
+
+export BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases"
+export BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds"
+
+function executable_not_found_in_dist_error {
+    1>&2 echo "Executable \"$1\" not found in specified dist dir. Exiting."
+    exit 1
+}
+
+# Returns the directory of the lib.sh script
+function get_test_utls_dir {
+    ## local script_path="${BASH_SOURCE[0]}"
+    ## local script_dir
+    ## if [[ -n "$script_path" ]]; then
+    ##     script_dir="$(cd "$(dirname "$script_path")" >/dev/null && pwd)"
+    ## else
+    ##     script_dir="$(cd "$(dirname "$0")" >/dev/null && pwd)"
+    ## fi
+    ## echo "$script_dir"
+    echo "$SCRIPT_DIR"
+}
+
+# Infer stable version from GitHub repo
+RELEASES=$(curl -sf https://api.github.com/repos/mullvad/mullvadvpn-app/releases | jq -r '[.[] | select(((.tag_name|(startswith("android") or startswith("ios"))) | not))]')
+LATEST_STABLE_RELEASE=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_name' <<<"$RELEASES")
+
+function get_current_version {
+    local app_dir
+    app_dir="$REPO_ROOT"
+    if [ -n "${TEST_DIST_DIR+x}" ]; then
+        if [ ! -x "${TEST_DIST_DIR%/}/mullvad-version" ]; then
+            executable_not_found_in_dist_error mullvad-version
+        fi
+        "${TEST_DIST_DIR%/}/mullvad-version"
+    else
+        cargo run -q --manifest-path="$app_dir/Cargo.toml" --bin mullvad-version
+    fi
+}
+
+CURRENT_VERSION=$(get_current_version)
+commit=$(git rev-parse HEAD^\{commit\})
+commit=${commit:0:6}
+
+TAG=$(git describe --exact-match HEAD 2>/dev/null || echo "")
+
+if [[ -n "$TAG" && ${CURRENT_VERSION} =~ -dev- ]]; then
+    # Remove disallowed version characters from the tag
+    CURRENT_VERSION+="+${TAG//[^0-9a-z_-]/}"
+fi
+
+export CURRENT_VERSION
+export LATEST_STABLE_RELEASE
+
+function print_available_releases {
+    for release in $(jq -r '.[].tag_name' <<<"$RELEASES"); do
+        echo "$release"
+    done
+}
+
+function get_package_dir {
+    local package_dir
+    if [[ -n "${PACKAGE_DIR+x}" ]]; then
+        # Resolve the package dir to an absolute path since cargo must be invoked from the test directory
+        package_dir=$(realpath "$PACKAGE_DIR")
+    elif [[ ("$(uname -s)" == "Darwin") ]]; then
+        package_dir="$HOME/Library/Caches/mullvad-test/packages"
+    elif [[ ("$(uname -s)" == "Linux") ]]; then
+        package_dir="$HOME/.cache/mullvad-test/packages"
+    else
+        echo "Unsupported OS" 1>&2
+        exit 1
+    fi
+
+    mkdir -p "$package_dir" || exit 1
+    # Clean up old packages
+    find "$package_dir" -type f -mtime +5 -delete || true
+
+    echo "$package_dir"
+    return 0
+}
+
+function nice_time {
+    SECONDS=0
+    if "$@"; then
+        result=0
+    else
+        result=$?
+    fi
+    s=$SECONDS
+    echo "\"$*\" completed in $((s / 60))m:$((s % 60))s"
+    return $result
+}
+# Matches $1 with a build version string and sets the following exported variables:
+# - BUILD_VERSION: The version part of the build string (e.g., "2024.3-beta1-dev-").
+# - COMMIT_HASH: The commit hash part of the build string (e.g., "abcdef").
+# - TAG: The tag part of the build string (e.g., "+tag").
+function parse_build_version {
+    if [[ "$1" =~ (^[0-9.]+(-beta[0-9]+)?-dev-)([0-9a-z]+)(\+[0-9a-z|-]+)?$ ]]; then
+        BUILD_VERSION="${BASH_REMATCH[1]}"
+        COMMIT_HASH="${BASH_REMATCH[3]}"
+        TAG="${BASH_REMATCH[4]}"
+        return 0
+    fi
+    return 1
+}
+
+# Returns 0 if $1 is a development build.
+function is_dev_version {
+    if [[ "$1" == *"-dev-"* ]]; then
+        return 0
+    fi
+    return 1
+}
+
+function get_app_filename {
+    local version=$1
+    local os=$2
+    if is_dev_version "$version"; then
+        parse_build_version "$version"
+        version="${BUILD_VERSION}${COMMIT_HASH}${TAG:-}"
+    fi
+    case $os in
+    debian* | ubuntu*)
+        echo "MullvadVPN-${version}_amd64.deb"
+        ;;
+    fedora*)
+        echo "MullvadVPN-${version}_x86_64.rpm"
+        ;;
+    windows*)
+        echo "MullvadVPN-${version}.exe"
+        ;;
+    macos*)
+        echo "MullvadVPN-${version}.pkg"
+        ;;
+    *)
+        echo "Unsupported target: $os" 1>&2
+        return 1
+        ;;
+    esac
+}
+
+function download_app_package {
+    local version=$1
+    local os=$2
+    local package_repo=""
+
+    if is_dev_version "$version"; then
+        package_repo="${BUILD_DEV_REPOSITORY}"
+    else
+        package_repo="${BUILD_RELEASE_REPOSITORY}"
+    fi
+
+    local filename
+    filename=$(get_app_filename "$version" "$os")
+    local url="${package_repo}/$version/$filename"
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    if [[ ! -f "$package_dir/$filename" ]]; then
+        echo "Downloading build for $version ($os) from $url"
+        if ! curl -sf -o "$package_dir/$filename" "$url"; then
+            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
+            exit 1
+        fi
+    else
+        echo "App package for version $version ($os) already exists at $package_dir/$filename, skipping download"
+    fi
+}
+
+function get_e2e_filename {
+    local version=$1
+    local os=$2
+    if is_dev_version "$version"; then
+        parse_build_version "$version"
+        version="${BUILD_VERSION}${COMMIT_HASH}"
+    fi
+    case $os in
+    debian* | ubuntu* | fedora*)
+        echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu"
+        ;;
+    windows*)
+        echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe"
+        ;;
+    macos*)
+        echo "app-e2e-tests-${version}-aarch64-apple-darwin"
+        ;;
+    *)
+        echo "Unsupported target: $os" 1>&2
+        return 1
+        ;;
+    esac
+}
+
+function download_e2e_executable {
+    local version=${1:?Error: version not set}
+    local os=${2:?Error: os not set}
+    local package_repo
+
+    if is_dev_version "$version"; then
+        package_repo="${BUILD_DEV_REPOSITORY}"
+    else
+        package_repo="${BUILD_RELEASE_REPOSITORY}"
+    fi
+
+    local filename
+    filename=$(get_e2e_filename "$version" "$os")
+    local url="${package_repo}/$version/additional-files/$filename"
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    if [[ ! -f "$package_dir/$filename" ]]; then
+        echo "Downloading e2e executable for $version ($os) from $url"
+        if ! curl -sf -o "$package_dir/$filename" "$url"; then
+            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
+            exit 1
+        fi
+    else
+        echo "GUI e2e executable for version $version ($os) already exists at $package_dir/$filename, skipping download"
+    fi
+}
diff --git a/test/scripts/utils/lib.sh b/test/scripts/utils/lib.sh
index 939289d3bf48..6825cfffa332 100755
--- a/test/scripts/utils/lib.sh
+++ b/test/scripts/utils/lib.sh
@@ -235,10 +235,15 @@ function run_tests_for_os {
         if [ ! -x "${TEST_DIST_DIR%/}/test-manager" ]; then
             executable_not_found_in_dist_error test-manager
         fi
+        # TODO: Assume test-manager is in path.
+        #test_manager="${TEST_DIST_DIR%/}/test-manager"
         test_manager="${TEST_DIST_DIR%/}/test-manager"
+        # test_manager="$(which test-manager)"
         runner_dir_flag=("--runner-dir" "$TEST_DIST_DIR")
     else
-        test_manager="cargo run --bin test-manager"
+        # TODO: Assume test-manager is in path.
+        #test_manager="cargo run --bin test-manager"
+        test_manager="$(which test-manager)"
         runner_dir_flag=()
     fi
 

From 91469cd4f7436961dc7e07366db31ffd58c4c0e0 Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Thu, 13 Feb 2025 14:45:34 +0100
Subject: [PATCH 07/15] Build `test-runner` binaries as a separate step

---
 .github/workflows/desktop-e2e.yml | 66 +++++++++++++++++++++++++------
 1 file changed, 55 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index 36ab128fe8a4..73c2eadc5dc3 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -143,7 +143,6 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: ${{ needs.prepare-linux.outputs.container_image }}
-    continue-on-error: true
     steps:
       # Fix for HOME path overridden by GH runners when building in containers, see:
       # https://github.com/actions/runner/issues/863
@@ -171,10 +170,13 @@ jobs:
   build-test-manager-linux:
     name: Build Test Manager
     needs: prepare-linux
-    runs-on: [self-hosted, desktop-test, Linux]
+    # Note: libssl-dev is installed on the test server, so build test-manager there for the sake of simplicity
+    #runs-on: [self-hosted, desktop-test, Linux]
+    runs-on: [self-hosted, se-got-github-runner-013]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
       - name: Build test-manager
         run: ./test/scripts/container-run.sh cargo build --package test-manager --release
       - uses: actions/upload-artifact@v4
@@ -183,18 +185,48 @@ jobs:
           name: linux-test-manager-build
           path: |
             ./test/target/release/test-manager
+          if-no-files-found: error
       - name: Clean up Cargo artifacts
         run: |
           cargo clean
 
+  build-test-runner-binaries-linux:
+    name: Build Test Runner Binaries
+    needs: prepare-linux
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ needs.prepare-linux.outputs.container_image }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Build binaries
+        run: |
+          # TODO: Remove
+          mkdir bin/
+          test/scripts/build/test-runner.sh linux
+          mv -t bin/ \
+            "$CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release/test-runner" \
+            "$CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release/connection-checker"
+      - uses: actions/upload-artifact@v4
+        if: '!cancelled()'
+        with:
+          name: linux-test-runner-binaries
+          path: bin/*
+          if-no-files-found: error
+
   e2e-test-linux:
     name: Linux end-to-end tests
-    needs: [prepare-matrices, build-linux-app, build-mullvad-version-linux, build-test-manager-linux]
+    # yamllint disable-line rule:line-length
+    needs: [prepare-matrices, build-linux-app, build-mullvad-version-linux, build-test-manager-linux, build-test-runner-binaries-linux]
     if: |
       !cancelled() &&
       needs.prepare-matrices.outputs.linux_matrix != '[]' &&
       needs.prepare-matrices.outputs.linux_matrix != ''
+    # runs-on: [self-hosted, se-got-github-runner-013] # Magnus<3
     runs-on: [self-hosted, desktop-test, Linux] # app-test-linux
+    # runs-on: ubuntu-latest
+    # container:
+    #   image: ${{ needs.prepare-linux.outputs.container_image }}
     timeout-minutes: 240
     strategy:
       fail-fast: false
@@ -221,15 +253,26 @@ jobs:
         with:
           name: mullvad-version-linux
           path: ${{ github.workspace }}/bin
-      - name: Check binaries
-        run: ls -la ${{ github.workspace }}/bin
-        shell: bash
-      - name: Download App
+      - name: Download Test Runner binaries
         uses: actions/download-artifact@v4
-        if: ${{ needs.build-linux-app.result == 'success' }}
+        if: ${{ needs.build-test-runner-binaries-linux.result == 'success' }}
         with:
-          name: linux-build
-          path: ~/.cache/mullvad-test/packages
+          name: linux-test-runner-binaries
+          path: ${{ github.workspace }}/bin
+      - name: chmod binaries
+        run: |
+          chmod +x ${{ github.workspace }}/bin/*
+        shell: bash
+      - name: Check binaries
+        run: |
+          ls -la ${{ github.workspace }}/bin
+        shell: bash
+# - name: Download App
+#uses: actions/download-artifact@v4
+#if: ${{ needs.build-linux-app.result == 'success' }}
+        #with:
+          #name: linux-build
+          #path: ~/.cache/mullvad-test/packages
       - name: Run end-to-end tests
         shell: bash -ieo pipefail {0}
         run: |
@@ -237,7 +280,8 @@ jobs:
           export TEST_DIST_DIR="${{ github.workspace }}/bin/"
           git fetch --tags --prune-tags --force
           export TEST_FILTERS="${{ github.event.inputs.tests }}"
-          # TEST_DIST_DIR
+          ls -la "$TEST_DIST_DIR"
+          ${{ github.workspace }}/bin/mullvad-version
           ./test/scripts/run/ci.sh ${{ matrix.os }}
       - name: Upload test report
         uses: actions/upload-artifact@v4

From 846109643a54cd2556fc10cc25e2ab0e45ae5287 Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Fri, 14 Feb 2025 10:12:18 +0100
Subject: [PATCH 08/15] Do not relaunch as root if only formatting test reports

---
 test/test-manager/src/main.rs | 49 +++++++++++++++--------------------
 1 file changed, 21 insertions(+), 28 deletions(-)

diff --git a/test/test-manager/src/main.rs b/test/test-manager/src/main.rs
index 2a043b4276ac..c64d8d416595 100644
--- a/test/test-manager/src/main.rs
+++ b/test/test-manager/src/main.rs
@@ -183,25 +183,12 @@ enum VmConfig {
     List,
 }
 
-#[cfg(target_os = "linux")]
-impl Args {
-    fn get_vnc_port(&self) -> Option<u16> {
-        match self.cmd {
-            Commands::RunTests { vnc, .. } | Commands::RunVm { vnc, .. } => vnc,
-            _ => None,
-        }
-    }
-}
-
 #[tokio::main]
 async fn main() -> Result<()> {
     logging::Logger::get_or_init();
 
     let args = Args::parse();
 
-    #[cfg(target_os = "linux")]
-    container::relaunch_with_rootlesskit(args.get_vnc_port()).await;
-
     let mut config = config::ConfigFile::load_or_default()
         .await
         .context("Failed to load config")?;
@@ -250,11 +237,29 @@ async fn main() -> Result<()> {
                 }
             },
         },
+        Commands::ListTests => {
+            println!("priority\tname");
+            for test in tests::get_test_descriptions() {
+                println!(
+                    "{priority:8}\t{name}",
+                    name = test.name,
+                    priority = test.priority.unwrap_or(0),
+                );
+            }
+            Ok(())
+        }
+        Commands::FormatTestReports { reports } => {
+            summary::print_summary_table(&reports).await;
+            Ok(())
+        }
         Commands::RunVm {
             vm,
             vnc,
             keep_changes,
         } => {
+            #[cfg(target_os = "linux")]
+            container::relaunch_with_rootlesskit(vnc).await;
+
             let mut config = config.clone();
             config.runtime_opts.keep_changes = keep_changes;
             config.runtime_opts.display = if vnc.is_some() {
@@ -269,17 +274,6 @@ async fn main() -> Result<()> {
 
             Ok(())
         }
-        Commands::ListTests => {
-            println!("priority\tname");
-            for test in tests::get_test_descriptions() {
-                println!(
-                    "{priority:8}\t{name}",
-                    name = test.name,
-                    priority = test.priority.unwrap_or(0),
-                );
-            }
-            Ok(())
-        }
         Commands::RunTests {
             vm,
             display,
@@ -296,6 +290,9 @@ async fn main() -> Result<()> {
             test_report,
             runner_dir,
         } => {
+            #[cfg(target_os = "linux")]
+            container::relaunch_with_rootlesskit(vnc).await;
+
             let mut config = config.clone();
             config.runtime_opts.display = match (display, vnc.is_some()) {
                 (false, false) => config::Display::None,
@@ -410,10 +407,6 @@ async fn main() -> Result<()> {
             // Propagate any error from the test run if applicable
             result?.anyhow()
         }
-        Commands::FormatTestReports { reports } => {
-            summary::print_summary_table(&reports).await;
-            Ok(())
-        }
         Commands::Update { name } => {
             let vm_config = vm::get_vm_config(&config, &name).context("Cannot get VM config")?;
 

From 4ac7490a7757f0d6355b984069186a3337d74ba9 Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Thu, 13 Feb 2025 10:50:16 +0100
Subject: [PATCH 09/15] Do not compile `test-manager` when compiling the test
 report

---
 .github/workflows/desktop-e2e.yml | 41 ++++++++++++++++++++-----------
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index 73c2eadc5dc3..1a83acc485a9 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -453,25 +453,38 @@ jobs:
     name: Result matrix
     needs: [e2e-test-linux, e2e-test-windows, e2e-test-macos]
     if: '!cancelled()'
-    runs-on: [self-hosted, desktop-test, Linux]
-    timeout-minutes: 240
-    strategy:
-      fail-fast: false
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ needs.prepare-linux.outputs.container_image }}
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
+      - name: Download test report
+        uses: actions/download-artifact@v4
         with:
-          path: ./test/.ci-logs/artifacts
+          pattern: '*_report'
+          merge-multiple: true
+      - name: Create binaries directory
+        shell: bash -ieo pipefail {0}
+        run: |
+          mkdir "${{ github.workspace }}/bin"
+      - name: Download report compiler
+        uses: actions/download-artifact@v4
+        with:
+          name: linux-test-manager-build
+          path: ${{ github.workspace }}/bin
+      - name: chmod binaries
+        run: |
+          chmod +x ${{ github.workspace }}/bin/*
+      - name: Check binaries
+        run: |
+          ls -la ${{ github.workspace }}/bin
+        shell: bash
       - name: Generate test result matrix
         shell: bash -ieo pipefail {0}
         run: |
-          cd test
-          # "Unpack" the downloaded report artifacts: https://github.com/actions/download-artifact/issues/141
-          cp ./.ci-logs/artifacts/*_report/*_report ./.ci-logs/
-          cargo run --bin test-manager format-test-reports ./.ci-logs/*_report \
-            | tee summary.html >> $GITHUB_STEP_SUMMARY
+          ${{ github.workspace }}/bin/test-manager \
+          format-test-reports ${{ github.workspace }}/*_report \
+          | tee summary.html >> $GITHUB_STEP_SUMMARY
       - uses: actions/upload-artifact@v4
         with:
           name: summary.html
-          path: test/summary.html
+          path: summary.html

From bd5841de6407e3ac3950be6b9f6e4ddf337f9f9d Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Mon, 17 Feb 2025 16:28:06 +0100
Subject: [PATCH 10/15] Cache Cargo artifacts between builds

---
 .github/workflows/desktop-e2e.yml | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index 1a83acc485a9..efc9d8bbdf56 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -123,8 +123,21 @@ jobs:
         uses: actions/checkout@v4
       - name: Checkout repository
         uses: ./.github/actions/checkout-app
+      - uses: actions/cache@v4
+        id: cache-app-cargo-artifacts
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
       - name: Build app
-        run: ./build.sh
+        run: |
+          export CARGO_TARGET_DIR=target/
+          ./build.sh
       - name: Build test executable
         run: ./desktop/packages/mullvad-vpn/scripts/build-test-executable.sh
       - name: Upload app
@@ -163,9 +176,6 @@ jobs:
           name: mullvad-version-linux
           path: ./bin/mullvad-version
           if-no-files-found: error
-      - name: Clean up Cargo artifacts
-        run: |
-          cargo clean
 
   build-test-manager-linux:
     name: Build Test Manager
@@ -267,12 +277,12 @@ jobs:
         run: |
           ls -la ${{ github.workspace }}/bin
         shell: bash
-# - name: Download App
-#uses: actions/download-artifact@v4
-#if: ${{ needs.build-linux-app.result == 'success' }}
-        #with:
-          #name: linux-build
-          #path: ~/.cache/mullvad-test/packages
+      - name: Download App
+        uses: actions/download-artifact@v4
+        if: ${{ needs.build-linux-app.result == 'success' }}
+        with:
+          name: linux-build
+          path: ~/.cache/mullvad-test/packages
       - name: Run end-to-end tests
         shell: bash -ieo pipefail {0}
         run: |

From 5b5f6ed6c59302011055c3c854b1dac4db9dd946 Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Tue, 18 Feb 2025 13:17:08 +0100
Subject: [PATCH 11/15] Remove custom checkout action

Use `submodule: recursive` option of `actions/checkout` to checkout all
required submodules instead.
---
 .github/actions/checkout-app/action.yml | 14 --------------
 .github/workflows/desktop-e2e.yml       | 12 ++++++------
 2 files changed, 6 insertions(+), 20 deletions(-)
 delete mode 100644 .github/actions/checkout-app/action.yml

diff --git a/.github/actions/checkout-app/action.yml b/.github/actions/checkout-app/action.yml
deleted file mode 100644
index 3c9290201ff5..000000000000
--- a/.github/actions/checkout-app/action.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-name: "Checkout app"
-description: "Checkout Mullvad VPN app repository (including submodules)"
-# TODO: Define input for path ?
-runs:
-  using: "composite"
-  steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-    - name: Checkout submodules
-      run: |
-        git config --global --add safe.directory '*'
-        git submodule update --init --depth=1 dist-assets/binaries
-        git submodule update --init wireguard-go-rs/libwg/wireguard-go
-      shell: bash
diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index efc9d8bbdf56..87ed61da8409 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -121,8 +121,8 @@ jobs:
         run: echo "HOME=/root" >> $GITHUB_ENV
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Checkout repository
-        uses: ./.github/actions/checkout-app
+        with:
+          submodules: recursive
       - uses: actions/cache@v4
         id: cache-app-cargo-artifacts
         with:
@@ -310,8 +310,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Checkout repository
-        uses: ./.github/actions/checkout-app
+        with:
+          submodules: recursive
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
         with:
@@ -397,8 +397,8 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Checkout repository
-        uses: ./.github/actions/checkout-app
+        with:
+          submodules: recursive
       - name: Install Go
         uses: actions/setup-go@v3
         with:

From d09abbc194d7607cf485046dece60805e42d9e28 Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Wed, 19 Feb 2025 09:11:53 +0100
Subject: [PATCH 12/15] fixup! Remove custom checkout action

---
 .github/workflows/desktop-e2e.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index 87ed61da8409..130031e95cd3 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -122,7 +122,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
-          submodules: recursive
+          submodules: true
       - uses: actions/cache@v4
         id: cache-app-cargo-artifacts
         with:
@@ -311,7 +311,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
-          submodules: recursive
+          submodules: true
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
         with:
@@ -398,7 +398,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
-          submodules: recursive
+          submodules: true
       - name: Install Go
         uses: actions/setup-go@v3
         with:

From a3b4dbc3528fa3bb5cde260d498de7e8d4af19fa Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Wed, 19 Feb 2025 12:30:14 +0100
Subject: [PATCH 13/15] fixup! Build test-manager once per e2e-test run

---
 .github/workflows/desktop-e2e.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index 130031e95cd3..3eac26afa48d 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -166,9 +166,6 @@ jobs:
       - name: Build mullvad-version
         run: |
           cargo build --package mullvad-version --release
-          # TODO: Remove
-          mkdir bin
-          mv -t ./bin/ "$CARGO_TARGET_DIR/release/mullvad-version"
         shell: bash
       - name: Upload mullvad-version
         uses: actions/upload-artifact@v4

From 50fd010fb66667152cbc1685730fa253cb30be06 Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Wed, 19 Feb 2025 12:30:14 +0100
Subject: [PATCH 14/15] fixup! Build test-manager once per e2e-test run

---
 .github/workflows/desktop-e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index 3eac26afa48d..a8af2327fa07 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -171,7 +171,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: mullvad-version-linux
-          path: ./bin/mullvad-version
+          path: ${{ env.CARGO_TARGET_DIR }}/release/mullvad-version
           if-no-files-found: error
 
   build-test-manager-linux:

From c0e2be44587183ab043b7e8038077c32df524f0d Mon Sep 17 00:00:00 2001
From: Markus Pettersson <markus.pettersson@mullvad.net>
Date: Wed, 19 Feb 2025 12:34:37 +0100
Subject: [PATCH 15/15] fixup! fixup! Remove custom checkout action

---
 .github/workflows/desktop-e2e.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index a8af2327fa07..5007aa6e20d9 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -123,6 +123,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           submodules: true
+          set-safe-directory: "*"
       - uses: actions/cache@v4
         id: cache-app-cargo-artifacts
         with: