From 2176fdb805a25c2fbee508a9f1ab0f82298be8c6 Mon Sep 17 00:00:00 2001 From: Hector Cao <122458375+hector-cao@users.noreply.github.com> Date: Wed, 15 Jan 2025 21:48:56 +0100 Subject: [PATCH] tests reorganization and add main test script (#307) * tests reorganization and add main test script 1 - reorganization of the tests folder tests having same category live in a subfolder 2 - expose pytest capabilities to users currently, tox is masking all pytest features and only expose what we specify in tox.ini file since tox is only here to manage the test environment it should be transparent to the users for tests execution and management. the users should be able to take full advantage of what pytest offers. we introduced a new script tdtest that hides tox and exopose all pytest features to users * Update tests/README.md Co-authored-by: Will French * update tdtest param --------- Co-authored-by: Will French --- tests/README.md | 82 +++---------------- tests/{tests => lib}/guest/test_tdreport.py | 0 tests/tdtest | 80 ++++++++++++++++++ tests/tests/{ => boot}/test_boot_basic.py | 0 tests/tests/{ => boot}/test_boot_coexist.py | 0 tests/tests/{ => boot}/test_boot_multiple.py | 0 .../tests/{ => boot}/test_boot_td_creation.py | 0 .../{ => eventlog}/test_guest_eventlog.py | 0 .../{ => eventlog}/test_guest_measurement.py | 0 tests/tests/{ => guest}/test_guest_cpu_off.py | 0 tests/tests/{ => guest}/test_guest_fail.py | 0 tests/tests/{ => guest}/test_guest_memory.py | 0 .../{ => guest}/test_guest_tsc_config.py | 0 tests/tests/{ => guest}/test_nmi_debug_off.py | 0 tests/tests/{ => host}/test_host_kdump.sh | 0 tests/tests/{ => host}/test_host_kexec.sh | 0 .../{ => host}/test_host_tdx_hardware.py | 0 .../{ => host}/test_host_tdx_software.py | 0 tests/tests/{ => perf}/test_perf_benchmark.py | 0 tests/tests/{ => perf}/test_perf_boot_time.py | 0 tests/tests/{ => quote}/test_guest_ita.py | 0 .../tests/{ => quote}/test_guest_tdxattest.py | 0 .../{ => quote}/test_quote_configfs_tsm.py | 0 tests/tests/{ => report}/test_guest_report.py | 2 +- tests/tests/{ => stress}/test_stress_boot.py | 0 tests/tests/{ => stress}/test_stress_quote.py | 0 .../{ => stress}/test_stress_resources.py | 0 tests/tests/{ => vsock}/test_vsock_vm.py | 0 tests/tox.ini | 42 +--------- 29 files changed, 96 insertions(+), 110 deletions(-) rename tests/{tests => lib}/guest/test_tdreport.py (100%) create mode 100755 tests/tdtest rename tests/tests/{ => boot}/test_boot_basic.py (100%) rename tests/tests/{ => boot}/test_boot_coexist.py (100%) rename tests/tests/{ => boot}/test_boot_multiple.py (100%) rename tests/tests/{ => boot}/test_boot_td_creation.py (100%) rename tests/tests/{ => eventlog}/test_guest_eventlog.py (100%) rename tests/tests/{ => eventlog}/test_guest_measurement.py (100%) rename tests/tests/{ => guest}/test_guest_cpu_off.py (100%) rename tests/tests/{ => guest}/test_guest_fail.py (100%) rename tests/tests/{ => guest}/test_guest_memory.py (100%) rename tests/tests/{ => guest}/test_guest_tsc_config.py (100%) rename tests/tests/{ => guest}/test_nmi_debug_off.py (100%) rename tests/tests/{ => host}/test_host_kdump.sh (100%) rename tests/tests/{ => host}/test_host_kexec.sh (100%) rename tests/tests/{ => host}/test_host_tdx_hardware.py (100%) rename tests/tests/{ => host}/test_host_tdx_software.py (100%) rename tests/tests/{ => perf}/test_perf_benchmark.py (100%) rename tests/tests/{ => perf}/test_perf_boot_time.py (100%) rename tests/tests/{ => quote}/test_guest_ita.py (100%) rename tests/tests/{ => quote}/test_guest_tdxattest.py (100%) rename tests/tests/{ => quote}/test_quote_configfs_tsm.py (100%) rename tests/tests/{ => report}/test_guest_report.py (92%) rename tests/tests/{ => stress}/test_stress_boot.py (100%) rename tests/tests/{ => stress}/test_stress_quote.py (100%) rename tests/tests/{ => stress}/test_stress_resources.py (100%) rename tests/tests/{ => vsock}/test_vsock_vm.py (100%) diff --git a/tests/README.md b/tests/README.md index 018d73b7..2d3e8431 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,88 +4,30 @@ This folder contains Intel TDX tests. ### Pre-requisites -- The tests must be executed a host that has been setup properly for Intel TDX. +- The tests must be executed on a host that has been set up properly for Intel TDX. -- Tox must be installed along with python3: -``` -$ sudo apt install tox -$ sudo apt install python3 -``` - -- You must specify a path to the guest image with `TDXTEST_GUEST_IMG` environment variable. - This is for both pytest and checkbox tests. - -- The guest image must enable ssh server with password-based authentication for `root` user. - The root user password must be `123456` - -### Run tests with tox/pytest - -Go to the `tests` folder. +### Run tests -- Run sanity tests to check the host setup: +The script `tdtest` is the test runner, it is a wrapper around `pytest`. +Please run `tdtest -h` for more details about its usage. -``` -$ sudo -E tox -e test_host -``` +The tests are organized in different categories and this organization is +reflected in the structure of the `tests`. -- Run guest tests: +You can choose to run a category of tests by specifying the appropriate sudirectory under `tests`. For example, to run the boot tests: ``` -$ sudo -E tox -e test_guest +$ sudo ./tdtest tests/boot ``` -- Run boot tests: +You can run all tests except the performance: ``` -$ sudo -E tox -e test_boot +$ sudo ./tdtest -k 'not test_perf' ``` -- Run perf tests: - -``` -$ sudo -E tox -e test_perf -``` - -- Run quote tests: - -``` -$ sudo -E tox -e test_quote -``` - -- Run stress tests: - -``` -$ sudo -E tox -e test_stress -``` - -- Run all tests: - -Please note that the performance tests can take a long time (order of magnitude of a few hours per `pytest.test_perf_benchmark`) to run. - -``` -$ sudo -E tox -e test_all -``` - -To list tests without running: - -``` -sudo -E tox -e collect_tests -- EXPRESSION -``` - -`EXPRESSION` is the expression provided to the argument `-k` of `pytest`. -For more details on its format, you can refer to `pytest` help page. - -For example: - -``` -sudo -E tox -e collect_tests -- 'test_guest or test_stress' -``` - -To run tests matching an expression: - -``` -sudo -E tox -e test_specify -- EXPRESSION -``` +Since `tdtest` is a wrapper of pytest, it exposes all the features of `pytest` +that you can use to run, manage and inspect the tests. ### Run tests with checkbox: diff --git a/tests/tests/guest/test_tdreport.py b/tests/lib/guest/test_tdreport.py similarity index 100% rename from tests/tests/guest/test_tdreport.py rename to tests/lib/guest/test_tdreport.py diff --git a/tests/tdtest b/tests/tdtest new file mode 100755 index 00000000..5a1584a9 --- /dev/null +++ b/tests/tdtest @@ -0,0 +1,80 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +usage() { + cat < Ubuntu guest image + -h|--help Show this help + +Examples: + To run tests in boot folder: + $ sudo $(basename "$0") tests/boot + To list tests in boot folder: + $ sudo $(basename "$0") --co tests/boot + To run all the tests: + $ sudo $(basename "$0") + +--- +EOM +} + +parse_params() { + while :; do + case "${1-}" in + -h | --help) + usage + exit 0 + ;; + -i | --td-image) + TDXTEST_GUEST_IMG=$(realpath "${2-}") + shift + ;; + "") + break + ;; + *) + # other arguments will be passed to pytest + pytest_args="$@" + break + ;; + esac + shift + done +} + +parse_params "$@" + +if [ "$EUID" -ne 0 ] + then echo "Please run as root" + exit +fi + +if [[ -z "${UBUNTU_VER}" ]]; then + UBUNTU_VER=$(lsb_release -rs) +fi + +if [[ ! -f "${TDXTEST_GUEST_IMG}" ]]; then + TDXTEST_GUEST_IMG=$PWD/../guest-tools/image/tdx-guest-ubuntu-${UBUNTU_VER}-generic.qcow2 +fi + +apt install -y python3 tox + +export TDXTEST_GUEST_IMG + +echo "Run tests with TD image: ${TDXTEST_GUEST_IMG}" +echo " pytest args: ${pytest_args}" +tox -- --ignore=lib/ "${pytest_args}" diff --git a/tests/tests/test_boot_basic.py b/tests/tests/boot/test_boot_basic.py similarity index 100% rename from tests/tests/test_boot_basic.py rename to tests/tests/boot/test_boot_basic.py diff --git a/tests/tests/test_boot_coexist.py b/tests/tests/boot/test_boot_coexist.py similarity index 100% rename from tests/tests/test_boot_coexist.py rename to tests/tests/boot/test_boot_coexist.py diff --git a/tests/tests/test_boot_multiple.py b/tests/tests/boot/test_boot_multiple.py similarity index 100% rename from tests/tests/test_boot_multiple.py rename to tests/tests/boot/test_boot_multiple.py diff --git a/tests/tests/test_boot_td_creation.py b/tests/tests/boot/test_boot_td_creation.py similarity index 100% rename from tests/tests/test_boot_td_creation.py rename to tests/tests/boot/test_boot_td_creation.py diff --git a/tests/tests/test_guest_eventlog.py b/tests/tests/eventlog/test_guest_eventlog.py similarity index 100% rename from tests/tests/test_guest_eventlog.py rename to tests/tests/eventlog/test_guest_eventlog.py diff --git a/tests/tests/test_guest_measurement.py b/tests/tests/eventlog/test_guest_measurement.py similarity index 100% rename from tests/tests/test_guest_measurement.py rename to tests/tests/eventlog/test_guest_measurement.py diff --git a/tests/tests/test_guest_cpu_off.py b/tests/tests/guest/test_guest_cpu_off.py similarity index 100% rename from tests/tests/test_guest_cpu_off.py rename to tests/tests/guest/test_guest_cpu_off.py diff --git a/tests/tests/test_guest_fail.py b/tests/tests/guest/test_guest_fail.py similarity index 100% rename from tests/tests/test_guest_fail.py rename to tests/tests/guest/test_guest_fail.py diff --git a/tests/tests/test_guest_memory.py b/tests/tests/guest/test_guest_memory.py similarity index 100% rename from tests/tests/test_guest_memory.py rename to tests/tests/guest/test_guest_memory.py diff --git a/tests/tests/test_guest_tsc_config.py b/tests/tests/guest/test_guest_tsc_config.py similarity index 100% rename from tests/tests/test_guest_tsc_config.py rename to tests/tests/guest/test_guest_tsc_config.py diff --git a/tests/tests/test_nmi_debug_off.py b/tests/tests/guest/test_nmi_debug_off.py similarity index 100% rename from tests/tests/test_nmi_debug_off.py rename to tests/tests/guest/test_nmi_debug_off.py diff --git a/tests/tests/test_host_kdump.sh b/tests/tests/host/test_host_kdump.sh similarity index 100% rename from tests/tests/test_host_kdump.sh rename to tests/tests/host/test_host_kdump.sh diff --git a/tests/tests/test_host_kexec.sh b/tests/tests/host/test_host_kexec.sh similarity index 100% rename from tests/tests/test_host_kexec.sh rename to tests/tests/host/test_host_kexec.sh diff --git a/tests/tests/test_host_tdx_hardware.py b/tests/tests/host/test_host_tdx_hardware.py similarity index 100% rename from tests/tests/test_host_tdx_hardware.py rename to tests/tests/host/test_host_tdx_hardware.py diff --git a/tests/tests/test_host_tdx_software.py b/tests/tests/host/test_host_tdx_software.py similarity index 100% rename from tests/tests/test_host_tdx_software.py rename to tests/tests/host/test_host_tdx_software.py diff --git a/tests/tests/test_perf_benchmark.py b/tests/tests/perf/test_perf_benchmark.py similarity index 100% rename from tests/tests/test_perf_benchmark.py rename to tests/tests/perf/test_perf_benchmark.py diff --git a/tests/tests/test_perf_boot_time.py b/tests/tests/perf/test_perf_boot_time.py similarity index 100% rename from tests/tests/test_perf_boot_time.py rename to tests/tests/perf/test_perf_boot_time.py diff --git a/tests/tests/test_guest_ita.py b/tests/tests/quote/test_guest_ita.py similarity index 100% rename from tests/tests/test_guest_ita.py rename to tests/tests/quote/test_guest_ita.py diff --git a/tests/tests/test_guest_tdxattest.py b/tests/tests/quote/test_guest_tdxattest.py similarity index 100% rename from tests/tests/test_guest_tdxattest.py rename to tests/tests/quote/test_guest_tdxattest.py diff --git a/tests/tests/test_quote_configfs_tsm.py b/tests/tests/quote/test_quote_configfs_tsm.py similarity index 100% rename from tests/tests/test_quote_configfs_tsm.py rename to tests/tests/quote/test_quote_configfs_tsm.py diff --git a/tests/tests/test_guest_report.py b/tests/tests/report/test_guest_report.py similarity index 92% rename from tests/tests/test_guest_report.py rename to tests/tests/report/test_guest_report.py index 5d9f9a5b..f360aadf 100644 --- a/tests/tests/test_guest_report.py +++ b/tests/tests/report/test_guest_report.py @@ -34,6 +34,6 @@ def test_guest_report(qm): deploy_and_setup(m) - m.check_exec(f'python3 {guest_workdir}/tests/guest/test_tdreport.py') + m.check_exec(f'python3 {guest_workdir}/lib/guest/test_tdreport.py') qm.stop() diff --git a/tests/tests/test_stress_boot.py b/tests/tests/stress/test_stress_boot.py similarity index 100% rename from tests/tests/test_stress_boot.py rename to tests/tests/stress/test_stress_boot.py diff --git a/tests/tests/test_stress_quote.py b/tests/tests/stress/test_stress_quote.py similarity index 100% rename from tests/tests/test_stress_quote.py rename to tests/tests/stress/test_stress_quote.py diff --git a/tests/tests/test_stress_resources.py b/tests/tests/stress/test_stress_resources.py similarity index 100% rename from tests/tests/test_stress_resources.py rename to tests/tests/stress/test_stress_resources.py diff --git a/tests/tests/test_vsock_vm.py b/tests/tests/vsock/test_vsock_vm.py similarity index 100% rename from tests/tests/test_vsock_vm.py rename to tests/tests/vsock/test_vsock_vm.py diff --git a/tests/tox.ini b/tests/tox.ini index 3045f8a8..6e34ee51 100644 --- a/tests/tox.ini +++ b/tests/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - test_host + tdx isolated_build = True skip_missing_interpreters = True @@ -22,42 +22,6 @@ commands_pre = bash -c 'if [ `id -u` -ne 0 ]; then echo "Must run all tests with sudo" 1>&2; (exit -1); fi' {toxinidir}/tox/setup-env-tox.sh -[testenv:test_guest] -commands = - python3 -m pytest -s -v --junitxml=test_guest_report.xml --ignore=tests/guest -k test_guest - -[testenv:test_host] -commands = - python3 -m pytest -s -v --junitxml=test_host_report.xml --ignore=tests/guest -k test_host - -[testenv:test_boot] -commands = - python3 -m pytest -s -v --junitxml=test_boot_report.xml --ignore=tests/guest -k test_boot - -[testenv:test_perf] -commands = - python3 -m pytest -s -v --junitxml=test_perf_report.xml --ignore=tests/guest -k test_perf - -[testenv:test_quote] -commands = - python3 -m pytest -s -v --junitxml=test_quote_report.xml --ignore=tests/guest -k test_quote - -[testenv:test_stress] -commands = - python3 -m pytest -s -v --junitxml=test_stress_report.xml --ignore=tests/guest -k test_stress - -[testenv:test_all_except_perf] -commands = - python3 -m pytest -s -v --junitxml=test_all_report.xml --ignore=tests/guest -k 'not test_perf' - -[testenv:test_all] -commands = - python3 -m pytest -s -v --junitxml=test_all_report.xml --ignore=tests/guest - -[testenv:test_specify] -commands = - python3 -m pytest -s -v --junitxml=test_specify_report.xml --ignore=tests/guest -k {posargs} - -[testenv:collect_tests] +[testenv:tdx] commands = - python3 -m pytest -s -v --collect-only --ignore=tests/guest -k {posargs} + python3 -m pytest -s -v {posargs}