From fea6902fb800a62825d8ee9fcb48cc53bd7a0df5 Mon Sep 17 00:00:00 2001 From: ykim-1 Date: Mon, 20 Jan 2025 10:20:31 -0800 Subject: [PATCH] Refactor make test commands and update related workflows --- .github/workflows/e2e-suite-windows.yml | 2 +- .github/workflows/e2e-suite.yml | 20 +++++++-------- .github/workflows/nightly-smoke-tests.yml | 2 +- Makefile | 30 ++++++++++------------- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/.github/workflows/e2e-suite-windows.yml b/.github/workflows/e2e-suite-windows.yml index 0c347f74..c3e030a6 100644 --- a/.github/workflows/e2e-suite-windows.yml +++ b/.github/workflows/e2e-suite-windows.yml @@ -73,7 +73,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - run: make MODULE="${{ inputs.module }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" testint + - run: make MODULE="${{ inputs.module }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" test-int env: LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN_2 }} diff --git a/.github/workflows/e2e-suite.yml b/.github/workflows/e2e-suite.yml index e58b1fde..c9af831b 100644 --- a/.github/workflows/e2e-suite.yml +++ b/.github/workflows/e2e-suite.yml @@ -4,14 +4,14 @@ on: workflow_dispatch: inputs: use_minimal_test_account: - description: 'Use minimal test account' + description: 'Indicate whether to use a minimal test account with limited resources for testing. Defaults to "false"' required: false default: 'false' - module: - description: "The module from 'test/integration' to the target to be tested, e.g. 'cli, domains, events, etc'" + test_suite: + description: "Specify test suite to run from the 'tests/integration' directory. Examples: 'cli', 'domains', 'events', etc. If not provided, all suites are executed" required: false run_long_tests: - description: "Select True to run long tests, e.g. database, rebuild, etc" + description: "Select 'True' to include long-running tests (e.g., database provisioning, server rebuilds). Defaults to 'False'" required: false type: choice options: @@ -19,21 +19,21 @@ on: - "False" default: "False" sha: - description: 'The hash value of the commit.' + description: 'Specify commit hash to test. This value is mandatory to ensure the tests run against a specific commit' required: true default: '' pull_request_number: - description: 'The number of the PR. Ensure sha value is provided' + description: 'Specify pull request number associated with the commit. Optional, but recommended when providing a commit hash (sha)' required: false openapi_spec_url: - description: 'URL of the OpenAPI spec to use for the tests' + description: 'Specify URL of the OpenAPI specification file to use for testing. Useful for validating tests against a specific API version or custom specification' required: false default: '' python-version: - description: 'Specify Python version to use' + description: 'Specify the Python version to use for running tests. Leave empty to use the default Python version configured in the environment' required: false run-eol-python-version: - description: 'Run EOL python version?' + description: 'Indicates whether to run tests using an End-of-Life (EOL) Python version. Defaults to "false". Choose "true" to include tests for deprecated Python versions' required: false default: 'false' type: choice @@ -124,7 +124,7 @@ jobs: run: | timestamp=$(date +'%Y%m%d%H%M') report_filename="${timestamp}_cli_test_report.xml" - make testint TEST_ARGS="--junitxml=${report_filename}" MODULE="${{ inputs.module }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" + make test-int TEST_ARGS="--junitxml=${report_filename}" TEST_SUITE="${{ inputs.test_suite }}" RUN_LONG_TESTS="${{ inputs.run_long_tests }}" env: LINODE_CLI_TOKEN: ${{ env.LINODE_CLI_TOKEN }} diff --git a/.github/workflows/nightly-smoke-tests.yml b/.github/workflows/nightly-smoke-tests.yml index 10cbad35..372de28c 100644 --- a/.github/workflows/nightly-smoke-tests.yml +++ b/.github/workflows/nightly-smoke-tests.yml @@ -40,7 +40,7 @@ jobs: - name: Run smoke tests id: smoke_tests run: | - make smoketest + make test-smoke env: LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }} diff --git a/Makefile b/Makefile index 04963b9d..27eb9355 100644 --- a/Makefile +++ b/Makefile @@ -2,15 +2,6 @@ # Makefile for more convenient building of the Linode CLI and its baked content # -# Test-related arguments -MODULE := -TEST_CASE_COMMAND := -TEST_ARGS := - -ifdef TEST_CASE -TEST_CASE_COMMAND = -k $(TEST_CASE) -endif - SPEC_VERSION ?= latest ifndef SPEC override SPEC = $(shell ./resolve_spec_url ${SPEC_VERSION}) @@ -66,8 +57,8 @@ clean: rm -f data-* rm -rf dist linode_cli.egg-info build -.PHONY: testunit -testunit: +.PHONY: test-unit +test-unit: @mkdir -p /tmp/linode/.config @orig_xdg_config_home=$${XDG_CONFIG_HOME:-}; \ export LINODE_CLI_TEST_MODE=1 XDG_CONFIG_HOME=/tmp/linode/.config; \ @@ -76,9 +67,14 @@ testunit: export XDG_CONFIG_HOME=$$orig_xdg_config_home; \ exit $$exit_code -.PHONY: testint -testint: - pytest tests/integration/${MODULE} ${TEST_CASE_COMMAND} ${TEST_ARGS} +# Integration Test Arguments +# TEST_SUITE: Optional, specify a test suite (e.g. domains), Default to run everything if not set +# TEST_CASE: Optional, specify a test case (e.g. 'test_create_a_domain') +# TEST_ARGS: Optional, additional arguments for pytest (e.g. '-v' for verbose mode) + +.PHONY: test-int +test-int: + pytest tests/integration/$(TEST_SUITE) $(if $(TEST_CASE),-k $(TEST_CASE)) $(TEST_ARGS) .PHONY: testall testall: @@ -86,7 +82,7 @@ testall: # Alias for unit; integration tests should be explicit .PHONY: test -test: testunit +test: test-unit .PHONY: black black: @@ -103,6 +99,6 @@ autoflake: .PHONY: format format: black isort autoflake -@PHONEY: smoketest -smoketest: +@PHONEY: test-smoke +test-smoke: pytest -m smoke tests/integration