From ddfe307ecade81835ce22cb5ff4c9fc089eb68a4 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Thu, 30 Jan 2025 16:36:57 +0100 Subject: [PATCH 1/5] Timeout, updated test-r, external epoch loop --- Cargo.lock | 12 ++++---- Cargo.toml | 2 +- golem-worker-executor-base/src/lib.rs | 30 +++++++++---------- .../tests/guest_languages2.rs | 3 +- 4 files changed, 23 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dac686ef7..bb1294392 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9554,9 +9554,9 @@ dependencies = [ [[package]] name = "test-r" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d8cb299f722f145e6379e79c8c406f923340f28b33497c3957ef9cf9fa2b225" +checksum = "5c752de9d8ad6913f1c869438aa9a3a5ce40fa65d5c0319e77f58d3208b492dc" dependencies = [ "ctor", "test-r-core", @@ -9566,9 +9566,9 @@ dependencies = [ [[package]] name = "test-r-core" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "065e701871bdcf15c796c9c12c7806b7f67b75603f54ddcf9531f927a54919bb" +checksum = "bc0d07b4ec88b97a69cb45b6a9cfb302872f7eb1269387370ad98555454d0bbd" dependencies = [ "anstream", "anstyle", @@ -9589,9 +9589,9 @@ dependencies = [ [[package]] name = "test-r-macro" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3356b06edde5f3c37f868e1113fb86347a0f4837814fe45c98d4dc778a2f2eb2" +checksum = "196317fa7e132bc12e40927b4e17f70e0379ed80505c753fd607b5936cc3cbeb" dependencies = [ "darling", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index a9e18f52b..1acb43013 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -188,7 +188,7 @@ testcontainers-modules = { version = "0.11.4", features = [ "redis", "minio", ] } -test-r = { version = "2.0.1", default-features = true } +test-r = { version = "2.1.0", default-features = true } thiserror = "2.0.6" tokio = { version = "1.42", features = [ "macros", diff --git a/golem-worker-executor-base/src/lib.rs b/golem-worker-executor-base/src/lib.rs index d1d768341..aa31fdb2d 100644 --- a/golem-worker-executor-base/src/lib.rs +++ b/golem-worker-executor-base/src/lib.rs @@ -99,6 +99,7 @@ const VERSION: &str = golem_version!(); pub struct RunDetails { pub http_port: u16, pub grpc_port: u16, + pub epoch_thread: std::thread::JoinHandle<()>, } /// The Bootstrap trait should be implemented by all Worker Executors to customize the initialization @@ -249,12 +250,11 @@ pub trait Bootstrap { let lazy_worker_activator = Arc::new(LazyWorkerActivator::new()); - let worker_executor_impl = create_worker_executor_impl::( + let (worker_executor_impl, epoch_thread) = create_worker_executor_impl::( golem_config.clone(), self, runtime.clone(), &lazy_worker_activator, - join_set, ) .await?; @@ -272,6 +272,7 @@ pub trait Bootstrap { Ok(RunDetails { http_port, grpc_port: addr.port(), + epoch_thread, }) } } @@ -281,8 +282,7 @@ async fn create_worker_executor_impl + ?Sized> bootstrap: &A, runtime: Handle, lazy_worker_activator: &Arc>, - join_set: &mut JoinSet>, -) -> Result, anyhow::Error> { +) -> Result<(All, std::thread::JoinHandle<()>), anyhow::Error> { let (redis, sqlite, key_value_storage): ( Option, Option, @@ -473,17 +473,13 @@ async fn create_worker_executor_impl + ?Sized> let engine = Arc::new(Engine::new(&config)?); let linker = bootstrap.create_wasmtime_linker(&engine)?; - let mut epoch_interval = tokio::time::interval(golem_config.limits.epoch_interval); let engine_ref: Arc = engine.clone(); - join_set.spawn( - async move { - loop { - epoch_interval.tick().await; - engine_ref.increment_epoch(); - } - } - .in_current_span(), - ); + + let epoch_interval = golem_config.limits.epoch_interval; + let epoch_thread = std::thread::spawn(move || loop { + std::thread::sleep(epoch_interval); + engine_ref.increment_epoch(); + }); let linker = Arc::new(linker); @@ -539,7 +535,7 @@ async fn create_worker_executor_impl + ?Sized> golem_config.scheduler.refresh_interval, ); - bootstrap + let all = bootstrap .create_services( active_workers, engine, @@ -564,5 +560,7 @@ async fn create_worker_executor_impl + ?Sized> plugins, oplog_processor_plugin, ) - .await + .await?; + + Ok((all, epoch_thread)) } diff --git a/golem-worker-executor-base/tests/guest_languages2.rs b/golem-worker-executor-base/tests/guest_languages2.rs index 557bd1381..0e620ba3d 100644 --- a/golem-worker-executor-base/tests/guest_languages2.rs +++ b/golem-worker-executor-base/tests/guest_languages2.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use test_r::{inherit_test_dep, test}; +use test_r::{inherit_test_dep, test, timeout}; use crate::common::{start, TestContext}; use crate::{LastUniqueId, Tracing, WorkerExecutorTestDependencies}; @@ -28,6 +28,7 @@ inherit_test_dep!(Tracing); #[test] #[tracing::instrument] +#[timeout(180_000)] async fn javascript_example_3( last_unique_id: &LastUniqueId, deps: &WorkerExecutorTestDependencies, From a9dcc0b7b53ed55acb2a0755f35bc6fb079282e2 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Thu, 30 Jan 2025 17:23:00 +0100 Subject: [PATCH 2/5] Reenabled parallel tests --- Makefile.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile.toml b/Makefile.toml index 868afb19b..0baa709db 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -221,8 +221,8 @@ dependencies = ["wit"] description = "Runs worker executor tests only (group 1/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' -cargo test --package golem-worker-executor-base --test integration :tag: -- --report-time -cargo test --package golem-worker-executor-base --test integration :tag:group1 -- --report-time +cargo test --package golem-worker-executor-base --test integration :tag: -- --report-time --nocapture $JUNIT_OPTS +cargo test --package golem-worker-executor-base --test integration :tag:group1 -- --report-time --nocapture $JUNIT_OPTS ''' [tasks.worker-executor-tests-group2] @@ -230,7 +230,7 @@ dependencies = ["wit"] description = "Runs worker executor tests only (group 2/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' -cargo test --package golem-worker-executor-base --test integration :tag:group2 -- --report-time +cargo test --package golem-worker-executor-base --test integration :tag:group2 -- --report-time --nocapture $JUNIT_OPTS ''' [tasks.worker-executor-tests-group3] @@ -238,7 +238,7 @@ dependencies = ["wit"] description = "Runs worker executor tests only (group 3/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' -cargo test --package golem-worker-executor-base --test integration :tag:group3 -- --report-time +cargo test --package golem-worker-executor-base --test integration :tag:group3 -- --report-time --nocapture $JUNIT_OPTS ''' [tasks.worker-executor-tests-group4] @@ -246,7 +246,7 @@ dependencies = ["wit"] description = "Runs worker executor tests only (group 4/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' -cargo test --package golem-worker-executor-base --test integration :tag:group4 -- --report-time $JUNIT_OPTS +cargo test --package golem-worker-executor-base --test integration :tag:group4 -- --report-time --nocapture $JUNIT_OPTS ''' [tasks.worker-executor-tests-group5] @@ -254,7 +254,7 @@ dependencies = ["wit"] description = "Runs worker executor tests only (group 5/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' -cargo test --package golem-worker-executor-base --test integration :tag:group5 -- --report-time +cargo test --package golem-worker-executor-base --test integration :tag:group5 -- --report-time --nocapture $JUNIT_OPTS ''' [tasks.worker-executor-tests-group6] @@ -262,7 +262,7 @@ dependencies = ["wit"] description = "Runs worker executor tests only (group 6/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' -cargo test --package golem-worker-executor-base --test integration :tag:group6 -- --report-time +cargo test --package golem-worker-executor-base --test integration :tag:group6 -- --report-time --nocapture $JUNIT_OPTS ''' [tasks.worker-executor-tests-group7] @@ -270,7 +270,7 @@ dependencies = ["wit"] description = "Runs worker executor tests only (group 7/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' -cargo test --package golem-worker-executor-base --test integration :tag:group7 -- --report-time +cargo test --package golem-worker-executor-base --test integration :tag:group7 -- --report-time --nocapture $JUNIT_OPTS ''' [tasks.worker-executor-tests-group8] @@ -278,7 +278,7 @@ dependencies = ["wit"] description = "Runs worker executor tests only (group 8/8)" env = { "RUST_BACKTRACE" = "1", "WASMTIME_BACKTRACE_DETAILS" = "1", "RUST_LOG" = "info", "RUST_TEST_TIME_INTEGRATION" = "5000,30000" } script = ''' -cargo test --package golem-worker-executor-base --test integration :tag:group8 -- --report-time +cargo test --package golem-worker-executor-base --test integration :tag:group8 -- --report-time --nocapture $JUNIT_OPTS ''' [tasks.integration-tests] From 118adb1e769bae22def5c764c91f7b9f539cc332 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Thu, 30 Jan 2025 17:44:34 +0100 Subject: [PATCH 3/5] Increase timeout, reduce parallelism --- .github/workflows/ci.yaml | 2 ++ golem-worker-executor-base/tests/guest_languages2.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6bf718f99..92602e357 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -411,6 +411,8 @@ jobs: include_passed: true cli-tests: runs-on: ubuntu-latest-xlarge + env: + CARGO_BUILD_JOBS: 6 steps: - name: Checkout uses: actions/checkout@v4 diff --git a/golem-worker-executor-base/tests/guest_languages2.rs b/golem-worker-executor-base/tests/guest_languages2.rs index 0e620ba3d..a495d6a8d 100644 --- a/golem-worker-executor-base/tests/guest_languages2.rs +++ b/golem-worker-executor-base/tests/guest_languages2.rs @@ -28,7 +28,7 @@ inherit_test_dep!(Tracing); #[test] #[tracing::instrument] -#[timeout(180_000)] +#[timeout(300_000)] async fn javascript_example_3( last_unique_id: &LastUniqueId, deps: &WorkerExecutorTestDependencies, From 7878f1953a3a3464ec275e521e292c6d9370f056 Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Thu, 30 Jan 2025 19:26:33 +0100 Subject: [PATCH 4/5] Removed huge debug print --- golem-worker-executor-base/tests/observability.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/golem-worker-executor-base/tests/observability.rs b/golem-worker-executor-base/tests/observability.rs index 21be74cb4..418b32272 100644 --- a/golem-worker-executor-base/tests/observability.rs +++ b/golem-worker-executor-base/tests/observability.rs @@ -244,7 +244,7 @@ async fn get_oplog_with_api_changing_updates( .filter(|entry| !matches!(entry, PublicOplogEntry::PendingWorkerInvocation(_))) .collect::>(); - println!("oplog\n{:#?}", oplog); + // println!("oplog\n{:#?}", oplog); check!(result[0] == Value::U64(11)); assert_eq!(oplog.len(), 17); From c574be9c373585dd7091f42981ff662e5e7b5c1b Mon Sep 17 00:00:00 2001 From: Daniel Vigovszky Date: Thu, 30 Jan 2025 19:26:42 +0100 Subject: [PATCH 5/5] Always collect jUnit reports even if cancelled (by timeout) --- .github/workflows/ci.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 92602e357..dd3a9a697 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -242,7 +242,7 @@ jobs: run: cargo make --profile ci check - name: Publish Test Report uses: mikepenz/action-junit-report@v4 - if: success() || failure() # always run even if the previous step fails + if: always() with: report_paths: '**/target/report-*.xml' detailed_summary: true @@ -310,7 +310,7 @@ jobs: timeout-minutes: 20 - name: Publish Test Report uses: mikepenz/action-junit-report@v4 - if: success() || failure() # always run even if the previous step fails + if: always() with: report_paths: '**/target/report-*.xml' detailed_summary: true @@ -360,7 +360,7 @@ jobs: timeout-minutes: 40 - name: Publish Test Report uses: mikepenz/action-junit-report@v4 - if: success() || failure() # always run even if the previous step fails + if: always() with: report_paths: '**/target/report-*.xml' detailed_summary: true @@ -404,7 +404,7 @@ jobs: timeout-minutes: 30 - name: Publish Test Report uses: mikepenz/action-junit-report@v4 - if: success() || failure() # always run even if the previous step fails + if: always() with: report_paths: '**/target/report-*.xml' detailed_summary: true @@ -445,7 +445,7 @@ jobs: timeout-minutes: 35 - name: Publish Test Report uses: mikepenz/action-junit-report@v4 - if: success() || failure() # always run even if the previous step fails + if: always() with: report_paths: '**/target/report-*.xml' detailed_summary: true @@ -486,7 +486,7 @@ jobs: timeout-minutes: 40 - name: Publish Test Report uses: mikepenz/action-junit-report@v4 - if: success() || failure() # always run even if the previous step fails + if: always() with: report_paths: '**/target/report-*.xml' detailed_summary: true