From 6c5010c1961d9b313812053cec7a8028738b878b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?D=C3=A1vid=20Istv=C3=A1n=20B=C3=ADr=C3=B3?= Date: Tue, 4 Feb 2025 10:56:01 +0100 Subject: [PATCH] upload ifs archive in test component service when using HTTP --- Cargo.lock | 2 + golem-test-framework/Cargo.toml | 2 + .../component_service/filesystem.rs | 7 +- .../src/components/component_service/mod.rs | 148 ++++++++++++------ golem-test-framework/src/dsl/mod.rs | 21 ++- integration-tests/tests/worker.rs | 2 +- 6 files changed, 126 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 107dc8cbf..9b8dc7824 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4068,6 +4068,7 @@ dependencies = [ "async-dropper-simple", "async-scoped", "async-trait", + "async_zip", "bytes 1.9.0", "chrono", "clap", @@ -4094,6 +4095,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "tempfile", "test-r", "testcontainers", "testcontainers-modules", diff --git a/golem-test-framework/Cargo.toml b/golem-test-framework/Cargo.toml index 7ba3e2769..c694bb26c 100644 --- a/golem-test-framework/Cargo.toml +++ b/golem-test-framework/Cargo.toml @@ -23,6 +23,7 @@ async-dropper = { version = "0.3.1", features = ["simple", "tokio"] } async-dropper-simple = { version = "0.2.6", features = ["no-default-bound"] } async-scoped = "0.9.0" async-trait = { workspace = true } +async_zip = { workspace = true, features = ["tokio", "tokio-fs", "deflate"] } bytes = { workspace = true } chrono = { workspace = true } clap = { workspace = true } @@ -43,6 +44,7 @@ reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_yaml = { workspace = true } +tempfile = { workspace = true } testcontainers = { workspace = true } testcontainers-modules = { workspace = true } tokio = { workspace = true } diff --git a/golem-test-framework/src/components/component_service/filesystem.rs b/golem-test-framework/src/components/component_service/filesystem.rs index 1cbd26ca4..918a0cbb3 100644 --- a/golem-test-framework/src/components/component_service/filesystem.rs +++ b/golem-test-framework/src/components/component_service/filesystem.rs @@ -210,6 +210,8 @@ impl ComponentService for FileSystemComponentService { component_id: &ComponentId, local_path: &Path, component_type: ComponentType, + files: Option<&[InitialComponentFile]>, + dynamic_linking: Option<&HashMap>, ) -> u64 { let target_dir = &self.root; @@ -226,14 +228,15 @@ impl ComponentService for FileSystemComponentService { let last_version = self.get_latest_version(component_id).await; let new_version = last_version + 1; + let empty_linking = HashMap::::new(); self.write_component_to_filesystem( local_path, component_id, new_version, component_type, - &[], + files.unwrap_or_default(), false, - &HashMap::new(), + dynamic_linking.unwrap_or(&empty_linking), ) .await .expect("Failed to write component to filesystem"); diff --git a/golem-test-framework/src/components/component_service/mod.rs b/golem-test-framework/src/components/component_service/mod.rs index 8ff812bf5..06d595b41 100644 --- a/golem-test-framework/src/components/component_service/mod.rs +++ b/golem-test-framework/src/components/component_service/mod.rs @@ -19,6 +19,8 @@ use crate::components::{ use crate::config::GolemClientProtocol; use anyhow::anyhow; use async_trait::async_trait; +use async_zip::base::write::ZipFileWriter; +use async_zip::{Compression, ZipEntryBuilder}; use futures_util::TryStreamExt; use golem_api_grpc::proto::golem::component::v1::component_service_client::ComponentServiceClient as ComponentServiceGrpcClient; use golem_api_grpc::proto::golem::component::v1::plugin_service_client::PluginServiceClient as PluginServiceGrpcClient; @@ -46,9 +48,11 @@ use golem_common::model::{ use std::collections::HashMap; use std::error::Error; use std::fmt::{Display, Formatter}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; +use tempfile::TempDir; +use tokio::fs; use tokio::fs::File; use tokio::io::AsyncReadExt; use tokio::time::sleep; @@ -297,14 +301,30 @@ pub trait ComponentService { } } ComponentServiceClient::Http(client) => { + let archive = build_ifs_archive(Some(files)).await.map_err(|error| { + AddComponentError::Other(format!( + "Failed to build IFS archive golem-component-service add component: {error:?}" + )) + })?; + + let archive_file = match &archive { + Some((_, path)) => + Some(File::open(path).await.map_err(|error| { + AddComponentError::Other(format!( + "Failed to open IFS archive golem-component-service add component: {error:?}" + )) + })?), + None => None, + }; + match client .create_component( name, Some(&component_type), file, to_http_file_permissions(files).as_ref(), - None::, // TODO: zipped files - to_http_dynamic_linking(dynamic_linking).as_ref(), + archive_file, + to_http_dynamic_linking(Some(dynamic_linking)).as_ref(), ) .await { @@ -332,24 +352,8 @@ pub trait ComponentService { component_id: &ComponentId, local_path: &Path, component_type: ComponentType, - ) -> u64 { - self.update_component_with_files( - component_id, - local_path, - component_type, - &None, - &HashMap::new(), - ) - .await - } - - async fn update_component_with_files( - &self, - component_id: &ComponentId, - local_path: &Path, - component_type: ComponentType, - files: &Option>, - dynamic_linking: &HashMap, + files: Option<&[InitialComponentFile]>, + dynamic_linking: Option<&HashMap>, ) -> u64 { let mut file = File::open(local_path) .await @@ -364,7 +368,7 @@ pub trait ComponentService { let files: Vec = files - .iter() + .into_iter() .flatten() .map(|f| f.clone().into()) .collect::>(); @@ -378,7 +382,8 @@ pub trait ComponentService { files, dynamic_linking: HashMap::from_iter( dynamic_linking - .iter() + .into_iter() + .flatten() .map(|(k, v)| (k.clone(), v.clone().into())), ), }, @@ -424,6 +429,24 @@ pub trait ComponentService { } } ComponentServiceClient::Http(client) => { + let archive = match build_ifs_archive(files).await { + Ok(archive) => archive, + Err(error) => panic!( + "Failed to build IFS archive in golem-component-service update component: {error:?}" + ) + }; + + let archive_file = match &archive { + Some((_, path)) => + match File::open(path).await { + Ok(file) => Some(file), + Err(error) => panic!( + "Failed to open IFS archive in golem-component-service update component: {error:?}" + ) + } + None => None, + }; + match client .update_component( &component_id.0, @@ -433,7 +456,7 @@ pub trait ComponentService { .as_ref() .and_then(|files| to_http_file_permissions(files)) .as_ref(), - None::, // TODO: zipped files + archive_file, to_http_dynamic_linking(dynamic_linking).as_ref(), ) .await @@ -852,29 +875,62 @@ fn to_http_file_permissions( } fn to_http_dynamic_linking( - dynamic_linking: &HashMap, + dynamic_linking: Option<&HashMap>, ) -> Option { + let Some(dynamic_linking) = dynamic_linking else { + return None; + }; if dynamic_linking.is_empty() { - None - } else { - Some(golem_client::model::DynamicLinking { - dynamic_linking: dynamic_linking - .iter() - .map(|(k, v)| { - ( - k.clone(), - match v { - DynamicLinkedInstance::WasmRpc(link) => { - golem_client::model::DynamicLinkedInstance::WasmRpc( - golem_client::model::DynamicLinkedWasmRpc { - target_interface_name: link.target_interface_name.clone(), - }, - ) - } - }, - ) - }) - .collect(), - }) + return None; + } + + Some(golem_client::model::DynamicLinking { + dynamic_linking: dynamic_linking + .iter() + .map(|(k, v)| { + ( + k.clone(), + match v { + DynamicLinkedInstance::WasmRpc(link) => { + golem_client::model::DynamicLinkedInstance::WasmRpc( + golem_client::model::DynamicLinkedWasmRpc { + target_interface_name: link.target_interface_name.clone(), + }, + ) + } + }, + ) + }) + .collect(), + }) +} + +async fn build_ifs_archive( + files: Option<&[InitialComponentFile]>, +) -> crate::Result> { + static ARCHIVE_NAME: &str = "ifs.zip"; + + let Some(files) = files else { return Ok(None) }; + if files.is_empty() { + return Ok(None); } + + let temp_dir = tempfile::Builder::new() + .prefix("golem-test-framework-ifs-zip") + .tempdir()?; + let temp_file = File::create(temp_dir.path().join(ARCHIVE_NAME)).await?; + let mut zip_writer = ZipFileWriter::with_tokio(temp_file); + + for file in files { + zip_writer + .write_entry_whole( + ZipEntryBuilder::new(file.key.0.clone().into(), Compression::Deflate), + &(fs::read(Path::new(file.path.as_path())).await?), + ) + .await?; + } + + zip_writer.close().await?; + let file_path = temp_dir.path().join(ARCHIVE_NAME); + Ok(Some((temp_dir, file_path))) } diff --git a/golem-test-framework/src/dsl/mod.rs b/golem-test-framework/src/dsl/mod.rs index 6a63d1a31..05b6dd4c5 100644 --- a/golem-test-framework/src/dsl/mod.rs +++ b/golem-test-framework/src/dsl/mod.rs @@ -187,7 +187,7 @@ pub trait TestDsl { &self, component_id: &ComponentId, name: &str, - files: &Option>, + files: Option<&[InitialComponentFile]>, ) -> ComponentVersion; async fn add_initial_component_file( @@ -432,6 +432,7 @@ impl TestDsl for T { account_id: &AccountId, path: &Path, ) -> InitialComponentFileKey { + // TODO: should not upload when using HTTP client with component service let source_path = self.component_directory().join(path); let data = tokio::fs::read(&source_path) .await @@ -446,7 +447,13 @@ impl TestDsl for T { async fn update_component(&self, component_id: &ComponentId, name: &str) -> ComponentVersion { let source_path = self.component_directory().join(format!("{name}.wasm")); self.component_service() - .update_component(component_id, &source_path, ComponentType::Durable) + .update_component( + component_id, + &source_path, + ComponentType::Durable, + None, + None, + ) .await } @@ -454,16 +461,16 @@ impl TestDsl for T { &self, component_id: &ComponentId, name: &str, - files: &Option>, + files: Option<&[InitialComponentFile]>, ) -> ComponentVersion { let source_path = self.component_directory().join(format!("{name}.wasm")); self.component_service() - .update_component_with_files( + .update_component( component_id, &source_path, ComponentType::Durable, files, - &HashMap::new(), + None, ) .await } @@ -1570,7 +1577,7 @@ pub trait TestDslUnsafe { &self, component_id: &ComponentId, name: &str, - files: &Option>, + files: Option<&[InitialComponentFile]>, ) -> ComponentVersion; async fn add_initial_component_file( &self, @@ -1757,7 +1764,7 @@ impl TestDslUnsafe for T { &self, component_id: &ComponentId, name: &str, - files: &Option>, + files: Option<&[InitialComponentFile]>, ) -> ComponentVersion { ::update_component_with_files(self, component_id, name, files).await } diff --git a/integration-tests/tests/worker.rs b/integration-tests/tests/worker.rs index ce180f142..a2622cc1c 100644 --- a/integration-tests/tests/worker.rs +++ b/integration-tests/tests/worker.rs @@ -1722,7 +1722,7 @@ async fn worker_initial_files_after_automatic_worker_update( .update_component_with_files( &component_id, "initial-file-read-write", - &Some(component_files2), + Some(&component_files2), ) .await; deps.auto_update_worker(&worker_id, target_version).await;