Skip to content

Commit

Permalink
upload ifs archive in test component service when using HTTP
Browse files Browse the repository at this point in the history
  • Loading branch information
@noise64
noise64 committed Feb 4, 2025
1 parent a048882 commit 6c5010c
Show file tree
Hide file tree
Showing 6 changed files with 126 additions and 56 deletions.
2 changes: 2 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions golem-test-framework/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ async-dropper = { version = "0.3.1", features = ["simple", "tokio"] }
async-dropper-simple = { version = "0.2.6", features = ["no-default-bound"] }
async-scoped = "0.9.0"
async-trait = { workspace = true }
async_zip = { workspace = true, features = ["tokio", "tokio-fs", "deflate"] }
bytes = { workspace = true }
chrono = { workspace = true }
clap = { workspace = true }
Expand All @@ -43,6 +44,7 @@ reqwest = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
serde_yaml = { workspace = true }
tempfile = { workspace = true }
testcontainers = { workspace = true }
testcontainers-modules = { workspace = true }
tokio = { workspace = true }
Expand Down
7 changes: 5 additions & 2 deletions golem-test-framework/src/components/component_service/filesystem.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,8 @@ impl ComponentService for FileSystemComponentService {
component_id: &ComponentId,
local_path: &Path,
component_type: ComponentType,
files: Option<&[InitialComponentFile]>,
dynamic_linking: Option<&HashMap<String, DynamicLinkedInstance>>,
) -> u64 {
let target_dir = &self.root;

Expand All @@ -226,14 +228,15 @@ impl ComponentService for FileSystemComponentService {
let last_version = self.get_latest_version(component_id).await;
let new_version = last_version + 1;

let empty_linking = HashMap::<String, DynamicLinkedInstance>::new();
self.write_component_to_filesystem(
local_path,
component_id,
new_version,
component_type,
&[],
files.unwrap_or_default(),
false,
&HashMap::new(),
dynamic_linking.unwrap_or(&empty_linking),
)
.await
.expect("Failed to write component to filesystem");
Expand Down
148 changes: 102 additions & 46 deletions golem-test-framework/src/components/component_service/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ use crate::components::{
use crate::config::GolemClientProtocol;
use anyhow::anyhow;
use async_trait::async_trait;
use async_zip::base::write::ZipFileWriter;
use async_zip::{Compression, ZipEntryBuilder};
use futures_util::TryStreamExt;
use golem_api_grpc::proto::golem::component::v1::component_service_client::ComponentServiceClient as ComponentServiceGrpcClient;
use golem_api_grpc::proto::golem::component::v1::plugin_service_client::PluginServiceClient as PluginServiceGrpcClient;
Expand Down Expand Up @@ -46,9 +48,11 @@ use golem_common::model::{
use std::collections::HashMap;
use std::error::Error;
use std::fmt::{Display, Formatter};
use std::path::Path;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use tempfile::TempDir;
use tokio::fs;
use tokio::fs::File;
use tokio::io::AsyncReadExt;
use tokio::time::sleep;
Expand Down Expand Up @@ -297,14 +301,30 @@ pub trait ComponentService {
}
}
ComponentServiceClient::Http(client) => {
let archive = build_ifs_archive(Some(files)).await.map_err(|error| {
AddComponentError::Other(format!(
"Failed to build IFS archive golem-component-service add component: {error:?}"
))
})?;

let archive_file = match &archive {
Some((_, path)) =>
Some(File::open(path).await.map_err(|error| {
AddComponentError::Other(format!(
"Failed to open IFS archive golem-component-service add component: {error:?}"
))
})?),
None => None,
};

match client
.create_component(
name,
Some(&component_type),
file,
to_http_file_permissions(files).as_ref(),
None::<File>, // TODO: zipped files
to_http_dynamic_linking(dynamic_linking).as_ref(),
archive_file,
to_http_dynamic_linking(Some(dynamic_linking)).as_ref(),
)
.await
{
Expand Down Expand Up @@ -332,24 +352,8 @@ pub trait ComponentService {
component_id: &ComponentId,
local_path: &Path,
component_type: ComponentType,
) -> u64 {
self.update_component_with_files(
component_id,
local_path,
component_type,
&None,
&HashMap::new(),
)
.await
}

async fn update_component_with_files(
&self,
component_id: &ComponentId,
local_path: &Path,
component_type: ComponentType,
files: &Option<Vec<InitialComponentFile>>,
dynamic_linking: &HashMap<String, DynamicLinkedInstance>,
files: Option<&[InitialComponentFile]>,
dynamic_linking: Option<&HashMap<String, DynamicLinkedInstance>>,
) -> u64 {
let mut file = File::open(local_path)
.await
Expand All @@ -364,7 +368,7 @@ pub trait ComponentService {

let files: Vec<golem_api_grpc::proto::golem::component::InitialComponentFile> =
files
.iter()
.into_iter()
.flatten()
.map(|f| f.clone().into())
.collect::<Vec<_>>();
Expand All @@ -378,7 +382,8 @@ pub trait ComponentService {
files,
dynamic_linking: HashMap::from_iter(
dynamic_linking
.iter()
.into_iter()
.flatten()
.map(|(k, v)| (k.clone(), v.clone().into())),
),
},
Expand Down Expand Up @@ -424,6 +429,24 @@ pub trait ComponentService {
}
}
ComponentServiceClient::Http(client) => {
let archive = match build_ifs_archive(files).await {
Ok(archive) => archive,
Err(error) => panic!(
"Failed to build IFS archive in golem-component-service update component: {error:?}"
)
};

let archive_file = match &archive {
Some((_, path)) =>
match File::open(path).await {
Ok(file) => Some(file),
Err(error) => panic!(
"Failed to open IFS archive in golem-component-service update component: {error:?}"
)
}
None => None,
};

match client
.update_component(
&component_id.0,
Expand All @@ -433,7 +456,7 @@ pub trait ComponentService {
.as_ref()
.and_then(|files| to_http_file_permissions(files))
.as_ref(),
None::<File>, // TODO: zipped files
archive_file,
to_http_dynamic_linking(dynamic_linking).as_ref(),
)
.await
Expand Down Expand Up @@ -852,29 +875,62 @@ fn to_http_file_permissions(
}

fn to_http_dynamic_linking(
dynamic_linking: &HashMap<String, DynamicLinkedInstance>,
dynamic_linking: Option<&HashMap<String, DynamicLinkedInstance>>,
) -> Option<golem_client::model::DynamicLinking> {
let Some(dynamic_linking) = dynamic_linking else {
return None;
};
if dynamic_linking.is_empty() {
None
} else {
Some(golem_client::model::DynamicLinking {
dynamic_linking: dynamic_linking
.iter()
.map(|(k, v)| {
(
k.clone(),
match v {
DynamicLinkedInstance::WasmRpc(link) => {
golem_client::model::DynamicLinkedInstance::WasmRpc(
golem_client::model::DynamicLinkedWasmRpc {
target_interface_name: link.target_interface_name.clone(),
},
)
}
},
)
})
.collect(),
})
return None;
}

Some(golem_client::model::DynamicLinking {
dynamic_linking: dynamic_linking
.iter()
.map(|(k, v)| {
(
k.clone(),
match v {
DynamicLinkedInstance::WasmRpc(link) => {
golem_client::model::DynamicLinkedInstance::WasmRpc(
golem_client::model::DynamicLinkedWasmRpc {
target_interface_name: link.target_interface_name.clone(),
},
)
}
},
)
})
.collect(),
})
}

async fn build_ifs_archive(
files: Option<&[InitialComponentFile]>,
) -> crate::Result<Option<(TempDir, PathBuf)>> {
static ARCHIVE_NAME: &str = "ifs.zip";

let Some(files) = files else { return Ok(None) };
if files.is_empty() {
return Ok(None);
}

let temp_dir = tempfile::Builder::new()
.prefix("golem-test-framework-ifs-zip")
.tempdir()?;
let temp_file = File::create(temp_dir.path().join(ARCHIVE_NAME)).await?;
let mut zip_writer = ZipFileWriter::with_tokio(temp_file);

for file in files {
zip_writer
.write_entry_whole(
ZipEntryBuilder::new(file.key.0.clone().into(), Compression::Deflate),
&(fs::read(Path::new(file.path.as_path())).await?),
)
.await?;
}

zip_writer.close().await?;
let file_path = temp_dir.path().join(ARCHIVE_NAME);
Ok(Some((temp_dir, file_path)))
}
21 changes: 14 additions & 7 deletions golem-test-framework/src/dsl/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ pub trait TestDsl {
&self,
component_id: &ComponentId,
name: &str,
files: &Option<Vec<InitialComponentFile>>,
files: Option<&[InitialComponentFile]>,
) -> ComponentVersion;

async fn add_initial_component_file(
Expand Down Expand Up @@ -432,6 +432,7 @@ impl<T: TestDependencies + Send + Sync> TestDsl for T {
account_id: &AccountId,
path: &Path,
) -> InitialComponentFileKey {
// TODO: should not upload when using HTTP client with component service
let source_path = self.component_directory().join(path);
let data = tokio::fs::read(&source_path)
.await
Expand All @@ -446,24 +447,30 @@ impl<T: TestDependencies + Send + Sync> TestDsl for T {
async fn update_component(&self, component_id: &ComponentId, name: &str) -> ComponentVersion {
let source_path = self.component_directory().join(format!("{name}.wasm"));
self.component_service()
.update_component(component_id, &source_path, ComponentType::Durable)
.update_component(
component_id,
&source_path,
ComponentType::Durable,
None,
None,
)
.await
}

async fn update_component_with_files(
&self,
component_id: &ComponentId,
name: &str,
files: &Option<Vec<InitialComponentFile>>,
files: Option<&[InitialComponentFile]>,
) -> ComponentVersion {
let source_path = self.component_directory().join(format!("{name}.wasm"));
self.component_service()
.update_component_with_files(
.update_component(
component_id,
&source_path,
ComponentType::Durable,
files,
&HashMap::new(),
None,
)
.await
}
Expand Down Expand Up @@ -1570,7 +1577,7 @@ pub trait TestDslUnsafe {
&self,
component_id: &ComponentId,
name: &str,
files: &Option<Vec<InitialComponentFile>>,
files: Option<&[InitialComponentFile]>,
) -> ComponentVersion;
async fn add_initial_component_file(
&self,
Expand Down Expand Up @@ -1757,7 +1764,7 @@ impl<T: TestDsl + Sync> TestDslUnsafe for T {
&self,
component_id: &ComponentId,
name: &str,
files: &Option<Vec<InitialComponentFile>>,
files: Option<&[InitialComponentFile]>,
) -> ComponentVersion {
<T as TestDsl>::update_component_with_files(self, component_id, name, files).await
}
Expand Down
2 changes: 1 addition & 1 deletion integration-tests/tests/worker.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1722,7 +1722,7 @@ async fn worker_initial_files_after_automatic_worker_update(
.update_component_with_files(
&component_id,
"initial-file-read-write",
&Some(component_files2),
Some(&component_files2),
)
.await;
deps.auto_update_worker(&worker_id, target_version).await;
Expand Down

0 comments on commit 6c5010c

Please sign in to comment.