From ac7f0c360b58dad8d62eeaa82401effc2eef6ead Mon Sep 17 00:00:00 2001 From: Facundo Villa Date: Fri, 18 Aug 2023 00:43:01 -0300 Subject: [PATCH] Resource manager revamp. --- .gitignore | 7 +- {resources => assets}/Box.gltf | 0 {resources => assets}/cube.json | 0 {resources => assets}/shaders/fragment.besl | 0 {resources => assets}/shaders/vertex.besl | 0 src/camera.rs | 2 +- src/file_tracker.rs | 2 +- src/render_domain.rs | 37 +- src/render_system.rs | 6 + .../image_resource_handler.rs | 4 +- .../material_resource_handler.rs | 94 ++--- src/resource_manager/mesh_resource_handler.rs | 4 +- src/resource_manager/mod.rs | 354 +++++++++++------- 13 files changed, 329 insertions(+), 181 deletions(-) rename {resources => assets}/Box.gltf (100%) rename {resources => assets}/cube.json (100%) rename {resources => assets}/shaders/fragment.besl (100%) rename {resources => assets}/shaders/vertex.besl (100%) diff --git a/.gitignore b/.gitignore index a752bcf1..bb3e27b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ **/target -**/assets +**/resources *.db -*.db.* \ No newline at end of file +*.db.* +.byte +.byte-engine +.byte-editor \ No newline at end of file diff --git a/resources/Box.gltf b/assets/Box.gltf similarity index 100% rename from resources/Box.gltf rename to assets/Box.gltf diff --git a/resources/cube.json b/assets/cube.json similarity index 100% rename from resources/cube.json rename to assets/cube.json diff --git a/resources/shaders/fragment.besl b/assets/shaders/fragment.besl similarity index 100% rename from resources/shaders/fragment.besl rename to assets/shaders/fragment.besl diff --git a/resources/shaders/vertex.besl b/assets/shaders/vertex.besl similarity index 100% rename from resources/shaders/vertex.besl rename to assets/shaders/vertex.besl diff --git a/src/camera.rs b/src/camera.rs index b4643037..8024d72a 100644 --- a/src/camera.rs +++ b/src/camera.rs @@ -1,4 +1,4 @@ -use crate::{Vec3f, orchestrator::{Property, EntityHandle, self, Component, Entity}}; +use crate::{Vec3f, orchestrator::{Property, self, Component, Entity}}; #[derive(component_derive::Component)] /// Camera struct diff --git a/src/file_tracker.rs b/src/file_tracker.rs index 2ab5ba09..3552dd42 100644 --- a/src/file_tracker.rs +++ b/src/file_tracker.rs @@ -16,7 +16,7 @@ pub struct FileTracker { impl FileTracker { pub fn new(orchestrator: orchestrator::OrchestratorReference) -> FileTracker { - let db = polodb_core::Database::open_file("files.db").unwrap(); + let db = polodb_core::Database::open_file(".byte-editor/files.db").unwrap(); let (tx, rx) = std::sync::mpsc::channel(); diff --git a/src/render_domain.rs b/src/render_domain.rs index 659dca3c..03ee37fe 100644 --- a/src/render_domain.rs +++ b/src/render_domain.rs @@ -35,7 +35,8 @@ pub struct VisibilityWorldRenderDomain { mesh_resources: HashMap<&'static str, u32>, /// Maps resource ids to shaders - shaders: std::collections::HashMap, + /// The hash and the shader handle are stored to determine if the shader has changed + shaders: std::collections::HashMap, } impl VisibilityWorldRenderDomain { @@ -223,14 +224,14 @@ impl VisibilityWorldRenderDomain { let mut resource_manager = resource_manager.get_mut(); let resource_manager: &mut resource_manager::ResourceManager = resource_manager.downcast_mut().unwrap(); - let resource_request = resource_manager.get_resource_info(mesh.resource_id); + let resource_request = resource_manager.request_resource(mesh.resource_id); let resource_request = if let Some(resource_info) = resource_request { resource_info } else { return; }; let vertex_buffer = render_system.get_mut_buffer_slice(None, self.vertices_buffer); let index_buffer = render_system.get_mut_buffer_slice(None, self.indices_buffer); - let resource = resource_manager.load_resource_into_buffer(&resource_request, vertex_buffer, index_buffer); // TODO: add offset + let resource = resource_manager.load_resource(&resource_request, None, None); self.mesh_resources.insert(mesh.resource_id, self.index_count); @@ -238,10 +239,38 @@ impl VisibilityWorldRenderDomain { self.index_count += resource_info.index_count; - let material = resource_manager.get("cube.json"); + let material = resource_manager.get("cube"); let material = if let Some(s) = material { s } else { return; }; + for resource in material.0.get_array("resources").unwrap() { + let resource = resource.as_document().unwrap(); + let class = resource.get_str("class").unwrap(); + + match class { + "Shader" => { + let shader_name = resource.get_str("name").unwrap(); + let hash = resource.get_i64("hash").unwrap() as u64; + let resource_id = resource.get_i64("id").unwrap() as u64; + + if let Some((old_hash, old_shader)) = self.shaders.get(&resource_id) { + if *old_hash == hash { continue; } + } + + let offset = resource.get_i64("offset").unwrap() as usize; + let size = resource.get_i64("size").unwrap() as usize; + + let new_shader = render_system.add_shader(crate::render_system::ShaderSourceType::SPIRV, &material.1[offset..(offset + size)]); + + self.shaders.insert(resource_id, (hash, new_shader)); + } + "Material" => { + // TODO: update pipeline + } + _ => {} + } + } + let result = resource_manager.get("cube"); let result = if let Some(s) = result { s } else { return; }; diff --git a/src/render_system.rs b/src/render_system.rs index fc64928b..a003f032 100644 --- a/src/render_system.rs +++ b/src/render_system.rs @@ -26,6 +26,7 @@ pub fn select_by_score(values: &[T], score: impl Fn(&T) -> i64) -> Option<&T> best_value } +/// Handle to a Pipeline Layout #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct PipelineLayoutHandle(u32); @@ -59,6 +60,7 @@ struct Texture { role: String, } +/// Handle to a Sampler #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct SamplerHandle(u32); @@ -66,11 +68,15 @@ struct Sampler { sampler: render_backend::Sampler, } +/// Possible types of a shader source pub enum ShaderSourceType { + /// GLSL code string GLSL, + /// SPIR-V binary SPIRV, } +/// Primitive GPU/shader data types. #[derive(Hash, Clone, Copy)] pub enum DataTypes { Float, diff --git a/src/resource_manager/image_resource_handler.rs b/src/resource_manager/image_resource_handler.rs index 5daf681d..1ba91956 100644 --- a/src/resource_manager/image_resource_handler.rs +++ b/src/resource_manager/image_resource_handler.rs @@ -21,7 +21,7 @@ impl ResourceHandler for ImageResourceHandler { } } - fn process(&self, bytes: Vec) -> Result<(Document, Vec), String> { + fn process(&self, bytes: Vec) -> Result)>, String> { let mut decoder = png::Decoder::new(bytes.as_slice()); decoder.set_transformations(png::Transformations::EXPAND); let mut reader = decoder.read_info().unwrap(); @@ -59,7 +59,7 @@ impl ResourceHandler for ImageResourceHandler { "compression": "BC7", }; - Ok((resource_document, intel_tex_2::bc7::compress_blocks(&settings, &rgba_surface))) + Ok(vec![(resource_document, intel_tex_2::bc7::compress_blocks(&settings, &rgba_surface))]) } } diff --git a/src/resource_manager/material_resource_handler.rs b/src/resource_manager/material_resource_handler.rs index b19b86b2..2eb674a7 100644 --- a/src/resource_manager/material_resource_handler.rs +++ b/src/resource_manager/material_resource_handler.rs @@ -26,77 +26,85 @@ impl ResourceHandler for MaterialResourcerHandler { } } - fn process(&self, bytes: Vec) -> Result<(Document, Vec), String> { + fn process(&self, bytes: Vec) -> Result)>, String> { let material_json = json::parse(std::str::from_utf8(&bytes).unwrap()).unwrap(); let t = material_json["type"].as_str().unwrap(); let vertex = material_json["vertex"].as_str().unwrap(); let fragment = material_json["fragment"].as_str().unwrap(); - let vertex_path = "resources/".to_string() + vertex; - let fragment_path = "resources/".to_string() + fragment; + fn treat_shader(path: &str, t: &str) -> (Document, Vec) { + let path = "resources/".to_string() + path; + let shader_code = std::fs::read_to_string(path).unwrap(); + let shader = beshader_compiler::parse(beshader_compiler::tokenize(&shader_code)); - let vertex_shader_code = std::fs::read_to_string(vertex_path).unwrap(); - let fragment_shader_code = std::fs::read_to_string(fragment_path).unwrap(); + let mut shader_spec = json::object! { glsl: { version: "450" } }; - let vertex_shader = beshader_compiler::parse(beshader_compiler::tokenize(&vertex_shader_code)); - let fragment_shader = beshader_compiler::parse(beshader_compiler::tokenize(&fragment_shader_code)); + let common = crate::rendering::common_shader_generator::CommonShaderGenerator::new(); - let mut shader_spec = json::object! { glsl: { version: "450" } }; + let c = common.process(); - let common = crate::rendering::common_shader_generator::CommonShaderGenerator::new(); + shader_spec["root"][c.0] = c.1; - let c = common.process(); + let visibility = crate::rendering::visibility_shader_generator::VisibilityShaderGenerator::new(); - shader_spec["root"][c.0] = c.1; + let v = visibility.process(); - let visibility = crate::rendering::visibility_shader_generator::VisibilityShaderGenerator::new(); + shader_spec["root"][c.0][v.0] = v.1; - let v = visibility.process(); + let shader_generator = shader_generator::ShaderGenerator::new(); - shader_spec["root"][c.0][v.0] = v.1; + let glsl = shader_generator.generate(&shader_spec, &json::object!{ path: "Common.Visibility", type: t }); - let shader_generator = shader_generator::ShaderGenerator::new(); + let compiler = shaderc::Compiler::new().unwrap(); + let mut options = shaderc::CompileOptions::new().unwrap(); - let glsl = shader_generator.generate(&shader_spec, &json::object!{ path: "Common.Visibility", type: "fragment" }); + options.set_optimization_level(shaderc::OptimizationLevel::Performance); + options.set_target_env(shaderc::TargetEnv::Vulkan, shaderc::EnvVersion::Vulkan1_2 as u32); + options.set_generate_debug_info(); + options.set_target_spirv(shaderc::SpirvVersion::V1_5); + options.set_invert_y(true); - let compiler = shaderc::Compiler::new().unwrap(); - let mut options = shaderc::CompileOptions::new().unwrap(); + let binary = compiler.compile_into_spirv(&glsl, shaderc::ShaderKind::InferFromSource, "shader_name", "main", Some(&options)); - options.set_optimization_level(shaderc::OptimizationLevel::Performance); - options.set_target_env(shaderc::TargetEnv::Vulkan, shaderc::EnvVersion::Vulkan1_2 as u32); - options.set_generate_debug_info(); - options.set_target_spirv(shaderc::SpirvVersion::V1_5); - options.set_invert_y(true); + let compilation_artifact = match binary { Ok(binary) => { binary } Err(error) => { panic!(); } }; - let binary = compiler.compile_into_spirv(&glsl, shaderc::ShaderKind::InferFromSource, "shader_name", "main", Some(&options)); + if compilation_artifact.get_num_warnings() > 0 { + println!("Shader warnings: {}", compilation_artifact.get_warning_messages()); + } - let shader_stage: String = glsl.find("#pragma shader_stage(").map(|index| glsl[index + 21..].split(')').next().unwrap().to_string()).unwrap_or(String::from("")); + let result_shader_bytes = compilation_artifact.as_binary_u8(); - let shader_stage = match shader_stage.as_str() { - "vertex" => { crate::render_backend::ShaderTypes::Vertex }, - "fragment" => { crate::render_backend::ShaderTypes::Fragment }, - _ => { crate::render_backend::ShaderTypes::Vertex }, - }; - - let compilation_artifact = match binary { Ok(binary) => { binary } Err(error) => { return Err(error.to_string()); } }; + let mut hasher = DefaultHasher::new(); - if compilation_artifact.get_num_warnings() > 0 { - println!("Shader warnings: {}", compilation_artifact.get_warning_messages()); - } + std::hash::Hash::hash(&result_shader_bytes, &mut hasher); - let result_shader_bytes = compilation_artifact.as_binary_u8(); + let hash = hasher.finish() as i64; - let mut hasher = DefaultHasher::new(); + let resource = polodb_core::bson::doc!{ + "class": "Shader", + "hash": hash + }; - std::hash::Hash::hash(&result_shader_bytes, &mut hasher); - - let hash = hasher.finish() as i64; + (resource, Vec::from(result_shader_bytes)) + } - let resource = polodb_core::bson::doc!{ - "hash": hash + let a = treat_shader(vertex, "vertex"); + let b = treat_shader(fragment, "fragment"); + + let material_resource_document = polodb_core::bson::doc!{ + "class": "Material", + "required_resources": [ + { + "path": vertex, + }, + { + "path": fragment, + } + ], + "resource": {} }; - Ok((resource, Vec::from(result_shader_bytes))) + Ok(vec![a, b]) } } \ No newline at end of file diff --git a/src/resource_manager/mesh_resource_handler.rs b/src/resource_manager/mesh_resource_handler.rs index 65a060e7..d1848f24 100644 --- a/src/resource_manager/mesh_resource_handler.rs +++ b/src/resource_manager/mesh_resource_handler.rs @@ -22,7 +22,7 @@ impl ResourceHandler for MeshResourceHandler { } } - fn process(&self, bytes: Vec) -> Result<(Document, Vec), String> { + fn process(&self, bytes: Vec) -> Result)>, String> { let (gltf, buffers, _) = gltf::import_slice(bytes.as_slice()).unwrap(); let mut buf: Vec = Vec::with_capacity(4096 * 1024 * 3); @@ -101,7 +101,7 @@ impl ResourceHandler for MeshResourceHandler { let serialized_mesh = mesh.serialize(polodb_core::bson::Serializer::new()).unwrap(); - Ok((serialized_mesh.as_document().unwrap().clone(), buf)) + Ok(vec![(serialized_mesh.as_document().unwrap().clone(), buf)]) } } diff --git a/src/resource_manager/mod.rs b/src/resource_manager/mod.rs index 8396c8dd..54fcc163 100644 --- a/src/resource_manager/mod.rs +++ b/src/resource_manager/mod.rs @@ -6,7 +6,7 @@ mod image_resource_handler; mod mesh_resource_handler; mod material_resource_handler; -use std::{io::prelude::*, sync::Arc}; +use std::{io::prelude::*, str::FromStr, hash::{Hasher, Hash},}; use polodb_core::bson::{Document, doc}; @@ -16,14 +16,35 @@ use crate::orchestrator::{System, self}; trait ResourceHandler { fn can_handle_type(&self, resource_type: &str) -> bool; - fn process(&self, bytes: Vec) -> Result<(Document, Vec), String>; + + /// Returns a tuple containing the resource description and it's associated binary data.\ + /// + /// The returned document is like the following: + /// ```json + /// { "class": "X", "resource": { ... }, "hash": 0, "required_resources":[{ "path": "..." }] } + /// ``` + /// Fields: + /// - **class**: The resource class. This is used to identify the resource type. Needs to be meaningful and will be a public constant. + /// - **resource**: The resource data. Can look like anything. + /// - **hash**(optional): The resource hash. This is used to identify the resource data. If the resource handler wants to generate a hash for the resource it can do so else the resource manager will generate a hash for it. This is because some resources can generate hashes inteligently (EJ: code generators can output same hash for different looking code if the code is semantically identical). + /// - **required_resources**(optional): A list of resources that this resource depends on. This is used to load resources that depend on other resources. + fn process(&self, bytes: Vec) -> Result)>, String>; } /// Resource manager. /// Handles loading assets or resources from different origins (network, local, etc.). /// It also handles caching of resources. /// +/// Resource can be sourced from the local filesystem or from the network. /// When in a debug build it will lazily load resources from source and cache them. +/// When in a release build it will exclusively load resources from cache. +/// +/// If accessing the filesystem paths will be relative to the assets directory, and assets should omit the extension. +/// +/// The stored resource document is like the following: +/// ```json +/// { "_id":"OId" , "id": 01234, "path": "../..", "class": "X", "size": 0, "resource": { ... }, "hash": 0 } +/// ``` pub struct ResourceManager { db: polodb_core::Database, resource_handlers: Vec>, @@ -104,16 +125,28 @@ pub struct Request { impl ResourceManager { /// Creates a new resource manager. pub fn new() -> Self { - std::fs::create_dir_all("assets").unwrap(); + if let Err(error) = std::fs::create_dir_all("assets") { + match error.kind() { + std::io::ErrorKind::AlreadyExists => {}, + _ => panic!("Could not create assets directory"), + } + } + + if let Err(error) = std::fs::create_dir_all("resources") { + match error.kind() { + std::io::ErrorKind::AlreadyExists => {}, + _ => panic!("Could not create assets directory"), + } + } let mut args = std::env::args(); let memory_only = args.find(|arg| arg == "--ResourceManager.memory_only").is_some(); let db_res = if !memory_only { - polodb_core::Database::open_file("assets/resources.db") + polodb_core::Database::open_file("resources/resources.db") } else { - println!("\x1B[WARNING]Using memory database instead of file database."); + println!("\x1B[INFO]Using memory database instead of file database."); polodb_core::Database::open_memory() }; @@ -152,57 +185,132 @@ impl ResourceManager { Self::new() } - fn resolve_resource_path(&self, path: &str) -> String { "resources/".to_string() + path } - fn resolve_asset_path(&self, path: &str) -> String { "assets/".to_string() + path } + /// Tries to load a resource from cache or source.\ + /// This is a more advanced version of get() as it allows to load resources that depend on other resources.\ + /// + /// If the resource cannot be found (non existent file, unreacheble network address, fails to parse, etc.) it will return None.\ + /// If the resource is in cache but it's data cannot be parsed, it will return None. + /// Return is a tuple containing the resource description and it's associated binary data.\ + /// ```json + /// { ..., "resources":[{ "id": 01234, "size": 0, "offset": 0, "class": "X" "resource": { ... }, "hash": 0 }] } + /// ``` + /// Members: + /// - **id**: u64 - The resource id. This is used to identify the resource. Needs to be meaningful and will be a public constant. + /// - **size**: u64 - The resource size in bytes. + /// - **offset**: u64 - The resource offset in the resource bundle, relative to the start of the bundle. + /// - **class**: String - The resource class. This is used to identify the resource type. Needs to be meaningful and will be a public constant. + /// - **resource**: The resource data. Can look like anything. + /// - **hash**: u64 - The resource hash. This is used to identify the resource data. If the resource handler wants to generate a hash for the resource it can do so else the resource manager will generate a hash for it. This is because some resources can generate hashes inteligently (EJ: code generators can output same hash for different looking code if the code is semantically identical). + /// + /// The requested resource will always the last one in the array. With the previous resources being the ones it depends on. This way when iterating the array forward the dependencies will be loaded first. + pub fn get(&mut self, path: &str) -> Option<(Document, Vec)> { + let resource_description = if let Some(r) = self.get_document_from_cache(path) { + r + } else { + if let Some(r) = self.load_asset_from_source_and_cache_it(path) { r } else { return None; } + }; - fn get_document_from_cache(&mut self, path: &str) -> Option { - self.db.collection::("resources").find_one(doc!{ "id": path }).unwrap() - } + let mut buffer = Vec::with_capacity(8192); - fn load_resource_into_cache(&mut self, path: &str) -> Option { - let resource_origin = if path.starts_with("http://") || path.starts_with("https://") { "network" } else { "local" }; + let mut bundle_description = doc!{ "resources": [] }; - let mut source_bytes; - let format; + if let Some(polodb_core::bson::Bson::Array(required_resources)) = resource_description.get("required_resources") { + let mut native_id_array = polodb_core::bson::Array::new(); + let mut path_array = polodb_core::bson::Array::new(); - match resource_origin { - "network" => { - let request = if let Ok(request) = ureq::get(path).call() { request } else { return None; }; - let content_type = if let Some(e) = request.header("content-type") { e.to_string() } else { return None; }; - format = content_type; + for required_resource in required_resources { + let required_resource = if let polodb_core::bson::Bson::Document(required_resource) = required_resource { required_resource } else { break; }; - source_bytes = Vec::new(); + if let Some(polodb_core::bson::Bson::String(resource_path)) = required_resource.get("path") { + native_id_array.push(polodb_core::bson::oid::ObjectId::from_str(resource_path).unwrap().into()); + } else if let Some(polodb_core::bson::Bson::String(required_resource_path)) = required_resource.get("path") { + path_array.push(required_resource_path.into()); + } else { + panic!("Invalid required resource"); + } + } - request.into_reader().read_to_end(&mut source_bytes); - }, - "local" => { - let files = if let Ok(r) = std::fs::read_dir("resources") { r } else { return None; }; - let files = files.filter(|f| if let Ok(f) = f { f.path().to_str().unwrap().contains(path) } else { false }); - let p = files.last().unwrap().unwrap().path(); + let search_doc = polodb_core::bson::doc! { + "$or": [ + { "_id": { "$in": native_id_array } }, + { "path": { "$in": path_array } } + ] + }; - let mut file = std::fs::File::open(&p).unwrap(); - let extension = p.extension().unwrap().to_str().unwrap(); + // TODO: make recursive + let required_resources = self.db.collection::("resources").find(search_doc).unwrap().map(|e| e.unwrap()).collect::>(); - format = extension.to_string(); + for required_resource in &required_resources { + let size = required_resource.get_i64("size").unwrap(); + let offset = buffer.len() as i64; - source_bytes = Vec::with_capacity(file.metadata().unwrap().len() as usize); + bundle_description.get_array_mut("resources").unwrap().push(doc!{ "size": size, "offset": offset, "resource": required_resource.get("resource").unwrap() }.into()); - if let Err(_) = file.read_to_end(&mut source_bytes) { - return None; - } - }, - _ => { - // Could not resolve how to get raw resource, return empty bytes - return None; + self.load_data_from_cache(&required_resource, &mut buffer).unwrap(); } } - let mut bytes = None; + let data = self.load_data_from_cache(&resource_description, &mut buffer); + + if data.is_err() { + return None; + } + + return Some((resource_description, buffer)); + } + + /// Tries to load the information/metadata for a resource (and it's dependecies).\ + /// This is a more advanced version of get() as it allows to use your own buffer and/or apply some transformation to the resources when loading.\ + /// The result of this function can be later fed into `load_resource()` which will load the binary data. + pub fn request_resource(&mut self, path: &str) -> Option { + let resource_description = self.get_document_from_cache(path).or_else(|| self.load_asset_from_source_and_cache_it(path))?; + Some(resource_description) + } + + /// Loads the resource binary data from cache.\ + /// If a buffer range is provided it will load the data into the buffer.\ + /// If no buffer range is provided it will return the data in a vector. + pub fn load_resource(&self, resource: &Document, options: Option, buffer: Option<&mut [u8]>) -> Result>, LoadResults> { + if let Some(buffer) = buffer { + self.load_data_from_cache(resource, buffer); + Ok(None) + } else { + let mut b = Vec::new(); + + self.load_data_from_cache(resource, &mut b); + + Ok(Some(b)) + } + + //let resource: T = T::deserialize(polodb_core::bson::Deserializer::new(document.get("resource").unwrap().into())).unwrap(); + } + + fn resolve_resource_path(&self, path: &str) -> String { "resources/".to_string() + path } + fn resolve_asset_path(&self, path: &str) -> String { "assets/".to_string() + path } + + /// Tries to load a resource from cache.\ + /// The returned documents is like the following: + /// ```json + /// { "_id":"OId" , "id": 01234, "path":"../..", "size": 0, "class": "X", "resource": { ... }, "hash": 0 } + /// ``` + fn get_document_from_cache(&mut self, path: &str) -> Option { + self.db.collection::("resources").find_one(doc!{ "path": path }).unwrap() + } + + /// Tries to load a resource from source and cache it.\ + /// If the resource cannot be found (non existent file, unreacheble network address, fails to parse, etc.) it will return None. + fn load_asset_from_source_and_cache_it(&mut self, path: &str) -> Option { + let (source_bytes, format) = match get_asset_from_source(path) { + Ok(value) => value, + Err(value) => return value, + }; + + let mut resource_packages = None; for handler in &self.resource_handlers { if handler.can_handle_type(&format) { if let Ok(e) = handler.process(source_bytes) { - bytes = Some(e); + resource_packages = Some(e); break; } else { return None; @@ -210,124 +318,118 @@ impl ResourceManager { } }; - let (resource_document, bytes) = if let Some(bytes) = bytes { bytes } else { return None; }; - - let id = path.to_string(); - - let document = doc! { - "id": id, - "resource": resource_document - }; - - let insert_result = if let Ok(insert_result) = self.db.collection::("resources").insert_one(&document) { - insert_result - } else { - return None; - }; - - let resource_id = insert_result.inserted_id.as_object_id().unwrap(); + return self.cache_resources(resource_packages?, path); + } - let asset_path = self.resolve_asset_path(resource_id.to_string().as_str()); + /// Stores the asset as a resource. + /// Returns the resource document. + fn cache_resources(&mut self, resource_packages: Vec<(Document, Vec)>, path: &str) -> Option { + let mut document = None; - let mut file = if let Ok(file) = std::fs::File::create(asset_path) { file } else { return None; }; + for resource_package in resource_packages { + let mut full_resource_document = resource_package.0; - file.write_all(bytes.as_slice()).unwrap(); + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + path.hash(&mut hasher); - let document = self.db.collection::("resources").find_one(doc!{ "_id": resource_id }).unwrap().unwrap(); + full_resource_document.insert("id", hasher.finish() as i64); + full_resource_document.insert("path", path.to_string()); + full_resource_document.insert("size", resource_package.1.len() as i64); - return Some(document); - } + if let None = full_resource_document.get("hash") { // TODO: might be a good idea just to generate a random hash, since this method does not reflect changes to the document of the resource + let mut hasher = std::collections::hash_map::DefaultHasher::new(); - fn load_data_from_cache(&mut self, path: &str) -> Result, LoadResults> { - let result = self.db.collection::("resources").find_one(doc!{ "id": path })?; + std::hash::Hasher::write(&mut hasher, resource_package.1.as_slice()); - if let Some(resource) = result { - let native_db_resource_id = if let Some(polodb_core::bson::Bson::ObjectId(id)) = resource.get("_id") { - id - } else { - return Err(LoadResults::LoadFailed); - }; + full_resource_document.insert("hash", hasher.finish() as i64); + } - let native_db_resource_id = native_db_resource_id.to_string(); + let insert_result = self.db.collection::("resources").insert_one(&full_resource_document).ok()?; - let mut file = match std::fs::File::open("assets/".to_string() + native_db_resource_id.as_str()) { - Ok(it) => it, - Err(reason) => { - match reason { // TODO: handle specific errors - _ => return Err(LoadResults::CacheFileNotFound { document: resource }), - } - } - }; + let resource_id = insert_result.inserted_id.as_object_id()?; - let mut bytes = Vec::new(); + let resource_path = self.resolve_resource_path(resource_id.to_string().as_str()); - let res = file.read_to_end(&mut bytes); + let mut file = std::fs::File::create(resource_path).ok()?; - if res.is_err() { return Err(LoadResults::LoadFailed); } + file.write_all(resource_package.1.as_slice()).unwrap(); - return Ok(bytes); + document = Some(self.db.collection::("resources").find_one(doc!{ "_id": resource_id }).unwrap().unwrap()); } - return Err(LoadResults::ResourceNotFound); + return document; } - /// Tries to load a resource from cache or source.\ - /// If the resource cannot be found (non existent file, unreacheble network address, fails to parse, etc.) it will return None.\ - /// If the resource is in cache but it's data cannot be parsed, it will return None. - pub fn get(&mut self, path: &str) -> Option<(Document, Vec)> { - let document = if let Some(r) = self.get_document_from_cache(path) { - r - } else { - if let Some(r) = self.load_resource_into_cache(path) { - r - } else { - return None; - } - }; - - let data = self.load_data_from_cache(path); + /// Tries to load a resource from cache.\ + /// If the resource cannot be found/loaded or if it's become stale it will return None. + fn load_data_from_cache(&mut self, resource_document: &polodb_core::bson::Document, buffer: &mut [u8]) -> Result<(), LoadResults> { + let native_db_resource_id = if let Some(polodb_core::bson::Bson::ObjectId(oid)) = resource_document.get("_id") { oid } else { return Err(LoadResults::LoadFailed); }; - if data.is_err() { - return None; - } - - let data = data.unwrap(); - - return Some((document, data)); - } + let native_db_resource_id = native_db_resource_id.to_string(); - pub fn get_resource_info<'a, T: serde::Deserialize<'a>>(&mut self, path: &str) -> Option> { - let document = if let Some(r) = self.get_document_from_cache(path) { - r - } else { - if let Some(r) = self.load_resource_into_cache(path) { - r - } else { - return None; + let mut file = match std::fs::File::open(self.resolve_asset_path(&native_db_resource_id)) { + Ok(it) => it, + Err(reason) => { + match reason { // TODO: handle specific errors + _ => return Err(LoadResults::CacheFileNotFound { document: resource_document.clone() }), + } } }; - dbg!(&document); + let res = file.read_exact(buffer); - let resource: T = T::deserialize(polodb_core::bson::Deserializer::new(document.get("resource").unwrap().into())).unwrap(); + if res.is_err() { return Err(LoadResults::LoadFailed); } - return Some(Request { document, resource, id: path.to_string() }); + return Ok(()); } +} - pub fn load_resource_into_buffer<'a>(&mut self, request: &Request, vertex_buffer: &mut [u8], index_buffer: &mut [u8]) { - let _id = request.document.get("_id").unwrap().as_object_id().unwrap().to_string(); - let mut file = std::fs::File::open("assets/".to_string() + _id.as_str()).unwrap(); +/// Loads an asset from source.\ +/// Expects an asset name in the form of a path relative to the assets directory, or a network address.\ +/// If the asset is not found it will return None. +/// ```rust +/// let (bytes, format) = resource_manager::get_asset_from_source("textures/concrete").unwrap(); // Path relative to .../assets +/// ``` +fn get_asset_from_source(path: &str) -> Result<(Vec, String), Option> { + let resource_origin = if path.starts_with("http://") || path.starts_with("https://") { "network" } else { "local" }; + let mut source_bytes; + let format; + match resource_origin { + "network" => { + let request = if let Ok(request) = ureq::get(path).call() { request } else { return Err(None); }; + let content_type = if let Some(e) = request.header("content-type") { e.to_string() } else { return Err(None); }; + format = content_type; + + source_bytes = Vec::new(); + + request.into_reader().read_to_end(&mut source_bytes); + }, + "local" => { + let files = if let Ok(r) = std::fs::read_dir("assets") { r } else { return Err(None); }; + let files = files.filter(|f| if let Ok(f) = f { f.path().to_str().unwrap().contains(path) } else { false }); + let p = files.last().unwrap().unwrap().path(); - let mesh_info =& request.resource; + let (mut file, extension) = if let Ok(dir) = std::fs::read_dir("assets") { + let files = dir.filter(|f| if let Ok(f) = f { f.path().to_str().unwrap().contains(path) } else { false }); + let file_path = files.last().unwrap().unwrap().path(); + (std::fs::File::open(&p).unwrap(), p.extension().unwrap().to_str().unwrap().to_string()) + } else { return Err(None); }; - use mesh_resource_handler::Size; + format = extension.to_string(); - let vertex_size = mesh_info.vertex_components.size(); + source_bytes = Vec::with_capacity(file.metadata().unwrap().len() as usize); - file.read(&mut vertex_buffer[..(vertex_size * mesh_info.vertex_count as usize)]).unwrap(); - file.seek(std::io::SeekFrom::Start((vertex_size * mesh_info.vertex_count as usize).next_multiple_of(16) as u64)).unwrap(); - file.read(&mut index_buffer[..(mesh_info.index_count as usize * mesh_info.index_type.size())]).unwrap(); + if let Err(_) = file.read_to_end(&mut source_bytes) { + return Err(None); + } + }, + _ => { + // Could not resolve how to get raw resource, return empty bytes + return Err(None); + } } + + Ok((source_bytes, format)) } // TODO: test resource caching