From 7afe3366a8ea67b1165f84c999923de81f033cb9 Mon Sep 17 00:00:00 2001 From: Facundo Villa Date: Fri, 20 Oct 2023 20:17:23 -0300 Subject: [PATCH] Refactored res man code. Fixed render tests. --- .github/workflows/rust.yml | 2 +- src/application.rs | 7 +- src/render_domain.rs | 30 +- src/rendering/cct.rs | 15 - src/rendering/render_system.rs | 28 +- src/rendering/vulkan_render_system.rs | 228 +++--- .../material_resource_handler.rs | 35 +- src/resource_manager/mesh_resource_handler.rs | 90 ++- src/resource_manager/mod.rs | 739 +----------------- src/resource_manager/resource_handler.rs | 22 + src/resource_manager/resource_manager.rs | 438 +++++++++++ .../texture_resource_handler.rs | 53 +- src/window_system.rs | 21 +- 13 files changed, 800 insertions(+), 908 deletions(-) create mode 100644 src/resource_manager/resource_handler.rs create mode 100644 src/resource_manager/resource_manager.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d488aff8..42652c4d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -27,4 +27,4 @@ jobs: - name: Build run: cargo build - name: Run unit tests - run: RUST_BACKTRACE=1 cargo test -- --skip render \ No newline at end of file + run: RUST_BACKTRACE=1 cargo test -- --skip render --skip window --skip graphics \ No newline at end of file diff --git a/src/application.rs b/src/application.rs index d3aaa6b2..a42c22b5 100644 --- a/src/application.rs +++ b/src/application.rs @@ -60,7 +60,7 @@ impl Application for BaseApplication { use log::{info, trace}; use maths_rs::prelude::Base; -use crate::{orchestrator, rendering::{render_system}, window_system, input_manager, Vector2, rendering::{self, render_system::RenderSystem}, render_domain, resource_manager, file_tracker}; +use crate::{orchestrator, rendering::render_system, window_system, input_manager, Vector2, rendering::{self}, render_domain, resource_manager, file_tracker}; /// An orchestrated application is an application that uses the orchestrator to manage systems. /// It is the recommended way to create a simple application. @@ -139,7 +139,7 @@ impl Application for GraphicsApplication { let orchestrator = application.get_mut_orchestrator(); - orchestrator.spawn_entity(resource_manager::ResourceManager::new_as_system()); + orchestrator.spawn_entity(resource_manager::resource_manager::ResourceManager::new_as_system()); let window_system_handle = orchestrator.spawn_entity(window_system::WindowSystem::new_as_system()).unwrap(); let input_system_handle = orchestrator.spawn_entity(input_manager::InputManager::new_as_system()).unwrap(); @@ -279,7 +279,6 @@ mod tests { app.deinitialize(); } - #[ignore = "Ignore until we have a way to disable this test in CI where windows are not supported"] #[test] fn create_graphics_application() { let mut app = GraphicsApplication::new("Test"); @@ -292,8 +291,6 @@ mod tests { while !app.application.close { app.tick(); - println!("Tick!"); - if start_time.elapsed().as_secs() > 1 { app.close(); } diff --git a/src/render_domain.rs b/src/render_domain.rs index c34b14d4..28efd905 100644 --- a/src/render_domain.rs +++ b/src/render_domain.rs @@ -150,7 +150,7 @@ impl VisibilityWorldRenderDomain { let indices_buffer_handle = render_system.create_buffer(Some("Visibility Index Buffer"), 1024 * 1024 * 16, render_system::Uses::Index | render_system::Uses::Storage, render_system::DeviceAccesses::CpuWrite | render_system::DeviceAccesses::GpuRead, render_system::UseCases::STATIC); let debug_position = render_system.create_texture(Some("debug position"), Extent::new(1920, 1080, 1), render_system::TextureFormats::RGBAu16, None, render_system::Uses::RenderTarget | render_system::Uses::Storage | render_system::Uses::TransferDestination, render_system::DeviceAccesses::GpuRead, render_system::UseCases::DYNAMIC); - let debug_normal = render_system.create_texture(Some("debug normal"), Extent::new(1920, 1080, 1), render_system::TextureFormats::RGBAu16, None, render_system::Uses::RenderTarget | render_system::Uses::Storage | render_system::Uses::TransferDestination, render_system::DeviceAccesses::GpuRead, render_system::UseCases::DYNAMIC); + let debug_normals = render_system.create_texture(Some("debug normal"), Extent::new(1920, 1080, 1), render_system::TextureFormats::RGBAu16, None, render_system::Uses::RenderTarget | render_system::Uses::Storage | render_system::Uses::TransferDestination, render_system::DeviceAccesses::GpuRead, render_system::UseCases::DYNAMIC); let albedo = render_system.create_texture(Some("albedo"), Extent::new(1920, 1080, 1), render_system::TextureFormats::RGBAu16, None, render_system::Uses::RenderTarget | render_system::Uses::Storage | render_system::Uses::TransferDestination, render_system::DeviceAccesses::GpuRead, render_system::UseCases::DYNAMIC); let depth_target = render_system.create_texture(Some("depth_target"), Extent::new(1920, 1080, 1), render_system::TextureFormats::Depth32, None, render_system::Uses::DepthStencil, render_system::DeviceAccesses::GpuRead, render_system::UseCases::DYNAMIC); @@ -608,19 +608,19 @@ impl VisibilityWorldRenderDomain { descriptor_set: visibility_passes_descriptor_set, binding: 5, array_element: 0, - descriptor: render_system::Descriptor::Texture(material_id), + descriptor: render_system::Descriptor::Texture{ handle: material_id, layout: render_system::Layouts::General }, }, render_system::DescriptorWrite { // MaterialId descriptor_set: visibility_passes_descriptor_set, binding: 6, array_element: 0, - descriptor: render_system::Descriptor::Texture(primitive_index), + descriptor: render_system::Descriptor::Texture{ handle: primitive_index, layout: render_system::Layouts::General }, }, render_system::DescriptorWrite { // InstanceId descriptor_set: visibility_passes_descriptor_set, binding: 7, array_element: 0, - descriptor: render_system::Descriptor::Texture(instance_id), + descriptor: render_system::Descriptor::Texture{ handle: instance_id, layout: render_system::Layouts::General }, }, ]); @@ -730,7 +730,7 @@ impl VisibilityWorldRenderDomain { descriptor_set: material_evaluation_descriptor_set, binding: 0, array_element: 0, - descriptor: render_system::Descriptor::Texture(albedo), + descriptor: render_system::Descriptor::Texture{ handle: albedo, layout: render_system::Layouts::General }, }, render_system::DescriptorWrite { // MeshBuffer descriptor_set: material_evaluation_descriptor_set, @@ -772,13 +772,13 @@ impl VisibilityWorldRenderDomain { descriptor_set: material_evaluation_descriptor_set, binding: 7, array_element: 0, - descriptor: render_system::Descriptor::Texture(debug_position), + descriptor: render_system::Descriptor::Texture{ handle: debug_position, layout: render_system::Layouts::General } }, render_system::DescriptorWrite { // debug_normals descriptor_set: material_evaluation_descriptor_set, binding: 8, array_element: 0, - descriptor: render_system::Descriptor::Texture(debug_normal), + descriptor: render_system::Descriptor::Texture{ handle: debug_normals, layout: render_system::Layouts::General } }, render_system::DescriptorWrite { // LightData descriptor_set: material_evaluation_descriptor_set, @@ -894,13 +894,13 @@ impl VisibilityWorldRenderDomain { descriptor_set, binding: 0, array_element: 0, - descriptor: render_system::Descriptor::Texture(albedo), + descriptor: render_system::Descriptor::Texture{ handle: albedo, layout: render_system::Layouts::General }, }, render_system::DescriptorWrite { descriptor_set, binding: 1, array_element: 0, - descriptor: render_system::Descriptor::Texture(result), + descriptor: render_system::Descriptor::Texture{ handle: result, layout: render_system::Layouts::General }, }, ]); @@ -974,7 +974,7 @@ impl VisibilityWorldRenderDomain { instance_id, debug_position, - debug_normal, + debug_normal: debug_normals, material_count, material_offset, @@ -997,7 +997,7 @@ impl VisibilityWorldRenderDomain { .add_listener::() } - fn load_material(&mut self, resource_manager: &mut resource_manager::ResourceManager, render_system: &mut render_system::RenderSystemImplementation, asset_url: &str) { + fn load_material(&mut self, resource_manager: &mut resource_manager::resource_manager::ResourceManager, render_system: &mut render_system::RenderSystemImplementation, asset_url: &str) { let (response, buffer) = resource_manager.get(asset_url).unwrap(); for resource_document in &response.resources { @@ -1622,17 +1622,17 @@ impl orchestrator::EntitySubscriber for VisibilityWorldRenderDomain { orchestrator.tie_self(Self::transform, &handle, Mesh::transform); { - let resource_manager = orchestrator.get_by_class::(); + let resource_manager = orchestrator.get_by_class::(); let mut resource_manager = resource_manager.get_mut(); - let resource_manager: &mut resource_manager::ResourceManager = resource_manager.downcast_mut().unwrap(); + let resource_manager: &mut resource_manager::resource_manager::ResourceManager = resource_manager.downcast_mut().unwrap(); self.load_material(resource_manager, render_system, mesh.material_id); } if !self.mesh_resources.contains_key(mesh.resource_id) { // Load only if not already loaded - let resource_manager = orchestrator.get_by_class::(); + let resource_manager = orchestrator.get_by_class::(); let mut resource_manager = resource_manager.get_mut(); - let resource_manager: &mut resource_manager::ResourceManager = resource_manager.downcast_mut().unwrap(); + let resource_manager: &mut resource_manager::resource_manager::ResourceManager = resource_manager.downcast_mut().unwrap(); self.load_material(resource_manager, render_system, mesh.material_id); diff --git a/src/rendering/cct.rs b/src/rendering/cct.rs index 2aa2d683..b6d215c2 100644 --- a/src/rendering/cct.rs +++ b/src/rendering/cct.rs @@ -1,19 +1,4 @@ //! Taken from https://github.com/m-lima/tempergb -//! Convert a [color temperature](https://en.wikipedia.org/wiki/Color_temperature) into RGB -//! -//! This is a rust port of the work by [Tanner Helland](https://tannerhelland.com/2012/09/18/convert-temperature-rgb-algorithm-code.html) -//! -//! # Example -//! -//! ``` -//! use tempergb::{rgb_from_temperature, Color}; -//! let temperature = 2500; -//! let rgb = tempergb::rgb_from_temperature(temperature); -//! assert_eq!(rgb, (255, 159, 70)); -//! assert_eq!(rgb.r(), 255); -//! assert_eq!(rgb.g(), 159); -//! assert_eq!(rgb.b(), 70); -//! ``` use crate::Vector3; diff --git a/src/rendering/render_system.rs b/src/rendering/render_system.rs index ea59f3d0..d4206ff5 100644 --- a/src/rendering/render_system.rs +++ b/src/rendering/render_system.rs @@ -202,11 +202,14 @@ pub trait CommandBufferRecording { } pub enum Descriptor { - Buffer{ + Buffer { handle: BufferHandle, size: usize, }, - Texture(TextureHandle), + Texture { + handle: TextureHandle, + layout: Layouts, + }, Swapchain(SwapchainHandle), Sampler(SamplerHandle), } @@ -918,7 +921,7 @@ pub(super) mod tests { let vertex_shader = renderer.create_shader(ShaderSourceType::GLSL, ShaderTypes::Vertex, vertex_shader_code.as_bytes()); let fragment_shader = renderer.create_shader(ShaderSourceType::GLSL, ShaderTypes::Fragment, fragment_shader_code.as_bytes()); - let pipeline_layout = renderer.create_pipeline_layout(&[], &[]); + let pipeline_layout = renderer.create_pipeline_layout(&[], &[PushConstantRange{ offset: 0, size: 16 * 4 }]); // Use and odd width to make sure there is a middle/center pixel let extent = crate::Extent { width: 1920, height: 1080, depth: 1 }; @@ -1087,7 +1090,7 @@ pub(super) mod tests { let buffer = renderer.create_buffer(None, 64, Uses::Uniform | Uses::Storage, DeviceAccesses::CpuWrite | DeviceAccesses::GpuRead, UseCases::DYNAMIC); - let sampled_texture = renderer.create_texture(None, crate::Extent { width: 2, height: 2, depth: 1 }, TextureFormats::RGBAu8, None, Uses::Texture, DeviceAccesses::CpuWrite | DeviceAccesses::GpuRead, UseCases::STATIC); + let sampled_texture = renderer.create_texture(Some("sampled texture"), crate::Extent { width: 2, height: 2, depth: 1 }, TextureFormats::RGBAu8, None, Uses::Texture, DeviceAccesses::CpuWrite | DeviceAccesses::GpuRead, UseCases::STATIC); let pixels = vec![ RGBAu8 { r: 255, g: 0, b: 0, a: 255 }, @@ -1096,7 +1099,7 @@ pub(super) mod tests { RGBAu8 { r: 255, g: 255, b: 0, a: 255 }, ]; - let sampler = renderer.create_sampler(); + let sampler = renderer.create_sampler(); let bindings = [ DescriptorSetLayoutBinding { @@ -1132,7 +1135,7 @@ pub(super) mod tests { renderer.write(&[ DescriptorWrite { descriptor_set: descriptor_set, binding: 0, array_element: 0, descriptor: Descriptor::Sampler(sampler) }, DescriptorWrite { descriptor_set: descriptor_set, binding: 1, array_element: 0, descriptor: Descriptor::Buffer{ handle: buffer, size: 64 } }, - DescriptorWrite { descriptor_set: descriptor_set, binding: 2, array_element: 0, descriptor: Descriptor::Texture(sampled_texture) }, + DescriptorWrite { descriptor_set: descriptor_set, binding: 2, array_element: 0, descriptor: Descriptor::Texture{ handle: sampled_texture, layout: Layouts::Texture } }, ]); assert!(!renderer.has_errors()); @@ -1170,7 +1173,14 @@ pub(super) mod tests { command_buffer_recording.write_texture_data(sampled_texture, &pixels); - // command_buffer_recording.transition_textures(&[(sampled_texture, true, Layouts::Texture, Stages::SHADER_READ, AccessPolicies::READ)]); + command_buffer_recording.consume_resources(&[ + Consumption{ + handle: Handle::Texture(sampled_texture), + stages: Stages::FRAGMENT, + access: AccessPolicies::READ, + layout: Layouts::Texture, + } + ]); let attachments = [ AttachmentInformation { @@ -1392,8 +1402,6 @@ bitflags::bitflags! { const PRESENTATION = 0b1000000; /// The host stage. const HOST = 0b10000000; - /// The shader read stage. - const SHADER_READ = 0b100000000; /// The shader write stage. const SHADER_WRITE = 0b1000000000; /// The indirect commands evaluation stage. @@ -1529,7 +1537,7 @@ pub struct DescriptorWrite { /// The descriptor set to write to. pub descriptor_set: DescriptorSetHandle, /// The binding to write to. - pub binding: u32, + pub binding: u32, /// The index of the array element to write to in the binding(if the binding is an array). pub array_element: u32, /// Information describing the descriptor. diff --git a/src/rendering/vulkan_render_system.rs b/src/rendering/vulkan_render_system.rs index 858a8d84..19c8e086 100644 --- a/src/rendering/vulkan_render_system.rs +++ b/src/rendering/vulkan_render_system.rs @@ -1,7 +1,10 @@ -use std::{collections::HashMap, num::NonZeroU64, hash::Hasher}; +use std::{collections::HashMap,}; use crate::{orchestrator, window_system, render_debugger::RenderDebugger}; +#[cfg(test)] +use std::{println as error, println as warn}; + pub struct VulkanRenderSystem { entry: ash::Entry, instance: ash::Instance, @@ -11,6 +14,9 @@ pub struct VulkanRenderSystem { #[cfg(debug_assertions)] debug_utils_messenger: Option, + #[cfg(debug_assertions)] + debug_data: Box, + physical_device: vk::PhysicalDevice, device: ash::Device, queue_family_index: u32, @@ -29,6 +35,7 @@ pub struct VulkanRenderSystem { buffers: Vec, textures: Vec, allocations: Vec, + descriptor_sets_layouts: Vec, descriptor_sets: Vec, meshes: Vec, command_buffers: Vec, @@ -36,12 +43,6 @@ pub struct VulkanRenderSystem { swapchains: Vec, } -fn insert_return_length(collection: &mut Vec, value: T) -> usize { - let length = collection.len(); - collection.push(value); - return length; -} - impl orchestrator::Entity for VulkanRenderSystem {} impl orchestrator::System for VulkanRenderSystem {} @@ -121,7 +122,9 @@ impl render_system::RenderSystem for VulkanRenderSystem { } fn create_descriptor_set_layout(&mut self, bindings: &[render_system::DescriptorSetLayoutBinding]) -> render_system::DescriptorSetLayoutHandle { - fn m(rs: &mut VulkanRenderSystem, bindings: &[render_system::DescriptorSetLayoutBinding], layout_bindings: &mut Vec) -> vk::DescriptorSetLayout { + let mut map: HashMap = HashMap::new(); + + fn m(rs: &mut VulkanRenderSystem, bindings: &[render_system::DescriptorSetLayoutBinding], layout_bindings: &mut Vec, map: &mut HashMap) -> vk::DescriptorSetLayout { if let Some(binding) = bindings.get(0) { let b = vk::DescriptorSetLayoutBinding::default() .binding(binding.binding) @@ -143,9 +146,11 @@ impl render_system::RenderSystem for VulkanRenderSystem { b.immutable_samplers(&x); + map.insert(DSLB { binding: binding.binding, }, b.descriptor_type); + layout_bindings.push(b); - m(rs, &bindings[1..], layout_bindings) + m(rs, &bindings[1..], layout_bindings, map) } else { let descriptor_set_layout_create_info = vk::DescriptorSetLayoutCreateInfo::default().bindings(&layout_bindings); @@ -155,9 +160,16 @@ impl render_system::RenderSystem for VulkanRenderSystem { } } - let descriptor_set_layout = m(self, bindings, &mut Vec::new()); + let descriptor_set_layout = m(self, bindings, &mut Vec::new(), &mut map); - render_system::DescriptorSetLayoutHandle(descriptor_set_layout.as_raw()) + let handle = render_system::DescriptorSetLayoutHandle(self.descriptor_sets_layouts.len() as u64); + + self.descriptor_sets_layouts.push(DescriptorSetLayout { + map, + descriptor_set_layout, + }); + + handle } fn create_descriptor_set(&mut self, descriptor_set_layout_handle: &render_system::DescriptorSetLayoutHandle, bindings: &[render_system::DescriptorSetLayoutBinding]) -> render_system::DescriptorSetHandle { @@ -181,9 +193,11 @@ impl render_system::RenderSystem for VulkanRenderSystem { let descriptor_pool = unsafe { self.device.create_descriptor_pool(&descriptor_pool_create_info, None).expect("No descriptor pool") }; + let descriptor_set_layout = self.descriptor_sets_layouts[descriptor_set_layout_handle.0 as usize].descriptor_set_layout; + // Allocate 2 descriptor sets from our pool. // TODO: Tie this count to the number of frames. - let descriptor_set_layouts = [vk::DescriptorSetLayout::from_raw(descriptor_set_layout_handle.0), vk::DescriptorSetLayout::from_raw(descriptor_set_layout_handle.0)]; + let descriptor_set_layouts = [descriptor_set_layout, descriptor_set_layout]; let descriptor_set_allocate_info = vk::DescriptorSetAllocateInfo::default() .descriptor_pool(descriptor_pool) @@ -202,6 +216,7 @@ impl render_system::RenderSystem for VulkanRenderSystem { DescriptorSet { next: None, descriptor_set, + descriptor_set_layout: *descriptor_set_layout_handle, } ); @@ -217,29 +232,18 @@ impl render_system::RenderSystem for VulkanRenderSystem { fn write(&self, descriptor_set_writes: &[render_system::DescriptorWrite]) { for descriptor_set_write in descriptor_set_writes { - let descriptor_type = match descriptor_set_write.descriptor { - render_system::Descriptor::Buffer { handle: _, size: _ } => { - vk::DescriptorType::STORAGE_BUFFER - }, - render_system::Descriptor::Texture(handle) => { - let texture = &self.textures[handle.0 as usize]; - match texture.layout { - vk::ImageLayout::COLOR_ATTACHMENT_OPTIMAL => vk::DescriptorType::STORAGE_IMAGE, - _ => vk::DescriptorType::STORAGE_IMAGE, - } - }, - render_system::Descriptor::Sampler(_) => { - vk::DescriptorType::SAMPLER - }, - render_system::Descriptor::Swapchain(_) => { - unimplemented!() - } - }; + let descriptor_set = &self.descriptor_sets[descriptor_set_write.descriptor_set.0 as usize]; + + let layout = descriptor_set.descriptor_set_layout; + + let descriptor_set_layout = &self.descriptor_sets_layouts[layout.0 as usize]; + + let descriptor_type = *descriptor_set_layout.map.get(&DSLB { binding: descriptor_set_write.binding }).unwrap(); match descriptor_set_write.descriptor { render_system::Descriptor::Buffer { handle, size } => { let mut descriptor_set_handle_option = Some(descriptor_set_write.descriptor_set); - let mut buffer_handle_option = Some(handle); + let mut _buffer_handle_option = Some(handle); while let Some(descriptor_set_handle) = descriptor_set_handle_option { let descriptor_set = &self.descriptor_sets[descriptor_set_handle.0 as usize]; @@ -263,7 +267,7 @@ impl render_system::RenderSystem for VulkanRenderSystem { // } } }, - render_system::Descriptor::Texture(handle) => { + render_system::Descriptor::Texture{ handle, layout } => { let mut descriptor_set_handle_option = Some(descriptor_set_write.descriptor_set); let mut texture_handle_option = Some(handle); @@ -271,7 +275,7 @@ impl render_system::RenderSystem for VulkanRenderSystem { let descriptor_set = &self.descriptor_sets[descriptor_set_handle.0 as usize]; let texture = &self.textures[texture_handle.0 as usize]; - let images = [vk::DescriptorImageInfo::default().image_layout(vk::ImageLayout::GENERAL).image_view(texture.image_view)]; + let images = [vk::DescriptorImageInfo::default().image_layout(texture_format_and_resource_use_to_image_layout(texture.format_, layout, None)).image_view(texture.image_view)]; let write_info = vk::WriteDescriptorSet::default() .dst_set(descriptor_set.descriptor_set) @@ -319,9 +323,8 @@ impl render_system::RenderSystem for VulkanRenderSystem { } fn create_pipeline_layout(&mut self, descriptor_set_layout_handles: &[render_system::DescriptorSetLayoutHandle], push_constant_ranges: &[render_system::PushConstantRange]) -> render_system::PipelineLayoutHandle { - // self.create_vulkan_pipeline_layout(&descriptor_set_layout_handles.iter().map(|descriptor_set_layout_handle| vk::DescriptorSetLayout::from_raw(descriptor_set_layout_handle.0)).collect::>()) let push_constant_ranges = push_constant_ranges.iter().map(|push_constant_range| vk::PushConstantRange::default().size(push_constant_range.size).offset(push_constant_range.offset).stage_flags(vk::ShaderStageFlags::VERTEX | vk::ShaderStageFlags::MESH_EXT | vk::ShaderStageFlags::FRAGMENT | vk::ShaderStageFlags::COMPUTE)).collect::>(); - let set_layouts = descriptor_set_layout_handles.iter().map(|set_layout| vk::DescriptorSetLayout::from_raw(set_layout.0)).collect::>(); + let set_layouts = descriptor_set_layout_handles.iter().map(|set_layout| self.descriptor_sets_layouts[set_layout.0 as usize].descriptor_set_layout).collect::>(); let pipeline_layout_create_info = vk::PipelineLayoutCreateInfo::default() .set_layouts(&set_layouts) @@ -764,6 +767,8 @@ impl render_system::RenderSystem for VulkanRenderSystem { } use ash::{vk::{self, ValidationFeatureEnableEXT, Handle}, Entry}; + +#[cfg(not(test))] use log::{warn, error, debug}; use super::render_system::{self, CommandBufferRecording, TextureFormats}; @@ -775,10 +780,23 @@ pub(crate) struct Swapchain { swapchain: vk::SwapchainKHR, } +#[derive(Clone, Hash, PartialEq, Eq)] +struct DSLB { + binding: u32, + // position: u32, +} + +#[derive(Clone)] +pub(crate) struct DescriptorSetLayout { + map: std::collections::HashMap, + descriptor_set_layout: vk::DescriptorSetLayout, +} + #[derive(Clone, Copy)] pub(crate) struct DescriptorSet { next: Option, descriptor_set: vk::DescriptorSet, + descriptor_set_layout: render_system::DescriptorSetLayoutHandle, } #[derive(Clone, Copy)] @@ -843,9 +861,7 @@ unsafe impl Send for Texture {} // acceleration_structure: vk::AccelerationStructureKHR, // } -static mut COUNTER: u32 = 0; - -unsafe extern "system" fn vulkan_debug_utils_callback(message_severity: vk::DebugUtilsMessageSeverityFlagsEXT, _message_type: vk::DebugUtilsMessageTypeFlagsEXT, p_callback_data: *const vk::DebugUtilsMessengerCallbackDataEXT, _p_user_data: *mut std::ffi::c_void,) -> vk::Bool32 { +unsafe extern "system" fn vulkan_debug_utils_callback(message_severity: vk::DebugUtilsMessageSeverityFlagsEXT, _message_type: vk::DebugUtilsMessageTypeFlagsEXT, p_callback_data: *const vk::DebugUtilsMessengerCallbackDataEXT, p_user_data: *mut std::ffi::c_void,) -> vk::Bool32 { let message = std::ffi::CStr::from_ptr((*p_callback_data).p_message); match message_severity { @@ -854,7 +870,7 @@ unsafe extern "system" fn vulkan_debug_utils_callback(message_severity: vk::Debu } vk::DebugUtilsMessageSeverityFlagsEXT::ERROR => { error!("{}", message.to_str().unwrap()); - COUNTER += 1; + (*(p_user_data as *mut DebugCallbackData)).error_count += 1; } _ => {} } @@ -974,7 +990,7 @@ fn to_pipeline_stage_flags(stages: render_system::Stages) -> vk::PipelineStageFl } if stages.contains(render_system::Stages::FRAGMENT) { - pipeline_stage_flags |= vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT + pipeline_stage_flags |= vk::PipelineStageFlags2::FRAGMENT_SHADER } if stages.contains(render_system::Stages::COMPUTE) { @@ -989,10 +1005,6 @@ fn to_pipeline_stage_flags(stages: render_system::Stages) -> vk::PipelineStageFl pipeline_stage_flags |= vk::PipelineStageFlags2::BOTTOM_OF_PIPE } - if stages.contains(render_system::Stages::SHADER_READ) { - pipeline_stage_flags |= vk::PipelineStageFlags2::FRAGMENT_SHADER; // TODO: not really? - } - if stages.contains(render_system::Stages::INDIRECT) { pipeline_stage_flags |= vk::PipelineStageFlags2::DRAW_INDIRECT; } @@ -1000,7 +1012,7 @@ fn to_pipeline_stage_flags(stages: render_system::Stages) -> vk::PipelineStageFl pipeline_stage_flags } -fn to_pipeline_stage_flags_with_format(stages: render_system::Stages, format: render_system::TextureFormats) -> vk::PipelineStageFlags2 { +fn to_pipeline_stage_flags_with_format(stages: render_system::Stages, format: render_system::TextureFormats, access: render_system::AccessPolicies) -> vk::PipelineStageFlags2 { let mut pipeline_stage_flags = vk::PipelineStageFlags2::NONE; if stages.contains(render_system::Stages::VERTEX) { @@ -1009,7 +1021,13 @@ fn to_pipeline_stage_flags_with_format(stages: render_system::Stages, format: re if stages.contains(render_system::Stages::FRAGMENT) { if format != render_system::TextureFormats::Depth32 { - pipeline_stage_flags |= vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT + if access.contains(render_system::AccessPolicies::READ) { + pipeline_stage_flags |= vk::PipelineStageFlags2::FRAGMENT_SHADER + } + + if access.contains(render_system::AccessPolicies::WRITE) { + pipeline_stage_flags |= vk::PipelineStageFlags2::COLOR_ATTACHMENT_OUTPUT + } } else { pipeline_stage_flags |= vk::PipelineStageFlags2::EARLY_FRAGMENT_TESTS } @@ -1027,10 +1045,6 @@ fn to_pipeline_stage_flags_with_format(stages: render_system::Stages, format: re pipeline_stage_flags |= vk::PipelineStageFlags2::BOTTOM_OF_PIPE } - if stages.contains(render_system::Stages::SHADER_READ) { - pipeline_stage_flags |= vk::PipelineStageFlags2::FRAGMENT_SHADER; // TODO: not really? - } - if stages.contains(render_system::Stages::INDIRECT) { pipeline_stage_flags |= vk::PipelineStageFlags2::DRAW_INDIRECT; } @@ -1048,7 +1062,7 @@ fn to_access_flags(accesses: render_system::AccessPolicies, stages: render_syste if stages.intersects(render_system::Stages::PRESENTATION) { access_flags |= vk::AccessFlags2::NONE } - if stages.intersects(render_system::Stages::SHADER_READ) { + if stages.intersects(render_system::Stages::FRAGMENT) { access_flags |= vk::AccessFlags2::SHADER_SAMPLED_READ; } if stages.intersects(render_system::Stages::COMPUTE) { @@ -1084,12 +1098,16 @@ fn to_access_flags_with_format(accesses: render_system::AccessPolicies, stages: if stages.intersects(render_system::Stages::PRESENTATION) { access_flags |= vk::AccessFlags2::NONE } - if stages.intersects(render_system::Stages::SHADER_READ) { - access_flags |= vk::AccessFlags2::SHADER_SAMPLED_READ; - } if stages.intersects(render_system::Stages::COMPUTE) { access_flags |= vk::AccessFlags2::SHADER_READ } + if stages.intersects(render_system::Stages::FRAGMENT) { + if format != render_system::TextureFormats::Depth32 { + access_flags |= vk::AccessFlags2::SHADER_SAMPLED_READ + } else { + access_flags |= vk::AccessFlags2::DEPTH_STENCIL_ATTACHMENT_READ + } + } if stages.intersects(render_system::Stages::INDIRECT) { access_flags |= vk::AccessFlags2::INDIRECT_COMMAND_READ } @@ -1186,6 +1204,10 @@ pub struct Settings { ray_tracing: bool, } +struct DebugCallbackData { + error_count: u64, +} + impl VulkanRenderSystem { pub fn new(settings: &Settings) -> VulkanRenderSystem { let entry: ash::Entry = Entry::linked(); @@ -1232,13 +1254,18 @@ impl VulkanRenderSystem { let instance = unsafe { entry.create_instance(&instance_create_info, None).expect("No instance") }; + let mut debug_data = Box::new(DebugCallbackData { + error_count: 0, + }); + let (debug_utils, debug_utils_messenger) = if settings.validation { let debug_utils = ash::extensions::ext::DebugUtils::new(&entry, &instance); let debug_utils_create_info = vk::DebugUtilsMessengerCreateInfoEXT::default() .message_severity(vk::DebugUtilsMessageSeverityFlagsEXT::INFO | vk::DebugUtilsMessageSeverityFlagsEXT::WARNING | vk::DebugUtilsMessageSeverityFlagsEXT::ERROR,) .message_type(vk::DebugUtilsMessageTypeFlagsEXT::GENERAL | vk::DebugUtilsMessageTypeFlagsEXT::VALIDATION | vk::DebugUtilsMessageTypeFlagsEXT::PERFORMANCE,) - .pfn_user_callback(Some(vulkan_debug_utils_callback)); + .pfn_user_callback(Some(vulkan_debug_utils_callback)) + .user_data(debug_data.as_mut() as *mut DebugCallbackData as *mut std::ffi::c_void); let debug_utils_messenger = unsafe { debug_utils.create_debug_utils_messenger(&debug_utils_create_info, None).expect("Debug Utils Callback") }; @@ -1401,8 +1428,11 @@ impl VulkanRenderSystem { VulkanRenderSystem { entry, instance, + debug_utils, debug_utils_messenger, + debug_data, + physical_device, device, queue_family_index, @@ -1420,6 +1450,7 @@ impl VulkanRenderSystem { allocations: Vec::new(), buffers: Vec::new(), textures: Vec::new(), + descriptor_sets_layouts: Vec::new(), descriptor_sets: Vec::new(), meshes: Vec::new(), command_buffers: Vec::new(), @@ -1436,7 +1467,7 @@ impl VulkanRenderSystem { orchestrator::EntityReturn::new(render_system::RenderSystemImplementation::new(Box::new(VulkanRenderSystem::new(&settings)))) } - fn get_log_count(&self) -> u32 { unsafe { COUNTER } } + fn get_log_count(&self) -> u64 { self.debug_data.error_count } fn create_vulkan_shader(&self, stage: render_system::ShaderTypes, shader: &[u8]) -> render_system::ShaderHandle { let shader_module_create_info = vk::ShaderModuleCreateInfo::default() @@ -2476,8 +2507,8 @@ impl render_system::CommandBufferRecording for VulkanCommandBufferRecording<'_> let mut memory_barriers = Vec::new(); for consumption in consumptions { - let new_stage_mask = to_pipeline_stage_flags(consumption.stages); - let new_access_mask = to_access_flags(consumption.access, consumption.stages); + let mut new_stage_mask = to_pipeline_stage_flags(consumption.stages); + let mut new_access_mask = to_access_flags(consumption.access, consumption.stages); match consumption.handle { render_system::Handle::Texture(texture_handle) => { @@ -2485,8 +2516,8 @@ impl render_system::CommandBufferRecording for VulkanCommandBufferRecording<'_> let new_layout = texture_format_and_resource_use_to_image_layout(texture.format_, consumption.layout, Some(consumption.access)); - let new_stage_mask = to_pipeline_stage_flags_with_format(consumption.stages, texture.format_); - let new_access_mask = to_access_flags_with_format(consumption.access, consumption.stages, texture.format_); + new_stage_mask = to_pipeline_stage_flags_with_format(consumption.stages, texture.format_, consumption.access); + new_access_mask = to_access_flags_with_format(consumption.access, consumption.stages, texture.format_); let image_memory_barrier = if let Some(barrier_source) = self.states.get(&consumption.handle) { vk::ImageMemoryBarrier2KHR::default() @@ -2678,6 +2709,15 @@ impl render_system::CommandBufferRecording for VulkanCommandBufferRecording<'_> } fn write_texture_data(&mut self, texture_handle: render_system::TextureHandle, data: &[render_system::RGBAu8]) { + self.consume_resources( + &[render_system::Consumption{ + handle: render_system::Handle::Texture(texture_handle), + stages: render_system::Stages::TRANSFER, + access: render_system::AccessPolicies::WRITE, + layout: render_system::Layouts::Transfer, + }] + ); + let (texture_handle, texture) = self.get_texture(texture_handle); let staging_buffer_handle = texture.staging_buffer.expect("No staging buffer"); @@ -2706,38 +2746,6 @@ impl render_system::CommandBufferRecording for VulkanCommandBufferRecording<'_> } } - let command_buffer = self.get_command_buffer(); - - let image_memory_barriers = [ - vk::ImageMemoryBarrier2KHR::default() - .old_layout(vk::ImageLayout::UNDEFINED) - .src_stage_mask(vk::PipelineStageFlags2::empty()) - .src_access_mask(vk::AccessFlags2KHR::empty()) - .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) - .new_layout(vk::ImageLayout::TRANSFER_DST_OPTIMAL) - .dst_stage_mask(vk::PipelineStageFlags2::TRANSFER) - .dst_access_mask(vk::AccessFlags2KHR::TRANSFER_WRITE) - .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) - .image(texture.image) - .subresource_range(vk::ImageSubresourceRange { - aspect_mask: vk::ImageAspectFlags::COLOR, - base_mip_level: 0, - level_count: vk::REMAINING_MIP_LEVELS, - base_array_layer: 0, - layer_count: vk::REMAINING_ARRAY_LAYERS, - }) - /* .build() */, - ]; - - let dependency_info = vk::DependencyInfo::default() - .image_memory_barriers(&image_memory_barriers) - .dependency_flags(vk::DependencyFlags::BY_REGION) - /* .build() */; - - unsafe { - self.render_system.device.cmd_pipeline_barrier2(command_buffer.command_buffer, &dependency_info); - } - let regions = [vk::BufferImageCopy2::default() .buffer_offset(0) .buffer_row_length(0) @@ -2759,39 +2767,10 @@ impl render_system::CommandBufferRecording for VulkanCommandBufferRecording<'_> .dst_image_layout(vk::ImageLayout::TRANSFER_DST_OPTIMAL) .regions(®ions); - unsafe { - self.render_system.device.cmd_copy_buffer_to_image2(command_buffer.command_buffer, &buffer_image_copy); - } - - let image_memory_barriers = [ - vk::ImageMemoryBarrier2KHR::default() - .old_layout(vk::ImageLayout::TRANSFER_DST_OPTIMAL) - .src_stage_mask(vk::PipelineStageFlags2::TRANSFER) - .src_access_mask(vk::AccessFlags2KHR::TRANSFER_WRITE) - .src_queue_family_index(vk::QUEUE_FAMILY_IGNORED) - .new_layout(vk::ImageLayout::SHADER_READ_ONLY_OPTIMAL) - .dst_stage_mask(vk::PipelineStageFlags2::FRAGMENT_SHADER) - .dst_access_mask(vk::AccessFlags2KHR::SHADER_READ) - .dst_queue_family_index(vk::QUEUE_FAMILY_IGNORED) - .image(texture.image) - .subresource_range(vk::ImageSubresourceRange { - aspect_mask: vk::ImageAspectFlags::COLOR, - base_mip_level: 0, - level_count: vk::REMAINING_MIP_LEVELS, - base_array_layer: 0, - layer_count: vk::REMAINING_ARRAY_LAYERS, - }) - /* .build() */ - ]; - - // Transition destination texture to shader read - let dependency_info = vk::DependencyInfo::default() - .image_memory_barriers(&image_memory_barriers) - .dependency_flags(vk::DependencyFlags::BY_REGION) - /* .build() */; + let command_buffer = self.get_command_buffer(); unsafe { - self.render_system.device.cmd_pipeline_barrier2(command_buffer.command_buffer, &dependency_info); + self.render_system.device.cmd_copy_buffer_to_image2(command_buffer.command_buffer, &buffer_image_copy); } } @@ -3216,15 +3195,12 @@ mod tests { render_system::tests::render_triangle(&mut vulkan_render_system); } - #[ignore = "CI doesn't support presentation"] #[test] fn present() { let mut vulkan_render_system = VulkanRenderSystem::new(&Settings { validation: true, ray_tracing: false }); render_system::tests::present(&mut vulkan_render_system); } - - #[ignore = "CI doesn't support presentation"] #[test] fn multiframe_present() { let mut vulkan_render_system = VulkanRenderSystem::new(&Settings { validation: true, ray_tracing: false }); diff --git a/src/resource_manager/material_resource_handler.rs b/src/resource_manager/material_resource_handler.rs index d4a2ea32..bf975629 100644 --- a/src/resource_manager/material_resource_handler.rs +++ b/src/resource_manager/material_resource_handler.rs @@ -1,12 +1,11 @@ -use std::{collections::hash_map::DefaultHasher, hash::Hasher}; +use std::io::Read; use log::{warn, debug, error}; -use polodb_core::bson::Document; use serde::{Serialize, Deserialize}; use crate::{rendering::render_system, jspd::{self}}; -use super::{ResourceHandler, ResourceManager, SerializedResourceDocument, GenericResourceSerialization, Resource, ProcessedResources}; +use super::{SerializedResourceDocument, GenericResourceSerialization, Resource, ProcessedResources, resource_handler::ResourceHandler, resource_manager::ResourceManager}; pub struct MaterialResourcerHandler { @@ -80,6 +79,10 @@ impl ResourceHandler for MaterialResourcerHandler { } } + fn read(&self, _resource: &Box, file: &mut std::fs::File, buffers: &mut [super::Buffer]) { + file.read_exact(buffers[0].buffer).unwrap(); + } + fn process(&self, resource_manager: &ResourceManager, asset_url: &str, bytes: &[u8]) -> Result, String> { let asset_json = json::parse(std::str::from_utf8(&bytes).unwrap()).unwrap(); @@ -264,4 +267,30 @@ impl MaterialResourcerHandler { Some(Self::treat_shader("", domain, stage, material, shader_node,)?.unwrap()) } } +} + +#[cfg(test)] +mod tests { + use crate::resource_manager::resource_manager::ResourceManager; + + #[test] + fn load_material() { + let mut resource_manager = ResourceManager::new(); + + let (response, _) = resource_manager.get("solid").expect("Failed to load material"); + + assert_eq!(response.resources.len(), 3); // 1 material, 1 shader, 1 texture + + let resource_container = &response.resources[0]; + + assert_eq!(resource_container.class, "Shader"); + + let resource_container = &response.resources[1]; + + assert_eq!(resource_container.class, "Texture"); + + let resource_container = &response.resources[2]; + + assert_eq!(resource_container.class, "Material"); + } } \ No newline at end of file diff --git a/src/resource_manager/mesh_resource_handler.rs b/src/resource_manager/mesh_resource_handler.rs index 3b7f28dd..60686ee0 100644 --- a/src/resource_manager/mesh_resource_handler.rs +++ b/src/resource_manager/mesh_resource_handler.rs @@ -4,7 +4,7 @@ use log::error; use polodb_core::bson::{Document, doc}; use serde::{Serialize, Deserialize}; -use super::{ResourceHandler, SerializedResourceDocument, GenericResourceSerialization, Resource, ProcessedResources}; +use super::{SerializedResourceDocument, GenericResourceSerialization, Resource, ProcessedResources, resource_handler::ResourceHandler, resource_manager::ResourceManager}; pub struct MeshResourceHandler { @@ -25,7 +25,7 @@ impl ResourceHandler for MeshResourceHandler { } } - fn process(&self, _: &super::ResourceManager, asset_url: &str, bytes: &[u8]) -> Result, String> { + fn process(&self, _: &ResourceManager, asset_url: &str, bytes: &[u8]) -> Result, String> { let (gltf, buffers, _) = gltf::import_slice(bytes).unwrap(); let mut buf: Vec = Vec::with_capacity(4096 * 1024 * 3); @@ -271,7 +271,7 @@ impl Size for IntegralTypes { #[cfg(test)] mod tests { - use crate::{resource_manager::{ResourceManager, Options, OptionResource, Buffer}, Vector3}; + use crate::{resource_manager::{resource_manager::ResourceManager, Options, OptionResource, Buffer}, Vector3}; use super::*; @@ -291,20 +291,82 @@ mod tests { } #[test] - fn load_with_manager_buffer() { + fn load_local_mesh() { let mut resource_manager = ResourceManager::new(); - // Test loading from source + let (response, buffer) = resource_manager.get("Box").expect("Failed to get resource"); + + assert_eq!(response.resources.len(), 1); + + let resource_container = &response.resources[0]; + let resource = &resource_container.resource; + + assert_eq!(resource.type_id(), std::any::TypeId::of::()); + + assert_eq!(buffer.len(), (24 /* vertices */ * (3 /* components per position */ * 4 /* float size */ + 3/*normals */ * 4) as usize).next_multiple_of(16) + 6/* cube faces */ * 2 /* triangles per face */ * 3 /* indices per triangle */ * 2 /* bytes per index */); + + let mesh = resource.downcast_ref::().unwrap(); + + assert_eq!(mesh.bounding_box, [[-0.5f32, -0.5f32, -0.5f32], [0.5f32, 0.5f32, 0.5f32]]); + assert_eq!(mesh.vertex_count, 24); + assert_eq!(mesh.index_count, 36); + assert_eq!(mesh.index_type, IntegralTypes::U16); + assert_eq!(mesh.vertex_components.len(), 2); + assert_eq!(mesh.vertex_components[0].semantic, VertexSemantics::Position); + assert_eq!(mesh.vertex_components[0].format, "vec3f"); + assert_eq!(mesh.vertex_components[0].channel, 0); + assert_eq!(mesh.vertex_components[1].semantic, VertexSemantics::Normal); + assert_eq!(mesh.vertex_components[1].format, "vec3f"); + assert_eq!(mesh.vertex_components[1].channel, 1); + + let resource_request = resource_manager.request_resource("Box"); - let resource_result = resource_manager.get("Box"); + let resource_request = if let Some(resource_info) = resource_request { resource_info } else { return; }; - assert!(resource_result.is_some()); + let mut options = Options { resources: Vec::new(), }; - let (request, buffer) = resource_result.unwrap(); + let mut vertex_buffer = vec![0u8; 1024]; + let mut index_buffer = vec![0u8; 1024]; - assert_eq!(request.resources.len(), 1); + let resource = &resource_request.resources[0]; - let resource_container = &request.resources[0]; + match resource.class.as_str() { + "Mesh" => { + options.resources.push(OptionResource { + url: resource.url.clone(), + buffers: vec![Buffer{ buffer: vertex_buffer.as_mut_slice(), tag: "Vertex".to_string() }, Buffer{ buffer: index_buffer.as_mut_slice(), tag: "Index".to_string() }], + }); + } + _ => {} + } + + let resource = if let Ok(a) = resource_manager.load_resource(resource_request, Some(options), None) { a } else { return; }; + + let (response, _buffer) = (resource.0, resource.1.unwrap()); + + for resource in &response.resources { + match resource.class.as_str() { + "Mesh" => { + let mesh = resource.resource.downcast_ref::().unwrap(); + + assert_eq!(buffer[0..(mesh.vertex_count * mesh.vertex_components.size() as u32) as usize], vertex_buffer[0..(mesh.vertex_count * mesh.vertex_components.size() as u32) as usize]); + + assert_eq!(buffer[576..(576 + mesh.index_count * 2) as usize], index_buffer[0..(mesh.index_count * 2) as usize]); + } + _ => {} + } + } + } + + #[test] + fn load_with_manager_buffer() { + let mut resource_manager = ResourceManager::new(); + + let (response, buffer) = resource_manager.get("Box").expect("Failed to get resource"); + + assert_eq!(response.resources.len(), 1); + + let resource_container = &response.resources[0]; let resource = &resource_container.resource; assert_eq!(resource.type_id(), std::any::TypeId::of::()); @@ -406,10 +468,6 @@ mod tests { assert_eq!(index_buffer[0], 0); assert_eq!(index_buffer[1], 1); assert_eq!(index_buffer[2], 2); - - // assert_eq!(mesh_buffer[0..(mesh.vertex_count * mesh.vertex_components.size() as u32) as usize], vertex_buffer[0..(mesh.vertex_count * mesh.vertex_components.size() as u32) as usize]); - - // assert_eq!(mesh_buffer[576..(576 + mesh.index_count * 2) as usize], index_buffer[0..(mesh.index_count * 2) as usize]); } _ => {} } @@ -477,10 +535,6 @@ mod tests { assert_eq!(index_buffer[0], 0); assert_eq!(index_buffer[1], 1); assert_eq!(index_buffer[2], 2); - - // assert_eq!(mesh_buffer[0..(mesh.vertex_count * mesh.vertex_components.size() as u32) as usize], vertex_buffer[0..(mesh.vertex_count * mesh.vertex_components.size() as u32) as usize]); - - // assert_eq!(mesh_buffer[576..(576 + mesh.index_count * 2) as usize], index_buffer[0..(mesh.index_count * 2) as usize]); } _ => {} } diff --git a/src/resource_manager/mod.rs b/src/resource_manager/mod.rs index 300fdfcd..6a66cf86 100644 --- a/src/resource_manager/mod.rs +++ b/src/resource_manager/mod.rs @@ -2,77 +2,55 @@ //! Handles loading assets or resources from different origins (network, local, etc.). //! It also handles caching of resources. +pub mod resource_manager; + +pub mod resource_handler; pub mod texture_resource_handler; pub mod mesh_resource_handler; pub mod material_resource_handler; -use std::{io::prelude::*, hash::{Hasher, Hash}, ffi::OsString,}; - -use log::{warn, info, error, trace, debug}; -use polodb_core::bson::{Document, doc, to_vec}; - -use crate::orchestrator::{System, self}; - // https://www.yosoygames.com.ar/wp/2018/03/vertex-formats-part-1-compression/ +/// This is the struct resource handlers should return when processing a resource. #[derive(Debug, Clone)] -enum ProcessedResources { - Generated((GenericResourceSerialization, Vec)), - Ref(String), +pub struct GenericResourceSerialization { + /// The resource id. This is used to identify the resource. Needs to be meaningful and will be a public constant. + url: String, + /// The resource class (EJ: "Texture", "Mesh", "Material", etc.) + class: String, + /// List of resources that this resource depends on. + required_resources: Vec, + /// The resource data. + resource: polodb_core::bson::Document, } -trait ResourceHandler { - fn can_handle_type(&self, resource_type: &str) -> bool; - - /// Returns a tuple containing the resource description and it's associated binary data.\ - /// - /// The returned document is like the following: - /// ```json - /// { "class": "X", "resource": { ... }, "hash": 0, "required_resources":[{ "path": "..." }] } - /// ``` - /// Fields: - /// - **class**: The resource class. This is used to identify the resource type. Needs to be meaningful and will be a public constant. - /// - **resource**: The resource data. Can look like anything. - /// - **hash**(optional): The resource hash. This is used to identify the resource data. If the resource handler wants to generate a hash for the resource it can do so else the resource manager will generate a hash for it. This is because some resources can generate hashes inteligently (EJ: code generators can output same hash for different looking code if the code is semantically identical). - /// - **required_resources**(optional): A list of resources that this resource depends on. This is used to load resources that depend on other resources. - fn process(&self, resource_manager: &ResourceManager, asset_url: &str, bytes: &[u8]) -> Result, String>; - - fn get_deserializers(&self) -> Vec<(&'static str, Box Box + Send>)>; +impl GenericResourceSerialization { + pub fn new(url: String, resource: T) -> Self { + GenericResourceSerialization { + url, + required_resources: Vec::new(), + class: resource.get_class().to_string(), + resource: polodb_core::bson::to_document(&resource).unwrap(), + } + } - fn read(&self, _resource: &Box, file: &mut std::fs::File, buffers: &mut [Buffer]) { - file.read_exact(buffers[0].buffer).unwrap(); + pub fn required_resources(mut self, required_resources: &[ProcessedResources]) -> Self { + self.required_resources = required_resources.to_vec(); + self } } -/// Resource manager. -/// Handles loading assets or resources from different origins (network, local, etc.). -/// It also handles caching of resources. -/// -/// Resource can be sourced from the local filesystem or from the network. -/// When in a debug build it will lazily load resources from source and cache them. -/// When in a release build it will exclusively load resources from cache. -/// -/// If accessing the filesystem paths will be relative to the assets directory, and assets should omit the extension. -/// -/// The stored resource document is like the following: -/// ```json -/// { "_id":"OId" , "id": 01234, "path": "../..", "class": "X", "size": 0, "resource": { ... }, "hash": 0 } -/// ``` -pub struct ResourceManager { - db: polodb_core::Database, - resource_handlers: Vec>, - deserializers: std::collections::HashMap<&'static str, Box Box + Send>>, +#[derive(Debug, Clone)] +pub enum ProcessedResources { + Generated((GenericResourceSerialization, Vec)), + Ref(String), } -impl orchestrator::Entity for ResourceManager {} -impl System for ResourceManager {} - -impl From for LoadResults { - fn from(error: polodb_core::Error) -> Self { - match error { - _ => LoadResults::LoadFailed - } - } +pub struct Buffer<'a> { + /// The slice of the buffer to load the resource binary data into. + pub buffer: &'a mut [u8], + /// The subresource tag. This is used to identify the subresource. (EJ: "Vertex", "Index", etc.) + pub tag: String, } /// Enumaration for all the possible results of a resource load fails. @@ -119,35 +97,6 @@ pub trait Resource { fn get_class(&self) -> &'static str; } -/// This is the struct resource handlers should return when processing a resource. -#[derive(Debug, Clone)] -pub struct GenericResourceSerialization { - /// The resource id. This is used to identify the resource. Needs to be meaningful and will be a public constant. - url: String, - /// The resource class (EJ: "Texture", "Mesh", "Material", etc.) - class: String, - /// List of resources that this resource depends on. - required_resources: Vec, - /// The resource data. - resource: polodb_core::bson::Document, -} - -impl GenericResourceSerialization { - pub fn new(url: String, resource: T) -> Self { - GenericResourceSerialization { - url, - required_resources: Vec::new(), - class: resource.get_class().to_string(), - resource: polodb_core::bson::to_document(&resource).unwrap(), - } - } - - pub fn required_resources(mut self, required_resources: &[ProcessedResources]) -> Self { - self.required_resources = required_resources.to_vec(); - self - } -} - #[derive(Debug, Clone)] pub struct SerializedResourceDocument(polodb_core::bson::Document); @@ -159,13 +108,6 @@ pub struct Response { pub resources: Vec, } -pub struct Buffer<'a> { - /// The slice of the buffer to load the resource binary data into. - pub buffer: &'a mut [u8], - /// The subresource tag. This is used to identify the subresource. (EJ: "Vertex", "Index", etc.) - pub tag: String, -} - /// Options for loading a resource. pub struct OptionResource<'a> { /// The resource to apply this option to. @@ -177,617 +119,4 @@ pub struct OptionResource<'a> { /// Represents the options for performing a bundled/batch resource load. pub struct Options<'a> { pub resources: Vec>, -} - -impl ResourceManager { - /// Creates a new resource manager. - pub fn new() -> Self { - if let Err(error) = std::fs::create_dir_all("assets") { - match error.kind() { - std::io::ErrorKind::AlreadyExists => {}, - _ => panic!("Could not create assets directory"), - } - } - - if let Err(error) = std::fs::create_dir_all("resources") { - match error.kind() { - std::io::ErrorKind::AlreadyExists => {}, - _ => panic!("Could not create resources directory"), - } - } - - let mut args = std::env::args(); - - let mut memory_only = args.find(|arg| arg == "--ResourceManager.memory_only").is_some(); - - if cfg!(test) { // If we are running tests we want to use memory database. This way we can run tests in parallel. - memory_only = true; - } - - let db_res = if !memory_only { - polodb_core::Database::open_file("resources/resources.db") - } else { - info!("Using memory database instead of file database."); - polodb_core::Database::open_memory() - }; - - let db = match db_res { - Ok(db) => db, - Err(_) => { - // Delete file and try again - std::fs::remove_file("resources/resources.db").unwrap(); - - warn!("Database file was corrupted, deleting and trying again."); - - let db_res = polodb_core::Database::open_file("resources/resources.db"); - - match db_res { - Ok(db) => db, - Err(_) => match polodb_core::Database::open_memory() { // If we can't create a file database, create a memory database. This way we can still run the application. - Ok(db) => { - error!("Could not create database file, using memory database instead."); - db - }, - Err(_) => panic!("Could not create database"), - } - } - } - }; - - let resource_handlers: Vec> = vec![ - Box::new(texture_resource_handler::ImageResourceHandler::new()), - Box::new(mesh_resource_handler::MeshResourceHandler::new()), - Box::new(material_resource_handler::MaterialResourcerHandler::new()) - ]; - - let mut deserializers = std::collections::HashMap::new(); - - for resource_handler in resource_handlers.as_slice() { - for deserializer in resource_handler.get_deserializers() { - deserializers.insert(deserializer.0, deserializer.1); - } - } - - ResourceManager { - db, - resource_handlers, - deserializers, - } - } - - pub fn new_as_system() -> orchestrator::EntityReturn { - orchestrator::EntityReturn::new(Self::new()) - } - - /// Tries to load a resource from cache or source.\ - /// This is a more advanced version of get() as it allows to load resources that depend on other resources.\ - /// - /// If the resource cannot be found (non existent file, unreacheble network address, fails to parse, etc.) it will return None.\ - /// If the resource is in cache but it's data cannot be parsed, it will return None. - /// Return is a tuple containing the resource description and it's associated binary data.\ - /// The requested resource will always the last one in the array. With the previous resources being the ones it depends on. This way when iterating the array forward the dependencies will be loaded first. - pub fn get(&mut self, path: &str) -> Option<(Response, Vec)> { - let request = self.load_from_cache_or_source(path)?; - - let size = request.resources.iter().map(|r| r.size).sum::() as usize; - - let mut buffer = Vec::with_capacity(size); - - unsafe { buffer.set_len(size); } - - let response = self.load_data_from_cache(request, None, &mut buffer).ok()?; - Some((response, buffer)) - } - - /// Tries to load the information/metadata for a resource (and it's dependencies).\ - /// This is a more advanced version of get() as it allows to use your own buffer and/or apply some transformation to the resources when loading.\ - /// The result of this function can be later fed into `load_resource()` which will load the binary data. - pub fn request_resource(&mut self, path: &str) -> Option { - let request = self.load_from_cache_or_source(path)?; - Some(request) - } - - /// Loads the resource binary data from cache.\ - /// If a buffer range is provided it will load the data into the buffer.\ - /// If no buffer range is provided it will return the data in a vector. - /// - /// If a buffer is not provided for a resurce in the options parameters it will be either be loaded into the provided buffer or returned in a vector. - /// - /// Options: Let's you specify how to load the resources. - /// ```json - /// { "resources": [{ "path": "../..", "buffer":{ "index": 0, "offset": 0 } }]} - /// ``` - pub fn load_resource(&mut self, request: Request, options: Option, buffer: Option<&mut [u8]>) -> Result<(Response, Option>), LoadResults> { - if let Some(buffer) = buffer { - let response = self.load_data_from_cache(request, options, buffer)?; - Ok((response, None)) - } else { - let mut buffer = Vec::new(); - let response = self.load_data_from_cache(request, options, &mut buffer)?; - Ok((response, Some(buffer))) - } - } - - /// Recursively loads all the resources needed to load the resource at the given url. - /// **Will** load from source and cache the resources if they are not already cached. - fn gather(&self, db: &polodb_core::Database, url: &str) -> Option> { - let resource_documents = if let Some(resource_document) = db.collection::("resources").find_one(doc!{ "url": url }).unwrap() { - let mut documents = vec![]; - - if let Some(polodb_core::bson::Bson::Array(required_resources)) = resource_document.get("required_resources") { - for required_resource in required_resources { - if let polodb_core::bson::Bson::Document(required_resource) = required_resource { - let resource_path = required_resource.get("url").unwrap().as_str().unwrap(); - documents.append(&mut self.gather(db, resource_path)?); - } - - if let polodb_core::bson::Bson::String(required_resource) = required_resource { - let resource_path = required_resource.as_str(); - documents.append(&mut self.gather(db, resource_path)?); - } - } - } - - documents.push(resource_document); - - documents - } else { - let r = self.read_asset_from_source(url).unwrap(); - - let mut loaded_resource_documents = Vec::new(); - - let resource_handlers = self.resource_handlers.iter().filter(|h| h.can_handle_type(r.1.as_str())); - - for resource_handler in resource_handlers { - let gg = resource_handler.process(self, url, &r.0).unwrap(); - - for g in gg { - match g { - ProcessedResources::Generated(g) => { - for e in &g.0.required_resources { - match e { - ProcessedResources::Generated(g) => { - loaded_resource_documents.push(self.write_resource_to_cache(&g,)?); - }, - ProcessedResources::Ref(r) => { - loaded_resource_documents.append(&mut self.gather(db, &r)?); - } - } - } - - loaded_resource_documents.push(self.write_resource_to_cache(&g,)?); - }, - ProcessedResources::Ref(r) => { - loaded_resource_documents.append(&mut self.gather(db, &r)?); - } - } - } - } - - if loaded_resource_documents.len() == 0 { - warn!("No resource handler could handle resource: {}", url); - } - - loaded_resource_documents - }; - - - Some(resource_documents) - } - - /// Tries to load a resource from cache.\ - /// It also resolves all dependencies.\ - fn load_from_cache_or_source(&self, url: &str) -> Option { - let resource_descriptions = self.gather(&self.db, url).expect("Could not load resource"); - - for r in &resource_descriptions { - trace!("Loaded resource: {:#?}", r); - } - - let request = Request { - resources: resource_descriptions.iter().map(|r| - ResourceRequest { - _id: r.get_object_id("_id").unwrap().clone(), - id: r.get_i64("id").unwrap() as u64, - url: r.get_str("url").unwrap().to_string(), - size: r.get_i64("size").unwrap() as u64, - hash: r.get_i64("hash").unwrap() as u64, - class: r.get_str("class").unwrap().to_string(), - resource: self.deserializers[r.get_str("class").unwrap()](r.get_document("resource").unwrap()), - required_resources: if let Ok(rr) = r.get_array("required_resources") { rr.iter().map(|e| e.as_str().unwrap().to_string()).collect() } else { vec![] }, - } - ).collect(), - }; - - Some(request) - } - - /// Stores the asset as a resource. - /// Returns the resource document. - fn write_resource_to_cache(&self, resource_package: &(GenericResourceSerialization, Vec)) -> Option { - let mut resource_document = polodb_core::bson::Document::new(); - - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - - resource_document.insert("id", hasher.finish() as i64); - resource_document.insert("size", resource_package.1.len() as i64); - - resource_document.insert("url", resource_package.0.url.clone()); - resource_package.0.url.hash(&mut hasher); - - resource_document.insert("class", resource_package.0.class.clone()); - - let mut required_resources_json = polodb_core::bson::Array::new(); - - for required_resources in &resource_package.0.required_resources { // TODO: make new type that gives a guarantee that these resources have been loaded - match required_resources { - ProcessedResources::Generated(g) => { - required_resources_json.push(polodb_core::bson::Bson::String(g.0.url.clone())); - }, - ProcessedResources::Ref(r) => { - required_resources_json.push(polodb_core::bson::Bson::String(r.clone())); - } - } - } - - resource_document.insert("required_resources", required_resources_json); - - let json_resource = resource_package.0.resource.clone(); - - if let None = resource_document.get("hash") { - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - - std::hash::Hasher::write(&mut hasher, resource_package.1.as_slice()); // Hash binary data - - std::hash::Hasher::write(&mut hasher, &to_vec(&json_resource).unwrap()); // Hash resource metadata, since changing the resources description must also change the hash. (For caching purposes) - - resource_document.insert("hash", hasher.finish() as i64); - } - - resource_document.insert("resource", json_resource); - - debug!("Generated resource: {:#?}", &resource_document); - - let insert_result = self.db.collection::("resources").insert_one(&resource_document).ok()?; - - let resource_id = insert_result.inserted_id.as_object_id()?; - - let resource_path = self.resolve_resource_path(resource_id.to_string().as_str()); - - let mut file = std::fs::File::create(resource_path).ok()?; - - file.write_all(resource_package.1.as_slice()).unwrap(); - - resource_document.insert("_id", resource_id); - - return Some(resource_document); - } - - /// Tries to load a resource from cache.\ - /// If the resource cannot be found/loaded or if it's become stale it will return None. - fn load_data_from_cache(&mut self, request: Request, mut options: Option, buffer: &mut [u8]) -> Result { - let mut offset = 0usize; - - let resources = request.resources.into_iter().map(|resource_container| { - let native_db_resource_id = resource_container._id.to_string(); - - let mut file = match std::fs::File::open(self.resolve_resource_path(&native_db_resource_id)) { - Ok(it) => it, - Err(reason) => { - match reason { // TODO: handle specific errors - _ => return Err(LoadResults::CacheFileNotFound), - } - } - }; - - let response = ResourceResponse { - id: resource_container.id, - url: resource_container.url.clone(), - size: resource_container.size, - offset: offset as u64, - hash: resource_container.hash, - class: resource_container.class.clone(), - resource: resource_container.resource, - required_resources: resource_container.required_resources, - }; - - let _slice = if let Some(options) = &mut options { - if let Some(x) = options.resources.iter_mut().find(|e| e.url == resource_container.url) { - self.resource_handlers.iter().find(|h| h.can_handle_type(resource_container.class.as_str())).unwrap(). - read(&response.resource, &mut file, x.buffers.as_mut_slice()); - } else { - let range = &mut buffer[offset..(offset + resource_container.size as usize)]; - offset += resource_container.size as usize; - if let Err(_) = file.read_exact(range) { return Err(LoadResults::LoadFailed); } - } - } else { - let range = &mut buffer[offset..(offset + resource_container.size as usize)]; - offset += resource_container.size as usize; - if let Err(_) = file.read_exact(range) { return Err(LoadResults::LoadFailed); } - }; - - Ok(response) - }).collect::, LoadResults>>()?; - - return Ok(Response { resources }); - } - - fn resolve_resource_path(&self, path: &str) -> String { "resources/".to_string() + path } - fn resolve_asset_path(&self, path: &str) -> String { "assets/".to_string() + path } - - /// Loads an asset from source.\ - /// Expects an asset name in the form of a path relative to the assets directory, or a network address.\ - /// If the asset is not found it will return None. - /// ```ignore - /// let (bytes, format) = ResourceManager::read_asset_from_source("textures/concrete").unwrap(); // Path relative to .../assets - /// ``` - fn read_asset_from_source(&self, url: &str) -> Result<(Vec, String), Option> { - let resource_origin = if url.starts_with("http://") || url.starts_with("https://") { "network" } else { "local" }; - let mut source_bytes; - let format; - match resource_origin { - "network" => { - let request = if let Ok(request) = ureq::get(url).call() { request } else { return Err(None); }; - let content_type = if let Some(e) = request.header("content-type") { e.to_string() } else { return Err(None); }; - format = content_type; - - source_bytes = Vec::new(); - - request.into_reader().read_to_end(&mut source_bytes); - }, - "local" => { - let path = std::path::Path::new("assets/"); - - let url_as_path = std::path::Path::new(url); - - let url_as_path_parent = url_as_path.parent().unwrap(); - - let path = path.join(url_as_path_parent); - - let (mut file, extension) = if let Ok(dir) = std::fs::read_dir(path) { - let files = dir.filter(|f| if let Ok(f) = f { f.path().file_stem().unwrap().eq(url_as_path.file_name().unwrap()) } else { false }); - let file_path = files.last().unwrap().unwrap().path(); - (std::fs::File::open(&file_path).unwrap(), file_path.extension().unwrap().to_str().unwrap().to_string()) - } else { return Err(None); }; - - format = extension.to_string(); - - source_bytes = Vec::with_capacity(file.metadata().unwrap().len() as usize); - - if let Err(_) = file.read_to_end(&mut source_bytes) { - return Err(None); - } - }, - _ => { - // Could not resolve how to get raw resource, return empty bytes - return Err(None); - } - } - - Ok((source_bytes, format)) - } -} - -// TODO: test resource caching - -#[cfg(test)] -mod tests { - use crate::resource_manager::{texture_resource_handler::Texture, mesh_resource_handler::{Mesh, IntegralTypes, VertexSemantics, Size}}; - - /// Tests for the resource manager. - /// It is important to test the load twice as the first time it will be loaded from source and the second time it will be loaded from cache. - - // TODO: test resource load order - - use super::*; - - #[test] - fn load_net_image() { - std::env::set_var("--ResourceManager.memory_only", "true"); // Don't use file database - - let mut resource_manager = ResourceManager::new(); - - // Test loading from source - - let resource_result = resource_manager.get("https://camo.githubusercontent.com/dca6cdb597abc9c7ff4a0e066e6c35eb70b187683fbff2208d0440b4ef6c5a30/68747470733a2f2f692e696d6775722e636f6d2f56525261434f702e706e67"); - - assert!(resource_result.is_some()); - - let (request, _buffer) = resource_result.unwrap(); - - assert_eq!(request.resources.len(), 1); - - let resource_container = &request.resources[0]; - let resource = &resource_container.resource; - - assert_eq!(resource.type_id(), std::any::TypeId::of::()); - - let texture_info = resource.downcast_ref::().unwrap(); - - assert_eq!(texture_info.extent, crate::Extent{ width: 4096, height: 1024, depth: 1 }); - - // Test loading from cache - - let resource_result = resource_manager.get("https://camo.githubusercontent.com/dca6cdb597abc9c7ff4a0e066e6c35eb70b187683fbff2208d0440b4ef6c5a30/68747470733a2f2f692e696d6775722e636f6d2f56525261434f702e706e67"); - - assert!(resource_result.is_some()); - - let (request, _buffer) = resource_result.unwrap(); - - assert_eq!(request.resources.len(), 1); - - let resource_container = &request.resources[0]; - let resource = &resource_container.resource; - - assert_eq!(resource.type_id(), std::any::TypeId::of::()); - - let texture_info = resource.downcast_ref::().unwrap(); - - assert_eq!(texture_info.extent, crate::Extent{ width: 4096, height: 1024, depth: 1 }); - } - - #[ignore] - #[test] - fn load_local_image() { - std::env::set_var("--ResourceManager.memory_only", "true"); // Don't use file database - - let mut resource_manager = ResourceManager::new(); - - let resource_result = resource_manager.get("test"); - - assert!(resource_result.is_some()); - - let (request, _buffer) = resource_result.unwrap(); - - assert_eq!(request.resources.len(), 1); - - let resource_container = &request.resources[0]; - let resource = &resource_container.resource; - - assert_eq!(resource.type_id(), std::any::TypeId::of::()); - - let texture_info = resource.downcast_ref::().unwrap(); - - assert!(texture_info.extent.width == 4096 && texture_info.extent.height == 1024 && texture_info.extent.depth == 1); - } - - #[test] - fn load_local_mesh() { - std::env::set_var("--ResourceManager.memory_only", "true"); // Don't use file database - - let mut resource_manager = ResourceManager::new(); - - // Test loading from source - - let resource_result = resource_manager.get("Box"); - - assert!(resource_result.is_some()); - - let (request, buffer) = resource_result.unwrap(); - - assert_eq!(request.resources.len(), 1); - - let resource_container = &request.resources[0]; - let resource = &resource_container.resource; - - assert_eq!(resource.type_id(), std::any::TypeId::of::()); - - assert_eq!(buffer.len(), (24 /* vertices */ * (3 /* components per position */ * 4 /* float size */ + 3/*normals */ * 4) as usize).next_multiple_of(16) + 6/* cube faces */ * 2 /* triangles per face */ * 3 /* indices per triangle */ * 2 /* bytes per index */); - - let mesh = resource.downcast_ref::().unwrap(); - - assert_eq!(mesh.bounding_box, [[-0.5f32, -0.5f32, -0.5f32], [0.5f32, 0.5f32, 0.5f32]]); - assert_eq!(mesh.vertex_count, 24); - assert_eq!(mesh.index_count, 36); - assert_eq!(mesh.index_type, IntegralTypes::U16); - assert_eq!(mesh.vertex_components.len(), 2); - assert_eq!(mesh.vertex_components[0].semantic, VertexSemantics::Position); - assert_eq!(mesh.vertex_components[0].format, "vec3f"); - assert_eq!(mesh.vertex_components[0].channel, 0); - assert_eq!(mesh.vertex_components[1].semantic, VertexSemantics::Normal); - assert_eq!(mesh.vertex_components[1].format, "vec3f"); - assert_eq!(mesh.vertex_components[1].channel, 1); - - // Test loading from cache - - let resource_result = resource_manager.get("Box"); - - assert!(resource_result.is_some()); - - let (_resource, mesh_buffer) = resource_result.unwrap(); - - assert_eq!(request.resources.len(), 1); - - let resource_container = &request.resources[0]; - let resource = &resource_container.resource; - - assert_eq!(resource.type_id(), std::any::TypeId::of::()); - - assert_eq!(buffer.len(), (24 /* vertices */ * (3 /* components per position */ * 4 /* float size */ + 3/*normals */ * 4) as usize).next_multiple_of(16) + 6/* cube faces */ * 2 /* triangles per face */ * 3 /* indices per triangle */ * 2 /* bytes per index */); - - let mesh = resource.downcast_ref::().unwrap(); - - assert_eq!(mesh.bounding_box, [[-0.5f32, -0.5f32, -0.5f32], [0.5f32, 0.5f32, 0.5f32]]); - assert_eq!(mesh.vertex_count, 24); - assert_eq!(mesh.index_count, 36); - assert_eq!(mesh.index_type, IntegralTypes::U16); - assert_eq!(mesh.vertex_components.len(), 2); - assert_eq!(mesh.vertex_components[0].semantic, VertexSemantics::Position); - assert_eq!(mesh.vertex_components[0].format, "vec3f"); - assert_eq!(mesh.vertex_components[0].channel, 0); - assert_eq!(mesh.vertex_components[1].semantic, VertexSemantics::Normal); - assert_eq!(mesh.vertex_components[1].format, "vec3f"); - assert_eq!(mesh.vertex_components[1].channel, 1); - - let resource_request = resource_manager.request_resource("Box"); - - let resource_request = if let Some(resource_info) = resource_request { resource_info } else { return; }; - - let mut options = Options { resources: Vec::new(), }; - - let mut vertex_buffer = vec![0u8; 1024]; - let mut index_buffer = vec![0u8; 1024]; - - let resource = &resource_request.resources[0]; - - match resource.class.as_str() { - "Mesh" => { - options.resources.push(OptionResource { - url: resource.url.clone(), - buffers: vec![Buffer{ buffer: vertex_buffer.as_mut_slice(), tag: "Vertex".to_string() }, Buffer{ buffer: index_buffer.as_mut_slice(), tag: "Index".to_string() }], - }); - } - _ => {} - } - - let resource = if let Ok(a) = resource_manager.load_resource(resource_request, Some(options), None) { a } else { return; }; - - let (response, _buffer) = (resource.0, resource.1.unwrap()); - - for resource in &response.resources { - match resource.class.as_str() { - "Mesh" => { - let mesh = resource.resource.downcast_ref::().unwrap(); - - assert_eq!(mesh_buffer[0..(mesh.vertex_count * mesh.vertex_components.size() as u32) as usize], vertex_buffer[0..(mesh.vertex_count * mesh.vertex_components.size() as u32) as usize]); - - assert_eq!(mesh_buffer[576..(576 + mesh.index_count * 2) as usize], index_buffer[0..(mesh.index_count * 2) as usize]); - } - _ => {} - } - } - } - - #[ignore] - #[test] - fn load_material() { - std::env::set_var("--ResourceManager.memory_only", "true"); // Don't use file database - - // TODO: move this test to material resource handler - // Evertything being tested here are related to the material resource handler, not the resource manager. - // And the details are implementation specific. - - let mut resource_manager = ResourceManager::new(); - - // Test loading from source - - let resource_result = resource_manager.get("solid"); - - assert!(resource_result.is_some()); - - let (request, _buffer) = resource_result.unwrap(); - - assert_eq!(request.resources.len(), 2); // 1 material, 2 shaders - - let resource_container = &request.resources[0]; - - assert_eq!(resource_container.class, "Shader"); - - let id = resource_container.id; - - let resource_container = &request.resources[1]; - - assert_eq!(resource_container.class, "Material"); - assert_ne!(resource_container.id, id); - } } \ No newline at end of file diff --git a/src/resource_manager/resource_handler.rs b/src/resource_manager/resource_handler.rs new file mode 100644 index 00000000..9fe81bf7 --- /dev/null +++ b/src/resource_manager/resource_handler.rs @@ -0,0 +1,22 @@ +use super::{resource_manager, ProcessedResources, Buffer}; + +pub trait ResourceHandler { + fn can_handle_type(&self, resource_type: &str) -> bool; + + /// Returns a tuple containing the resource description and it's associated binary data.\ + /// + /// The returned document is like the following: + /// ```json + /// { "class": "X", "resource": { ... }, "hash": 0, "required_resources":[{ "path": "..." }] } + /// ``` + /// Fields: + /// - **class**: The resource class. This is used to identify the resource type. Needs to be meaningful and will be a public constant. + /// - **resource**: The resource data. Can look like anything. + /// - **hash**(optional): The resource hash. This is used to identify the resource data. If the resource handler wants to generate a hash for the resource it can do so else the resource manager will generate a hash for it. This is because some resources can generate hashes inteligently (EJ: code generators can output same hash for different looking code if the code is semantically identical). + /// - **required_resources**(optional): A list of resources that this resource depends on. This is used to load resources that depend on other resources. + fn process(&self, resource_manager: &resource_manager::ResourceManager, asset_url: &str, bytes: &[u8]) -> Result, String>; + + fn get_deserializers(&self) -> Vec<(&'static str, Box Box + Send>)>; + + fn read(&self, _resource: &Box, file: &mut std::fs::File, buffers: &mut [Buffer]); +} \ No newline at end of file diff --git a/src/resource_manager/resource_manager.rs b/src/resource_manager/resource_manager.rs new file mode 100644 index 00000000..219151ed --- /dev/null +++ b/src/resource_manager/resource_manager.rs @@ -0,0 +1,438 @@ +use std::{io::{Read, Write}, hash::{Hasher, Hash}}; + +use log::{info, warn, error, trace, debug}; + +use crate::orchestrator; + +use super::{resource_handler, texture_resource_handler, mesh_resource_handler, material_resource_handler, Request, Response, Options, LoadResults, ProcessedResources, ResourceRequest, GenericResourceSerialization, ResourceResponse}; + +/// Resource manager. +/// Handles loading assets or resources from different origins (network, local, etc.). +/// It also handles caching of resources. +/// +/// Resource can be sourced from the local filesystem or from the network. +/// When in a debug build it will lazily load resources from source and cache them. +/// When in a release build it will exclusively load resources from cache. +/// +/// If accessing the filesystem paths will be relative to the assets directory, and assets should omit the extension. +/// +/// The stored resource document is like the following: +/// ```json +/// { "_id":"OId" , "id": 01234, "path": "../..", "class": "X", "size": 0, "resource": { ... }, "hash": 0 } +/// ``` +pub struct ResourceManager { + db: polodb_core::Database, + resource_handlers: Vec>, + deserializers: std::collections::HashMap<&'static str, Box Box + Send>>, +} + +impl orchestrator::Entity for ResourceManager {} +impl orchestrator::System for ResourceManager {} + +impl From for super::LoadResults { + fn from(error: polodb_core::Error) -> Self { + match error { + _ => super::LoadResults::LoadFailed + } + } +} + +impl ResourceManager { + /// Creates a new resource manager. + pub fn new() -> Self { + if let Err(error) = std::fs::create_dir_all("assets") { + match error.kind() { + std::io::ErrorKind::AlreadyExists => {}, + _ => panic!("Could not create assets directory"), + } + } + + if let Err(error) = std::fs::create_dir_all("resources") { + match error.kind() { + std::io::ErrorKind::AlreadyExists => {}, + _ => panic!("Could not create resources directory"), + } + } + + let mut args = std::env::args(); + + let mut memory_only = args.find(|arg| arg == "--ResourceManager.memory_only").is_some(); + + if cfg!(test) { // If we are running tests we want to use memory database. This way we can run tests in parallel. + memory_only = true; + } + + let db_res = if !memory_only { + polodb_core::Database::open_file("resources/resources.db") + } else { + info!("Using memory database instead of file database."); + polodb_core::Database::open_memory() + }; + + let db = match db_res { + Ok(db) => db, + Err(_) => { + // Delete file and try again + std::fs::remove_file("resources/resources.db").unwrap(); + + warn!("Database file was corrupted, deleting and trying again."); + + let db_res = polodb_core::Database::open_file("resources/resources.db"); + + match db_res { + Ok(db) => db, + Err(_) => match polodb_core::Database::open_memory() { // If we can't create a file database, create a memory database. This way we can still run the application. + Ok(db) => { + error!("Could not create database file, using memory database instead."); + db + }, + Err(_) => panic!("Could not create database"), + } + } + } + }; + + let resource_handlers: Vec> = vec![ + Box::new(texture_resource_handler::ImageResourceHandler::new()), + Box::new(mesh_resource_handler::MeshResourceHandler::new()), + Box::new(material_resource_handler::MaterialResourcerHandler::new()) + ]; + + let mut deserializers = std::collections::HashMap::new(); + + for resource_handler in resource_handlers.as_slice() { + for deserializer in resource_handler.get_deserializers() { + deserializers.insert(deserializer.0, deserializer.1); + } + } + + ResourceManager { + db, + resource_handlers, + deserializers, + } + } + + pub fn new_as_system() -> orchestrator::EntityReturn { + orchestrator::EntityReturn::new(Self::new()) + } + + /// Tries to load a resource from cache or source.\ + /// This is a more advanced version of get() as it allows to load resources that depend on other resources.\ + /// + /// If the resource cannot be found (non existent file, unreacheble network address, fails to parse, etc.) it will return None.\ + /// If the resource is in cache but it's data cannot be parsed, it will return None. + /// Return is a tuple containing the resource description and it's associated binary data.\ + /// The requested resource will always the last one in the array. With the previous resources being the ones it depends on. This way when iterating the array forward the dependencies will be loaded first. + pub fn get(&mut self, path: &str) -> Option<(Response, Vec)> { + let request = self.load_from_cache_or_source(path)?; + + let size = request.resources.iter().map(|r| r.size).sum::() as usize; + + let mut buffer = Vec::with_capacity(size); + + unsafe { buffer.set_len(size); } + + let response = self.load_data_from_cache(request, None, &mut buffer).ok()?; + Some((response, buffer)) + } + + /// Tries to load the information/metadata for a resource (and it's dependencies).\ + /// This is a more advanced version of get() as it allows to use your own buffer and/or apply some transformation to the resources when loading.\ + /// The result of this function can be later fed into `load_resource()` which will load the binary data. + pub fn request_resource(&mut self, path: &str) -> Option { + let request = self.load_from_cache_or_source(path)?; + Some(request) + } + + /// Loads the resource binary data from cache.\ + /// If a buffer range is provided it will load the data into the buffer.\ + /// If no buffer range is provided it will return the data in a vector. + /// + /// If a buffer is not provided for a resurce in the options parameters it will be either be loaded into the provided buffer or returned in a vector. + /// + /// Options: Let's you specify how to load the resources. + /// ```json + /// { "resources": [{ "path": "../..", "buffer":{ "index": 0, "offset": 0 } }]} + /// ``` + pub fn load_resource(&mut self, request: Request, options: Option, buffer: Option<&mut [u8]>) -> Result<(Response, Option>), LoadResults> { + if let Some(buffer) = buffer { + let response = self.load_data_from_cache(request, options, buffer)?; + Ok((response, None)) + } else { + let mut buffer = Vec::new(); + let response = self.load_data_from_cache(request, options, &mut buffer)?; + Ok((response, Some(buffer))) + } + } + + /// Recursively loads all the resources needed to load the resource at the given url. + /// **Will** load from source and cache the resources if they are not already cached. + fn gather(&self, db: &polodb_core::Database, url: &str) -> Option> { + let resource_documents = if let Some(resource_document) = db.collection::("resources").find_one(polodb_core::bson::doc!{ "url": url }).unwrap() { + let mut documents = vec![]; + + if let Some(polodb_core::bson::Bson::Array(required_resources)) = resource_document.get("required_resources") { + for required_resource in required_resources { + if let polodb_core::bson::Bson::Document(required_resource) = required_resource { + let resource_path = required_resource.get("url").unwrap().as_str().unwrap(); + documents.append(&mut self.gather(db, resource_path)?); + } + + if let polodb_core::bson::Bson::String(required_resource) = required_resource { + let resource_path = required_resource.as_str(); + documents.append(&mut self.gather(db, resource_path)?); + } + } + } + + documents.push(resource_document); + + documents + } else { + let r = self.read_asset_from_source(url).unwrap(); + + let mut loaded_resource_documents = Vec::new(); + + let resource_handlers = self.resource_handlers.iter().filter(|h| h.can_handle_type(r.1.as_str())); + + for resource_handler in resource_handlers { + let gg = resource_handler.process(self, url, &r.0).unwrap(); + + for g in gg { + match g { + ProcessedResources::Generated(g) => { + for e in &g.0.required_resources { + match e { + ProcessedResources::Generated(g) => { + loaded_resource_documents.push(self.write_resource_to_cache(&g,)?); + }, + ProcessedResources::Ref(r) => { + loaded_resource_documents.append(&mut self.gather(db, &r)?); + } + } + } + + loaded_resource_documents.push(self.write_resource_to_cache(&g,)?); + }, + ProcessedResources::Ref(r) => { + loaded_resource_documents.append(&mut self.gather(db, &r)?); + } + } + } + } + + if loaded_resource_documents.len() == 0 { + warn!("No resource handler could handle resource: {}", url); + } + + loaded_resource_documents + }; + + + Some(resource_documents) + } + + /// Tries to load a resource from cache.\ + /// It also resolves all dependencies.\ + fn load_from_cache_or_source(&self, url: &str) -> Option { + let resource_descriptions = self.gather(&self.db, url).expect("Could not load resource"); + + for r in &resource_descriptions { + trace!("Loaded resource: {:#?}", r); + } + + let request = Request { + resources: resource_descriptions.iter().map(|r| + ResourceRequest { + _id: r.get_object_id("_id").unwrap().clone(), + id: r.get_i64("id").unwrap() as u64, + url: r.get_str("url").unwrap().to_string(), + size: r.get_i64("size").unwrap() as u64, + hash: r.get_i64("hash").unwrap() as u64, + class: r.get_str("class").unwrap().to_string(), + resource: self.deserializers[r.get_str("class").unwrap()](r.get_document("resource").unwrap()), + required_resources: if let Ok(rr) = r.get_array("required_resources") { rr.iter().map(|e| e.as_str().unwrap().to_string()).collect() } else { vec![] }, + } + ).collect(), + }; + + Some(request) + } + + /// Stores the asset as a resource. + /// Returns the resource document. + fn write_resource_to_cache(&self, resource_package: &(GenericResourceSerialization, Vec)) -> Option { + let mut resource_document = polodb_core::bson::Document::new(); + + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + + resource_document.insert("id", hasher.finish() as i64); + resource_document.insert("size", resource_package.1.len() as i64); + + resource_document.insert("url", resource_package.0.url.clone()); + resource_package.0.url.hash(&mut hasher); + + resource_document.insert("class", resource_package.0.class.clone()); + + let mut required_resources_json = polodb_core::bson::Array::new(); + + for required_resources in &resource_package.0.required_resources { // TODO: make new type that gives a guarantee that these resources have been loaded + match required_resources { + ProcessedResources::Generated(g) => { + required_resources_json.push(polodb_core::bson::Bson::String(g.0.url.clone())); + }, + ProcessedResources::Ref(r) => { + required_resources_json.push(polodb_core::bson::Bson::String(r.clone())); + } + } + } + + resource_document.insert("required_resources", required_resources_json); + + let json_resource = resource_package.0.resource.clone(); + + if let None = resource_document.get("hash") { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + + std::hash::Hasher::write(&mut hasher, resource_package.1.as_slice()); // Hash binary data + + std::hash::Hasher::write(&mut hasher, &polodb_core::bson::to_vec(&json_resource).unwrap()); // Hash resource metadata, since changing the resources description must also change the hash. (For caching purposes) + + resource_document.insert("hash", hasher.finish() as i64); + } + + resource_document.insert("resource", json_resource); + + debug!("Generated resource: {:#?}", &resource_document); + + let insert_result = self.db.collection::("resources").insert_one(&resource_document).ok()?; + + let resource_id = insert_result.inserted_id.as_object_id()?; + + let resource_path = self.resolve_resource_path(resource_id.to_string().as_str()); + + let mut file = std::fs::File::create(resource_path).ok()?; + + file.write_all(resource_package.1.as_slice()).unwrap(); + + resource_document.insert("_id", resource_id); + + return Some(resource_document); + } + + /// Tries to load a resource from cache.\ + /// If the resource cannot be found/loaded or if it's become stale it will return None. + fn load_data_from_cache(&mut self, request: Request, mut options: Option, buffer: &mut [u8]) -> Result { + let mut offset = 0usize; + + let resources = request.resources.into_iter().map(|resource_container| { + let native_db_resource_id = resource_container._id.to_string(); + + let mut file = match std::fs::File::open(self.resolve_resource_path(&native_db_resource_id)) { + Ok(it) => it, + Err(reason) => { + match reason { // TODO: handle specific errors + _ => return Err(LoadResults::CacheFileNotFound), + } + } + }; + + let response = ResourceResponse { + id: resource_container.id, + url: resource_container.url.clone(), + size: resource_container.size, + offset: offset as u64, + hash: resource_container.hash, + class: resource_container.class.clone(), + resource: resource_container.resource, + required_resources: resource_container.required_resources, + }; + + let _slice = if let Some(options) = &mut options { + if let Some(x) = options.resources.iter_mut().find(|e| e.url == resource_container.url) { + self.resource_handlers.iter().find(|h| h.can_handle_type(resource_container.class.as_str())).unwrap(). + read(&response.resource, &mut file, x.buffers.as_mut_slice()); + } else { + let range = &mut buffer[offset..(offset + resource_container.size as usize)]; + offset += resource_container.size as usize; + if let Err(_) = file.read_exact(range) { return Err(LoadResults::LoadFailed); } + } + } else { + let range = &mut buffer[offset..(offset + resource_container.size as usize)]; + offset += resource_container.size as usize; + if let Err(_) = file.read_exact(range) { return Err(LoadResults::LoadFailed); } + }; + + Ok(response) + }).collect::, LoadResults>>()?; + + return Ok(Response { resources }); + } + + fn resolve_resource_path(&self, path: &str) -> String { "resources/".to_string() + path } + fn resolve_asset_path(&self, path: &str) -> String { "assets/".to_string() + path } + + /// Loads an asset from source.\ + /// Expects an asset name in the form of a path relative to the assets directory, or a network address.\ + /// If the asset is not found it will return None. + /// ```ignore + /// let (bytes, format) = ResourceManager::read_asset_from_source("textures/concrete").unwrap(); // Path relative to .../assets + /// ``` + pub fn read_asset_from_source(&self, url: &str) -> Result<(Vec, String), Option> { + let resource_origin = if url.starts_with("http://") || url.starts_with("https://") { "network" } else { "local" }; + let mut source_bytes; + let format; + match resource_origin { + "network" => { + let request = if let Ok(request) = ureq::get(url).call() { request } else { return Err(None); }; + let content_type = if let Some(e) = request.header("content-type") { e.to_string() } else { return Err(None); }; + format = content_type; + + source_bytes = Vec::new(); + + request.into_reader().read_to_end(&mut source_bytes); + }, + "local" => { + let path = std::path::Path::new("assets/"); + + let url_as_path = std::path::Path::new(url); + + let url_as_path_parent = url_as_path.parent().ok_or(None)?; + + let path = path.join(url_as_path_parent); + + let (mut file, extension) = if let Ok(dir) = std::fs::read_dir(path) { + let files = dir.filter(|f| if let Ok(f) = f { f.path().file_stem().unwrap().eq(url_as_path.file_name().unwrap()) } else { false }); + + let file_path = files.last().ok_or(None)?.or(Err(None))?.path(); + + (std::fs::File::open(&file_path).unwrap(), file_path.extension().unwrap().to_str().unwrap().to_string()) + } else { return Err(None); }; + + format = extension.to_string(); + + source_bytes = Vec::with_capacity(file.metadata().unwrap().len() as usize); + + if let Err(_) = file.read_to_end(&mut source_bytes) { + return Err(None); + } + }, + _ => { + // Could not resolve how to get raw resource, return empty bytes + return Err(None); + } + } + + Ok((source_bytes, format)) + } +} + +// TODO: test resource caching + +#[cfg(test)] +mod tests { + // TODO: test resource load order + + use super::*; +} \ No newline at end of file diff --git a/src/resource_manager/texture_resource_handler.rs b/src/resource_manager/texture_resource_handler.rs index 22b259f2..9d02bb78 100644 --- a/src/resource_manager/texture_resource_handler.rs +++ b/src/resource_manager/texture_resource_handler.rs @@ -1,8 +1,10 @@ +use std::io::Read; + use serde::{Serialize, Deserialize}; use crate::resource_manager::GenericResourceSerialization; -use super::{ResourceHandler, SerializedResourceDocument, Resource, ProcessedResources}; +use super::{SerializedResourceDocument, Resource, ProcessedResources, resource_manager::ResourceManager, resource_handler::ResourceHandler}; pub(crate) struct ImageResourceHandler { @@ -23,7 +25,7 @@ impl ResourceHandler for ImageResourceHandler { } } - fn process(&self, _: &super::ResourceManager, asset_url: &str, bytes: &[u8]) -> Result, String> { + fn process(&self, _: &ResourceManager, asset_url: &str, bytes: &[u8]) -> Result, String> { let mut decoder = png::Decoder::new(bytes); decoder.set_transformations(png::Transformations::normalize_to_color8()); let mut reader = decoder.read_info().unwrap(); @@ -67,6 +69,10 @@ impl ResourceHandler for ImageResourceHandler { Ok(vec![ProcessedResources::Generated((resource_document, intel_tex_2::bc7::compress_blocks(&settings, &rgba_surface)))]) } + fn read(&self, _resource: &Box, file: &mut std::fs::File, buffers: &mut [super::Buffer]) { + file.read_exact(buffers[0].buffer).unwrap(); + } + fn get_deserializers(&self) -> Vec<(&'static str, Box Box + Send>)> { vec![("Texture", Box::new(|document| { let texture = Texture::deserialize(polodb_core::bson::Deserializer::new(document.into())).unwrap(); @@ -88,4 +94,47 @@ pub struct Texture { impl Resource for Texture { fn get_class(&self) -> &'static str { "Texture" } +} + +#[cfg(test)] +mod tests { + use crate::resource_manager::resource_manager::ResourceManager; + + use super::*; + + #[test] + fn load_net_image() { + let mut resource_manager = ResourceManager::new(); + + let (response, _) = resource_manager.get("https://camo.githubusercontent.com/dca6cdb597abc9c7ff4a0e066e6c35eb70b187683fbff2208d0440b4ef6c5a30/68747470733a2f2f692e696d6775722e636f6d2f56525261434f702e706e67").expect("Failed to load image"); + + assert_eq!(response.resources.len(), 1); + + let resource_container = &response.resources[0]; + let resource = &resource_container.resource; + + assert_eq!(resource.type_id(), std::any::TypeId::of::()); + + let texture_info = resource.downcast_ref::().unwrap(); + + assert_eq!(texture_info.extent, crate::Extent{ width: 4096, height: 1024, depth: 1 }); + } + + #[test] + fn load_local_image() { + let mut resource_manager = ResourceManager::new(); + + let (response, _) = resource_manager.get("patterned_brick_floor_02_diff_2k").expect("Failed to load image"); + + assert_eq!(response.resources.len(), 1); + + let resource_container = &response.resources[0]; + let resource = &resource_container.resource; + + assert_eq!(resource.type_id(), std::any::TypeId::of::()); + + let texture_info = resource.downcast_ref::().unwrap(); + + assert!(texture_info.extent.width == 2048 && texture_info.extent.height == 2048 && texture_info.extent.depth == 1); + } } \ No newline at end of file diff --git a/src/window_system.rs b/src/window_system.rs index e6dfa762..b6300f7d 100644 --- a/src/window_system.rs +++ b/src/window_system.rs @@ -585,6 +585,12 @@ impl WindowInternal { } } } + + pub fn close(&self) { + let connection = &self.connection; + + connection.send_request(&xcb::x::DestroyWindow { window: self.window }); + } } /// The window system. @@ -662,6 +668,10 @@ impl WindowSystem { } } + pub fn close_window(&mut self, window_handle: &WindowHandle) { + self.windows[window_handle.0 as usize].close(); + } + /// Gets the OS handles for a window. /// /// # Arguments @@ -724,19 +734,14 @@ mod tests { // assert_ne!(os_handles.xcb_window, 0); // } - #[ignore] #[test] fn test_window_loop() { let mut window_system = WindowSystem::new(); - let _window_handle = window_system.create_window("Main Window", Extent { width: 1920, height: 1080, depth: 1 }, "main_window"); + let window_handle = window_system.create_window("Main Window", Extent { width: 1920, height: 1080, depth: 1 }, "main_window"); - loop { - if window_system.update() == false { - break; - } + std::thread::sleep(std::time::Duration::from_millis(500)); - std::thread::sleep(std::time::Duration::from_millis(16)); - } + window_system.close_window(&window_handle); } } \ No newline at end of file