diff --git a/Cargo.lock b/Cargo.lock index 3cd3114..edd048f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -120,6 +120,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2707e3afba5e19b75d582d88bc79237418f2a2a2d673d01cf9b03633b46e98f3" + [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -298,6 +304,7 @@ version = "0.2.0" dependencies = [ "byteorder", "cmake", + "crc64", "fnv", "sha1", "strum", diff --git a/Cargo.toml b/Cargo.toml index 23a8393..58631b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" byteorder = "1.5" fnv = "1.0" sha1 = "0.10" +crc64 = "2.0" strum = "0.25" strum_macros = "0.25" walkdir = "2.4" diff --git a/src/archive.rs b/src/archive.rs deleted file mode 100644 index ef23a21..0000000 --- a/src/archive.rs +++ /dev/null @@ -1,748 +0,0 @@ -///////////////////////////////////////////////////////////////////////////////////////// -/// ARCHIVE -///////////////////////////////////////////////////////////////////////////////////////// -use std::cmp::Ordering; -use std::collections::{HashMap, HashSet}; -use std::fs::{create_dir_all, File}; -use std::io::{self, BufWriter, Read, Result, Seek, SeekFrom, Write}; -use std::mem; -use std::path::{Path, PathBuf}; - -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; -use strum::IntoEnumIterator; -use walkdir::WalkDir; - -use crate::cr2w::read_cr2w_header; -use crate::io::{read_null_terminated_string, write_null_terminated_string, FromReader}; -use crate::kraken::{ - self, compress, decompress, get_compressed_buffer_size_needed, CompressionLevel, -}; -use crate::{fnv1a64_hash_string, sha1_hash_file, ERedExtension}; - -#[derive(Debug, Clone, Default)] -pub struct Archive { - pub header: Header, - pub index: Index, - - // custom - pub file_names: HashMap, -} - -impl Archive { - // Function to read a Header from a file - pub fn from_file(file_path: &PathBuf) -> Result { - let mut file = File::open(file_path)?; - let mut buffer = Vec::with_capacity(mem::size_of::
()); - - file.read_to_end(&mut buffer)?; - - // Ensure that the buffer has enough bytes to represent a Header - if buffer.len() < mem::size_of::
() { - return Err(io::Error::new( - io::ErrorKind::InvalidData, - "File does not contain enough data to parse Header", - )); - } - - let mut cursor = io::Cursor::new(&buffer); - let header = Header::from_reader(&mut cursor)?; - - // read custom data - let mut file_names: HashMap = HashMap::default(); - if let Ok(custom_data_length) = cursor.read_u32::() { - if custom_data_length > 0 { - cursor.set_position(HEADER_EXTENDED_SIZE); - if let Ok(footer) = LxrsFooter::from_reader(&mut cursor) { - // add files to hashmap - for f in footer.files { - let hash = fnv1a64_hash_string(&f); - file_names.insert(hash, f); - } - } - } - } - - // move to offset Header.IndexPosition - cursor.set_position(header.index_position); - let index = Index::from_reader(&mut cursor)?; - - Ok(Archive { - header, - index, - file_names, - }) - } - - // get filehashes - pub fn get_file_hashes(&self) -> Vec { - self.index - .file_entries - .iter() - .map(|f| f.1.name_hash_64) - .collect::>() - } -} - -//static HEADER_MAGIC: u32 = 1380009042; -//static HEADER_SIZE: i32 = 40; -static HEADER_EXTENDED_SIZE: u64 = 0xAC; - -#[derive(Debug, Clone, Copy)] -pub struct Header { - pub magic: u32, - pub version: u32, - pub index_position: u64, - pub index_size: u32, - pub debug_position: u64, - pub debug_size: u32, - pub filesize: u64, -} - -impl Default for Header { - fn default() -> Self { - Self { - magic: 1380009042, - version: 12, - index_position: Default::default(), - index_size: Default::default(), - debug_position: Default::default(), - debug_size: Default::default(), - filesize: Default::default(), - } - } -} - -impl FromReader for Header { - fn from_reader(reader: &mut R) -> Result { - Ok(Header { - magic: reader.read_u32::()?, - version: reader.read_u32::()?, - index_position: reader.read_u64::()?, - index_size: reader.read_u32::()?, - debug_position: reader.read_u64::()?, - debug_size: reader.read_u32::()?, - filesize: reader.read_u64::()?, - }) - } -} -impl Header { - pub fn serialize(&self, writer: &mut W) -> Result<()> { - writer.write_u32::(self.magic)?; - writer.write_u32::(self.version)?; - writer.write_u64::(self.index_position)?; - writer.write_u32::(self.index_size)?; - writer.write_u64::(self.debug_position)?; - writer.write_u32::(self.debug_size)?; - writer.write_u64::(self.filesize)?; - - Ok(()) - } -} - -#[derive(Debug, Clone, Default)] -pub struct Index { - pub file_table_offset: u32, - pub file_table_size: u32, - pub crc: u64, - pub file_entry_count: u32, - pub file_segment_count: u32, - pub resource_dependency_count: u32, - - // not serialized - pub file_entries: HashMap, - pub file_segments: Vec, - pub dependencies: Vec, -} -impl Index { - pub fn serialize(&self, writer: &mut W) -> Result<()> { - writer.write_u32::(self.file_table_offset)?; - writer.write_u32::(self.file_table_size)?; - writer.write_u64::(self.crc)?; - writer.write_u32::(self.file_entry_count)?; - writer.write_u32::(self.file_segment_count)?; - writer.write_u32::(self.resource_dependency_count)?; - - Ok(()) - } -} -impl FromReader for Index { - fn from_reader(cursor: &mut R) -> io::Result { - let mut index = Index { - file_table_offset: cursor.read_u32::()?, - file_table_size: cursor.read_u32::()?, - crc: cursor.read_u64::()?, - file_entry_count: cursor.read_u32::()?, - file_segment_count: cursor.read_u32::()?, - resource_dependency_count: cursor.read_u32::()?, - - file_entries: HashMap::default(), - file_segments: vec![], - dependencies: vec![], - }; - - // read tables - for _i in 0..index.file_entry_count { - let entry = FileEntry::from_reader(cursor)?; - index.file_entries.insert(entry.name_hash_64, entry); - } - - for _i in 0..index.file_segment_count { - index.file_segments.push(FileSegment::from_reader(cursor)?); - } - - for _i in 0..index.resource_dependency_count { - index.dependencies.push(Dependency::from_reader(cursor)?); - } - - // ignore the rest of the archive - - Ok(index) - } -} - -#[derive(Debug, Clone, Copy)] -pub struct FileSegment { - pub offset: u64, - pub z_size: u32, - pub size: u32, -} - -impl FromReader for FileSegment { - fn from_reader(reader: &mut R) -> io::Result { - Ok(FileSegment { - offset: reader.read_u64::()?, - z_size: reader.read_u32::()?, - size: reader.read_u32::()?, - }) - } -} - -#[derive(Debug, Clone, Copy)] -pub struct FileEntry { - pub name_hash_64: u64, - pub timestamp: u64, //SystemTime, - pub num_inline_buffer_segments: u32, - pub segments_start: u32, - pub segments_end: u32, - pub resource_dependencies_start: u32, - pub resource_dependencies_end: u32, - pub sha1_hash: [u8; 20], -} - -impl FromReader for FileEntry { - fn from_reader(reader: &mut R) -> io::Result { - let mut entry = FileEntry { - name_hash_64: reader.read_u64::()?, - timestamp: reader.read_u64::()?, - num_inline_buffer_segments: reader.read_u32::()?, - segments_start: reader.read_u32::()?, - segments_end: reader.read_u32::()?, - resource_dependencies_start: reader.read_u32::()?, - resource_dependencies_end: reader.read_u32::()?, - sha1_hash: [0; 20], - }; - - reader.read_exact(&mut entry.sha1_hash[..])?; - - Ok(entry) - } -} - -#[derive(Debug, Clone, Copy)] -pub struct Dependency { - pub hash: u64, -} - -impl FromReader for Dependency { - fn from_reader(reader: &mut R) -> io::Result { - Ok(Dependency { - hash: reader.read_u64::()?, - }) - } -} - -#[derive(Debug, Clone)] -pub struct LxrsFooter { - pub files: Vec, -} - -impl LxrsFooter { - //const MINLEN: u32 = 20; - const MAGIC: u32 = 0x4C585253; - const VERSION: u32 = 1; - - pub fn serialize(&self, writer: &mut W) -> Result<()> { - writer.write_u32::(self.files.len() as u32)?; - writer.write_u32::(LxrsFooter::VERSION)?; - - // write strings to buffer - let mut buffer: Vec = Vec::new(); - for f in &self.files { - write_null_terminated_string(&mut buffer, f.to_owned())?; - } - - // compress - let size = buffer.len(); - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress(&buffer, &mut compressed_buffer, CompressionLevel::Normal); - assert!((zsize as u32) <= size as u32); - compressed_buffer.resize(zsize as usize, 0); - - // write to writer - writer.write_all(&compressed_buffer)?; - - Ok(()) - } -} -impl FromReader for LxrsFooter { - fn from_reader(reader: &mut R) -> io::Result { - let magic = reader.read_u32::()?; - if magic != LxrsFooter::MAGIC { - return Err(io::Error::new(io::ErrorKind::Other, "invalid magic")); - } - let _version = reader.read_u32::()?; - let size = reader.read_u32::()?; - let zsize = reader.read_u32::()?; - let count = reader.read_i32::()?; - - let mut files: Vec = vec![]; - match size.cmp(&zsize) { - Ordering::Greater => { - // buffer is compressed - let mut compressed_buffer = vec![0; zsize as usize]; - reader.read_exact(&mut compressed_buffer[..])?; - let mut output_buffer = vec![]; - let result = decompress(compressed_buffer, &mut output_buffer, size as usize); - assert_eq!(result as u32, size); - - // read from buffer - let mut inner_cursor = io::Cursor::new(&output_buffer); - for _i in 0..count { - // read NullTerminatedString - if let Ok(string) = read_null_terminated_string(&mut inner_cursor) { - files.push(string); - } - } - } - Ordering::Less => { - // error - return Err(io::Error::new(io::ErrorKind::Other, "invalid buffer")); - } - Ordering::Equal => { - // no compression - for _i in 0..count { - // read NullTerminatedString - if let Ok(string) = read_null_terminated_string(reader) { - files.push(string); - } - } - } - } - - let footer = LxrsFooter { files }; - - Ok(footer) - } -} - -fn pad_until_page(writer: &mut W) -> io::Result<()> { - let pos = writer.stream_position()?; - let modulo = pos / 4096; - let diff = ((modulo + 1) * 4096) - pos; - let padding = vec![0xD9; diff as usize]; - writer.write_all(padding.as_slice())?; - - Ok(()) -} - -/// Decompresses and writes a kraken-compressed segment from an archive to a stream -/// -/// # Errors -/// -/// This function will return an error if . -fn decompress_segment( - archive_reader: &mut R, - segment: &FileSegment, - file_writer: &mut W, -) -> Result<()> { - archive_reader.seek(SeekFrom::Start(segment.offset))?; - - let magic = archive_reader.read_u32::()?; - if magic == kraken::MAGIC { - // read metadata - let mut size = segment.size; - let size_in_header = archive_reader.read_u32::()?; - if size_in_header != size { - size = size_in_header; - } - let mut compressed_buffer = vec![0; segment.z_size as usize - 8]; - archive_reader.read_exact(&mut compressed_buffer[..])?; - let mut output_buffer = vec![]; - let result = decompress(compressed_buffer, &mut output_buffer, size as usize); - assert_eq!(result as u32, size); - - // write - file_writer.write_all(&output_buffer)?; - } else { - // incorrect data, fall back to direct copy - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - }; - - Ok(()) -} - -///////////////////////////////////////////////////////////////////////////////////////// -/// Lib -///////////////////////////////////////////////////////////////////////////////////////// - -/// Extracts all files from an archive and writes them to a folder -/// -/// # Panics -/// -/// Panics if file path operations fail -/// -/// # Errors -/// -/// This function will return an error if any parsing fails -pub fn extract_archive( - in_file: &PathBuf, - out_dir: &Path, - hash_map: &HashMap, -) -> io::Result<()> { - // parse archive headers - let archive = Archive::from_file(in_file)?; - - let archive_file = File::open(in_file)?; - let mut archive_reader = io::BufReader::new(archive_file); - - for (hash, file_entry) in archive.index.file_entries.iter() { - // get filename - let mut name_or_hash: String = format!("{}.bin", hash); - if let Some(name) = hash_map.get(hash) { - name_or_hash = name.to_owned(); - } - if let Some(name) = archive.file_names.get(hash) { - name_or_hash = name.to_owned(); - } - - // name or hash is a relative path - let outfile = out_dir.join(name_or_hash); - create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; - - // extract to stream - let mut fs = File::create(outfile)?; - let mut file_writer = BufWriter::new(&mut fs); - // decompress main file - let start_index = file_entry.segments_start; - let next_index = file_entry.segments_end; - if let Some(segment) = archive.index.file_segments.get(start_index as usize) { - // read and decompress from main archive stream - - // kraken decompress - if segment.size == segment.z_size { - // just copy over - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - } else { - decompress_segment(&mut archive_reader, segment, &mut file_writer)?; - } - } - - // extract additional buffers - for i in start_index + 1..next_index { - if let Some(segment) = archive.index.file_segments.get(i as usize) { - // do not decompress with oodle - archive_reader.seek(SeekFrom::Start(segment.offset))?; - let mut buffer = vec![0; segment.z_size as usize]; - archive_reader.read_exact(&mut buffer[..])?; - file_writer.write_all(&buffer)?; - } - } - } - - Ok(()) -} - -/// Packs redengine 4 resource file in a folder to an archive -/// -/// # Panics -/// -/// Panics if any path conversions fail -/// -/// # Errors -/// -/// This function will return an error if any parsing or IO fails -pub fn write_archive( - in_folder: &Path, - out_folder: &Path, - modname: Option<&str>, - hash_map: HashMap, -) -> io::Result<()> { - if !in_folder.exists() { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "")); - } - - if !out_folder.exists() { - return Err(io::Error::new(io::ErrorKind::InvalidInput, "")); - } - - let archive_name = if let Some(name) = modname { - format!("{}.archive", name) - } else { - let folder_name = in_folder.file_name().expect("Could not get in_dir name"); - format!("{}.archive", folder_name.to_string_lossy()) - }; - - // collect files - let mut included_extensions = ERedExtension::iter() - .map(|variant| variant.to_string()) - .collect::>(); - included_extensions.push(String::from("bin")); - - // get only resource files - let allfiles = WalkDir::new(in_folder) - .into_iter() - .filter_map(|e| e.ok()) - .map(|f| f.into_path()) - .filter(|p| { - if let Some(ext) = p.extension() { - if let Some(ext) = ext.to_str() { - return included_extensions.contains(&ext.to_owned()); - } - } - false - }) - .collect::>(); - - // sort by hash - let mut hashed_paths = allfiles - .iter() - .filter_map(|f| { - if let Ok(relative_path) = f.strip_prefix(in_folder) { - if let Some(path_str) = relative_path.to_str() { - let hash = fnv1a64_hash_string(&path_str.to_string()); - return Some((f.clone(), hash)); - } - } - None - }) - .collect::>(); - hashed_paths.sort_by_key(|k| k.1); - - let outfile = out_folder.join(archive_name); - let mut fs = File::create(outfile)?; - let mut archive_writer = BufWriter::new(&mut fs); - - // write temp header - let mut archive = Archive::default(); - let header = Header::default(); - header.serialize(&mut archive_writer)?; - archive_writer.write_all(&[0u8; 132])?; // some weird padding - - // write custom header - assert_eq!(HEADER_EXTENDED_SIZE, archive_writer.stream_position()?); - let custom_paths = hashed_paths - .iter() - .filter(|(_p, k)| hash_map.contains_key(k)) - .filter_map(|(f, _h)| { - if let Ok(path) = f.strip_prefix(in_folder) { - return Some(path.to_string_lossy().to_string()); - } - None - }) - .collect::>(); - - let mut custom_data_length = 0; - if !custom_paths.is_empty() { - let wfooter = LxrsFooter { - files: custom_paths, - }; - wfooter.serialize(&mut archive_writer)?; - custom_data_length = archive_writer.stream_position()? - HEADER_EXTENDED_SIZE; - } - - // write files - let mut imports_hash_set: HashSet = HashSet::new(); - for (path, hash) in hashed_paths { - // read file - let mut file = File::open(&path)?; - let mut file_buffer = Vec::new(); - file.read_to_end(&mut file_buffer)?; - - let firstimportidx = imports_hash_set.len(); - let mut lastimportidx = imports_hash_set.len(); - let firstoffsetidx = archive.index.file_segments.len(); - let mut lastoffsetidx = 0; - let mut flags = 0; - - let mut file_cursor = io::Cursor::new(&file_buffer); - if let Ok(info) = read_cr2w_header(&mut file_cursor) { - // get main file - file_cursor.seek(SeekFrom::Start(0))?; - let size = info.header.objects_end; - let mut resource_buffer = vec![0; size as usize]; - file_cursor.read_exact(&mut resource_buffer[..])?; - // get archive offset before writing - let archive_offset = archive_writer.stream_position()?; - - // kark file - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress( - &resource_buffer, - &mut compressed_buffer, - CompressionLevel::Normal, - ); - assert!((zsize as u32) <= size); - compressed_buffer.resize(zsize as usize, 0); - - // write compressed main file archive - // KARK header - archive_writer.write_u32::(kraken::MAGIC)?; //magic - archive_writer.write_u32::(size)?; //uncompressed buffer length - archive_writer.write_all(&compressed_buffer)?; - - // add metadata to archive - archive.index.file_segments.push(FileSegment { - offset: archive_offset, - size, - z_size: zsize as u32, - }); - - // write buffers (bytes after the main file) - for buffer_info in info.buffers_table.iter() { - let mut buffer = vec![0; buffer_info.disk_size as usize]; - file_cursor.read_exact(&mut buffer[..])?; - - let bsize = buffer_info.mem_size; - let bzsize = buffer_info.disk_size; - let boffset = archive_writer.stream_position()?; - archive_writer.write_all(buffer.as_slice())?; - - // add metadata to archive - archive.index.file_segments.push(FileSegment { - offset: boffset, - size: bsize, - z_size: bzsize, - }); - } - - //register imports - for import in info.imports.iter() { - // TODO fix flags - // if (cr2WImportWrapper.Flags is not InternalEnums.EImportFlags.Soft and not InternalEnums.EImportFlags.Embedded) - imports_hash_set.insert(import.depot_path.to_owned()); - } - - lastimportidx = imports_hash_set.len(); - lastoffsetidx = archive.index.file_segments.len(); - flags = if !info.buffers_table.is_empty() { - info.buffers_table.len() - 1 - } else { - 0 - }; - } else { - // write non-cr2w file - file_cursor.seek(SeekFrom::Start(0))?; - if let Some(os_ext) = path.extension() { - let ext = os_ext.to_ascii_lowercase().to_string_lossy().to_string(); - if get_aligned_file_extensions().contains(&ext) { - pad_until_page(&mut archive_writer)?; - } - - let offset = archive_writer.stream_position()?; - let size = file_buffer.len() as u32; - let mut final_zsize = file_buffer.len() as u32; - if get_uncompressed_file_extensions().contains(&ext) { - // direct copy - archive_writer.write_all(&file_buffer)?; - } else { - // kark file - let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); - let mut compressed_buffer = vec![0; compressed_size_needed as usize]; - let zsize = compress( - &file_buffer, - &mut compressed_buffer, - CompressionLevel::Normal, - ); - assert!((zsize as u32) <= size); - compressed_buffer.resize(zsize as usize, 0); - final_zsize = zsize as u32; - // write - archive_writer.write_all(&compressed_buffer)?; - } - - // add metadata to archive - archive.index.file_segments.push(FileSegment { - offset, - size, - z_size: final_zsize, - }); - } - } - - // update archive metadata - let sha1_hash = sha1_hash_file(&file_buffer); - - let entry = FileEntry { - name_hash_64: hash, - timestamp: 0, // TODO proper timestamps - num_inline_buffer_segments: flags as u32, - segments_start: firstoffsetidx as u32, - segments_end: lastoffsetidx as u32, - resource_dependencies_start: firstimportidx as u32, - resource_dependencies_end: lastimportidx as u32, - sha1_hash, - }; - archive.index.file_entries.insert(hash, entry); - } - - // write footers - // padding - pad_until_page(&mut archive_writer)?; - - // write tables - let tableoffset = archive_writer.stream_position()?; - archive.index.serialize(&mut archive_writer)?; - let tablesize = archive_writer.stream_position()? - tableoffset; - - // padding - pad_until_page(&mut archive_writer)?; - let filesize = archive_writer.stream_position()?; - - // write the header again - archive.header.index_position = tableoffset; - archive.header.index_size = tablesize as u32; - archive.header.filesize = filesize; - archive_writer.seek(SeekFrom::Start(0))?; - archive.header.serialize(&mut archive_writer)?; - archive_writer.write_u32::(custom_data_length as u32)?; - - Ok(()) -} - -/// . -fn get_aligned_file_extensions() -> Vec { - let files = vec![".bk2", ".bnk", ".opusinfo", ".wem", ".bin"]; - files.into_iter().map(|f| f.to_owned()).collect::>() -} - -/// . -fn get_uncompressed_file_extensions() -> Vec { - let files = vec![ - ".bk2", - ".bnk", - ".opusinfo", - ".wem", - ".bin", - ".dat", - ".opuspak", - ]; - files.into_iter().map(|f| f.to_owned()).collect::>() -} diff --git a/src/archive/dependency.rs b/src/archive/dependency.rs new file mode 100644 index 0000000..3ef29e0 --- /dev/null +++ b/src/archive/dependency.rs @@ -0,0 +1,31 @@ +use std::io::{Read, Result}; + +use byteorder::{LittleEndian, ReadBytesExt}; + +use crate::io::FromReader; + +#[allow(dead_code)] +#[derive(Debug, Clone, Copy)] +pub struct Dependency { + hash: u64, +} + +// impl Dependency { +// pub(crate) fn new(hash: u64) -> Self { +// Self { hash } +// } + +// pub(crate) fn write(&self, writer: &mut W) -> Result<()> { +// writer.write_u64::(self.hash)?; +// Ok(()) +// } +// } +#[warn(dead_code)] + +impl FromReader for Dependency { + fn from_reader(reader: &mut R) -> Result { + Ok(Dependency { + hash: reader.read_u64::()?, + }) + } +} diff --git a/src/archive/file_entry.rs b/src/archive/file_entry.rs new file mode 100644 index 0000000..372765a --- /dev/null +++ b/src/archive/file_entry.rs @@ -0,0 +1,92 @@ +use std::io::{Read, Result, Write}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +use crate::io::FromReader; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct FileEntry { + name_hash_64: u64, + timestamp: u64, //SystemTime, + num_inline_buffer_segments: u32, + segments_start: u32, + segments_end: u32, + resource_dependencies_start: u32, + resource_dependencies_end: u32, + sha1_hash: [u8; 20], +} + +impl FileEntry { + pub(crate) fn new( + name_hash_64: u64, + timestamp: u64, + num_inline_buffer_segments: u32, + segments_start: u32, + segments_end: u32, + resource_dependencies_start: u32, + resource_dependencies_end: u32, + sha1_hash: [u8; 20], + ) -> Self { + Self { + name_hash_64, + timestamp, + num_inline_buffer_segments, + segments_start, + segments_end, + resource_dependencies_start, + resource_dependencies_end, + sha1_hash, + } + } + + pub(crate) fn write(&self, writer: &mut W) -> Result<()> { + writer.write_u64::(self.name_hash_64)?; + writer.write_u64::(self.timestamp)?; + writer.write_u32::(self.num_inline_buffer_segments)?; + writer.write_u32::(self.segments_start)?; + writer.write_u32::(self.segments_end)?; + writer.write_u32::(self.resource_dependencies_start)?; + writer.write_u32::(self.resource_dependencies_end)?; + writer.write_all(self.sha1_hash.as_slice())?; + Ok(()) + } + + pub(crate) fn name_hash_64(&self) -> u64 { + self.name_hash_64 + } + + pub(crate) fn segments_start(&self) -> u32 { + self.segments_start + } + + pub(crate) fn segments_end(&self) -> u32 { + self.segments_end + } + + pub(crate) fn set_segments_start(&mut self, segments_start: u32) { + self.segments_start = segments_start; + } + + pub(crate) fn set_segments_end(&mut self, segments_end: u32) { + self.segments_end = segments_end; + } +} + +impl FromReader for FileEntry { + fn from_reader(reader: &mut R) -> Result { + let mut entry = FileEntry { + name_hash_64: reader.read_u64::()?, + timestamp: reader.read_u64::()?, + num_inline_buffer_segments: reader.read_u32::()?, + segments_start: reader.read_u32::()?, + segments_end: reader.read_u32::()?, + resource_dependencies_start: reader.read_u32::()?, + resource_dependencies_end: reader.read_u32::()?, + sha1_hash: [0; 20], + }; + + reader.read_exact(&mut entry.sha1_hash[..])?; + + Ok(entry) + } +} diff --git a/src/archive/file_segment.rs b/src/archive/file_segment.rs new file mode 100644 index 0000000..e065a74 --- /dev/null +++ b/src/archive/file_segment.rs @@ -0,0 +1,51 @@ +use std::io::{Read, Result, Write}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +use crate::io::FromReader; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct FileSegment { + offset: u64, + z_size: u32, + size: u32, +} + +impl FileSegment { + pub(crate) fn new(offset: u64, z_size: u32, size: u32) -> Self { + Self { + offset, + z_size, + size, + } + } + + pub(crate) fn write(&self, writer: &mut W) -> Result<()> { + writer.write_u64::(self.offset)?; + writer.write_u32::(self.z_size)?; + writer.write_u32::(self.size)?; + Ok(()) + } + + pub(crate) fn offset(&self) -> u64 { + self.offset + } + + pub(crate) fn z_size(&self) -> u32 { + self.z_size + } + + pub(crate) fn size(&self) -> u32 { + self.size + } +} + +impl FromReader for FileSegment { + fn from_reader(reader: &mut R) -> Result { + Ok(FileSegment { + offset: reader.read_u64::()?, + z_size: reader.read_u32::()?, + size: reader.read_u32::()?, + }) + } +} diff --git a/src/archive/header.rs b/src/archive/header.rs new file mode 100644 index 0000000..33d37f3 --- /dev/null +++ b/src/archive/header.rs @@ -0,0 +1,86 @@ +use std::io::{Read, Result, Write}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +use crate::io::FromReader; + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Header { + magic: u32, + version: u32, + index_position: u64, + index_size: u32, + debug_position: u64, + debug_size: u32, + filesize: u64, +} + +impl Header { + pub(crate) fn new( + index_position: u64, + index_size: u32, + debug_position: u64, + debug_size: u32, + filesize: u64, + ) -> Self { + Self { + magic: Header::HEADER_MAGIC, + version: Header::HEADER_VERSION, + index_position, + index_size, + debug_position, + debug_size, + filesize, + } + } + + pub(crate) const HEADER_MAGIC: u32 = 1380009042; + pub(crate) const HEADER_VERSION: u32 = 12; + pub(crate) const HEADER_SIZE: usize = 40; + pub(crate) const HEADER_EXTENDED_SIZE: u64 = 0xAC; + + pub(crate) fn index_position(&self) -> u64 { + self.index_position + } +} + +// impl Default for Header { +// fn default() -> Self { +// Self { +// magic: 1380009042, +// version: 12, +// index_position: Default::default(), +// index_size: Default::default(), +// debug_position: Default::default(), +// debug_size: Default::default(), +// filesize: Default::default(), +// } +// } +// } + +impl FromReader for Header { + fn from_reader(reader: &mut R) -> Result { + Ok(Header { + magic: reader.read_u32::()?, + version: reader.read_u32::()?, + index_position: reader.read_u64::()?, + index_size: reader.read_u32::()?, + debug_position: reader.read_u64::()?, + debug_size: reader.read_u32::()?, + filesize: reader.read_u64::()?, + }) + } +} +impl Header { + pub(crate) fn write(&self, writer: &mut W) -> Result<()> { + writer.write_u32::(self.magic)?; + writer.write_u32::(self.version)?; + writer.write_u64::(self.index_position)?; + writer.write_u32::(self.index_size)?; + writer.write_u64::(self.debug_position)?; + writer.write_u32::(self.debug_size)?; + writer.write_u64::(self.filesize)?; + + Ok(()) + } +} diff --git a/src/archive/index.rs b/src/archive/index.rs new file mode 100644 index 0000000..0b436c3 --- /dev/null +++ b/src/archive/index.rs @@ -0,0 +1,49 @@ +use std::io::{Read, Result}; + +use byteorder::{LittleEndian, ReadBytesExt}; + +use crate::io::FromReader; + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub(crate) struct Index { + /// Offset from the beginning of this struct, should be 8 + file_table_offset: u32, + /// byte size of the table + file_table_size: u32, + crc: u64, + file_entry_count: u32, + file_segment_count: u32, + resource_dependency_count: u32, +} + +#[warn(dead_code)] + +impl Index { + pub(crate) fn file_entry_count(&self) -> u32 { + self.file_entry_count + } + + pub(crate) fn file_segment_count(&self) -> u32 { + self.file_segment_count + } + + pub(crate) fn resource_dependency_count(&self) -> u32 { + self.resource_dependency_count + } +} + +impl FromReader for Index { + fn from_reader(cursor: &mut R) -> Result { + let index = Index { + file_table_offset: cursor.read_u32::()?, + file_table_size: cursor.read_u32::()?, + crc: cursor.read_u64::()?, + file_entry_count: cursor.read_u32::()?, + file_segment_count: cursor.read_u32::()?, + resource_dependency_count: cursor.read_u32::()?, + }; + + Ok(index) + } +} diff --git a/src/archive/lxrs.rs b/src/archive/lxrs.rs new file mode 100644 index 0000000..1ab2636 --- /dev/null +++ b/src/archive/lxrs.rs @@ -0,0 +1,101 @@ +use std::{ + cmp::Ordering, + io::{Cursor, Error, ErrorKind, Read, Result, Write}, +}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; + +use crate::{io::*, kraken::*}; + +#[derive(Debug, Clone)] +pub(crate) struct LxrsFooter { + files: Vec, +} + +impl LxrsFooter { + pub(crate) fn new(files: Vec) -> Self { + Self { files } + } + + //const MINLEN: u32 = 20; + const MAGIC: u32 = 0x4C585253; + const VERSION: u32 = 1; + + pub(crate) fn write(&self, writer: &mut W) -> Result<()> { + writer.write_u32::(self.files.len() as u32)?; + writer.write_u32::(LxrsFooter::VERSION)?; + + // write strings to buffer + let mut buffer: Vec = Vec::new(); + for f in &self.files { + write_null_terminated_string(&mut buffer, f.to_owned())?; + } + + // compress + let size = buffer.len(); + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress(&buffer, &mut compressed_buffer, CompressionLevel::Normal); + assert!((zsize as u32) <= size as u32); + compressed_buffer.resize(zsize as usize, 0); + + // write to writer + writer.write_all(&compressed_buffer)?; + + Ok(()) + } + + pub(crate) fn files(&self) -> &[String] { + self.files.as_ref() + } +} +impl FromReader for LxrsFooter { + fn from_reader(reader: &mut R) -> Result { + let magic = reader.read_u32::()?; + if magic != LxrsFooter::MAGIC { + return Err(Error::new(ErrorKind::Other, "invalid magic")); + } + let _version = reader.read_u32::()?; + let size = reader.read_u32::()?; + let zsize = reader.read_u32::()?; + let count = reader.read_i32::()?; + + let mut files: Vec = vec![]; + match size.cmp(&zsize) { + Ordering::Greater => { + // buffer is compressed + let mut compressed_buffer = vec![0; zsize as usize]; + reader.read_exact(&mut compressed_buffer[..])?; + let mut output_buffer = vec![]; + let result = decompress(compressed_buffer, &mut output_buffer, size as usize); + assert_eq!(result as u32, size); + + // read from buffer + let mut inner_cursor = Cursor::new(&output_buffer); + for _i in 0..count { + // read NullTerminatedString + if let Ok(string) = read_null_terminated_string(&mut inner_cursor) { + files.push(string); + } + } + } + Ordering::Less => { + // error + return Err(Error::new(ErrorKind::Other, "invalid buffer")); + } + Ordering::Equal => { + // no compression + for _i in 0..count { + // read NullTerminatedString + if let Ok(string) = read_null_terminated_string(reader) { + files.push(string); + } + } + } + } + + let footer = LxrsFooter { files }; + + Ok(footer) + } +} diff --git a/src/archive/mod.rs b/src/archive/mod.rs new file mode 100644 index 0000000..0e4d6c0 --- /dev/null +++ b/src/archive/mod.rs @@ -0,0 +1,998 @@ +///////////////////////////////////////////////////////////////////////////////////////// +// ARCHIVE +///////////////////////////////////////////////////////////////////////////////////////// + +use std::{ + borrow::BorrowMut, + collections::HashMap, + fs::{create_dir_all, File}, + io::{self, BufWriter, Cursor, Error, ErrorKind, Read, Result, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, +}; + +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use strum::IntoEnumIterator; +use walkdir::WalkDir; + +use crate::kraken::*; +use crate::{cr2w::*, *}; +use crate::{fnv1a64_hash_string, io::FromReader}; + +use self::{dependency::*, file_entry::*, file_segment::*, header::*, index::*, lxrs::*}; + +mod dependency; +mod file_entry; +mod file_segment; +mod header; +mod index; +mod lxrs; + +///////////////////////////////////////////////////////////////////////////////////////// +// ARCHIVE_FILE +// https://learn.microsoft.com/en-us/dotnet/api/system.io.compression.zipfile?view=net-8.0#methods +// ZipFile -> namespace +// Provides static methods for creating, extracting, and opening zip archives. +// +// ZipArchive -> Archive +// Represents a package of compressed files in the zip archive format. +// +// ZipArchiveEntry -> ArchiveEntry +// Represents a compressed file within a zip archive. +///////////////////////////////////////////////////////////////////////////////////////// + +// public static void CreateFromDirectory (string sourceDirectoryName, System.IO.Stream destination); + +/// Creates an archive in the specified stream that contains the files and directories from the specified directory. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn create_from_directory( + source_directory_name: &P, + destination: W, + hash_map: Option>, +) -> Result<()> +where + P: AsRef, + W: Write + Seek, +{ + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + write_archive(source_directory_name, destination, map) +} + +// public static void CreateFromDirectory (string sourceDirectoryName, string destinationArchiveFileName); + +/// Creates an archive that contains the files and directories from the specified directory. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn create_from_directory_path

( + source_directory_name: &P, + destination: &P, + hash_map: Option>, +) -> Result<()> +where + P: AsRef, +{ + let map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + let fs: File = File::create(destination)?; + write_archive(source_directory_name, fs, map) +} + +// public static void ExtractToDirectory (System.IO.Stream source, string destinationDirectoryName, bool overwriteFiles); + +/// Extracts all the files from the archive stored in the specified stream and places them in the specified destination directory on the file system, and optionally allows choosing if the files in the destination directory should be overwritten. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn extract_to_directory( + source: &mut R, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: Option>, +) -> Result<()> +where + P: AsRef, + R: Read + Seek + 'static, +{ + let mut archive = ZipArchive::from_reader_consume(source, ArchiveMode::Read)?; + archive.extract_to_directory(destination_directory_name, overwrite_files, hash_map) +} + +// public static void ExtractToDirectory (string sourceArchiveFileName, string destinationDirectoryName, bool overwriteFiles); + +/// Extracts all of the files in the specified archive to a directory on the file system. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn extract_to_directory_path( + source_archive_file_name: &P, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: Option>, +) -> Result<()> +where + P: AsRef, + R: Read + Seek, +{ + let mut archive = open_read(source_archive_file_name)?; + archive.extract_to_directory(destination_directory_name, overwrite_files, hash_map) +} + +// public static System.IO.Compression.ZipArchive Open (string archiveFileName, System.IO.Compression.ZipArchiveMode mode); + +/// Opens an archive at the specified path and in the specified mode. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn open

(archive_file_name: P, mode: ArchiveMode) -> Result> +where + P: AsRef, +{ + match mode { + ArchiveMode::Create => { + let file = File::create(archive_file_name)?; + ZipArchive::from_reader_consume(file, mode) + } + ArchiveMode::Read => open_read(archive_file_name), + ArchiveMode::Update => { + let file = File::open(archive_file_name)?; + ZipArchive::from_reader_consume(file, mode) + } + } +} + +// public static System.IO.Compression.ZipArchive OpenRead (string archiveFileName); + +/// Opens an archive for reading at the specified path. +/// +/// # Errors +/// +/// This function will return an error if any io fails. +pub fn open_read

(archive_file_name: P) -> Result> +where + P: AsRef, +{ + let file = File::open(archive_file_name)?; + ZipArchive::from_reader_consume(file, ArchiveMode::Read) +} + +/// Packs redengine 4 resource file in a folder to an archive +/// +/// # Panics +/// +/// Panics if any path conversions fail +/// +/// # Errors +/// +/// This function will return an error if any parsing or IO fails +fn write_archive(in_folder: &P, out_stream: W, hash_map: HashMap) -> Result<()> +where + P: AsRef, + W: Write + Seek, +{ + if !in_folder.as_ref().exists() { + return Err(Error::new( + ErrorKind::InvalidInput, + "Input folder does not exist", + )); + } + // get files + let resources = collect_resource_files(in_folder); + + // get paths and sort by hash + let mut file_info = resources + .iter() + .filter_map(|f| { + if let Ok(relative_path) = f.strip_prefix(in_folder) { + if let Some(path_str) = relative_path.to_str() { + let hash = fnv1a64_hash_string(&path_str.to_string()); + return Some((f.clone(), hash)); + } + } + None + }) + .collect::>(); + file_info.sort_by_key(|k| k.1); + + let custom_paths = file_info + .iter() + .filter(|(_p, k)| hash_map.contains_key(k)) + .filter_map(|(f, _h)| { + if let Ok(path) = f.strip_prefix(in_folder) { + return Some(path.to_string_lossy().to_string()); + } + None + }) + .collect::>(); + + // start write + + let mut archive_writer = BufWriter::new(out_stream); + + // write empty header + archive_writer.write_all(&[0u8; Header::HEADER_SIZE])?; //write empty header + archive_writer.write_all(&[0u8; 132])?; // padding + + // write custom header + let mut custom_data_length = 0; + if !custom_paths.is_empty() { + let wfooter = LxrsFooter::new(custom_paths); + wfooter.write(&mut archive_writer)?; + custom_data_length = archive_writer.stream_position()? - Header::HEADER_EXTENDED_SIZE; + } + + // write files + //let imports_hash_set: HashSet = HashSet::new(); + let mut entries = HashMap::default(); + for (path, hash) in file_info { + let wrapped_entry = make_entry(path, &mut archive_writer, hash)?; + + entries.insert(hash, wrapped_entry); + } + + // run through entries again and enumerate the segments + let mut file_segments_cnt = 0; + for (_hash, entry) in entries.iter_mut() { + let firstoffsetidx = file_segments_cnt; + file_segments_cnt += entry.buffers.len() + 1; + let lastoffsetidx = file_segments_cnt; + entry.entry.set_segments_start(firstoffsetidx as u32); + entry.entry.set_segments_end(lastoffsetidx as u32); + } + + // write footers + // let dependencies = imports_hash_set + // .iter() + // .map(|e| Dependency::new(fnv1a64_hash_string(e))) + // .collect::>(); + + // padding + pad_until_page(&mut archive_writer)?; + + // write tables + let tableoffset = archive_writer.stream_position()?; + write_index(&mut archive_writer, &entries /*, &dependencies */)?; + let tablesize = archive_writer.stream_position()? - tableoffset; + + // padding + pad_until_page(&mut archive_writer)?; + + // write the header again + let filesize = archive_writer.stream_position()?; + let header = Header::new(tableoffset, tablesize as u32, 0, 0, filesize); + archive_writer.seek(SeekFrom::Start(0))?; + header.write(&mut archive_writer)?; + archive_writer.write_u32::(custom_data_length as u32)?; + + Ok(()) +} + +fn make_entry( + path: PathBuf, + archive_writer: &mut BufWriter, + hash: u64, +) -> Result { + let mut file = File::open(&path)?; + let mut file_buffer = Vec::new(); + file.read_to_end(&mut file_buffer)?; + let mut file_cursor = Cursor::new(&file_buffer); + + let mut flags = 0; + let segment: FileSegment; + let mut buffers = vec![]; + + if let Ok(info) = read_cr2w_header(&mut file_cursor) { + // get main file + file_cursor.seek(SeekFrom::Start(0))?; + let size = info.header.objects_end; + let mut resource_buffer = vec![0; size as usize]; + file_cursor.read_exact(&mut resource_buffer[..])?; + // get archive offset before writing + let archive_offset = archive_writer.stream_position()?; + + // kark file + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress( + &resource_buffer, + &mut compressed_buffer, + CompressionLevel::Normal, + ); + assert!((zsize as u32) <= size); + compressed_buffer.resize(zsize as usize, 0); + + // write compressed main file archive + // KARK header + archive_writer.write_u32::(kraken::MAGIC)?; //magic + archive_writer.write_u32::(size)?; //uncompressed buffer length + archive_writer.write_all(&compressed_buffer)?; + + // add metadata to archive + segment = FileSegment::new(archive_offset, zsize as u32, size); + + // write buffers (bytes after the main file) + for buffer_info in info.buffers_table.iter() { + let mut buffer = vec![0; buffer_info.disk_size as usize]; + file_cursor.read_exact(&mut buffer[..])?; + + let bsize = buffer_info.mem_size; + let bzsize = buffer_info.disk_size; + let boffset = archive_writer.stream_position()?; + archive_writer.write_all(buffer.as_slice())?; + + // add metadata to archive + buffers.push(FileSegment::new(boffset, bzsize, bsize)); + } + + //register imports + // NOTE don't use a dependency list for mods + //for import in info.imports.iter() { + // if (cr2WImportWrapper.Flags is not InternalEnums.EImportFlags.Soft and not InternalEnums.EImportFlags.Embedded) + //imports_hash_set.insert(import.depot_path.to_owned()); + //} + + //lastimportidx = imports_hash_set.len(); + + flags = if !info.buffers_table.is_empty() { + info.buffers_table.len() - 1 + } else { + 0 + }; + } else { + // write non-cr2w file + file_cursor.seek(SeekFrom::Start(0))?; + let os_ext = path.extension().unwrap(); + let ext = os_ext.to_ascii_lowercase().to_string_lossy().to_string(); + if get_aligned_file_extensions().contains(&ext) { + pad_until_page(archive_writer)?; + } + + let offset = archive_writer.stream_position()?; + let size = file_buffer.len() as u32; + let final_zsize; + if get_uncompressed_file_extensions().contains(&ext) { + // direct copy + archive_writer.write_all(&file_buffer)?; + final_zsize = size; + } else { + // kark file + let compressed_size_needed = get_compressed_buffer_size_needed(size as u64); + let mut compressed_buffer = vec![0; compressed_size_needed as usize]; + let zsize = compress( + &file_buffer, + &mut compressed_buffer, + CompressionLevel::Normal, + ); + assert!((zsize as u32) <= size); + compressed_buffer.resize(zsize as usize, 0); + final_zsize = zsize as u32; + // write + archive_writer.write_all(&compressed_buffer)?; + } + + // add metadata to archive + segment = FileSegment::new(offset, final_zsize, size); + } + let sha1_hash = sha1_hash_file(&file_buffer); + let entry = FileEntry::new( + hash, + 0, + flags as u32, + 0, //firstoffsetidx as u32, + 0, //lastoffsetidx as u32, + 0, //firstimportidx as u32, + 0, //lastimportidx as u32, + sha1_hash, + ); + let wrapped_entry = ZipEntry { + hash, + name: None, + entry, + segment, + buffers, + }; + Ok(wrapped_entry) +} + +fn collect_resource_files>(in_folder: &P) -> Vec { + // collect files + let mut included_extensions = ERedExtension::iter() + .map(|variant| variant.to_string()) + .collect::>(); + included_extensions.push(String::from("bin")); + + // get only resource files + let allfiles = WalkDir::new(in_folder) + .into_iter() + .filter_map(|e| e.ok()) + .map(|f| f.into_path()) + .filter(|p| { + if let Some(ext) = p.extension() { + if let Some(ext) = ext.to_str() { + return included_extensions.contains(&ext.to_owned()); + } + } + false + }) + .collect::>(); + allfiles +} + +/// Decompresses and writes a kraken-compressed segment from an archive to a stream +/// +/// # Errors +/// +/// This function will return an error if . +fn decompress_segment( + archive_reader: &mut R, + segment: &FileSegment, + file_writer: &mut W, +) -> Result<()> { + archive_reader.seek(SeekFrom::Start(segment.offset()))?; + + let magic = archive_reader.read_u32::()?; + if magic == kraken::MAGIC { + // read metadata + let mut size = segment.size(); + let size_in_header = archive_reader.read_u32::()?; + if size_in_header != size { + size = size_in_header; + } + let mut compressed_buffer = vec![0; segment.z_size() as usize - 8]; + archive_reader.read_exact(&mut compressed_buffer[..])?; + let mut output_buffer = vec![]; + let result = decompress(compressed_buffer, &mut output_buffer, size as usize); + assert_eq!(result as u32, size); + + // write + file_writer.write_all(&output_buffer)?; + } else { + // incorrect data, fall back to direct copy + archive_reader.seek(SeekFrom::Start(segment.offset()))?; + let mut buffer = vec![0; segment.z_size() as usize]; + archive_reader.read_exact(&mut buffer[..])?; + file_writer.write_all(&buffer)?; + }; + + Ok(()) +} + +/// . +fn get_aligned_file_extensions() -> Vec { + let files = vec![".bk2", ".bnk", ".opusinfo", ".wem", ".bin"]; + files.into_iter().map(|f| f.to_owned()).collect::>() +} + +/// . +fn get_uncompressed_file_extensions() -> Vec { + let files = vec![ + ".bk2", + ".bnk", + ".opusinfo", + ".wem", + ".bin", + ".dat", + ".opuspak", + ]; + files.into_iter().map(|f| f.to_owned()).collect::>() +} + +fn pad_until_page(writer: &mut W) -> Result<()> { + let pos = writer.stream_position()?; + let modulo = pos / 4096; + let diff = ((modulo + 1) * 4096) - pos; + let padding = vec![0xD9; diff as usize]; + writer.write_all(padding.as_slice())?; + + Ok(()) +} + +///////////////////////////////////////////////////////////////////////////////////////// +// API +///////////////////////////////////////////////////////////////////////////////////////// + +#[derive(Debug, Clone, Default, PartialEq)] +pub enum ArchiveMode { + #[default] + Create, + Read, + Update, +} + +#[derive(Debug)] +pub struct ZipArchive { + /// wraps a stream + stream: S, + + /// The read-write mode of the archive + mode: ArchiveMode, + dirty: bool, + /// The files inside an archive + entries: HashMap, + pub dependencies: Vec, +} + +impl ZipArchive { + /// Get an entry in the archive by resource path. + pub fn get_entry(&self, name: &str) -> Option<&ZipEntry> { + self.entries.get(&fnv1a64_hash_string(&name.to_owned())) + } + + /// Get an entry in the archive by hash (FNV1a64 of resource path). + pub fn get_entry_by_hash(&self, hash: &u64) -> Option<&ZipEntry> { + self.entries.get(hash) + } +} + +impl ZipArchive +where + R: Read + Seek, +{ + /// Extracts a single entry to a directory path. + /// + /// # Errors + /// + /// This function will return an error if the entry cannot be found or any io fails. + pub fn extract_entry>( + &mut self, + entry: ZipEntry, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: &HashMap, + ) -> Result<()> { + let Some(info) = entry.get_resolved_name(&hash_map) else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Could not get entry info from archive.", + )); + }; + + // name or hash is a relative path + let outfile = destination_directory_name.as_ref().join(info); + create_dir_all(outfile.parent().expect("Could not create an out_dir"))?; + + // extract to stream + let mut fs = if overwrite_files { + File::create(outfile)? + } else { + File::options() + .read(true) + .write(true) + .create_new(true) + .open(outfile)? + }; + + let writer = BufWriter::new(&mut fs); + self.extract_segments(&entry, writer)?; + + Ok(()) + } + + /// Extracts a single entry by hash to a directory path. + /// + /// # Errors + /// + /// This function will return an error if the entry cannot be found or any io fails. + pub fn extract_entry_by_hash>( + &mut self, + hash: u64, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: &HashMap, + ) -> Result<()> { + if let Some(entry) = self.get_entry_by_hash(&hash) { + self.extract_entry( + entry.clone(), + destination_directory_name, + overwrite_files, + hash_map, + ) + } else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Could not find entry.", + )); + } + } + + /// Extracts a single entry by resource path to a directory path. + /// + /// # Errors + /// + /// This function will return an error if the entry cannot be found or any io fails. + pub fn extract_entry_by_name>( + &mut self, + name: String, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: &HashMap, + ) -> Result<()> { + if let Some(entry) = self.get_entry(&name) { + self.extract_entry( + entry.clone(), + destination_directory_name, + overwrite_files, + hash_map, + ) + } else { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Could not find entry.", + )); + } + } + + /// Returns an open read stream to an entry of this [`ZipArchive`]. + pub fn open_entry(&mut self, entry: ZipEntry, writer: W) -> Result<()> { + self.extract_segments(&entry, writer)?; + + Ok(()) + } + + /// Extracts all entries to the given directory. + /// + /// # Errors + /// + /// This function will return an error if io fails. + pub fn extract_to_directory>( + &mut self, + destination_directory_name: &P, + overwrite_files: bool, + hash_map: Option>, + ) -> Result<()> { + let hash_map = if let Some(hash_map) = hash_map { + hash_map + } else { + get_red4_hashes() + }; + + // collect info + let mut entries: Vec = vec![]; + for (_hash, entry) in &self.entries { + entries.push(entry.clone()); + } + + for entry in entries { + self.extract_entry( + entry, + destination_directory_name, + overwrite_files, + &hash_map, + )?; + } + + Ok(()) + } + + // getters + + fn reader_mut(&mut self) -> &mut R { + self.stream.borrow_mut() + } + + // methods + + /// Extracts segments to a writer, expects correct offset info. + /// + /// # Errors + /// + /// This function will return an error if io fails + fn extract_segments(&mut self, entry: &ZipEntry, mut writer: W) -> Result<()> { + let segment = entry.segment; + let buffers = entry.buffers.clone(); + + if segment.size() == segment.z_size() { + // just copy + self.reader_mut().seek(SeekFrom::Start(segment.offset()))?; + let mut buffer = vec![0; segment.z_size() as usize]; + self.reader_mut().read_exact(&mut buffer[..])?; + writer.write_all(&buffer)?; + } else { + decompress_segment(self.reader_mut(), &segment, &mut writer)?; + } + for segment in buffers { + self.reader_mut().seek(SeekFrom::Start(segment.offset()))?; + let mut buffer = vec![0; segment.z_size() as usize]; + self.reader_mut().read_exact(&mut buffer[..])?; + writer.write_all(&buffer)?; + } + + Ok(()) + } + + /// Opens an archive, needs to be read-only + fn from_reader_consume(mut reader: R, mode: ArchiveMode) -> Result> { + // checks + if mode == ArchiveMode::Create { + return Ok(ZipArchive:: { + stream: reader, + mode, + dirty: true, + entries: HashMap::default(), + dependencies: Vec::default(), + }); + } + + // read header + let header = Header::from_reader(&mut reader)?; + + // read custom data + let mut file_names: HashMap = HashMap::default(); + if let Ok(custom_data_length) = reader.read_u32::() { + if custom_data_length > 0 { + reader.seek(io::SeekFrom::Start(Header::HEADER_EXTENDED_SIZE))?; + if let Ok(footer) = LxrsFooter::from_reader(&mut reader) { + // add files to hashmap + for f in footer.files() { + let hash = fnv1a64_hash_string(f); + file_names.insert(hash, f.to_owned()); + } + } + } + } + + // read index + // move to offset Header.IndexPosition + reader.seek(io::SeekFrom::Start(header.index_position()))?; + let index = Index::from_reader(&mut reader)?; + + // read tables + let mut file_entries: HashMap = HashMap::default(); + for _i in 0..index.file_entry_count() { + let entry = FileEntry::from_reader(&mut reader)?; + file_entries.insert(entry.name_hash_64(), entry); + } + + let mut file_segments = Vec::default(); + for _i in 0..index.file_segment_count() { + file_segments.push(FileSegment::from_reader(&mut reader)?); + } + + // dependencies can't be connected to individual files anymore + let mut dependencies = Vec::default(); + for _i in 0..index.resource_dependency_count() { + dependencies.push(Dependency::from_reader(&mut reader)?); + } + + // construct wrapper + let mut entries = HashMap::default(); + for (hash, entry) in file_entries.iter() { + let resolved = if let Some(name) = file_names.get(hash) { + Some(name.to_owned()) + } else { + None + }; + + let start_index = entry.segments_start(); + let next_index = entry.segments_end(); + if let Some(segment) = file_segments.get(start_index as usize) { + let mut buffers: Vec = vec![]; + for i in start_index + 1..next_index { + if let Some(buffer) = file_segments.get(i as usize) { + buffers.push(*buffer); + } + } + + let zip_entry = ZipEntry { + hash: *hash, + name: resolved, + entry: *entry, + segment: *segment, + buffers, + }; + entries.insert(*hash, zip_entry); + } + } + + let archive = ZipArchive:: { + stream: reader, + mode, + entries, + dependencies, + dirty: false, + }; + Ok(archive) + } +} + +impl ZipArchive { + fn write(&mut self) { + todo!() + } + + /// Compresses and adds a file to the archive. + /// + /// # Errors + /// + /// This function will return an error if compression or io fails, or if the mode is Read. + pub fn create_entry>( + &mut self, + _file_path: P, + _compression_level: CompressionLevel, + ) -> Result { + // can only add entries in update mode + if self.mode != ArchiveMode::Update { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Archive is in read-only mode.", + )); + } + + // write? + + // set dirty + self.dirty = true; + + todo!() + } + + /// Deletes an entry from the archive + pub fn delete_entry(&mut self, hash: &u64) -> Option { + // can only delete entries in update mode + if self.mode != ArchiveMode::Update { + return None; + } + + // Set dirty + self.dirty = true; + + self.entries.remove(hash) + } +} + +#[derive(Debug, Clone)] +pub struct ZipEntry { + /// FNV1a64 hash of the entry name + hash: u64, + /// Resolved resource path of that entry, this may not be available + name: Option, + + /// wrapped internal struct + entry: FileEntry, + segment: FileSegment, + buffers: Vec, +} + +impl ZipEntry { + fn get_resolved_name(&self, hash_map: &HashMap) -> Option { + // get filename + let resolved = if let Some(name) = &self.name { + name.to_owned() + } else { + let mut name_or_hash: String = format!("{}.bin", self.hash); + // check vanilla hashes + if let Some(name) = hash_map.get(&self.hash) { + name_or_hash = name.to_owned(); + } + name_or_hash + }; + + Some(resolved) + } +} + +///////////////////////////////////////////////////////////////////////////////////////// +// INTERNAL +///////////////////////////////////////////////////////////////////////////////////////// + +fn write_index( + writer: &mut W, + entries: &HashMap, + //dependencies: &[Dependency], +) -> Result<()> { + let file_entry_count = entries.len() as u32; + let buffer_counts = entries.iter().map(|e| e.1.buffers.len() + 1); + let file_segment_count = buffer_counts.sum::() as u32; + let resource_dependency_count = 0; //dependencies.len() as u32; + + // write table to buffer + let mut buffer: Vec = Vec::new(); + buffer.write_u32::(file_entry_count)?; + buffer.write_u32::(file_segment_count)?; + buffer.write_u32::(resource_dependency_count)?; + let mut entries = entries.values().collect::>(); + entries.sort_by_key(|e| e.hash); + // write entries + let mut segments = Vec::default(); + for entry in entries { + entry.entry.write(&mut buffer)?; + // collect offsets + segments.push(entry.segment); + for buffer in &entry.buffers { + segments.push(buffer.clone()); + } + } + // write segments + for segment in segments { + segment.write(&mut buffer)?; + } + + // write dependencies + // for dependency in dependencies { + // dependency.write(&mut buffer)?; + // } + + // write to out stream + let crc = crc64::crc64(0, buffer.as_slice()); + writer.write_u32::(8)?; + writer.write_u32::(buffer.len() as u32 + 8)?; + writer.write_u64::(crc)?; + writer.write_all(buffer.as_slice())?; + + Ok(()) +} + +///////////////////////////////////////////////////////////////////////////////////////// +/// TESTS +///////////////////////////////////////////////////////////////////////////////////////// + +#[cfg(test)] +mod integration_tests { + use std::{ + fs::{self}, + io::{self, Read}, + path::PathBuf, + }; + + use crate::archive::open_read; + + use super::FromReader; + use super::LxrsFooter; + + #[test] + fn read_srxl() { + let file_path = PathBuf::from("tests").join("srxl.bin"); + let mut file = fs::File::open(file_path).expect("Could not open file"); + let mut buffer: Vec = vec![]; + file.read_to_end(&mut buffer).expect("Could not read file"); + + let mut cursor = io::Cursor::new(&buffer); + + let _srxl = LxrsFooter::from_reader(&mut cursor).unwrap(); + } + + #[test] + fn read_archive() { + let archive_path = PathBuf::from("tests").join("test1.archive"); + let result = open_read(archive_path); + assert!(result.is_ok()); + } + + #[test] + fn read_archive2() { + let file = PathBuf::from("tests").join("nci.archive"); + let result = open_read(file); + assert!(result.is_ok()); + } + + #[test] + fn read_custom_data() { + let file = PathBuf::from("tests").join("test1.archive"); + let archive = open_read(file).expect("Could not parse archive"); + let mut file_names = archive + .entries + .values() + .map(|f| f.name.to_owned()) + .flatten() + .collect::>(); + file_names.sort(); + + let expected: Vec = vec!["base\\cycleweapons\\localization\\en-us.json".to_owned()]; + assert_eq!(expected, file_names); + } +} diff --git a/src/io.rs b/src/io.rs index 4bf5194..cacd754 100644 --- a/src/io.rs +++ b/src/io.rs @@ -5,7 +5,7 @@ use std::io::{self, Read, Write}; use byteorder::WriteBytesExt; -pub trait FromReader: Sized { +pub(crate) trait FromReader: Sized { fn from_reader(reader: &mut R) -> io::Result; } diff --git a/src/lib.rs b/src/lib.rs index acf07f2..d8ba151 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,20 +1,23 @@ #![warn(clippy::all, rust_2018_idioms)] +mod cr2w; +mod io; + pub mod archive; -pub mod cr2w; -pub mod io; pub mod kraken; -use std::collections::HashMap; -use std::fs::{self}; -use std::hash::Hasher; -use std::path::{Path, PathBuf}; +use std::{ + collections::HashMap, + hash::Hasher, + io::{BufRead, BufReader}, + path::Path, +}; use sha1::{Digest, Sha1}; use strum_macros::{Display, EnumIter}; ///////////////////////////////////////////////////////////////////////////////////////// -/// RED4 LIB +// RED4 LIB ///////////////////////////////////////////////////////////////////////////////////////// #[allow(non_camel_case_types)] @@ -163,33 +166,9 @@ enum ERedExtension { } #[warn(non_camel_case_types)] ///////////////////////////////////////////////////////////////////////////////////////// -/// HELPERS +// HELPERS ///////////////////////////////////////////////////////////////////////////////////////// -/// Get top-level files of a folder with given extension -pub fn get_files(folder_path: &Path, extension: &str) -> Vec { - let mut files = Vec::new(); - if !folder_path.exists() { - return files; - } - - if let Ok(entries) = fs::read_dir(folder_path) { - for entry in entries.flatten() { - if let Ok(file_type) = entry.file_type() { - if file_type.is_file() { - if let Some(ext) = entry.path().extension() { - if ext == extension { - files.push(entry.path()); - } - } - } - } - } - } - - files -} - /// Calculate FNV1a64 hash of a String pub fn fnv1a64_hash_string(str: &String) -> u64 { let mut hasher = fnv::FnvHasher::default(); @@ -217,8 +196,8 @@ pub fn get_red4_hashes() -> HashMap { let csv_data = include_bytes!("metadata-resources.csv"); let mut map: HashMap = HashMap::new(); - let reader = std::io::BufReader::new(&csv_data[..]); - for line in std::io::BufRead::lines(reader).flatten() { + let reader = BufReader::new(&csv_data[..]); + for line in BufRead::lines(reader).flatten() { let mut split = line.split(','); if let Some(name) = split.next() { if let Some(hash_str) = split.next() { @@ -233,7 +212,7 @@ pub fn get_red4_hashes() -> HashMap { } ///////////////////////////////////////////////////////////////////////////////////////// -/// TESTS +// TESTS ///////////////////////////////////////////////////////////////////////////////////////// #[cfg(test)] diff --git a/tests/functional_tests.rs b/tests/functional_tests.rs index d08e81c..e81032f 100644 --- a/tests/functional_tests.rs +++ b/tests/functional_tests.rs @@ -1,17 +1,14 @@ ///////////////////////////////////////////////////////////////////////////////////////// -/// TESTS +// TESTS ///////////////////////////////////////////////////////////////////////////////////////// #[cfg(test)] mod tests { - use std::fs::create_dir_all; - use std::io::{self, Read}; + use std::fs::{create_dir_all, File}; use std::path::Path; use std::time::Instant; use std::{fs, path::PathBuf}; - use red4lib::archive::*; - use red4lib::io::FromReader; use red4lib::*; #[test] @@ -24,47 +21,6 @@ mod tests { println!("Execution time csv: {:?}", duration); } - #[test] - fn read_srxl() { - let file_path = PathBuf::from("tests").join("srxl.bin"); - let mut file = fs::File::open(file_path).expect("Could not open file"); - let mut buffer: Vec = vec![]; - file.read_to_end(&mut buffer).expect("Could not read file"); - - let mut cursor = io::Cursor::new(&buffer); - - let _srxl = LxrsFooter::from_reader(&mut cursor).unwrap(); - } - - #[test] - fn read_archive() { - let archive_path = PathBuf::from("tests").join("test1.archive"); - let result = Archive::from_file(&archive_path); - assert!(result.is_ok()); - } - - #[test] - fn read_archive2() { - let archive_path = PathBuf::from("tests").join("nci.archive"); - let result = Archive::from_file(&archive_path); - assert!(result.is_ok()); - } - - #[test] - fn read_custom_data() { - let archive_path = PathBuf::from("tests").join("test1.archive"); - let archive = Archive::from_file(&archive_path).expect("Could not parse archive"); - let mut file_names = archive - .file_names - .values() - .map(|f| f.to_owned()) - .collect::>(); - file_names.sort(); - - let expected: Vec = vec!["base\\cycleweapons\\localization\\en-us.json".to_owned()]; - assert_eq!(expected, file_names); - } - #[test] fn test_extract_archive() { let archive_path = PathBuf::from("tests").join("test1.archive"); @@ -77,7 +33,12 @@ mod tests { assert!(fs::remove_dir_all(&dst_path).is_ok()); } - let result = extract_archive(&archive_path, &dst_path, &hashes); + let result = archive::extract_to_directory_path::( + &archive_path, + &dst_path, + true, + Some(hashes), + ); assert!(result.is_ok()); // check @@ -143,7 +104,7 @@ mod tests { // pack test data let data_path = PathBuf::from("tests").join("data"); let dst_path = PathBuf::from("tests").join("out2"); - let hash_map = get_red4_hashes(); + let dst_file = dst_path.join("data.archive"); // delete folder if exists if dst_path.exists() { @@ -151,12 +112,11 @@ mod tests { } create_dir_all(&dst_path).expect("Could not create folder"); - let result = write_archive(&data_path, &dst_path, None, hash_map); + let result = archive::create_from_directory_path(&data_path, &dst_file, None); assert!(result.is_ok()); // checks - let created_path = dst_path.join("data.archive"); - assert!(created_path.exists()); + assert!(dst_file.exists()); // TODO binary equality // let existing_path = PathBuf::from("tests").join("test1.archive"); @@ -169,7 +129,7 @@ mod tests { } ///////////////////////////////////////////////////////////////////////////////////////// - /// HELPERS + // HELPERS ///////////////////////////////////////////////////////////////////////////////////////// fn assert_binary_equality(e: &PathBuf, f: &PathBuf) {