Merge pull request #365 from RagnarokResearchLab/153-grf-metadata-cac…

…hing

Enable compilation and caching of GRF file tables

.gitignore

@@ -1,6 +1,7 @@
 *.log
 data.grf
 *.grf.extracted/*
+Cache/
 Exports
 Screenshots/
 *.mp3

Core/FileFormats/Optimized/CompiledGRF.lua

@@ -0,0 +1,143 @@
+local BinaryReader = require("Core.FileFormats.BinaryReader")
+
+local ffi = require("ffi")
+local uv = require("uv")
+local validation = require("validation")
+
+local new = ffi.new
+local sizeof = ffi.sizeof
+local tonumber = tonumber
+local table_new = table.new
+
+local CompiledGRF = {
+	CGRF_CACHE_DIRECTORY = "Cache",
+	errorStrings = {
+		INVALID_GRF_PATH = "Not a GRF file: %s",
+	},
+	cdefs = [[
+		typedef struct cgrf_version_t {
+			uint32_t major;
+			uint32_t minor;
+			uint32_t patch;
+		} cgrf_version_t;
+
+		typedef struct cgrf_header_t {
+			char signature[4];
+			cgrf_version_t semanticVersion;
+			uint32_t fileCount;
+		} cgrf_header_t;
+
+		typedef struct cgrf_entry_t {
+			uint8_t typeID;
+			uint32_t alignedSizeInBytes;
+			uint32_t compressedSizeInBytes;
+			uint32_t decompressedSizeInBytes;
+			uint32_t offsetRelativeToHeader;
+			uint8_t variableLengthPathBufferSize;
+		} cgrf_entry_t;
+	]],
+}
+
+-- No support for this in the runtime, yet :/
+local function FileSystem_GetLastModifiedTimestamp(fileSystemPath)
+	local fileAttributes, errorMessage = uv.fs_stat(fileSystemPath)
+	assert(fileAttributes, errorMessage)
+
+	return fileAttributes.mtime
+end
+
+function CompiledGRF:IsCacheUpdated(grfFilePath)
+	if not C_FileSystem.IsDirectory(CompiledGRF.CGRF_CACHE_DIRECTORY) then
+		C_FileSystem.MakeDirectoryTree(CompiledGRF.CGRF_CACHE_DIRECTORY)
+	end
+
+	validation.validateString(grfFilePath, "grfFilePath")
+
+	local extension = path.extname(grfFilePath)
+	local isGRF = string.lower(extension) ~= ".grf"
+	if not C_FileSystem.Exists(grfFilePath) or isGRF then
+		error(format(CompiledGRF.errorStrings.INVALID_GRF_PATH, grfFilePath), 0)
+	end
+
+	local grfFileName = path.basename(grfFilePath, extension)
+	local cgrfFilePath = path.join(CompiledGRF.CGRF_CACHE_DIRECTORY, grfFileName .. ".cgrf")
+
+	if not C_FileSystem.Exists(cgrfFilePath) then
+		return false
+	end
+
+	local grfModifiedDate = FileSystem_GetLastModifiedTimestamp(grfFilePath)
+	local cgrfModifiedDate = FileSystem_GetLastModifiedTimestamp(cgrfFilePath)
+
+	local isCachedVersionMoreRecent = cgrfModifiedDate.sec >= grfModifiedDate.sec
+		or (cgrfModifiedDate.sec == grfModifiedDate.sec and cgrfModifiedDate.nsec >= grfModifiedDate.nsec)
+
+	return isCachedVersionMoreRecent
+end
+
+function CompiledGRF:CompileTableOfContents(grf)
+	local fileList = grf:GetFileList()
+	local cgrfBuffer = buffer.new(1024)
+
+	local header = new("cgrf_header_t")
+	header.signature = "CGRF"
+	header.semanticVersion.major = 1
+	header.semanticVersion.minor = 0
+	header.semanticVersion.patch = 0
+	header.fileCount = grf.fileCount
+
+	cgrfBuffer:putcdata(header, sizeof(header))
+
+	for index, entry in ipairs(fileList) do
+		local cgrfEntry = new("cgrf_entry_t")
+		cgrfEntry.typeID = entry.typeID
+		cgrfEntry.alignedSizeInBytes = entry.alignedSizeInBytes
+		cgrfEntry.compressedSizeInBytes = entry.compressedSizeInBytes
+		cgrfEntry.decompressedSizeInBytes = entry.decompressedSizeInBytes
+		cgrfEntry.offsetRelativeToHeader = entry.offsetRelativeToHeader
+		cgrfEntry.variableLengthPathBufferSize = #entry.name
+
+		local variableLengthFilePath = new("char[?]", #entry.name + 1, entry.name)
+		cgrfBuffer:putcdata(cgrfEntry, sizeof(cgrfEntry))
+		cgrfBuffer:putcdata(variableLengthFilePath, #entry.name)
+	end
+
+	return tostring(cgrfBuffer)
+end
+
+function CompiledGRF:RestoreTableOfContents(grf, cgrfBuffer)
+	local reader = BinaryReader(cgrfBuffer)
+
+	local header = new("cgrf_header_t")
+	header.signature = reader:GetCountedString(sizeof(header.signature))
+	header.semanticVersion.major = reader:GetUnsignedInt32()
+	header.semanticVersion.minor = reader:GetUnsignedInt32()
+	header.semanticVersion.patch = reader:GetUnsignedInt32()
+	header.fileCount = reader:GetUnsignedInt32()
+
+	local entries = table_new(header.fileCount, 0)
+	for index = 1, tonumber(header.fileCount), 1 do
+		local cgrfEntry = reader:GetTypedArray("cgrf_entry_t", 1)[0]
+		local normalizedFilePath = reader:GetCountedString(cgrfEntry.variableLengthPathBufferSize)
+
+		-- Copying the data here isn't ideal, but the GRF interface expects Lua types and not cdata
+		local backwardsCompatibleLuaFileEntry = {
+			alignedSizeInBytes = tonumber(cgrfEntry.alignedSizeInBytes),
+			compressedSizeInBytes = tonumber(cgrfEntry.compressedSizeInBytes),
+			decompressedSizeInBytes = tonumber(cgrfEntry.decompressedSizeInBytes),
+			name = normalizedFilePath,
+			offsetRelativeToHeader = tonumber(cgrfEntry.offsetRelativeToHeader),
+			typeID = tonumber(cgrfEntry.typeID),
+		}
+		entries[normalizedFilePath] = backwardsCompatibleLuaFileEntry
+		entries[index] = backwardsCompatibleLuaFileEntry
+	end
+
+	grf.fileTable.compressedSizeInBytes = 0
+	grf.fileTable.decompressedSizeInBytes = 0
+	grf.fileTable.entries = entries
+end
+
+ffi.cdef(CompiledGRF.cdefs)
+
+return CompiledGRF

Core/FileFormats/RagnarokGRF.lua

@@ -1,3 +1,5 @@
+local CompiledGRF = require("Core.FileFormats.Optimized.CompiledGRF")
+
 local cstring = require("Core.RuntimeExtensions.cstring")
 
 local bit = require("bit")
@@ -58,16 +60,16 @@ function RagnarokGRF:Construct()
 		fileTable = {},
 	}
 
-	setmetatable(instance, self)
+	setmetatable(instance, {
+		__index = self,
+	})
 
 	return instance
 end
 
-RagnarokGRF.__index = RagnarokGRF
-RagnarokGRF.__call = RagnarokGRF.Construct
-setmetatable(RagnarokGRF, RagnarokGRF)
+class("RagnarokGRF", RagnarokGRF)
 
-function RagnarokGRF:Open(pathToGRF)
+function RagnarokGRF:Open(pathToGRF, cgrfFileContents)
 	local isValidPath = C_FileSystem.Exists(pathToGRF)
 	if not isValidPath then
 		error(format("Failed to open archive %s (No such file exists)", pathToGRF), 0)
@@ -82,6 +84,7 @@ function RagnarokGRF:Open(pathToGRF)
 	self.fileHandle = assert(io.open(pathToGRF, "rb"))
 
 	self.pathToGRF = pathToGRF
+	self.cgrfFileContents = cgrfFileContents
 
 	self:DecodeArchiveMetadata()
 end
@@ -128,6 +131,11 @@ function RagnarokGRF:DecodeHeader()
 end
 
 function RagnarokGRF:DecodeFileTable()
+	if rawget(self, "cgrfFileContents") then
+		CompiledGRF:RestoreTableOfContents(self, self.cgrfFileContents)
+		return
+	end
+
 	self:DecodeTableHeader()
 	self:DecodeFileEntries()
 end

Core/NativeClient/C_Resources.lua

@@ -1,3 +1,4 @@
+local CompiledGRF = require("Core.FileFormats.Optimized.CompiledGRF")
 local FogParameters = require("Core.FileFormats.FogParameters")
 local RagnarokGRF = require("Core.FileFormats.RagnarokGRF")
 
@@ -8,13 +9,33 @@ local C_Resources = {
 		["data/sprite/cursors.spr"] = false,
 		["data/fogparametertable.txt"] = FogParameters,
 	},
+	ENABLE_CGRF_CACHING = true,
 }
 
 local self = C_Resources
 
 function C_Resources.PreloadPersistentResources()
+	local grfFilePath = self.GRF_FILE_PATH
+
+	local hasUpdatedCacheEntry = self.ENABLE_CGRF_CACHING and CompiledGRF:IsCacheUpdated(grfFilePath) or false
+	local grfName = path.basename(grfFilePath, path.extname(grfFilePath))
+	local cgrfFilePath = path.join(CompiledGRF.CGRF_CACHE_DIRECTORY, grfName .. ".cgrf")
+
+	local cgrfFileContents
+	if hasUpdatedCacheEntry then
+		printf("CGRF_CACHE_HIT: %s (Restoring table of contents)", cgrfFilePath)
+		cgrfFileContents = C_FileSystem.ReadFile(cgrfFilePath)
+	end
+
 	local grf = RagnarokGRF()
-	grf:Open(self.GRF_FILE_PATH)
+	grf:Open(self.GRF_FILE_PATH, cgrfFileContents)
+
+	local needsCacheWrite = self.ENABLE_CGRF_CACHING and not hasUpdatedCacheEntry
+	if needsCacheWrite then
+		cgrfFileContents = CompiledGRF:CompileTableOfContents(grf)
+		printf("CGRF_CACHE_WRITE: %s (%s)", cgrfFilePath, string.filesize(#cgrfFileContents))
+		C_FileSystem.WriteFile(cgrfFilePath, cgrfFileContents)
+	end
 
 	printf("Preloading %d persistent resources from %s", table.count(self.PERSISTENT_RESOURCES), self.GRF_FILE_PATH)
 	for filePath, decoder in pairs(self.PERSISTENT_RESOURCES) do

Tests/FileFormats/Optimized/CompiledGRF.spec.lua

@@ -0,0 +1,121 @@
+local CompiledGRF = require("Core.FileFormats.Optimized.CompiledGRF")
+local RagnarokGRF = require("Core.FileFormats.RagnarokGRF")
+
+local uv = require("uv")
+
+local TEST_GRF_PATH = path.join("Tests", "Fixtures", "test.grf")
+
+describe("CompiledGRF", function()
+	describe("IsCacheUpdated", function()
+		it("should throw if no GRF file path was provided", function()
+			local expectedErrorMessage =
+				"Expected argument grfFilePath to be a string value, but received a nil value instead"
+			assertThrows(function()
+				CompiledGRF:IsCacheUpdated(nil)
+			end, expectedErrorMessage)
+		end)
+
+		it("should throw if no GRF with the provided file path exists", function()
+			assertThrows(function()
+				CompiledGRF:IsCacheUpdated("does-not-exist.grf")
+			end, format(CompiledGRF.errorStrings.INVALID_GRF_PATH, "does-not-exist.grf"))
+		end)
+
+		it("should throw if the provided file path is not referencing a GRF file", function()
+			assertThrows(function()
+				CompiledGRF:IsCacheUpdated("README.md")
+			end, format(CompiledGRF.errorStrings.INVALID_GRF_PATH, "README.md"))
+		end)
+
+		it("should return false if no CGRF entry exists for the provided GRF", function()
+			local cgrfFilePath = path.join(CompiledGRF.CGRF_CACHE_DIRECTORY, "test.cgrf")
+			C_FileSystem.Delete(cgrfFilePath)
+
+			assertFalse(CompiledGRF:IsCacheUpdated(TEST_GRF_PATH))
+		end)
+
+		it("should return false if the provided GRF is more recent than the cached CGRF", function()
+			local cgrfFilePath = path.join(CompiledGRF.CGRF_CACHE_DIRECTORY, "test.cgrf")
+			C_FileSystem.Delete(cgrfFilePath)
+
+			C_FileSystem.WriteFile(cgrfFilePath, "") -- File contents shouldn't be relevant for mtime detection
+			assert(uv.fs_utime(cgrfFilePath, 1, 1)) -- Set modified date to January 1st, 1970
+
+			assertFalse(CompiledGRF:IsCacheUpdated(TEST_GRF_PATH))
+
+			C_FileSystem.Delete(cgrfFilePath)
+		end)
+
+		it("should return true if the cached CGRF is more recent than the provided GRF", function()
+			local cgrfFilePath = path.join(CompiledGRF.CGRF_CACHE_DIRECTORY, "test.cgrf")
+			C_FileSystem.Delete(cgrfFilePath)
+
+			C_FileSystem.WriteFile(cgrfFilePath, "") -- File contents shouldn't be relevant for mtime detection
+
+			local fileAttributes, errorMessage = uv.fs_stat(cgrfFilePath)
+			assert(fileAttributes, errorMessage)
+			assert(uv.fs_utime(cgrfFilePath, 1, fileAttributes.mtime.sec + 1))
+
+			assertTrue(CompiledGRF:IsCacheUpdated(TEST_GRF_PATH))
+
+			C_FileSystem.Delete(cgrfFilePath)
+		end)
+
+		it("should create the CGRF cache directory if it doesn't yet exist", function()
+			local cacheDirectory = "DoesNotExistProbably"
+			local originalCacheDirectory = CompiledGRF.CGRF_CACHE_DIRECTORY
+
+			C_FileSystem.Delete(cacheDirectory)
+			assertFalse(C_FileSystem.Exists(cacheDirectory))
+			assertFalse(C_FileSystem.IsDirectory(cacheDirectory))
+
+			CompiledGRF.CGRF_CACHE_DIRECTORY = cacheDirectory
+			CompiledGRF:IsCacheUpdated(TEST_GRF_PATH)
+			CompiledGRF.CGRF_CACHE_DIRECTORY = originalCacheDirectory
+
+			assertTrue(C_FileSystem.Exists(cacheDirectory))
+			assertTrue(C_FileSystem.IsDirectory(cacheDirectory))
+
+			C_FileSystem.Delete(cacheDirectory)
+		end)
+	end)
+
+	describe("CompileTableOfContents", function()
+		it("should generate a CGRF buffer storing the provided GRF instance's file table", function()
+			local grf = RagnarokGRF()
+			grf:Open(TEST_GRF_PATH)
+			grf:Close()
+
+			local cgrfFileContents = CompiledGRF:CompileTableOfContents(grf)
+
+			local cgrfFilePath = path.join("Tests", "Fixtures", "test.cgrf")
+			local expectedFileContents = C_FileSystem.ReadFile(cgrfFilePath)
+			assertEquals(cgrfFileContents, expectedFileContents)
+		end)
+	end)
+
+	describe("RestoreTableOfContents", function()
+		it("should restore the provided GRF instance's file table from the CGRF buffer", function()
+			local grf = RagnarokGRF()
+			grf:Open(TEST_GRF_PATH)
+			grf:Close()
+			local expectedFileEntries = grf.fileTable.entries
+
+			local unOpenedGRF = RagnarokGRF()
+
+			local cgrfFilePath = path.join("Tests", "Fixtures", "test.cgrf")
+			local cgrfBuffer = C_FileSystem.ReadFile(cgrfFilePath)
+
+			CompiledGRF:RestoreTableOfContents(unOpenedGRF, cgrfBuffer)
+
+			assertEquals(#unOpenedGRF.fileTable.entries, #expectedFileEntries)
+			assertEquals(#unOpenedGRF.fileTable.entries, 4)
+			assertEquals(unOpenedGRF.fileTable.compressedSizeInBytes, 0)
+			assertEquals(unOpenedGRF.fileTable.decompressedSizeInBytes, 0)
+			assertEquals(unOpenedGRF.fileTable.entries[1], expectedFileEntries[1])
+			assertEquals(unOpenedGRF.fileTable.entries[2], expectedFileEntries[2])
+			assertEquals(unOpenedGRF.fileTable.entries[3], expectedFileEntries[3])
+			assertEquals(unOpenedGRF.fileTable.entries[4], expectedFileEntries[4])
+		end)
+	end)
+end)

Tests/FileFormats/RagnarokGRF.spec.lua

@@ -54,6 +54,19 @@ describe("RagnarokGRF", function()
 			assertEquals(grf.fileTable.entries["uppercase.png"].typeID, RagnarokGRF.COMPRESSED_FILE_ENTRY_TYPE)
 			assertEquals(grf.fileTable.entries["uppercase.png"].offsetRelativeToHeader, 160)
 		end)
+
+		it("should be able to restore the file table from a CGRF buffer", function()
+			local cgrfFilePath = path.join("Tests", "Fixtures", "test.cgrf")
+			local cgrfFileContents = C_FileSystem.ReadFile(cgrfFilePath)
+
+			local grf = RagnarokGRF()
+			grf:Open("Tests/Fixtures/test.grf", cgrfFileContents)
+			grf:Close()
+
+			-- These aren't stored in the CGRF (because they're useless on their own)
+			assertEquals(grf.fileTable.compressedSizeInBytes, 0)
+			assertEquals(grf.fileTable.decompressedSizeInBytes, 0)
+		end)
 	end)
 
 	describe("FindLargestFileEntry", function()