Merge pull request #7 from JohnImril/chunk-for-diabdat

Chunk for diabdat

src/App.tsx

@@ -5,7 +5,6 @@ import Peer from "peerjs";
 
 import getPlayerName from "./api/savefile";
 import load_game from "./api/loader";
-import { SpawnSizes } from "./api/load_spawn";
 import create_fs from "./fs";
 import { reportLink, isDropFile, getDropFile, findKeyboardRule } from "./utils";
 import CompressMpq from "./mpqcmp/CompressMpq";
@@ -34,7 +33,6 @@ const App: React.FC = () => {
 	const [started, setStarted] = useState(false);
 	const [loading, setLoading] = useState(false);
 	const [dropping, setDropping] = useState(0);
-	const [hasSpawn, setHasSpawn] = useState(false);
 	const [error, setError] = useState<IError | undefined>(undefined);
 	const [progress, setProgress] = useState<IProgress | undefined>(undefined);
 	const [saveNames, setSaveNames] = useState<boolean | Record<string, IPlayerInfo | null>>(false);
@@ -607,10 +605,6 @@ const App: React.FC = () => {
 		document.addEventListener("dragleave", handleDragLeave, true);
 
 		fs.current.then((fsInstance) => {
-			const spawn = fsInstance.files.get("spawn.mpq");
-			if (spawn && SpawnSizes.includes(spawn.byteLength)) {
-				setHasSpawn(true);
-			}
 			if ([...fsInstance.files.keys()].some((name) => /\.sv$/i.test(name))) {
 				setSaveNames(true);
 			}
@@ -736,10 +730,8 @@ const App: React.FC = () => {
 				{loading && !started && !error && <LoadingComponent title="Loading..." progress={progress} />}
 				{!started && !compress && !loading && !error && !showSaves && (
 					<StartScreen
-						hasSpawn={hasSpawn}
 						start={start}
 						saveNames={saveNames}
-						setCompress={setCompress}
 						setShowSaves={setShowSaves}
 						updateSaves={updateSaves}
 					/>

src/api/loader.ts

@@ -63,7 +63,7 @@ function testOffscreen() {
 async function do_load_game(api: IApi, audio: IAudioApi, mpq: File | null, spawn: boolean) {
 	const fs = await api.fs;
 
-	if (!navigator.userAgent.includes("Firefox")) await load_diabdat(api, fs); //fix this
+	await load_diabdat(api, fs);
 
 	let context: CanvasRenderingContext2D | ImageBitmapRenderingContext | null = null;
 	let offscreen = false;

src/components/StartScreen/StartScreen.tsx

@@ -3,10 +3,8 @@ import { IPlayerInfo } from "../../types";
 import "./StartScreen.css";
 
 const StartScreen: React.FC<{
-	hasSpawn: boolean;
 	start: (file?: File | null) => void;
 	saveNames: boolean | Record<string, IPlayerInfo | null>;
-	setCompress: React.Dispatch<React.SetStateAction<boolean>>;
 	setShowSaves: React.Dispatch<React.SetStateAction<boolean>>;
 	updateSaves: () => Promise<void>;
 }> = ({ start, saveNames, setShowSaves, updateSaves }) => {

src/fs.ts

@@ -1,15 +1,22 @@
 import { openDB, IDBPDatabase } from "idb";
 import { IFileSystem } from "./types";
 
+const CHUNK_SIZE = 5 * 1024 * 1024;
+
 export async function downloadFile(db: IDBPDatabase<unknown>, name: string) {
-	const file = await db.get("files", name.toLowerCase());
-	if (!file) {
-		console.error(`File ${name} does not exist`);
-		return;
+	try {
+		const fileName = name.toLowerCase();
+		const data = await downloadFileChunks(db, fileName);
+		if (!data) {
+			console.error(`File ${name} does not exist`);
+			return;
+		}
+		const blob = new Blob([data], { type: "binary/octet-stream" });
+		const url = URL.createObjectURL(blob);
+		triggerDownload(url, name);
+	} catch (e) {
+		console.error(`Failed to download file ${name}:`, e);
 	}
-	const blob = new Blob([file], { type: "binary/octet-stream" });
-	const url = URL.createObjectURL(blob);
-	triggerDownload(url, name);
 }
 
 function triggerDownload(url: string, name: string) {
@@ -40,10 +47,47 @@ const readFile = (file: File): Promise<ArrayBuffer> =>
 		reader.readAsArrayBuffer(file);
 	});
 
-async function uploadFile(db: IDBPDatabase<unknown>, files: Map<string, Uint8Array>, file: File) {
+async function uploadFileChunks(db: IDBPDatabase<unknown>, files: Map<string, Uint8Array>, file: File) {
 	const data = new Uint8Array(await readFile(file));
-	files.set(file.name.toLowerCase(), data);
-	await db.put("files", data, file.name.toLowerCase());
+	const fileName = file.name.toLowerCase();
+
+	if (data.length <= CHUNK_SIZE) {
+		// Store small files without chunking
+		await db.put("files", data, fileName);
+		files.set(fileName, data);
+		console.log(`File ${fileName} uploaded without chunking.`);
+	} else {
+		const totalChunks = Math.ceil(data.length / CHUNK_SIZE);
+		console.log(`File ${fileName} uploaded in ${totalChunks} chunks.`);
+
+		for (let i = 0; i < totalChunks; i++) {
+			const chunk = data.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
+			const chunkKey = `${fileName}_chunk_${i}`;
+			await db.put("files", chunk, chunkKey);
+		}
+
+		await db.put("files", { chunks: totalChunks }, `${fileName}_metadata`);
+		files.set(fileName, data);
+	}
+}
+
+async function downloadFileChunks(db: IDBPDatabase<unknown>, fileName: string): Promise<Uint8Array> {
+	const metadata = await db.get("files", `${fileName}_metadata`);
+	if (!metadata || !metadata.chunks) {
+		// File was stored without chunking
+		const data = await db.get("files", fileName);
+		if (!data) throw new Error(`File ${fileName} is missing.`);
+		return new Uint8Array(data);
+	}
+
+	const chunks = [];
+	for (let i = 0; i < metadata.chunks; i++) {
+		const chunkKey = `${fileName}_chunk_${i}`;
+		const chunk = await db.get("files", chunkKey);
+		if (!chunk) throw new Error(`Chunk ${i} of file ${fileName} is missing.`);
+		chunks.push(chunk);
+	}
+	return new Uint8Array(chunks.flat());
 }
 
 export default async function create_fs(): Promise<IFileSystem> {
@@ -68,9 +112,9 @@ export default async function create_fs(): Promise<IFileSystem> {
 
 		const keys = await db.getAllKeys("files");
 		for (const key of keys) {
-			const value = await db.get("files", key);
-			if (value) {
-				files.set(key as string, value as Uint8Array);
+			if (!(key as string).endsWith("_metadata") && !(key as string).includes("_chunk_")) {
+				const value = await downloadFileChunks(db, key as string);
+				files.set(key as string, value);
 			}
 		}
 
@@ -79,17 +123,31 @@ export default async function create_fs(): Promise<IFileSystem> {
 
 		return {
 			files,
-			update: (name: string, data: Uint8Array) => db.put("files", data, name),
-			delete: (name: string) => db.delete("files", name),
-			clear: () => db.clear("files"),
+			update: async (name: string, data: Uint8Array) => {
+				await uploadFileChunks(db, files, new File([data], name));
+			},
+			delete: async (name: string) => {
+				const metadata = await db.get("files", `${name}_metadata`);
+				if (metadata && metadata.chunks) {
+					for (let i = 0; i < metadata.chunks; i++) {
+						await db.delete("files", `${name}_chunk_${i}`);
+					}
+					await db.delete("files", `${name}_metadata`);
+				} else {
+					await db.delete("files", name);
+				}
+				files.delete(name);
+			},
+			clear: async () => {
+				await db.clear("files");
+				files.clear();
+			},
 			download: (name: string) => downloadFile(db, name),
-			upload: (file: File) => uploadFile(db, files, file),
+			upload: (file: File) => uploadFileChunks(db, files, file),
 			fileUrl: async (name: string) => {
-				const file = await db.get("files", name.toLowerCase());
-				if (file) {
-					const blob = new Blob([file], {
-						type: "binary/octet-stream",
-					});
+				const data = files.get(name.toLowerCase());
+				if (data) {
+					const blob = new Blob([data], { type: "binary/octet-stream" });
 					return URL.createObjectURL(blob);
 				}
 				return undefined;