HexEditor diff (#522)

* feat: simple decorators

Applies custom css styles on data cells by specifying a decorator style
and its offset range.

* refactor: move parseQuery to its own util file

* feat: compareSelected command

Opens two selected files into hex diff view. Works with any file
extension by changing the file scheme.

* synchronizes hex documents creation for diffing

* feat: basic Myers diff

Introduces the O(d^2) space version and its decorators. With this
diff algorithm, the diffs should be more insightful compared to
using a simple per-offset comparison.

* Adds alignment in original file

Inserted bytes are show in the original file by a stripe data cell.

* fix: negative k-index and wrong alignment

* optimize: use binary search for page decorators

Finds the lower and upper bound of the decorators that fit in the
page, instead of using .filter(). In addition to that, it also
removes the need to re-initialize the decorator's ranges.

* Makes diff read-only

Done to simplify diff, because recomputing diffs after a file change
is too complex to tackle at the moment.

* refactor: use diff package for myers diff

Although still O(d^2), it improves memory usage in other ways,
provides some optimization for edge cases and has some useful
built-in options to exit on large diffs.

* fix: diff failing to open in web

* perf: moved myers diff into a web-only worker

* adds node js worker

* adds worker transferable objects

* Use binary search in data row content

* removes web-only code

* fix: decorator's binary search gives wrong index

* Adds lazy init & better diff lifecycle

.esbuild.config.js

@@ -46,12 +46,23 @@ build({
 	tsconfig: "./tsconfig.json",
 	bundle: true,
 	format: "cjs",
-	external: ["vscode", "fs"],
+	external: ["vscode", "fs", "worker_threads"],
 	minify,
 	platform: "browser",
 	outfile: "dist/web/extension.js",
 });
 
+build({
+	entryPoints: ["shared/diffWorker.ts"],
+	tsconfig: "./tsconfig.json",
+	bundle: true,
+	format: "cjs",
+	external: ["vscode", "worker_threads"],
+	minify,
+	platform: "browser",
+	outfile: "dist/diffWorker.js",
+});
+
 // Build the data inspector
 build({
 	entryPoints: ["media/data_inspector/inspector.ts"],

media/editor/dataDisplay.tsx

@@ -3,6 +3,7 @@
 
 import React, { Suspense, useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { useRecoilValue, useSetRecoilState } from "recoil";
+import { HexDecorator } from "../../shared/decorators";
 import { EditRangeOp, HexDocumentEditOp } from "../../shared/hexDocumentModel";
 import {
 	CopyFormat,
@@ -14,8 +15,8 @@ import { Range } from "../../shared/util/range";
 import { PastePopup } from "./copyPaste";
 import _style from "./dataDisplay.css";
 import {
-	FocusedElement,
 	dataCellCls,
+	FocusedElement,
 	useDisplayContext,
 	useIsFocused,
 	useIsHovered,
@@ -31,9 +32,11 @@ import {
 	clsx,
 	getAsciiCharacter,
 	getScrollDimensions,
+	HexDecoratorStyles,
 	parseHexDigit,
 	throwOnUndefinedAccessInDev,
 } from "./util";
+import { binarySearch } from "../../shared/util/binarySearch";
 
 const style = throwOnUndefinedAccessInDev(_style);
 
@@ -428,8 +431,9 @@ const LoadingDataRows: React.FC<IDataPageProps> = props => (
 );
 
 const DataPageContents: React.FC<IDataPageProps> = props => {
-	const pageSelector = select.editedDataPages(props.pageNo);
-	const [data] = useLastAsyncRecoilValue(pageSelector);
+	const decorators = useRecoilValue(select.decoratorsPage(props.pageNo));
+	const dataPageSelector = select.editedDataPages(props.pageNo);
+	const [data] = useLastAsyncRecoilValue(dataPageSelector);
 
 	return (
 		<>
@@ -443,6 +447,7 @@ const DataPageContents: React.FC<IDataPageProps> = props => {
 					width={props.columnWidth}
 					showDecodedText={props.showDecodedText}
 					isRowWithInsertDataCell={isRowWithInsertDataCell}
+					decorators={decorators}
 				/>
 			))}
 		</>
@@ -688,7 +693,8 @@ const DataRowContents: React.FC<{
 	showDecodedText: boolean;
 	rawBytes: Uint8Array;
 	isRowWithInsertDataCell: boolean;
-}> = ({ offset, width, showDecodedText, rawBytes, isRowWithInsertDataCell }) => {
+	decorators: HexDecorator[];
+}> = ({ offset, width, showDecodedText, rawBytes, isRowWithInsertDataCell, decorators }) => {
 	let memoValue = "";
 	const ctx = useDisplayContext();
 	for (const byte of rawBytes) {
@@ -698,9 +704,21 @@ const DataRowContents: React.FC<{
 	const { bytes, chars } = useMemo(() => {
 		const bytes: React.ReactChild[] = [];
 		const chars: React.ReactChild[] = [];
+		const searcher = binarySearch<HexDecorator>(d => d.range.end);
+		let j = searcher(offset, decorators);
 		for (let i = 0; i < width; i++) {
 			const boffset = offset + i;
 			const value = rawBytes[i];
+			let decorator: HexDecorator | undefined = undefined;
+			// Searches for the decorator, if any. Leverages the fact that
+			// the decorators are sorted by range.
+			while (j < decorators.length && decorators[j].range.start <= boffset) {
+				if (boffset >= decorators[j].range.start && boffset < decorators[j].range.end) {
+					decorator = decorators[j];
+					break;
+				}
+				j++;
+			}
 
 			if (value === undefined) {
 				if (isRowWithInsertDataCell && !ctx.isReadonly) {
@@ -723,7 +741,14 @@ const DataRowContents: React.FC<{
 			}
 
 			bytes.push(
-				<DataCell key={i} offset={boffset} isChar={false} isAppend={false} value={value}>
+				<DataCell
+					key={i}
+					className={clsx(decorator !== undefined && HexDecoratorStyles[decorator.type])}
+					offset={boffset}
+					isChar={false}
+					isAppend={false}
+					value={value}
+				>
 					{value.toString(16).padStart(2, "0").toUpperCase()}
 				</DataCell>,
 			);
@@ -736,7 +761,10 @@ const DataRowContents: React.FC<{
 						offset={boffset}
 						isChar={true}
 						isAppend={false}
-						className={char === undefined ? style.nonGraphicChar : undefined}
+						className={clsx(
+							char === undefined ? style.nonGraphicChar : undefined,
+							decorator !== undefined && HexDecoratorStyles[decorator.type],
+						)}
 						value={value}
 					>
 						{char === undefined ? "." : char}

media/editor/state.ts

@@ -3,7 +3,14 @@
  *--------------------------------------------------------*/
 
 import { atom, DefaultValue, selector, selectorFamily } from "recoil";
-import { buildEditTimeline, HexDocumentEdit, readUsingRanges } from "../../shared/hexDocumentModel";
+import { HexDecorator, HexDecoratorType } from "../../shared/decorators";
+import {
+	buildEditTimeline,
+	HexDocumentEdit,
+	HexDocumentEditOp,
+	HexDocumentEmptyInsertEdit,
+	readUsingRanges,
+} from "../../shared/hexDocumentModel";
 import {
 	FromWebviewMessage,
 	InspectorLocation,
@@ -15,6 +22,7 @@ import {
 	ToWebviewMessage,
 } from "../../shared/protocol";
 import { deserializeEdits, serializeEdits } from "../../shared/serialization";
+import { binarySearch } from "../../shared/util/binarySearch";
 import { Range } from "../../shared/util/range";
 import { clamp } from "./util";
 
@@ -103,6 +111,11 @@ export const codeSettings = selector({
 	get: ({ get }) => get(readyQuery).codeSettings,
 });
 
+export const decorators = selector({
+	key: "decorators",
+	get: ({ get }) => get(readyQuery).decorators,
+});
+
 export const showReadonlyWarningForEl = atom<HTMLElement | null>({
 	key: "showReadonlyWarningForEl",
 	default: null,
@@ -136,7 +149,7 @@ export const fileSize = selector({
 	key: "fileSize",
 	get: ({ get }) => {
 		const initial = get(diskFileSize);
-		const sizeDelta = get(unsavedEditTimeline).sizeDelta;
+		const sizeDelta = get(unsavedAndDecoratorEditTimeline).sizeDelta;
 		return initial === undefined ? initial : initial + sizeDelta;
 	},
 });
@@ -372,13 +385,41 @@ export const unsavedEditTimeline = selector({
 	},
 });
 
+const emptyDecoratorEdits = selector({
+	key: "emptyDecoratorEdits",
+	get: ({ get }) => {
+		return get(decorators)
+			.filter(record => record.type === HexDecoratorType.Empty)
+			.map(value => {
+				return {
+					op: HexDocumentEditOp.EmptyInsert,
+					offset: value.range.start,
+					length: value.range.end - value.range.start,
+				} as HexDocumentEmptyInsertEdit;
+			});
+	},
+});
+
+/**
+ * Creates the edit timeline for the unsaved edits and empty decorators.
+ */
+export const unsavedAndDecoratorEditTimeline = selector({
+	key: "unsavedAndDecoratorEditTimeline",
+	get: ({ get }) => {
+		return buildEditTimeline([
+			...get(edits).slice(get(unsavedEditIndex)),
+			...get(emptyDecoratorEdits),
+		]);
+	},
+});
+
 export const editedDataPages = selectorFamily({
 	key: "editedDataPages",
 	get:
 		(pageNumber: number) =>
 		async ({ get }) => {
 			const pageSize = get(dataPageSize);
-			const { ranges } = get(unsavedEditTimeline);
+			const { ranges } = get(unsavedAndDecoratorEditTimeline);
 			const target = new Uint8Array(pageSize);
 			const it = readUsingRanges(
 				{
@@ -414,6 +455,25 @@ export const editedDataPages = selectorFamily({
 	},
 });
 
+/** Returns the decorators in a page */
+export const decoratorsPage = selectorFamily({
+	key: "decoratorsPage",
+	get:
+		(pageNumber: number) =>
+		async ({ get }) => {
+			const allDecorators = get(decorators);
+			if (allDecorators.length === 0) {
+				return [];
+			}
+			const pageSize = get(dataPageSize);
+			const searcherByEnd = binarySearch<HexDecorator>(decorator => decorator.range.end);
+			const startIndex = searcherByEnd(pageSize * pageNumber, allDecorators);
+			const searcherByStart = binarySearch<HexDecorator>(d => d.range.start);
+			const endIndex = searcherByStart(pageSize * pageNumber + pageSize+1, allDecorators);
+			return allDecorators.slice(startIndex, endIndex);
+		},
+});
+
 const rawDataPages = selectorFamily({
 	key: "rawDataPages",
 	get:

media/editor/util.css

@@ -5,3 +5,29 @@
 	width: 100px;
 	height: 100px;
 }
+
+/* Decorators */
+
+.diff-insert {
+	background-color: rgba(156, 204, 44, 0.4);
+	filter: opacity(100%);
+}
+
+.diff-delete {
+	background-color: rgba(255, 0, 0, 0.4);
+}
+
+.diff-empty {
+	background-image: linear-gradient(
+		-45deg,
+		var(--vscode-diffEditor-diagonalFill) 12.5%,
+		#0000 12.5%,
+		#0000 50%,
+		var(--vscode-diffEditor-diagonalFill) 50%,
+		var(--vscode-diffEditor-diagonalFill) 62.5%,
+		#0000 62.5%,
+		#0000 100%
+	);
+	background-size: 8px 8px;
+	color: transparent !important;
+}

media/editor/util.ts

@@ -3,6 +3,7 @@
 
 // Assorted helper functions
 
+import { HexDecoratorType } from "../../shared/decorators";
 import { Range } from "../../shared/util/range";
 import _style from "./util.css";
 
@@ -180,3 +181,9 @@ export const getScrollDimensions = (() => {
 		return value;
 	};
 })();
+
+export const HexDecoratorStyles: { [key in HexDecoratorType]: string } = {
+	[HexDecoratorType.Insert]: style.diffInsert,
+	[HexDecoratorType.Delete]: style.diffDelete,
+	[HexDecoratorType.Empty]: style.diffEmpty,
+};