From da6ff2bf49e60454d2045db7b8067ef60bd92c8c Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Sun, 30 Jun 2024 22:19:04 +0100 Subject: [PATCH 01/18] feat: simple decorators Applies custom css styles on data cells by specifying a decorator style and its offset range. --- media/editor/dataDisplay.tsx | 42 ++++++++++++++++++++++++++++++------ media/editor/state.ts | 29 ++++++++++++++++++++++++- media/editor/util.css | 2 ++ media/editor/util.ts | 4 ++++ shared/decorators.ts | 8 +++++++ shared/protocol.ts | 2 ++ src/hexDocument.ts | 11 +++++++++- src/hexEditorProvider.ts | 1 + 8 files changed, 91 insertions(+), 8 deletions(-) create mode 100644 shared/decorators.ts diff --git a/media/editor/dataDisplay.tsx b/media/editor/dataDisplay.tsx index 6c3569a4..4117baee 100644 --- a/media/editor/dataDisplay.tsx +++ b/media/editor/dataDisplay.tsx @@ -3,6 +3,7 @@ import React, { Suspense, useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useRecoilValue, useSetRecoilState } from "recoil"; +import { HexDecorator } from "../../shared/decorators"; import { EditRangeOp, HexDocumentEditOp } from "../../shared/hexDocumentModel"; import { CopyFormat, @@ -14,8 +15,8 @@ import { Range } from "../../shared/util/range"; import { PastePopup } from "./copyPaste"; import _style from "./dataDisplay.css"; import { - FocusedElement, dataCellCls, + FocusedElement, useDisplayContext, useIsFocused, useIsHovered, @@ -31,6 +32,7 @@ import { clsx, getAsciiCharacter, getScrollDimensions, + HexDecoratorStyles, parseHexDigit, throwOnUndefinedAccessInDev, } from "./util"; @@ -428,8 +430,9 @@ const LoadingDataRows: React.FC = props => ( ); const DataPageContents: React.FC = props => { - const pageSelector = select.editedDataPages(props.pageNo); - const [data] = useLastAsyncRecoilValue(pageSelector); + const dataPageSelector = select.editedDataPages(props.pageNo); + const [data] = useLastAsyncRecoilValue(dataPageSelector); + const decorators = useRecoilValue(select.decoratorsPage(props.pageNo)); return ( <> @@ -443,6 +446,11 @@ const DataPageContents: React.FC = props => { width={props.columnWidth} showDecodedText={props.showDecodedText} isRowWithInsertDataCell={isRowWithInsertDataCell} + decorators={decorators.filter(decorator => + decorator.range.overlaps( + new Range(offset - props.pageStart, offset - props.pageStart + props.columnWidth), + ), + )} /> ))} @@ -688,7 +696,8 @@ const DataRowContents: React.FC<{ showDecodedText: boolean; rawBytes: Uint8Array; isRowWithInsertDataCell: boolean; -}> = ({ offset, width, showDecodedText, rawBytes, isRowWithInsertDataCell }) => { + decorators: HexDecorator[]; +}> = ({ offset, width, showDecodedText, rawBytes, isRowWithInsertDataCell, decorators }) => { let memoValue = ""; const ctx = useDisplayContext(); for (const byte of rawBytes) { @@ -698,9 +707,20 @@ const DataRowContents: React.FC<{ const { bytes, chars } = useMemo(() => { const bytes: React.ReactChild[] = []; const chars: React.ReactChild[] = []; + let j = 0; for (let i = 0; i < width; i++) { const boffset = offset + i; const value = rawBytes[i]; + let decorator: HexDecorator | undefined = undefined; + // Searches for the decorator, if any. Leverages the fact that + // the decorators are sorted by range. + while (j < decorators.length && decorators[j].range.start <= boffset) { + if (decorators[j].range.includes(boffset)) { + decorator = decorators[j]; + break; + } + j++; + } if (value === undefined) { if (isRowWithInsertDataCell && !ctx.isReadonly) { @@ -723,7 +743,14 @@ const DataRowContents: React.FC<{ } bytes.push( - + {value.toString(16).padStart(2, "0").toUpperCase()} , ); @@ -736,7 +763,10 @@ const DataRowContents: React.FC<{ offset={boffset} isChar={true} isAppend={false} - className={char === undefined ? style.nonGraphicChar : undefined} + className={clsx( + char === undefined ? style.nonGraphicChar : undefined, + decorator !== undefined && HexDecoratorStyles[decorator.type], + )} value={value} > {char === undefined ? "." : char} diff --git a/media/editor/state.ts b/media/editor/state.ts index d2aced1f..c0a6f5c0 100644 --- a/media/editor/state.ts +++ b/media/editor/state.ts @@ -65,7 +65,21 @@ window.addEventListener("message", ev => messageHandler.handleMessage(ev.data)); const readyQuery = selector({ key: "ready", - get: () => messageHandler.sendRequest({ type: MessageType.ReadyRequest }), + get: async () => { + const resp = await messageHandler.sendRequest({ + type: MessageType.ReadyRequest, + }); + + // Due to the message exchange, decorator.range looses their class methods + // so re-initialize. + resp.decorators = resp.decorators.map(dec => { + return { + type: dec.type, + range: new Range(dec.range.start, dec.range.end), + }; + }); + return resp; + }, }); /** @@ -414,6 +428,19 @@ export const editedDataPages = selectorFamily({ }, }); +export const decoratorsPage = selectorFamily({ + key: "decorators", + get: + (pageNumber: number) => + async ({ get }) => { + const allDecorators = get(readyQuery).decorators; + const pageSize = get(dataPageSize); + const pageRange = new Range(pageSize * pageNumber, pageSize); + const pageDecorators = allDecorators.filter(decorator => decorator.range.overlaps(pageRange)); + return pageDecorators; + }, +}); + const rawDataPages = selectorFamily({ key: "rawDataPages", get: diff --git a/media/editor/util.css b/media/editor/util.css index bb06f714..71f66007 100644 --- a/media/editor/util.css +++ b/media/editor/util.css @@ -5,3 +5,5 @@ width: 100px; height: 100px; } + +/* Decorators */ \ No newline at end of file diff --git a/media/editor/util.ts b/media/editor/util.ts index b26788ab..07bb31a5 100644 --- a/media/editor/util.ts +++ b/media/editor/util.ts @@ -3,6 +3,7 @@ // Assorted helper functions +import { HexDecoratorType } from "../../shared/decorators"; import { Range } from "../../shared/util/range"; import _style from "./util.css"; @@ -180,3 +181,6 @@ export const getScrollDimensions = (() => { return value; }; })(); + +export const HexDecoratorStyles: { [key in HexDecoratorType]: string } = { +}; diff --git a/shared/decorators.ts b/shared/decorators.ts new file mode 100644 index 00000000..22146501 --- /dev/null +++ b/shared/decorators.ts @@ -0,0 +1,8 @@ +import { Range } from "./util/range"; +export enum HexDecoratorType { +} + +export interface HexDecorator { + type: HexDecoratorType; + range: Range; +} diff --git a/shared/protocol.ts b/shared/protocol.ts index 3be1d03b..2c5f0bac 100644 --- a/shared/protocol.ts +++ b/shared/protocol.ts @@ -2,6 +2,7 @@ * Copyright (C) Microsoft Corporation. All rights reserved. *--------------------------------------------------------*/ +import { HexDecorator } from "./decorators"; import { HexDocumentEditOp } from "./hexDocumentModel"; import { ISerializedEdits } from "./serialization"; @@ -80,6 +81,7 @@ export interface ReadyResponseMessage { isReadonly: boolean; isLargeFile: boolean; editMode: HexDocumentEditOp.Insert | HexDocumentEditOp.Replace; + decorators: HexDecorator[]; } export interface SetEditModeMessage { diff --git a/src/hexDocument.ts b/src/hexDocument.ts index 7402d1e9..ea621a2c 100644 --- a/src/hexDocument.ts +++ b/src/hexDocument.ts @@ -3,6 +3,7 @@ import TelemetryReporter from "@vscode/extension-telemetry"; import * as vscode from "vscode"; +import { HexDecorator } from "../shared/decorators"; import { FileAccessor } from "../shared/fileAccessor"; import { HexDocumentEdit, @@ -95,7 +96,15 @@ export class HexDocument extends Disposable implements vscode.CustomDocument { public get uri(): vscode.Uri { return vscode.Uri.parse(this.model.uri); } - + + /** + * Reads decorators from models, returning an array of all + * decorators. + */ + public async readDecorators(): Promise { + return []; + } + /** * Reads data including unsaved edits from the model, returning an iterable * of Uint8Array chunks. diff --git a/src/hexEditorProvider.ts b/src/hexEditorProvider.ts index 8cae05e2..88db3ecd 100644 --- a/src/hexEditorProvider.ts +++ b/src/hexEditorProvider.ts @@ -334,6 +334,7 @@ export class HexEditorProvider implements vscode.CustomEditorProvider Date: Sun, 30 Jun 2024 22:31:18 +0100 Subject: [PATCH 02/18] refactor: move parseQuery to its own util file --- shared/util/uri.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 shared/util/uri.ts diff --git a/shared/util/uri.ts b/shared/util/uri.ts new file mode 100644 index 00000000..bb1e82b1 --- /dev/null +++ b/shared/util/uri.ts @@ -0,0 +1,21 @@ +export interface HexEditorUriQuery { + baseAddress?: string; +} + +/** + * Utility function to convert a Uri query string into a map + */ +export function parseQuery(queryString: string): HexEditorUriQuery { + const queries: HexEditorUriQuery = {}; + if (queryString) { + const pairs = (queryString[0] === "?" ? queryString.substr(1) : queryString).split("&"); + for (const q of pairs) { + const pair = q.split("="); + const name = pair.shift(); + if (name) { + queries[name as keyof HexEditorUriQuery] = pair.join("="); + } + } + } + return queries; +} From d8d31e53dac679dd104652c37301a152469d2521 Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Mon, 1 Jul 2024 17:55:08 +0100 Subject: [PATCH 03/18] feat: compareSelected command Opens two selected files into hex diff view. Works with any file extension by changing the file scheme. --- package.json | 21 +++++++++++++++++++++ package.nls.json | 1 + shared/hexDiffModel.ts | 12 ++++++++++++ src/compareSelected.ts | 16 ++++++++++++++++ src/extension.ts | 23 ++++++++++++++++++++++- src/fileSystemAdaptor.ts | 2 +- src/hexDiffFS.ts | 39 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 shared/hexDiffModel.ts create mode 100644 src/compareSelected.ts create mode 100644 src/hexDiffFS.ts diff --git a/package.json b/package.json index 78e3c54b..afdf0eac 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,11 @@ } } ], + "configurationDefaults": { + "workbench.editorAssociations": { + "{hexdiff}:/**/*.*": "hexEditor.hexedit" + } + }, "customEditors": [ { "viewType": "hexEditor.hexedit", @@ -132,6 +137,11 @@ "command": "hexEditor.copyOffsetAsDec", "category": "%name%", "title": "%hexEditor.copyOffsetAsDec%" + }, + { + "command": "hexEditor.compareSelected", + "category": "%name%", + "title": "%hexEditor.compareSelected%" } ], "viewsContainers": { @@ -163,6 +173,10 @@ { "command": "hexEditor.switchEditMode", "when": "hexEditor:isActive" + }, + { + "command": "hexEditor.compareSelected", + "when": "false" } ], "editor/title": [ @@ -188,6 +202,13 @@ "command": "hexEditor.copyAs", "group": "9_cutcopypaste" } + ], + "explorer/context": [ + { + "command": "hexEditor.compareSelected", + "group": "3_compare", + "when": "listDoubleSelection" + } ] }, "keybindings": [ diff --git a/package.nls.json b/package.nls.json index 76932ba2..6f3b7123 100644 --- a/package.nls.json +++ b/package.nls.json @@ -16,5 +16,6 @@ "hexEditor.switchEditMode": "Switch Edit Mode", "hexEditor.copyOffsetAsDec": "Copy Offset as Decimal", "hexEditor.copyOffsetAsHex": "Copy Offset as Hex", + "hexEditor.compareSelected": "Compare Selected in HexEditor (Experimental)", "dataInspectorView": "Data Inspector" } diff --git a/shared/hexDiffModel.ts b/shared/hexDiffModel.ts new file mode 100644 index 00000000..80087b6d --- /dev/null +++ b/shared/hexDiffModel.ts @@ -0,0 +1,12 @@ +import * as vscode from "vscode"; +import { HexDocument } from "../src/hexDocument"; +import { HexDocumentModel } from "./hexDocumentModel"; + +export class HexDiffModel { + constructor( + private readonly originalModel: HexDocumentModel, + private readonly modifiedModel: HexDocumentModel, + ) {} + + public async computeDecorators(doc: HexDocument) {} +} diff --git a/src/compareSelected.ts b/src/compareSelected.ts new file mode 100644 index 00000000..bde4b69b --- /dev/null +++ b/src/compareSelected.ts @@ -0,0 +1,16 @@ +import * as vscode from "vscode"; + +// Initializes our custom editor with diff capabilities +// @see https://github.com/microsoft/vscode/issues/97683 +// @see https://github.com/microsoft/vscode/issues/138525 +export const openCompareSelected = async (originalFile: vscode.Uri, modifiedFile: vscode.Uri) => { + const diffOriginalUri = originalFile.with({ + scheme: "hexdiff", + }); + + const diffModifiedUri = modifiedFile.with({ + scheme: "hexdiff", + }); + + await vscode.commands.executeCommand("vscode.diff", diffOriginalUri, diffModifiedUri); +}; diff --git a/src/extension.ts b/src/extension.ts index ca96d7ff..6afe8efa 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -4,9 +4,11 @@ import TelemetryReporter from "@vscode/extension-telemetry"; import * as vscode from "vscode"; import { HexDocumentEditOp } from "../shared/hexDocumentModel"; +import { openCompareSelected } from "./compareSelected"; import { copyAs } from "./copyAs"; import { DataInspectorView } from "./dataInspectorView"; import { showGoToOffset } from "./goToOffset"; +import { HexDiffFSProvider } from "./hexDiffFS"; import { HexEditorProvider } from "./hexEditorProvider"; import { HexEditorRegistry } from "./hexEditorRegistry"; import { showSelectBetweenOffsets } from "./selectBetweenOffsets"; @@ -81,7 +83,6 @@ export function activate(context: vscode.ExtensionContext): void { } }); - const switchEditModeCommand = vscode.commands.registerCommand("hexEditor.switchEditMode", () => { if (registry.activeDocument) { registry.activeDocument.editMode = @@ -109,6 +110,20 @@ export function activate(context: vscode.ExtensionContext): void { } }); + const compareSelectedCommand = vscode.commands.registerCommand( + "hexEditor.compareSelected", + async (...args) => { + if (args.length !== 2 && !(args[1] instanceof Array)) { + return; + } + const [leftFile, rightFile] = args[1]; + if (!(leftFile instanceof vscode.Uri && rightFile instanceof vscode.Uri)) { + return; + } + openCompareSelected(leftFile, rightFile); + }, + ); + context.subscriptions.push(new StatusEditMode(registry)); context.subscriptions.push(new StatusFocus(registry)); context.subscriptions.push(new StatusHoverAndSelection(registry)); @@ -119,6 +134,12 @@ export function activate(context: vscode.ExtensionContext): void { context.subscriptions.push(openWithCommand); context.subscriptions.push(telemetryReporter); context.subscriptions.push(copyOffsetAsDec, copyOffsetAsHex); + context.subscriptions.push(compareSelectedCommand); + context.subscriptions.push( + vscode.workspace.registerFileSystemProvider("hexdiff", new HexDiffFSProvider(), { + isCaseSensitive: true, + }), + ); context.subscriptions.push( HexEditorProvider.register(context, telemetryReporter, dataInspectorProvider, registry), ); diff --git a/src/fileSystemAdaptor.ts b/src/fileSystemAdaptor.ts index fdf31cba..c49c94f4 100644 --- a/src/fileSystemAdaptor.ts +++ b/src/fileSystemAdaptor.ts @@ -24,7 +24,7 @@ export const accessFile = async ( // try to use native file access for local files to allow large files to be handled efficiently // todo@connor4312/lramos: push forward extension host API for this. - if (uri.scheme === "file") { + if (uri.scheme === "file" || uri.scheme === "hexdiff") { try { // eslint-disable @typescript-eslint/no-var-requires const fs = require("fs"); diff --git a/src/hexDiffFS.ts b/src/hexDiffFS.ts new file mode 100644 index 00000000..da55edf1 --- /dev/null +++ b/src/hexDiffFS.ts @@ -0,0 +1,39 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ + +import * as vscode from "vscode"; + +// Workaround to open our files in diff mode. +export class HexDiffFSProvider implements vscode.FileSystemProvider { + readDirectory(uri: vscode.Uri): [string, vscode.FileType][] | Thenable<[string, vscode.FileType][]> { + throw new Error("Method not implemented."); + } + createDirectory(uri: vscode.Uri): void | Thenable { + throw new Error("Method not implemented."); + } + readFile(uri: vscode.Uri): Uint8Array | Thenable { + throw new Error("Method not implemented."); + } + writeFile(uri: vscode.Uri, content: Uint8Array, options: { readonly create: boolean; readonly overwrite: boolean; }): void | Thenable { + throw new Error("Method not implemented."); + } + delete(uri: vscode.Uri, options: { readonly recursive: boolean; }): void | Thenable { + throw new Error("Method not implemented."); + } + rename(oldUri: vscode.Uri, newUri: vscode.Uri, options: { readonly overwrite: boolean; }): void | Thenable { + throw new Error("Method not implemented."); + } + copy?(source: vscode.Uri, destination: vscode.Uri, options: { readonly overwrite: boolean; }): void | Thenable { + throw new Error("Method not implemented."); + } + private _emitter = new vscode.EventEmitter(); + onDidChangeFile: vscode.Event = this._emitter.event; + public watch( + uri: vscode.Uri, + options: { readonly recursive: boolean; readonly excludes: readonly string[] }, + ): vscode.Disposable { + return new vscode.Disposable(() => {}); + } + stat(uri: vscode.Uri): vscode.FileStat | Thenable { + return vscode.workspace.fs.stat(uri); + } +} From 41428e6e9dd48711c00d279734a6a7946624dde0 Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Mon, 1 Jul 2024 18:54:37 +0100 Subject: [PATCH 04/18] synchronizes hex documents creation for diffing --- shared/hexDiffModel.ts | 62 ++++++++++++++++++++++++++++++++++++++-- src/compareSelected.ts | 10 ++++++- src/extension.ts | 2 +- src/hexDocument.ts | 11 ++++++- src/hexEditorProvider.ts | 1 + src/hexEditorRegistry.ts | 16 +++++++++++ 6 files changed, 97 insertions(+), 5 deletions(-) diff --git a/shared/hexDiffModel.ts b/shared/hexDiffModel.ts index 80087b6d..6b727008 100644 --- a/shared/hexDiffModel.ts +++ b/shared/hexDiffModel.ts @@ -1,12 +1,70 @@ import * as vscode from "vscode"; -import { HexDocument } from "../src/hexDocument"; +import { HexDecorator } from "./decorators"; import { HexDocumentModel } from "./hexDocumentModel"; +export type HexDiffModelBuilder = typeof HexDiffModel.Builder.prototype; + export class HexDiffModel { constructor( private readonly originalModel: HexDocumentModel, private readonly modifiedModel: HexDocumentModel, ) {} - public async computeDecorators(doc: HexDocument) {} + public async computeDecorators(): Promise { + return []; + } + + /** + * Class to coordinate the creation of HexDiffModel + * with both HexDocumentModels + */ + static Builder = class { + private originalModel: Promise; + private modifiedModel: Promise; + private resolveOriginalModel!: ( + value: HexDocumentModel | PromiseLike, + ) => void; + private resolveModifiedModel!: ( + value: HexDocumentModel | PromiseLike, + ) => void; + private builtModel?: HexDiffModel; + public onBuild?: () => void; + + constructor( + public readonly originalUri: vscode.Uri, + public readonly modifiedUri: vscode.Uri, + ) { + this.originalModel = new Promise(resolve => { + this.resolveOriginalModel = resolve; + }); + this.modifiedModel = new Promise(resolve => { + this.resolveModifiedModel = resolve; + }); + } + + public setModel(model: HexDocumentModel) { + if (this.originalUri.toString() === model.uri.toString()) { + this.resolveOriginalModel(model); + } else if (this.modifiedUri.toString() === model.uri.toString()) { + this.resolveModifiedModel(model); + } else { + throw new Error("Provided doc does not match uris."); + } + return this; + } + + public async build() { + const [originalModel, modifiedModel] = await Promise.all([ + this.originalModel, + this.modifiedModel, + ]); + if (this.builtModel === undefined) { + this.builtModel = new HexDiffModel(originalModel, modifiedModel); + if (this.onBuild) { + this.onBuild(); + } + } + return this.builtModel; + } + }; } diff --git a/src/compareSelected.ts b/src/compareSelected.ts index bde4b69b..937c5c74 100644 --- a/src/compareSelected.ts +++ b/src/compareSelected.ts @@ -1,9 +1,15 @@ import * as vscode from "vscode"; +import { HexEditorRegistry } from "./hexEditorRegistry"; +import { HexDiffModel } from "../shared/hexDiffModel"; // Initializes our custom editor with diff capabilities // @see https://github.com/microsoft/vscode/issues/97683 // @see https://github.com/microsoft/vscode/issues/138525 -export const openCompareSelected = async (originalFile: vscode.Uri, modifiedFile: vscode.Uri) => { +export const openCompareSelected = async ( + originalFile: vscode.Uri, + modifiedFile: vscode.Uri, + registry: HexEditorRegistry, +) => { const diffOriginalUri = originalFile.with({ scheme: "hexdiff", }); @@ -12,5 +18,7 @@ export const openCompareSelected = async (originalFile: vscode.Uri, modifiedFile scheme: "hexdiff", }); + const diffModelBuilder = new HexDiffModel.Builder(diffOriginalUri, diffModifiedUri); + registry.addDiff(diffModelBuilder); await vscode.commands.executeCommand("vscode.diff", diffOriginalUri, diffModifiedUri); }; diff --git a/src/extension.ts b/src/extension.ts index 6afe8efa..9d10ad19 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -120,7 +120,7 @@ export function activate(context: vscode.ExtensionContext): void { if (!(leftFile instanceof vscode.Uri && rightFile instanceof vscode.Uri)) { return; } - openCompareSelected(leftFile, rightFile); + openCompareSelected(leftFile, rightFile, registry); }, ); diff --git a/src/hexDocument.ts b/src/hexDocument.ts index ea621a2c..702b6cbc 100644 --- a/src/hexDocument.ts +++ b/src/hexDocument.ts @@ -5,6 +5,7 @@ import TelemetryReporter from "@vscode/extension-telemetry"; import * as vscode from "vscode"; import { HexDecorator } from "../shared/decorators"; import { FileAccessor } from "../shared/fileAccessor"; +import { HexDiffModel, HexDiffModelBuilder } from "../shared/hexDiffModel"; import { HexDocumentEdit, HexDocumentEditOp, @@ -28,6 +29,7 @@ export class HexDocument extends Disposable implements vscode.CustomDocument { uri: vscode.Uri, { backupId, untitledDocumentData }: vscode.CustomDocumentOpenContext, telemetryReporter: TelemetryReporter, + diffModelBuilder: HexDiffModelBuilder | undefined, ): Promise<{ document: HexDocument; accessor: FileAccessor }> { const accessor = await accessFile(uri, untitledDocumentData); const model = new HexDocumentModel({ @@ -56,7 +58,10 @@ export class HexDocument extends Disposable implements vscode.CustomDocument { (vscode.workspace.getConfiguration().get("hexeditor.maxFileSize") as number) * 1000000; const isLargeFile = !backupId && !accessor.supportsIncremetalAccess && (fileSize ?? 0) > maxFileSize; - return { document: new HexDocument(model, isLargeFile, baseAddress), accessor }; + + const diffModel = diffModelBuilder ? await diffModelBuilder.setModel(model).build() : undefined; + + return { document: new HexDocument(model, isLargeFile, baseAddress, diffModel), accessor }; } // Last save time @@ -74,6 +79,7 @@ export class HexDocument extends Disposable implements vscode.CustomDocument { private model: HexDocumentModel, public readonly isLargeFile: boolean, public readonly baseAddress: number, + private diffModel?: HexDiffModel, ) { super(); } @@ -102,6 +108,9 @@ export class HexDocument extends Disposable implements vscode.CustomDocument { * decorators. */ public async readDecorators(): Promise { + if (this.diffModel) { + await this.diffModel.computeDecorators(); + } return []; } diff --git a/src/hexEditorProvider.ts b/src/hexEditorProvider.ts index 88db3ecd..503434fa 100644 --- a/src/hexEditorProvider.ts +++ b/src/hexEditorProvider.ts @@ -73,6 +73,7 @@ export class HexEditorProvider implements vscode.CustomEditorProvider>(); + private readonly diffsBuilder = new Map(); private onChangeEmitter = new vscode.EventEmitter(); private _activeDocument?: HexDocument; @@ -68,6 +70,20 @@ export class HexEditorRegistry extends Disposable { }; } + /** adds a diff model */ + public addDiff(diffModelBuilder: HexDiffModelBuilder) { + this.diffsBuilder.set(diffModelBuilder.modifiedUri.toString(), diffModelBuilder); + this.diffsBuilder.set(diffModelBuilder.originalUri.toString(), diffModelBuilder); + diffModelBuilder.onBuild = () => { + this.diffsBuilder.delete(diffModelBuilder.modifiedUri.toString()); + this.diffsBuilder.delete(diffModelBuilder.originalUri.toString()); + }; + } + /** returns a diff model using the file uri */ + public getDiff(uri: vscode.Uri): HexDiffModelBuilder | undefined { + return this.diffsBuilder.get(uri.toString()); + } + private onChangedTabs() { const input = vscode.window.tabGroups.activeTabGroup.activeTab?.input; const uri = input instanceof vscode.TabInputCustom ? input.uri : undefined; From 393d697decd43a4a818831a54ef58519c4c65c13 Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Wed, 3 Jul 2024 00:46:15 +0100 Subject: [PATCH 05/18] feat: basic Myers diff Introduces the O(d^2) space version and its decorators. With this diff algorithm, the diffs should be more insightful compared to using a simple per-offset comparison. --- media/editor/util.css | 10 ++- media/editor/util.ts | 2 + shared/decorators.ts | 3 + shared/hexDiffModel.ts | 18 ++++- shared/util/myers.ts | 149 +++++++++++++++++++++++++++++++++++++++++ src/hexDocument.ts | 2 +- 6 files changed, 180 insertions(+), 4 deletions(-) create mode 100644 shared/util/myers.ts diff --git a/media/editor/util.css b/media/editor/util.css index 71f66007..49396c15 100644 --- a/media/editor/util.css +++ b/media/editor/util.css @@ -6,4 +6,12 @@ height: 100px; } -/* Decorators */ \ No newline at end of file +/* Decorators */ + +.diff-insert { + background-color: green; +} + +.diff-delete { + background-color: red; +} diff --git a/media/editor/util.ts b/media/editor/util.ts index 07bb31a5..25ac1345 100644 --- a/media/editor/util.ts +++ b/media/editor/util.ts @@ -183,4 +183,6 @@ export const getScrollDimensions = (() => { })(); export const HexDecoratorStyles: { [key in HexDecoratorType]: string } = { + [HexDecoratorType.Insert]: style.diffInsert, + [HexDecoratorType.Delete]: style.diffDelete, }; diff --git a/shared/decorators.ts b/shared/decorators.ts index 22146501..8ce87cf3 100644 --- a/shared/decorators.ts +++ b/shared/decorators.ts @@ -1,5 +1,8 @@ import { Range } from "./util/range"; + export enum HexDecoratorType { + Insert, + Delete, } export interface HexDecorator { diff --git a/shared/hexDiffModel.ts b/shared/hexDiffModel.ts index 6b727008..eaa0cc29 100644 --- a/shared/hexDiffModel.ts +++ b/shared/hexDiffModel.ts @@ -1,17 +1,31 @@ +import { bulkhead } from "cockatiel"; import * as vscode from "vscode"; import { HexDecorator } from "./decorators"; import { HexDocumentModel } from "./hexDocumentModel"; +import { MyersDiff } from "./util/myers"; export type HexDiffModelBuilder = typeof HexDiffModel.Builder.prototype; export class HexDiffModel { + /** Guard to make sure only one computation operation happens */ + private readonly saveGuard = bulkhead(1, Infinity); + private decorators?: { original: HexDecorator[]; modified: HexDecorator[] }; + constructor( private readonly originalModel: HexDocumentModel, private readonly modifiedModel: HexDocumentModel, ) {} - public async computeDecorators(): Promise { - return []; + public async computeDecorators(uri: vscode.Uri): Promise { + return await this.saveGuard.execute(async () => { + if (this.decorators === undefined) { + const editScript = await MyersDiff.lcs(this.originalModel, this.modifiedModel); + this.decorators = MyersDiff.toDecorator(editScript); + } + return uri.toString() === this.originalModel.uri.toString() + ? this.decorators.original + : this.decorators.modified; + }); } /** diff --git a/shared/util/myers.ts b/shared/util/myers.ts new file mode 100644 index 00000000..3a130ef7 --- /dev/null +++ b/shared/util/myers.ts @@ -0,0 +1,149 @@ +import { HexDecorator, HexDecoratorType } from "../decorators"; +import { HexDocumentModel } from "../hexDocumentModel"; +import { Range } from "./range"; + +type ScriptType = InsertScript | DeleteScript; + +interface InsertScript { + type: "insert"; + valuePositionModified: number; + atPositionOriginal: number; +} + +interface DeleteScript { + type: "delete"; + atPositionOriginal: number; +} + +/** + * O(d^2) implementation + */ +export class MyersDiff { + public static async chunkified(model: HexDocumentModel) { + const size = await model.size(); + if (size === undefined) { + throw new Error("undefined size"); + } + let range = new Range(0, 1024); + const chunk = new Uint8Array(1024); + await model.readInto(0, chunk); + + return { + get: async (offset: number) => { + if (!range.includes(offset)) { + range = new Range( + 1024 * Math.floor(offset / 1024), + 1024 * Math.floor(offset / 1024) + 1024, + ); + await model.readInto(range.start, chunk); + } + return chunk[offset % 1024]; + }, + }; + } + + public static async lcs( + original: HexDocumentModel, + modified: HexDocumentModel, + ): Promise> { + const originalChunk = await this.chunkified(original); + const modifiedChunk = await this.chunkified(modified); + + const N = await original.size(); + const M = await modified.size(); + if (N === undefined || M === undefined) { + return []; + } + const max = N + M; + const V = new Array(2 * max + 2).fill(0); + V[1] = 0; + const trace: number[][] = []; + + for (let d = 0; d <= max; d++) { + trace.push(V.slice()); + for (let k = -d; k <= d; k += 2) { + let x: number; + if (k === -d || (k !== d && V.at(k - 1) < V.at(k + 1))) { + x = V.at(k + 1); + } else { + x = V.at(k - 1) + 1; + } + + let y = x - k; + while (x < N && y < M && (await originalChunk.get(x)) === (await modifiedChunk.get(y))) { + x++; + y++; + } + + V[k] = x; + if (x >= N && y >= M) { + return this.traceback(trace, N, M); + } + } + } + return []; + } + + private static traceback(trace: number[][], n: number, m: number): Array { + let d = trace.length - 1; + let x = n; + let y = m; + const script: Array = []; + + while (d >= 0) { + const v = trace[d]; + const k = x - y; + let prevK: number; + if (k === -d || (k !== d && v.at(k - 1)! < v.at(k + 1)!)) { + prevK = k + 1; + } else { + prevK = k - 1; + } + + const prevX = v.at(prevK)!; + const prevY = prevX - prevK; + while (x > prevX && y > prevY) { + x--; + y--; + } + + if (d > 0) { + if (x == prevX) { + script.push({ type: "insert", valuePositionModified: y - 1, atPositionOriginal: x - 1 }); + } else { + script.push({ type: "delete", atPositionOriginal: x - 1 }); + } + x = prevX; + y = prevY; + } + d--; + } + + return script.reverse(); + } + + public static toDecorator(script: ScriptType[]) { + const out: { + original: HexDecorator[]; + modified: HexDecorator[]; + } = { original: [], modified: [] }; + for (const diffType of script) { + if (diffType.type === "delete") { + out.original.push({ + type: HexDecoratorType.Delete, + range: new Range(diffType.atPositionOriginal, diffType.atPositionOriginal + 1), + }); + } else { + //out.modified.push({ + // type: HexDecoratorType.Insert, + // range: new Range(diffType.atPositionOriginal, diffType.atPositionOriginal + 1), + //}); + out.modified.push({ + type: HexDecoratorType.Insert, + range: new Range(diffType.valuePositionModified, diffType.valuePositionModified + 1), + }); + } + } + return out; + } +} diff --git a/src/hexDocument.ts b/src/hexDocument.ts index 702b6cbc..2def0555 100644 --- a/src/hexDocument.ts +++ b/src/hexDocument.ts @@ -109,7 +109,7 @@ export class HexDocument extends Disposable implements vscode.CustomDocument { */ public async readDecorators(): Promise { if (this.diffModel) { - await this.diffModel.computeDecorators(); + return await this.diffModel.computeDecorators(this.uri); } return []; } From 6312f63f1f7353bfab85005a9b18b6f24723a25d Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Wed, 3 Jul 2024 18:00:33 +0100 Subject: [PATCH 06/18] Adds alignment in original file Inserted bytes are show in the original file by a stripe data cell. --- media/editor/dataDisplay.tsx | 2 +- media/editor/state.ts | 41 ++++++++++++++++++++++-- media/editor/util.css | 20 ++++++++++-- media/editor/util.ts | 1 + shared/decorators.ts | 1 + shared/hexDiffModel.ts | 2 +- shared/hexDocumentModel.ts | 62 ++++++++++++++++++++++++++++++++---- shared/util/myers.ts | 8 ++--- 8 files changed, 120 insertions(+), 17 deletions(-) diff --git a/media/editor/dataDisplay.tsx b/media/editor/dataDisplay.tsx index 4117baee..41fb2edb 100644 --- a/media/editor/dataDisplay.tsx +++ b/media/editor/dataDisplay.tsx @@ -430,9 +430,9 @@ const LoadingDataRows: React.FC = props => ( ); const DataPageContents: React.FC = props => { + const decorators = useRecoilValue(select.decoratorsPage(props.pageNo)); const dataPageSelector = select.editedDataPages(props.pageNo); const [data] = useLastAsyncRecoilValue(dataPageSelector); - const decorators = useRecoilValue(select.decoratorsPage(props.pageNo)); return ( <> diff --git a/media/editor/state.ts b/media/editor/state.ts index c0a6f5c0..8311f886 100644 --- a/media/editor/state.ts +++ b/media/editor/state.ts @@ -3,7 +3,14 @@ *--------------------------------------------------------*/ import { atom, DefaultValue, selector, selectorFamily } from "recoil"; -import { buildEditTimeline, HexDocumentEdit, readUsingRanges } from "../../shared/hexDocumentModel"; +import { HexDecoratorType } from "../../shared/decorators"; +import { + buildEditTimeline, + HexDocumentEdit, + HexDocumentEditOp, + HexDocumentEmptyInsertEdit, + readUsingRanges, +} from "../../shared/hexDocumentModel"; import { FromWebviewMessage, InspectorLocation, @@ -150,7 +157,7 @@ export const fileSize = selector({ key: "fileSize", get: ({ get }) => { const initial = get(diskFileSize); - const sizeDelta = get(unsavedEditTimeline).sizeDelta; + const sizeDelta = get(unsavedAndDecoratorEditTimeline).sizeDelta; return initial === undefined ? initial : initial + sizeDelta; }, }); @@ -386,13 +393,41 @@ export const unsavedEditTimeline = selector({ }, }); +const emptyDecoratorEdits = selector({ + key: "emptyDecoratorEdits", + get: ({ get }) => { + return get(readyQuery) + .decorators.filter(record => record.type === HexDecoratorType.Empty) + .map(value => { + return { + op: HexDocumentEditOp.EmptyInsert, + offset: value.range.start, + length: value.range.size, + } as HexDocumentEmptyInsertEdit; + }); + }, +}); + +/** + * Creates the edit timeline for the unsaved edits and empty decorators. + */ +export const unsavedAndDecoratorEditTimeline = selector({ + key: "unsavedAndDecoratorEditTimeline", + get: ({ get }) => { + return buildEditTimeline([ + ...get(edits).slice(get(unsavedEditIndex)), + ...get(emptyDecoratorEdits), + ]); + }, +}); + export const editedDataPages = selectorFamily({ key: "editedDataPages", get: (pageNumber: number) => async ({ get }) => { const pageSize = get(dataPageSize); - const { ranges } = get(unsavedEditTimeline); + const { ranges } = get(unsavedAndDecoratorEditTimeline); const target = new Uint8Array(pageSize); const it = readUsingRanges( { diff --git a/media/editor/util.css b/media/editor/util.css index 49396c15..dbc67e73 100644 --- a/media/editor/util.css +++ b/media/editor/util.css @@ -9,9 +9,25 @@ /* Decorators */ .diff-insert { - background-color: green; + background-color: rgba(156, 204, 44, 0.4); + filter: opacity(100%); } .diff-delete { - background-color: red; + background-color: rgba(255, 0, 0, 0.4); +} + +.diff-empty { + background-image: linear-gradient( + -45deg, + var(--vscode-diffEditor-diagonalFill) 12.5%, + #0000 12.5%, + #0000 50%, + var(--vscode-diffEditor-diagonalFill) 50%, + var(--vscode-diffEditor-diagonalFill) 62.5%, + #0000 62.5%, + #0000 100% + ); + background-size: 8px 8px; + color: transparent !important; } diff --git a/media/editor/util.ts b/media/editor/util.ts index 25ac1345..79d2cea2 100644 --- a/media/editor/util.ts +++ b/media/editor/util.ts @@ -185,4 +185,5 @@ export const getScrollDimensions = (() => { export const HexDecoratorStyles: { [key in HexDecoratorType]: string } = { [HexDecoratorType.Insert]: style.diffInsert, [HexDecoratorType.Delete]: style.diffDelete, + [HexDecoratorType.Empty]: style.diffEmpty, }; diff --git a/shared/decorators.ts b/shared/decorators.ts index 8ce87cf3..bc668462 100644 --- a/shared/decorators.ts +++ b/shared/decorators.ts @@ -3,6 +3,7 @@ import { Range } from "./util/range"; export enum HexDecoratorType { Insert, Delete, + Empty, } export interface HexDecorator { diff --git a/shared/hexDiffModel.ts b/shared/hexDiffModel.ts index eaa0cc29..02c1d636 100644 --- a/shared/hexDiffModel.ts +++ b/shared/hexDiffModel.ts @@ -17,7 +17,7 @@ export class HexDiffModel { ) {} public async computeDecorators(uri: vscode.Uri): Promise { - return await this.saveGuard.execute(async () => { + return this.saveGuard.execute(async () => { if (this.decorators === undefined) { const editScript = await MyersDiff.lcs(this.originalModel, this.modifiedModel); this.decorators = MyersDiff.toDecorator(editScript); diff --git a/shared/hexDocumentModel.ts b/shared/hexDocumentModel.ts index c0355fd8..22acc460 100644 --- a/shared/hexDocumentModel.ts +++ b/shared/hexDocumentModel.ts @@ -11,6 +11,7 @@ export const enum HexDocumentEditOp { Insert, Delete, Replace, + EmptyInsert, } export interface GenericHexDocumentEdit { @@ -35,6 +36,19 @@ export interface HexDocumentInsertEdit extends GenericHexDocumentEdit { value: Uint8Array; } +/** + * Similar to {@link HexDocumentInsertEdit} but only intended to be used + * when comparing files. Unlike {@link HexDocumentInsertEdit}, this allows us + * to align files without creating unnecessary Uint8Arrays. + * + * note: this is only a visual edit, so shouldn't be treated as a normal + * edit (that's why HexDocumentEdit does not include it) + */ +export interface HexDocumentEmptyInsertEdit extends GenericHexDocumentEdit { + op: HexDocumentEditOp.EmptyInsert; + length: number; +} + export type HexDocumentEdit = | HexDocumentInsertEdit | HexDocumentDeleteEdit @@ -74,6 +88,7 @@ export const enum EditRangeOp { Read, Skip, Insert, + EmptyInsert, } export type EditRange = @@ -82,7 +97,8 @@ export type EditRange = /** Skip starting at "offset" in the edited version of the file */ | { op: EditRangeOp.Skip; editIndex: number; offset: number } /** Insert "value" at the "offset" in th edited version of the file */ - | { op: EditRangeOp.Insert; editIndex: number; offset: number; value: Uint8Array }; + | { op: EditRangeOp.Insert; editIndex: number; offset: number; value: Uint8Array } + | { op: EditRangeOp.EmptyInsert; editIndex: number; offset: number; length: number }; export interface IEditTimeline { /** Instructions on how to read the file, in order. */ @@ -332,6 +348,21 @@ export async function* readUsingRanges( continue; } + if (range.op === EditRangeOp.EmptyInsert) { + const readLast = range.offset + range.length - fromOffset; + if (readLast <= 0) { + continue; + } + const toYield = + readLast < range.length + ? buf.fill(0, -readLast).subarray(-readLast) + : buf.fill(0, -range.length).subarray(-range.length); + if (toYield.length > 0) { + yield toYield; + } + continue; + } + if (range.op === EditRangeOp.Insert) { const readLast = range.offset + range.value.length - fromOffset; if (readLast <= 0) { @@ -363,7 +394,9 @@ export async function* readUsingRanges( } } -export const buildEditTimeline = (edits: readonly HexDocumentEdit[]): IEditTimeline => { +export const buildEditTimeline = ( + edits: readonly (HexDocumentEdit | HexDocumentEmptyInsertEdit)[], +): IEditTimeline => { // Serialize all edits to a single, continuous "timeline", which we'll // iterate through in order to read data and yield bytes. const ranges: EditRange[] = [{ op: EditRangeOp.Read, editIndex: -1, roffset: 0, offset: 0 }]; @@ -389,7 +422,7 @@ export const buildEditTimeline = (edits: readonly HexDocumentEdit[]): IEditTimel before: { op: EditRangeOp.Skip, editIndex, offset: split.offset }, after: { op: EditRangeOp.Skip, editIndex, offset: split.offset + atByte }, }; - } else { + } else if (split.op === EditRangeOp.Insert) { return { before: { op: EditRangeOp.Insert, @@ -404,6 +437,21 @@ export const buildEditTimeline = (edits: readonly HexDocumentEdit[]): IEditTimel value: split.value.subarray(atByte), }, }; + } else { + return { + before: { + op: EditRangeOp.EmptyInsert, + editIndex, + offset: split.offset, + length: atByte, + }, + after: { + op: EditRangeOp.EmptyInsert, + editIndex, + offset: split.offset + atByte, + length: split.length - atByte, + }, + }; } }; @@ -426,16 +474,18 @@ export const buildEditTimeline = (edits: readonly HexDocumentEdit[]): IEditTimel } const split = ranges[i]; - if (edit.op === HexDocumentEditOp.Insert) { + if (edit.op === HexDocumentEditOp.Insert || edit.op === HexDocumentEditOp.EmptyInsert) { const { before, after } = getSplit(editIndex, split, edit.offset - split.offset); ranges.splice( i, 1, before, - { op: EditRangeOp.Insert, editIndex, offset: edit.offset, value: edit.value }, + edit.op === HexDocumentEditOp.Insert + ? { op: EditRangeOp.Insert, editIndex, offset: edit.offset, value: edit.value } + : { op: EditRangeOp.EmptyInsert, editIndex, offset: edit.offset, length: edit.length }, after, ); - shiftAfter(i + 2, edit.value.length); + shiftAfter(i + 2, edit.op === HexDocumentEditOp.Insert ? edit.value.length : edit.length); } else if (edit.op === HexDocumentEditOp.Delete || edit.op === HexDocumentEditOp.Replace) { const { before } = getSplit(editIndex, split, edit.offset - split.offset); let until = searcher(edit.offset + edit.previous.length, ranges); diff --git a/shared/util/myers.ts b/shared/util/myers.ts index 3a130ef7..9ca7450a 100644 --- a/shared/util/myers.ts +++ b/shared/util/myers.ts @@ -134,10 +134,10 @@ export class MyersDiff { range: new Range(diffType.atPositionOriginal, diffType.atPositionOriginal + 1), }); } else { - //out.modified.push({ - // type: HexDecoratorType.Insert, - // range: new Range(diffType.atPositionOriginal, diffType.atPositionOriginal + 1), - //}); + out.original.push({ + type: HexDecoratorType.Empty, + range: new Range(diffType.atPositionOriginal, diffType.atPositionOriginal + 1), + }); out.modified.push({ type: HexDecoratorType.Insert, range: new Range(diffType.valuePositionModified, diffType.valuePositionModified + 1), From 8013725185f1ecbb2e3fe476b4f0ef5dfa186745 Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Thu, 4 Jul 2024 20:17:12 +0100 Subject: [PATCH 07/18] fix: negative k-index and wrong alignment --- shared/util/myers.ts | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/shared/util/myers.ts b/shared/util/myers.ts index 9ca7450a..a8a5d4f6 100644 --- a/shared/util/myers.ts +++ b/shared/util/myers.ts @@ -6,13 +6,12 @@ type ScriptType = InsertScript | DeleteScript; interface InsertScript { type: "insert"; - valuePositionModified: number; - atPositionOriginal: number; + position: number; } interface DeleteScript { type: "delete"; - atPositionOriginal: number; + position: number; } /** @@ -75,7 +74,8 @@ export class MyersDiff { y++; } - V[k] = x; + V[k < 0 ? V.length + k : k] = x; + if (x >= N && y >= M) { return this.traceback(trace, N, M); } @@ -108,10 +108,13 @@ export class MyersDiff { } if (d > 0) { - if (x == prevX) { - script.push({ type: "insert", valuePositionModified: y - 1, atPositionOriginal: x - 1 }); - } else { - script.push({ type: "delete", atPositionOriginal: x - 1 }); + if (x === prevX) { + script.push({ + type: "insert", + position: Math.max(y - 1, prevX), + }); + } else if (y === prevY) { + script.push({ type: "delete", position: Math.max(x - 1, prevY) }); } x = prevX; y = prevY; @@ -131,16 +134,20 @@ export class MyersDiff { if (diffType.type === "delete") { out.original.push({ type: HexDecoratorType.Delete, - range: new Range(diffType.atPositionOriginal, diffType.atPositionOriginal + 1), + range: new Range(diffType.position, diffType.position + 1), + }); + out.modified.push({ + type: HexDecoratorType.Empty, + range: new Range(diffType.position, diffType.position + 1), }); } else { out.original.push({ type: HexDecoratorType.Empty, - range: new Range(diffType.atPositionOriginal, diffType.atPositionOriginal + 1), + range: new Range(diffType.position, diffType.position + 1), }); out.modified.push({ type: HexDecoratorType.Insert, - range: new Range(diffType.valuePositionModified, diffType.valuePositionModified + 1), + range: new Range(diffType.position, diffType.position + 1), }); } } From eb766017d0c7bca305f448a684e975175cec459c Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Fri, 5 Jul 2024 18:04:37 +0100 Subject: [PATCH 08/18] optimize: use binary search for page decorators Finds the lower and upper bound of the decorators that fit in the page, instead of using .filter(). In addition to that, it also removes the need to re-initialize the decorator's ranges. --- media/editor/dataDisplay.tsx | 8 ++----- media/editor/state.ts | 45 +++++++++++++++++------------------- shared/decorators.ts | 4 +--- 3 files changed, 24 insertions(+), 33 deletions(-) diff --git a/media/editor/dataDisplay.tsx b/media/editor/dataDisplay.tsx index 41fb2edb..957e3475 100644 --- a/media/editor/dataDisplay.tsx +++ b/media/editor/dataDisplay.tsx @@ -446,11 +446,7 @@ const DataPageContents: React.FC = props => { width={props.columnWidth} showDecodedText={props.showDecodedText} isRowWithInsertDataCell={isRowWithInsertDataCell} - decorators={decorators.filter(decorator => - decorator.range.overlaps( - new Range(offset - props.pageStart, offset - props.pageStart + props.columnWidth), - ), - )} + decorators={decorators} /> ))} @@ -715,7 +711,7 @@ const DataRowContents: React.FC<{ // Searches for the decorator, if any. Leverages the fact that // the decorators are sorted by range. while (j < decorators.length && decorators[j].range.start <= boffset) { - if (decorators[j].range.includes(boffset)) { + if (boffset >= decorators[j].range.start && boffset < decorators[j].range.end) { decorator = decorators[j]; break; } diff --git a/media/editor/state.ts b/media/editor/state.ts index 8311f886..db16fb94 100644 --- a/media/editor/state.ts +++ b/media/editor/state.ts @@ -3,7 +3,7 @@ *--------------------------------------------------------*/ import { atom, DefaultValue, selector, selectorFamily } from "recoil"; -import { HexDecoratorType } from "../../shared/decorators"; +import { HexDecorator, HexDecoratorType } from "../../shared/decorators"; import { buildEditTimeline, HexDocumentEdit, @@ -22,6 +22,7 @@ import { ToWebviewMessage, } from "../../shared/protocol"; import { deserializeEdits, serializeEdits } from "../../shared/serialization"; +import { binarySearch } from "../../shared/util/binarySearch"; import { Range } from "../../shared/util/range"; import { clamp } from "./util"; @@ -72,21 +73,7 @@ window.addEventListener("message", ev => messageHandler.handleMessage(ev.data)); const readyQuery = selector({ key: "ready", - get: async () => { - const resp = await messageHandler.sendRequest({ - type: MessageType.ReadyRequest, - }); - - // Due to the message exchange, decorator.range looses their class methods - // so re-initialize. - resp.decorators = resp.decorators.map(dec => { - return { - type: dec.type, - range: new Range(dec.range.start, dec.range.end), - }; - }); - return resp; - }, + get: () => messageHandler.sendRequest({ type: MessageType.ReadyRequest }), }); /** @@ -124,6 +111,11 @@ export const codeSettings = selector({ get: ({ get }) => get(readyQuery).codeSettings, }); +export const decorators = selector({ + key: "decorators", + get: ({ get }) => get(readyQuery).decorators, +}); + export const showReadonlyWarningForEl = atom({ key: "showReadonlyWarningForEl", default: null, @@ -396,13 +388,13 @@ export const unsavedEditTimeline = selector({ const emptyDecoratorEdits = selector({ key: "emptyDecoratorEdits", get: ({ get }) => { - return get(readyQuery) - .decorators.filter(record => record.type === HexDecoratorType.Empty) + return get(decorators) + .filter(record => record.type === HexDecoratorType.Empty) .map(value => { return { op: HexDocumentEditOp.EmptyInsert, offset: value.range.start, - length: value.range.size, + length: value.range.end - value.range.start, } as HexDocumentEmptyInsertEdit; }); }, @@ -463,16 +455,21 @@ export const editedDataPages = selectorFamily({ }, }); +/** Returns the starting decorator index for the given page number */ export const decoratorsPage = selectorFamily({ - key: "decorators", + key: "decoratorsPage", get: (pageNumber: number) => async ({ get }) => { - const allDecorators = get(readyQuery).decorators; + const allDecorators = get(decorators); + if (allDecorators.length === 0) { + return []; + } const pageSize = get(dataPageSize); - const pageRange = new Range(pageSize * pageNumber, pageSize); - const pageDecorators = allDecorators.filter(decorator => decorator.range.overlaps(pageRange)); - return pageDecorators; + const searcher = binarySearch(decorator => decorator.range.start); + const startIndex = searcher(pageSize * pageNumber, allDecorators); + const endIndex = searcher(pageSize * pageNumber + pageSize, allDecorators); + return allDecorators.slice(startIndex, endIndex); }, }); diff --git a/shared/decorators.ts b/shared/decorators.ts index bc668462..3ee0348f 100644 --- a/shared/decorators.ts +++ b/shared/decorators.ts @@ -1,5 +1,3 @@ -import { Range } from "./util/range"; - export enum HexDecoratorType { Insert, Delete, @@ -8,5 +6,5 @@ export enum HexDecoratorType { export interface HexDecorator { type: HexDecoratorType; - range: Range; + range: { start: number; end: number }; } From 821aa5dcf846b164f1768c1f12ae56c465ad1855 Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Fri, 19 Jul 2024 20:46:07 +0100 Subject: [PATCH 09/18] Makes diff read-only Done to simplify diff, because recomputing diffs after a file change is too complex to tackle at the moment. --- src/fileSystemAdaptor.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fileSystemAdaptor.ts b/src/fileSystemAdaptor.ts index c49c94f4..12396a67 100644 --- a/src/fileSystemAdaptor.ts +++ b/src/fileSystemAdaptor.ts @@ -42,7 +42,8 @@ export const accessFile = async ( : !(fileStats.mode & 0o002); // other if (fileStats.isFile()) { - return new NativeFileAccessor(uri, isReadonly, fs); + // Diff is readonly since the diff is only computed at the beginning once + return new NativeFileAccessor(uri, uri.scheme === "hexdiff" ? true : isReadonly, fs); } } catch { // probably not node.js, or file does not exist From 954158d873d75e224f5d2ee4c957cc59f2496519 Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Fri, 19 Jul 2024 21:39:35 +0100 Subject: [PATCH 10/18] refactor: use diff package for myers diff Although still O(d^2), it improves memory usage in other ways, provides some optimization for edge cases and has some useful built-in options to exit on large diffs. --- package-lock.json | 45 ++++++++++-- package.json | 2 + shared/hexDiffModel.ts | 1 - shared/util/myers.ts | 161 ++++++++++------------------------------- src/hexDocument.ts | 8 +- 5 files changed, 85 insertions(+), 132 deletions(-) diff --git a/package-lock.json b/package-lock.json index 40aef374..136bc30c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@vscode/codicons": "0.0.27", "@vscode/extension-telemetry": "0.6.2", "cockatiel": "^3.1.2", + "diff": "^5.2.0", "js-base64": "^3.7.2", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -22,6 +23,7 @@ }, "devDependencies": { "@types/chai": "^4.3.0", + "@types/diff": "^5.2.1", "@types/mocha": "^9.0.0", "@types/node": "^18.11.9", "@types/react": "^17.0.38", @@ -1220,6 +1222,12 @@ "integrity": "sha512-/ceqdqeRraGolFTcfoXNiqjyQhZzbINDngeoAq9GoHa8PPK1yNzTaxWjA6BFWp5Ua9JpXEMSS4s5i9tS0hOJtw==", "dev": true }, + "node_modules/@types/diff": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/@types/diff/-/diff-5.2.1.tgz", + "integrity": "sha512-uxpcuwWJGhe2AR1g8hD9F5OYGCqjqWnBUQFD8gMZsDbv8oPHzxJF6iMO6n8Tk0AdzlxoaaoQhOYlIg/PukVU8g==", + "dev": true + }, "node_modules/@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -2065,10 +2073,10 @@ } }, "node_modules/diff": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", - "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", - "dev": true, + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", + "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", + "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" } @@ -3464,6 +3472,16 @@ "url": "https://opencollective.com/mochajs" } }, + "node_modules/mocha/node_modules/diff": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/mocha/node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -5112,6 +5130,12 @@ "integrity": "sha512-/ceqdqeRraGolFTcfoXNiqjyQhZzbINDngeoAq9GoHa8PPK1yNzTaxWjA6BFWp5Ua9JpXEMSS4s5i9tS0hOJtw==", "dev": true }, + "@types/diff": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/@types/diff/-/diff-5.2.1.tgz", + "integrity": "sha512-uxpcuwWJGhe2AR1g8hD9F5OYGCqjqWnBUQFD8gMZsDbv8oPHzxJF6iMO6n8Tk0AdzlxoaaoQhOYlIg/PukVU8g==", + "dev": true + }, "@types/json-schema": { "version": "7.0.15", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", @@ -5687,10 +5711,9 @@ "dev": true }, "diff": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", - "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", - "dev": true + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", + "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==" }, "dir-glob": { "version": "3.0.1", @@ -6666,6 +6689,12 @@ "yargs-unparser": "2.0.0" }, "dependencies": { + "diff": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", + "integrity": "sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w==", + "dev": true + }, "escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", diff --git a/package.json b/package.json index afdf0eac..2320fe55 100644 --- a/package.json +++ b/package.json @@ -238,6 +238,7 @@ }, "devDependencies": { "@types/chai": "^4.3.0", + "@types/diff": "^5.2.1", "@types/mocha": "^9.0.0", "@types/node": "^18.11.9", "@types/react": "^17.0.38", @@ -260,6 +261,7 @@ "@vscode/codicons": "0.0.27", "@vscode/extension-telemetry": "0.6.2", "cockatiel": "^3.1.2", + "diff": "^5.2.0", "js-base64": "^3.7.2", "react": "^18.2.0", "react-dom": "^18.2.0", diff --git a/shared/hexDiffModel.ts b/shared/hexDiffModel.ts index 02c1d636..2ceb9f21 100644 --- a/shared/hexDiffModel.ts +++ b/shared/hexDiffModel.ts @@ -3,7 +3,6 @@ import * as vscode from "vscode"; import { HexDecorator } from "./decorators"; import { HexDocumentModel } from "./hexDocumentModel"; import { MyersDiff } from "./util/myers"; - export type HexDiffModelBuilder = typeof HexDiffModel.Builder.prototype; export class HexDiffModel { diff --git a/shared/util/myers.ts b/shared/util/myers.ts index a8a5d4f6..e6b2a064 100644 --- a/shared/util/myers.ts +++ b/shared/util/myers.ts @@ -1,155 +1,72 @@ +import { ArrayChange, diffArrays } from "diff"; +import * as vscode from "vscode"; import { HexDecorator, HexDecoratorType } from "../decorators"; import { HexDocumentModel } from "../hexDocumentModel"; import { Range } from "./range"; -type ScriptType = InsertScript | DeleteScript; - -interface InsertScript { - type: "insert"; - position: number; -} - -interface DeleteScript { - type: "delete"; - position: number; -} - /** * O(d^2) implementation */ export class MyersDiff { - public static async chunkified(model: HexDocumentModel) { - const size = await model.size(); - if (size === undefined) { - throw new Error("undefined size"); - } - let range = new Range(0, 1024); - const chunk = new Uint8Array(1024); - await model.readInto(0, chunk); - - return { - get: async (offset: number) => { - if (!range.includes(offset)) { - range = new Range( - 1024 * Math.floor(offset / 1024), - 1024 * Math.floor(offset / 1024) + 1024, - ); - await model.readInto(range.start, chunk); - } - return chunk[offset % 1024]; - }, - }; - } - - public static async lcs( - original: HexDocumentModel, - modified: HexDocumentModel, - ): Promise> { - const originalChunk = await this.chunkified(original); - const modifiedChunk = await this.chunkified(modified); - - const N = await original.size(); - const M = await modified.size(); - if (N === undefined || M === undefined) { - return []; - } - const max = N + M; - const V = new Array(2 * max + 2).fill(0); - V[1] = 0; - const trace: number[][] = []; - - for (let d = 0; d <= max; d++) { - trace.push(V.slice()); - for (let k = -d; k <= d; k += 2) { - let x: number; - if (k === -d || (k !== d && V.at(k - 1) < V.at(k + 1))) { - x = V.at(k + 1); - } else { - x = V.at(k - 1) + 1; - } - - let y = x - k; - while (x < N && y < M && (await originalChunk.get(x)) === (await modifiedChunk.get(y))) { - x++; - y++; - } - - V[k < 0 ? V.length + k : k] = x; - - if (x >= N && y >= M) { - return this.traceback(trace, N, M); - } - } + public static async lcs(original: HexDocumentModel, modified: HexDocumentModel) { + const oSize = await original.sizeWithEdits(); + const mSize = await modified.sizeWithEdits(); + if (oSize === undefined || mSize === undefined) { + throw new Error(vscode.l10n.t("HexEditor Diff: Failed to get file sizes.")); } - return []; - } - - private static traceback(trace: number[][], n: number, m: number): Array { - let d = trace.length - 1; - let x = n; - let y = m; - const script: Array = []; - while (d >= 0) { - const v = trace[d]; - const k = x - y; - let prevK: number; - if (k === -d || (k !== d && v.at(k - 1)! < v.at(k + 1)!)) { - prevK = k + 1; - } else { - prevK = k - 1; - } - - const prevX = v.at(prevK)!; - const prevY = prevX - prevK; - while (x > prevX && y > prevY) { - x--; - y--; - } - - if (d > 0) { - if (x === prevX) { - script.push({ - type: "insert", - position: Math.max(y - 1, prevX), - }); - } else if (y === prevY) { - script.push({ type: "delete", position: Math.max(x - 1, prevY) }); - } - x = prevX; - y = prevY; - } - d--; + const oArray = new Uint8Array(oSize); + const mArray = new Uint8Array(mSize); + await original.readInto(0, oArray); + await modified.readInto(0, mArray); + // the types in @types/diff are incomplete. + const changes: ArrayChange[] | undefined = diffArrays( + oArray as any, + mArray as any, + { + timeout: 30000, // timeout in milliseconds + } as any, + ); + + // Triggered timeout + if (changes === undefined) { + throw new Error( + vscode.l10n.t( + "HexEditor Diff: Reached maximum computation time to compute diff. This usually happens when comparing large files.", + ), + ); } - - return script.reverse(); + return changes; } - public static toDecorator(script: ScriptType[]) { + public static toDecorator(script: ArrayChange[]) { const out: { original: HexDecorator[]; modified: HexDecorator[]; } = { original: [], modified: [] }; - for (const diffType of script) { - if (diffType.type === "delete") { + let offset = 0; + for (const change of script) { + const r = new Range(offset, offset + change.count!); + if (change.removed) { out.original.push({ type: HexDecoratorType.Delete, - range: new Range(diffType.position, diffType.position + 1), + range: r, }); out.modified.push({ type: HexDecoratorType.Empty, - range: new Range(diffType.position, diffType.position + 1), + range: r, }); - } else { + } else if (change.added) { out.original.push({ type: HexDecoratorType.Empty, - range: new Range(diffType.position, diffType.position + 1), + range: r, }); out.modified.push({ type: HexDecoratorType.Insert, - range: new Range(diffType.position, diffType.position + 1), + range: r, }); } + offset += change.count!; } return out; } diff --git a/src/hexDocument.ts b/src/hexDocument.ts index 2def0555..7ad2db29 100644 --- a/src/hexDocument.ts +++ b/src/hexDocument.ts @@ -109,7 +109,13 @@ export class HexDocument extends Disposable implements vscode.CustomDocument { */ public async readDecorators(): Promise { if (this.diffModel) { - return await this.diffModel.computeDecorators(this.uri); + try { + return await this.diffModel.computeDecorators(this.uri); + } catch (e: unknown) { + vscode.window.showErrorMessage( + e instanceof Error ? e.message : vscode.l10n.t("Unknown Error in HexEditor Diff"), + ); + } } return []; } From f721459eb6e0c64c8dcf70295f4164a25aee8d32 Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Tue, 23 Jul 2024 20:37:20 +0100 Subject: [PATCH 11/18] fix: diff failing to open in web --- src/compareSelected.ts | 2 +- src/hexDiffFS.ts | 47 ++++++++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/compareSelected.ts b/src/compareSelected.ts index 937c5c74..af866c6c 100644 --- a/src/compareSelected.ts +++ b/src/compareSelected.ts @@ -20,5 +20,5 @@ export const openCompareSelected = async ( const diffModelBuilder = new HexDiffModel.Builder(diffOriginalUri, diffModifiedUri); registry.addDiff(diffModelBuilder); - await vscode.commands.executeCommand("vscode.diff", diffOriginalUri, diffModifiedUri); + vscode.commands.executeCommand("vscode.diff", diffOriginalUri, diffModifiedUri); }; diff --git a/src/hexDiffFS.ts b/src/hexDiffFS.ts index da55edf1..65d0ce44 100644 --- a/src/hexDiffFS.ts +++ b/src/hexDiffFS.ts @@ -2,27 +2,48 @@ import * as vscode from "vscode"; -// Workaround to open our files in diff mode. +/** changes our scheme to file so we can use workspace.fs */ +function toFileUri(uri: vscode.Uri) { + return uri.with({ scheme: "file" }); +} +// Workaround to open our files in diff mode. Used by both web and node +// to create a diff model, but the methods are only used in web whereas +// in node we use node's fs. export class HexDiffFSProvider implements vscode.FileSystemProvider { - readDirectory(uri: vscode.Uri): [string, vscode.FileType][] | Thenable<[string, vscode.FileType][]> { + readDirectory( + uri: vscode.Uri, + ): [string, vscode.FileType][] | Thenable<[string, vscode.FileType][]> { throw new Error("Method not implemented."); } createDirectory(uri: vscode.Uri): void | Thenable { throw new Error("Method not implemented."); } readFile(uri: vscode.Uri): Uint8Array | Thenable { - throw new Error("Method not implemented."); - } - writeFile(uri: vscode.Uri, content: Uint8Array, options: { readonly create: boolean; readonly overwrite: boolean; }): void | Thenable { - throw new Error("Method not implemented."); + return vscode.workspace.fs.readFile(toFileUri(uri)); } - delete(uri: vscode.Uri, options: { readonly recursive: boolean; }): void | Thenable { - throw new Error("Method not implemented."); - } - rename(oldUri: vscode.Uri, newUri: vscode.Uri, options: { readonly overwrite: boolean; }): void | Thenable { - throw new Error("Method not implemented."); + writeFile( + uri: vscode.Uri, + content: Uint8Array, + options: { readonly create: boolean; readonly overwrite: boolean }, + ): void | Thenable { + return vscode.workspace.fs.writeFile(toFileUri(uri), content); } - copy?(source: vscode.Uri, destination: vscode.Uri, options: { readonly overwrite: boolean; }): void | Thenable { + + delete(uri: vscode.Uri, options: { readonly recursive: boolean }): void | Thenable { + return vscode.workspace.fs.delete(toFileUri(uri), options); + } + rename( + oldUri: vscode.Uri, + newUri: vscode.Uri, + options: { readonly overwrite: boolean }, + ): void | Thenable { + throw new Error("Method not implemented"); + } + copy?( + source: vscode.Uri, + destination: vscode.Uri, + options: { readonly overwrite: boolean }, + ): void | Thenable { throw new Error("Method not implemented."); } private _emitter = new vscode.EventEmitter(); @@ -34,6 +55,6 @@ export class HexDiffFSProvider implements vscode.FileSystemProvider { return new vscode.Disposable(() => {}); } stat(uri: vscode.Uri): vscode.FileStat | Thenable { - return vscode.workspace.fs.stat(uri); + return vscode.workspace.fs.stat(toFileUri(uri)); } } From 56163642c34db951d7f90b5eb6918a8d7303865d Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Wed, 24 Jul 2024 16:54:18 +0100 Subject: [PATCH 12/18] perf: moved myers diff into a web-only worker --- .esbuild.config.js | 11 +++++++++++ shared/diffWorker.ts | 23 +++++++++++++++++++++++ shared/diffWorkerProtocol.ts | 31 +++++++++++++++++++++++++++++++ shared/hexDiffModel.ts | 29 +++++++++++++++++++++++++---- shared/util/myers.ts | 30 +++--------------------------- src/compareSelected.ts | 10 ++++++++-- src/extension.ts | 31 +++++++++++++++++++++++++++++-- 7 files changed, 130 insertions(+), 35 deletions(-) create mode 100644 shared/diffWorker.ts create mode 100644 shared/diffWorkerProtocol.ts diff --git a/.esbuild.config.js b/.esbuild.config.js index 7d3a5851..cd1e5743 100644 --- a/.esbuild.config.js +++ b/.esbuild.config.js @@ -52,6 +52,17 @@ build({ outfile: "dist/web/extension.js", }); +build({ + entryPoints: ["shared/diffWorker.ts"], + tsconfig: "./tsconfig.json", + bundle: true, + format: "cjs", + external: ["vscode"], + minify, + platform: "browser", + outfile: "dist/web/diffWorker.js", +}); + // Build the data inspector build({ entryPoints: ["media/data_inspector/inspector.ts"], diff --git a/shared/diffWorker.ts b/shared/diffWorker.ts new file mode 100644 index 00000000..39d16b31 --- /dev/null +++ b/shared/diffWorker.ts @@ -0,0 +1,23 @@ +import { DiffMessageType, FromDiffWorkerMessage, ToDiffWorkerMessage } from "./diffWorkerProtocol"; +import { MessageHandler } from "./protocol"; +import { MyersDiff } from "./util/myers"; + +function onMessage(message: ToDiffWorkerMessage): undefined | FromDiffWorkerMessage { + switch (message.type) { + case DiffMessageType.DiffDecoratorRequest: + const script = MyersDiff.lcs(message.original, message.modified); + const decorators = MyersDiff.toDecorator(script); + return { + type: DiffMessageType.DiffDecoratorResponse, + original: decorators.original, + modified: decorators.modified, + }; + } +} + +const messageHandler = new MessageHandler( + async message => onMessage(message), + message => postMessage(message), +); + +onmessage = e => messageHandler.handleMessage(e.data); \ No newline at end of file diff --git a/shared/diffWorkerProtocol.ts b/shared/diffWorkerProtocol.ts new file mode 100644 index 00000000..50372f09 --- /dev/null +++ b/shared/diffWorkerProtocol.ts @@ -0,0 +1,31 @@ +import { HexDecorator } from "./decorators"; +import { MessageHandler } from "./protocol"; + +export type DiffExtensionHostMessageHandler = MessageHandler< + ToDiffWorkerMessage, + FromDiffWorkerMessage +>; +export type DiffWorkerMessageHandler = MessageHandler; + +export type ToDiffWorkerMessage = DiffDecoratorsRequestMessage; +export type FromDiffWorkerMessage = DiffDecoratorResponseMessage; +export enum DiffMessageType { + // #region to diffworker + DiffDecoratorRequest, + // #endregion + // #region from diff worker + DiffDecoratorResponse, + // #endregion +} + +export interface DiffDecoratorsRequestMessage { + type: DiffMessageType.DiffDecoratorRequest; + original: Uint8Array; + modified: Uint8Array; +} + +export interface DiffDecoratorResponseMessage { + type: DiffMessageType.DiffDecoratorResponse; + original: HexDecorator[]; + modified: HexDecorator[]; +} diff --git a/shared/hexDiffModel.ts b/shared/hexDiffModel.ts index 2ceb9f21..7ad17958 100644 --- a/shared/hexDiffModel.ts +++ b/shared/hexDiffModel.ts @@ -1,8 +1,12 @@ import { bulkhead } from "cockatiel"; import * as vscode from "vscode"; import { HexDecorator } from "./decorators"; +import { + DiffDecoratorResponseMessage, + DiffExtensionHostMessageHandler, + DiffMessageType, +} from "./diffWorkerProtocol"; import { HexDocumentModel } from "./hexDocumentModel"; -import { MyersDiff } from "./util/myers"; export type HexDiffModelBuilder = typeof HexDiffModel.Builder.prototype; export class HexDiffModel { @@ -13,13 +17,29 @@ export class HexDiffModel { constructor( private readonly originalModel: HexDocumentModel, private readonly modifiedModel: HexDocumentModel, + private readonly messageHandler: DiffExtensionHostMessageHandler, ) {} public async computeDecorators(uri: vscode.Uri): Promise { return this.saveGuard.execute(async () => { if (this.decorators === undefined) { - const editScript = await MyersDiff.lcs(this.originalModel, this.modifiedModel); - this.decorators = MyersDiff.toDecorator(editScript); + //TODO: Add a warning if the file sizes are too large? + const oSize = await this.originalModel.sizeWithEdits(); + const mSize = await this.modifiedModel.sizeWithEdits(); + if (oSize === undefined || mSize === undefined) { + throw new Error(vscode.l10n.t("HexEditor Diff: Failed to get file sizes.")); + } + + const oArray = new Uint8Array(oSize); + const mArray = new Uint8Array(mSize); + await this.originalModel.readInto(0, oArray); + await this.modifiedModel.readInto(0, mArray); + const decorators = await this.messageHandler.sendRequest({ + type: DiffMessageType.DiffDecoratorRequest, + original: oArray, + modified: mArray, + }); + this.decorators = decorators; } return uri.toString() === this.originalModel.uri.toString() ? this.decorators.original @@ -46,6 +66,7 @@ export class HexDiffModel { constructor( public readonly originalUri: vscode.Uri, public readonly modifiedUri: vscode.Uri, + private readonly messageHandler: DiffExtensionHostMessageHandler, ) { this.originalModel = new Promise(resolve => { this.resolveOriginalModel = resolve; @@ -72,7 +93,7 @@ export class HexDiffModel { this.modifiedModel, ]); if (this.builtModel === undefined) { - this.builtModel = new HexDiffModel(originalModel, modifiedModel); + this.builtModel = new HexDiffModel(originalModel, modifiedModel, this.messageHandler); if (this.onBuild) { this.onBuild(); } diff --git a/shared/util/myers.ts b/shared/util/myers.ts index e6b2a064..33c6e52d 100644 --- a/shared/util/myers.ts +++ b/shared/util/myers.ts @@ -1,41 +1,17 @@ import { ArrayChange, diffArrays } from "diff"; -import * as vscode from "vscode"; import { HexDecorator, HexDecoratorType } from "../decorators"; -import { HexDocumentModel } from "../hexDocumentModel"; import { Range } from "./range"; /** * O(d^2) implementation */ export class MyersDiff { - public static async lcs(original: HexDocumentModel, modified: HexDocumentModel) { - const oSize = await original.sizeWithEdits(); - const mSize = await modified.sizeWithEdits(); - if (oSize === undefined || mSize === undefined) { - throw new Error(vscode.l10n.t("HexEditor Diff: Failed to get file sizes.")); - } - - const oArray = new Uint8Array(oSize); - const mArray = new Uint8Array(mSize); - await original.readInto(0, oArray); - await modified.readInto(0, mArray); + public static lcs(original: Uint8Array, modified: Uint8Array) { // the types in @types/diff are incomplete. const changes: ArrayChange[] | undefined = diffArrays( - oArray as any, - mArray as any, - { - timeout: 30000, // timeout in milliseconds - } as any, + original as any, + modified as any, ); - - // Triggered timeout - if (changes === undefined) { - throw new Error( - vscode.l10n.t( - "HexEditor Diff: Reached maximum computation time to compute diff. This usually happens when comparing large files.", - ), - ); - } return changes; } diff --git a/src/compareSelected.ts b/src/compareSelected.ts index af866c6c..1f2091ca 100644 --- a/src/compareSelected.ts +++ b/src/compareSelected.ts @@ -1,6 +1,7 @@ import * as vscode from "vscode"; -import { HexEditorRegistry } from "./hexEditorRegistry"; +import { DiffExtensionHostMessageHandler } from "../shared/diffWorkerProtocol"; import { HexDiffModel } from "../shared/hexDiffModel"; +import { HexEditorRegistry } from "./hexEditorRegistry"; // Initializes our custom editor with diff capabilities // @see https://github.com/microsoft/vscode/issues/97683 @@ -9,6 +10,7 @@ export const openCompareSelected = async ( originalFile: vscode.Uri, modifiedFile: vscode.Uri, registry: HexEditorRegistry, + diffExtensionHostMessageHandler: DiffExtensionHostMessageHandler, ) => { const diffOriginalUri = originalFile.with({ scheme: "hexdiff", @@ -18,7 +20,11 @@ export const openCompareSelected = async ( scheme: "hexdiff", }); - const diffModelBuilder = new HexDiffModel.Builder(diffOriginalUri, diffModifiedUri); + const diffModelBuilder = new HexDiffModel.Builder( + diffOriginalUri, + diffModifiedUri, + diffExtensionHostMessageHandler, + ); registry.addDiff(diffModelBuilder); vscode.commands.executeCommand("vscode.diff", diffOriginalUri, diffModifiedUri); }; diff --git a/src/extension.ts b/src/extension.ts index 9d10ad19..9bb366c2 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -3,7 +3,9 @@ import TelemetryReporter from "@vscode/extension-telemetry"; import * as vscode from "vscode"; +import { FromDiffWorkerMessage, ToDiffWorkerMessage } from "../shared/diffWorkerProtocol"; import { HexDocumentEditOp } from "../shared/hexDocumentModel"; +import { MessageHandler } from "../shared/protocol"; import { openCompareSelected } from "./compareSelected"; import { copyAs } from "./copyAs"; import { DataInspectorView } from "./dataInspectorView"; @@ -39,7 +41,7 @@ function reopenWithHexEditor() { } } -export function activate(context: vscode.ExtensionContext): void { +export async function activate(context: vscode.ExtensionContext) { const registry = new HexEditorRegistry(); // Register the data inspector as a separate view on the side const dataInspectorProvider = new DataInspectorView(context.extensionUri, registry); @@ -110,6 +112,25 @@ export function activate(context: vscode.ExtensionContext): void { } }); + // Initializes web-only worker for diffing + let worker: Worker; + let workerMessageHandler: MessageHandler | undefined = + undefined; + try { + worker = new Worker( + vscode.Uri.joinPath(context.extensionUri, "dist", "web", "diffWorker.js").toString(), + ); + workerMessageHandler = new MessageHandler( + // Always return undefined as the diff worker + // does not request anything from extension host + async () => undefined, + message => worker.postMessage(message), + ); + worker.onmessage = e => workerMessageHandler!.handleMessage(e.data); + } catch { + // not a vscode web instance + } + const compareSelectedCommand = vscode.commands.registerCommand( "hexEditor.compareSelected", async (...args) => { @@ -120,7 +141,10 @@ export function activate(context: vscode.ExtensionContext): void { if (!(leftFile instanceof vscode.Uri && rightFile instanceof vscode.Uri)) { return; } - openCompareSelected(leftFile, rightFile, registry); + // Web-only + if (workerMessageHandler) { + openCompareSelected(leftFile, rightFile, registry, workerMessageHandler!); + } }, ); @@ -143,6 +167,9 @@ export function activate(context: vscode.ExtensionContext): void { context.subscriptions.push( HexEditorProvider.register(context, telemetryReporter, dataInspectorProvider, registry), ); + context.subscriptions.push({ + dispose: () => worker.terminate(), + }); } export function deactivate(): void { From 27833038a2fcbd52f448d2485eb3e163a33993fa Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Thu, 25 Jul 2024 17:52:02 +0100 Subject: [PATCH 13/18] adds node js worker --- .esbuild.config.js | 6 +++--- shared/diffWorker.ts | 26 +++++++++++++++++++++----- src/extension.ts | 42 ++++++++++++++++++++++++++++-------------- 3 files changed, 52 insertions(+), 22 deletions(-) diff --git a/.esbuild.config.js b/.esbuild.config.js index cd1e5743..f4d0324f 100644 --- a/.esbuild.config.js +++ b/.esbuild.config.js @@ -46,7 +46,7 @@ build({ tsconfig: "./tsconfig.json", bundle: true, format: "cjs", - external: ["vscode", "fs"], + external: ["vscode", "fs", "worker_threads"], minify, platform: "browser", outfile: "dist/web/extension.js", @@ -57,10 +57,10 @@ build({ tsconfig: "./tsconfig.json", bundle: true, format: "cjs", - external: ["vscode"], + external: ["vscode", "worker_threads"], minify, platform: "browser", - outfile: "dist/web/diffWorker.js", + outfile: "dist/diffWorker.js", }); // Build the data inspector diff --git a/shared/diffWorker.ts b/shared/diffWorker.ts index 39d16b31..a4e3e1c1 100644 --- a/shared/diffWorker.ts +++ b/shared/diffWorker.ts @@ -15,9 +15,25 @@ function onMessage(message: ToDiffWorkerMessage): undefined | FromDiffWorkerMess } } -const messageHandler = new MessageHandler( - async message => onMessage(message), - message => postMessage(message), -); +try { + // Web worker + const messageHandler = new MessageHandler( + async message => onMessage(message), + message => postMessage(message), + ); + onmessage = e => messageHandler.handleMessage(e.data); +} catch { + // node worker -onmessage = e => messageHandler.handleMessage(e.data); \ No newline at end of file + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { parentPort } = require("worker_threads") as typeof import("worker_threads"); + if (parentPort) { + const messageHandler = new MessageHandler( + async message => onMessage(message), + message => parentPort.postMessage(message), + ); + parentPort.on("message", e => { + messageHandler.handleMessage(e); + }); + } +} diff --git a/src/extension.ts b/src/extension.ts index 9bb366c2..0bb9cf40 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -112,25 +112,39 @@ export async function activate(context: vscode.ExtensionContext) { } }); - // Initializes web-only worker for diffing + // Initializes worker for diffing let worker: Worker; - let workerMessageHandler: MessageHandler | undefined = - undefined; + const workerFilePath = vscode.Uri.joinPath( + context.extensionUri, + "dist", + "diffWorker.js", + ).toString(); + try { - worker = new Worker( - vscode.Uri.joinPath(context.extensionUri, "dist", "web", "diffWorker.js").toString(), - ); - workerMessageHandler = new MessageHandler( - // Always return undefined as the diff worker - // does not request anything from extension host - async () => undefined, - message => worker.postMessage(message), - ); - worker.onmessage = e => workerMessageHandler!.handleMessage(e.data); + worker = new Worker(workerFilePath); } catch { - // not a vscode web instance + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { Worker } = require("worker_threads") as typeof import("worker_threads"); + const nodeWorker = new Worker(new URL(workerFilePath)); + // Web and node js have different worker interfaces, so we share a function + // to initialize both workers the same way. + const ref = nodeWorker.addListener; + (nodeWorker as any).addEventListener = ref; + worker = nodeWorker as any; } + const workerMessageHandler = new MessageHandler( + // Always return undefined as the diff worker + // does not request anything from extension host + async () => undefined, + message => worker.postMessage(message), + ); + + worker.addEventListener("message", e => + // e.data is used in web worker and e is used in node js worker + e.data ? workerMessageHandler.handleMessage(e.data) : workerMessageHandler.handleMessage(e as any), + ); + const compareSelectedCommand = vscode.commands.registerCommand( "hexEditor.compareSelected", async (...args) => { From 6f54c8ca0f028d34d37fd14dca2c563d44948846 Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:21:03 +0100 Subject: [PATCH 14/18] adds worker transferable objects --- shared/hexDiffModel.ts | 13 ++++++++----- shared/protocol.ts | 13 ++++++++----- src/extension.ts | 4 +++- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/shared/hexDiffModel.ts b/shared/hexDiffModel.ts index 7ad17958..16cc9ea1 100644 --- a/shared/hexDiffModel.ts +++ b/shared/hexDiffModel.ts @@ -34,11 +34,14 @@ export class HexDiffModel { const mArray = new Uint8Array(mSize); await this.originalModel.readInto(0, oArray); await this.modifiedModel.readInto(0, mArray); - const decorators = await this.messageHandler.sendRequest({ - type: DiffMessageType.DiffDecoratorRequest, - original: oArray, - modified: mArray, - }); + const decorators = await this.messageHandler.sendRequest( + { + type: DiffMessageType.DiffDecoratorRequest, + original: oArray, + modified: mArray, + }, + [oArray.buffer, mArray.buffer], + ); this.decorators = decorators; } return uri.toString() === this.originalModel.uri.toString() diff --git a/shared/protocol.ts b/shared/protocol.ts index 2c5f0bac..0cff2c93 100644 --- a/shared/protocol.ts +++ b/shared/protocol.ts @@ -312,18 +312,21 @@ export class MessageHandler { constructor( public messageHandler: (msg: TFrom) => Promise, - private readonly postMessage: (msg: WebviewMessage) => void, + private readonly postMessage: (msg: WebviewMessage, transfer?: Transferable[]) => void, ) {} /** Sends a request without waiting for a response */ - public sendEvent(body: TTo): void { - this.postMessage({ body, messageId: this.messageIdCounter++ }); + public sendEvent(body: TTo, transfer?: Transferable[]): void { + this.postMessage({ body, messageId: this.messageIdCounter++ }, transfer); } /** Sends a request that expects a response */ - public sendRequest(msg: TTo): Promise { + public sendRequest( + msg: TTo, + transfer?: Transferable[], + ): Promise { const id = this.messageIdCounter++; - this.postMessage({ body: msg, messageId: id }); + this.postMessage({ body: msg, messageId: id }, transfer); return new Promise((resolve, reject) => { this.pendingMessages.set(id, { resolve: resolve as (msg: TFrom) => void, reject }); }); diff --git a/src/extension.ts b/src/extension.ts index 0bb9cf40..5f94886b 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -137,7 +137,9 @@ export async function activate(context: vscode.ExtensionContext) { // Always return undefined as the diff worker // does not request anything from extension host async () => undefined, - message => worker.postMessage(message), + // worker.postMessage's transfer parameter type looks to be wrong because + // it should be set as optional. + (message, transfer) => worker.postMessage(message, transfer!), ); worker.addEventListener("message", e => From 970ad7bfca945a4df949fef3520295684ab6de2d Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:46:27 +0100 Subject: [PATCH 15/18] Use binary search in data row content --- media/editor/dataDisplay.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/media/editor/dataDisplay.tsx b/media/editor/dataDisplay.tsx index 957e3475..e775c960 100644 --- a/media/editor/dataDisplay.tsx +++ b/media/editor/dataDisplay.tsx @@ -36,6 +36,7 @@ import { parseHexDigit, throwOnUndefinedAccessInDev, } from "./util"; +import { binarySearch } from "../../shared/util/binarySearch"; const style = throwOnUndefinedAccessInDev(_style); @@ -703,7 +704,8 @@ const DataRowContents: React.FC<{ const { bytes, chars } = useMemo(() => { const bytes: React.ReactChild[] = []; const chars: React.ReactChild[] = []; - let j = 0; + const searcher = binarySearch(r => r.range.start); + let j = searcher(offset, decorators); for (let i = 0; i < width; i++) { const boffset = offset + i; const value = rawBytes[i]; From d8befa208ffdaaea77a76abae70b6e8eb4b86045 Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:53:29 +0100 Subject: [PATCH 16/18] removes web-only code --- src/extension.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/extension.ts b/src/extension.ts index 5f94886b..413d928b 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -157,10 +157,7 @@ export async function activate(context: vscode.ExtensionContext) { if (!(leftFile instanceof vscode.Uri && rightFile instanceof vscode.Uri)) { return; } - // Web-only - if (workerMessageHandler) { - openCompareSelected(leftFile, rightFile, registry, workerMessageHandler!); - } + openCompareSelected(leftFile, rightFile, registry, workerMessageHandler!); }, ); @@ -177,7 +174,7 @@ export async function activate(context: vscode.ExtensionContext) { context.subscriptions.push(compareSelectedCommand); context.subscriptions.push( vscode.workspace.registerFileSystemProvider("hexdiff", new HexDiffFSProvider(), { - isCaseSensitive: true, + isCaseSensitive: typeof process !== 'undefined' && process.platform !== 'win32' && process.platform !== 'darwin', }), ); context.subscriptions.push( From ce668a1b83fd8fad5ebe47b33463c4c4273c36e6 Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Sat, 3 Aug 2024 21:18:35 +0100 Subject: [PATCH 17/18] fix: decorator's binary search gives wrong index --- media/editor/dataDisplay.tsx | 2 +- media/editor/state.ts | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/media/editor/dataDisplay.tsx b/media/editor/dataDisplay.tsx index e775c960..21188672 100644 --- a/media/editor/dataDisplay.tsx +++ b/media/editor/dataDisplay.tsx @@ -704,7 +704,7 @@ const DataRowContents: React.FC<{ const { bytes, chars } = useMemo(() => { const bytes: React.ReactChild[] = []; const chars: React.ReactChild[] = []; - const searcher = binarySearch(r => r.range.start); + const searcher = binarySearch(d => d.range.end); let j = searcher(offset, decorators); for (let i = 0; i < width; i++) { const boffset = offset + i; diff --git a/media/editor/state.ts b/media/editor/state.ts index db16fb94..bad9444d 100644 --- a/media/editor/state.ts +++ b/media/editor/state.ts @@ -455,7 +455,7 @@ export const editedDataPages = selectorFamily({ }, }); -/** Returns the starting decorator index for the given page number */ +/** Returns the decorators in a page */ export const decoratorsPage = selectorFamily({ key: "decoratorsPage", get: @@ -466,9 +466,10 @@ export const decoratorsPage = selectorFamily({ return []; } const pageSize = get(dataPageSize); - const searcher = binarySearch(decorator => decorator.range.start); - const startIndex = searcher(pageSize * pageNumber, allDecorators); - const endIndex = searcher(pageSize * pageNumber + pageSize, allDecorators); + const searcherByEnd = binarySearch(decorator => decorator.range.end); + const startIndex = searcherByEnd(pageSize * pageNumber, allDecorators); + const searcherByStart = binarySearch(d => d.range.start); + const endIndex = searcherByStart(pageSize * pageNumber + pageSize+1, allDecorators); return allDecorators.slice(startIndex, endIndex); }, }); From f03b8429901f3c9e5500dcc457527d2345a1de85 Mon Sep 17 00:00:00 2001 From: Tomas Silva <136352470+tomilho@users.noreply.github.com> Date: Sun, 4 Aug 2024 09:34:16 +0100 Subject: [PATCH 18/18] Adds lazy init & better diff lifecycle --- shared/hexDiffModel.ts | 64 ++++++++++++++++++---------------------- shared/util/uri.ts | 27 +++++++++++++++-- src/compareSelected.ts | 34 +++++++++++---------- src/extension.ts | 49 +++++------------------------- src/hexDocument.ts | 30 +++++-------------- src/hexEditorProvider.ts | 6 ++-- src/hexEditorRegistry.ts | 53 ++++++++++++++++++++++++--------- src/initWorker.ts | 62 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 192 insertions(+), 133 deletions(-) create mode 100644 src/initWorker.ts diff --git a/shared/hexDiffModel.ts b/shared/hexDiffModel.ts index 16cc9ea1..b4b40590 100644 --- a/shared/hexDiffModel.ts +++ b/shared/hexDiffModel.ts @@ -55,53 +55,45 @@ export class HexDiffModel { * with both HexDocumentModels */ static Builder = class { - private originalModel: Promise; - private modifiedModel: Promise; - private resolveOriginalModel!: ( - value: HexDocumentModel | PromiseLike, - ) => void; - private resolveModifiedModel!: ( - value: HexDocumentModel | PromiseLike, - ) => void; - private builtModel?: HexDiffModel; - public onBuild?: () => void; + private original: { + promise: Promise; + resolve: (model: HexDocumentModel) => void; + }; + private modified: { + promise: Promise; + resolve: (model: HexDocumentModel) => void; + }; - constructor( - public readonly originalUri: vscode.Uri, - public readonly modifiedUri: vscode.Uri, - private readonly messageHandler: DiffExtensionHostMessageHandler, - ) { - this.originalModel = new Promise(resolve => { - this.resolveOriginalModel = resolve; - }); - this.modifiedModel = new Promise(resolve => { - this.resolveModifiedModel = resolve; - }); + private built?: HexDiffModel; + + constructor(private readonly messageHandler: DiffExtensionHostMessageHandler) { + let promise: Promise; + let res: (model: HexDocumentModel) => void; + + promise = new Promise(resolve => (res = resolve)); + this.original = { promise: promise, resolve: res! }; + promise = new Promise(resolve => (res = resolve)); + this.modified = { promise: promise, resolve: res! }; } - public setModel(model: HexDocumentModel) { - if (this.originalUri.toString() === model.uri.toString()) { - this.resolveOriginalModel(model); - } else if (this.modifiedUri.toString() === model.uri.toString()) { - this.resolveModifiedModel(model); + public setModel(side: "original" | "modified", document: HexDocumentModel) { + if (side === "original") { + this.original.resolve(document); } else { - throw new Error("Provided doc does not match uris."); + this.modified.resolve(document); } return this; } public async build() { - const [originalModel, modifiedModel] = await Promise.all([ - this.originalModel, - this.modifiedModel, + const [original, modified] = await Promise.all([ + this.original.promise, + this.modified.promise, ]); - if (this.builtModel === undefined) { - this.builtModel = new HexDiffModel(originalModel, modifiedModel, this.messageHandler); - if (this.onBuild) { - this.onBuild(); - } + if (this.built === undefined) { + this.built = new HexDiffModel(original, modified, this.messageHandler); } - return this.builtModel; + return this.built; } }; } diff --git a/shared/util/uri.ts b/shared/util/uri.ts index bb1e82b1..49dc2afa 100644 --- a/shared/util/uri.ts +++ b/shared/util/uri.ts @@ -1,5 +1,7 @@ export interface HexEditorUriQuery { baseAddress?: string; + token?: string; + side?: "modified" | "original"; } /** @@ -11,11 +13,32 @@ export function parseQuery(queryString: string): HexEditorUriQuery { const pairs = (queryString[0] === "?" ? queryString.substr(1) : queryString).split("&"); for (const q of pairs) { const pair = q.split("="); - const name = pair.shift(); + const name = pair.shift() as keyof HexEditorUriQuery; if (name) { - queries[name as keyof HexEditorUriQuery] = pair.join("="); + const value = pair.join("="); + if (name === "side") { + if (value === "modified" || value === "original" || value === undefined) { + queries.side = value; + } + } else { + queries[name] = value; + } } } } return queries; } + +/** + * Forms a valid HexEditor Query to be used in vscode.Uri + */ +export function formQuery(queries: HexEditorUriQuery): string { + const query: string[] = []; + for (const q in queries) { + const queryValue = queries[q as keyof HexEditorUriQuery]; + if (queryValue !== undefined && queryValue !== "") { + query.push(`${q}=${queryValue}`); + } + } + return query.join("&"); +} diff --git a/src/compareSelected.ts b/src/compareSelected.ts index 1f2091ca..1175220a 100644 --- a/src/compareSelected.ts +++ b/src/compareSelected.ts @@ -1,30 +1,34 @@ import * as vscode from "vscode"; -import { DiffExtensionHostMessageHandler } from "../shared/diffWorkerProtocol"; -import { HexDiffModel } from "../shared/hexDiffModel"; -import { HexEditorRegistry } from "./hexEditorRegistry"; +import { formQuery, parseQuery } from "../shared/util/uri"; + +const uuidGenerator = () => { + let uuid = 0; + return () => (uuid++).toString(); +}; +const uuid = uuidGenerator(); // Initializes our custom editor with diff capabilities // @see https://github.com/microsoft/vscode/issues/97683 // @see https://github.com/microsoft/vscode/issues/138525 -export const openCompareSelected = async ( - originalFile: vscode.Uri, - modifiedFile: vscode.Uri, - registry: HexEditorRegistry, - diffExtensionHostMessageHandler: DiffExtensionHostMessageHandler, -) => { +export const openCompareSelected = (originalFile: vscode.Uri, modifiedFile: vscode.Uri) => { + const token = uuid(); const diffOriginalUri = originalFile.with({ scheme: "hexdiff", + query: formQuery({ + ...parseQuery(originalFile.query), + side: "original", + token: token, + }), }); const diffModifiedUri = modifiedFile.with({ scheme: "hexdiff", + query: formQuery({ + ...parseQuery(originalFile.query), + side: "modified", + token: token, + }), }); - const diffModelBuilder = new HexDiffModel.Builder( - diffOriginalUri, - diffModifiedUri, - diffExtensionHostMessageHandler, - ); - registry.addDiff(diffModelBuilder); vscode.commands.executeCommand("vscode.diff", diffOriginalUri, diffModifiedUri); }; diff --git a/src/extension.ts b/src/extension.ts index 413d928b..717d783f 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -3,9 +3,7 @@ import TelemetryReporter from "@vscode/extension-telemetry"; import * as vscode from "vscode"; -import { FromDiffWorkerMessage, ToDiffWorkerMessage } from "../shared/diffWorkerProtocol"; import { HexDocumentEditOp } from "../shared/hexDocumentModel"; -import { MessageHandler } from "../shared/protocol"; import { openCompareSelected } from "./compareSelected"; import { copyAs } from "./copyAs"; import { DataInspectorView } from "./dataInspectorView"; @@ -13,6 +11,7 @@ import { showGoToOffset } from "./goToOffset"; import { HexDiffFSProvider } from "./hexDiffFS"; import { HexEditorProvider } from "./hexEditorProvider"; import { HexEditorRegistry } from "./hexEditorRegistry"; +import { prepareLazyInitDiffWorker } from "./initWorker"; import { showSelectBetweenOffsets } from "./selectBetweenOffsets"; import StatusEditMode from "./statusEditMode"; import StatusFocus from "./statusFocus"; @@ -42,7 +41,11 @@ function reopenWithHexEditor() { } export async function activate(context: vscode.ExtensionContext) { - const registry = new HexEditorRegistry(); + // Prepares the worker to be lazily initialized + const initWorker = prepareLazyInitDiffWorker(context.extensionUri, workerDispose => + context.subscriptions.push(workerDispose), + ); + const registry = new HexEditorRegistry(initWorker); // Register the data inspector as a separate view on the side const dataInspectorProvider = new DataInspectorView(context.extensionUri, registry); const configValues = readConfigFromPackageJson(context.extension); @@ -112,41 +115,6 @@ export async function activate(context: vscode.ExtensionContext) { } }); - // Initializes worker for diffing - let worker: Worker; - const workerFilePath = vscode.Uri.joinPath( - context.extensionUri, - "dist", - "diffWorker.js", - ).toString(); - - try { - worker = new Worker(workerFilePath); - } catch { - // eslint-disable-next-line @typescript-eslint/no-var-requires - const { Worker } = require("worker_threads") as typeof import("worker_threads"); - const nodeWorker = new Worker(new URL(workerFilePath)); - // Web and node js have different worker interfaces, so we share a function - // to initialize both workers the same way. - const ref = nodeWorker.addListener; - (nodeWorker as any).addEventListener = ref; - worker = nodeWorker as any; - } - - const workerMessageHandler = new MessageHandler( - // Always return undefined as the diff worker - // does not request anything from extension host - async () => undefined, - // worker.postMessage's transfer parameter type looks to be wrong because - // it should be set as optional. - (message, transfer) => worker.postMessage(message, transfer!), - ); - - worker.addEventListener("message", e => - // e.data is used in web worker and e is used in node js worker - e.data ? workerMessageHandler.handleMessage(e.data) : workerMessageHandler.handleMessage(e as any), - ); - const compareSelectedCommand = vscode.commands.registerCommand( "hexEditor.compareSelected", async (...args) => { @@ -157,7 +125,7 @@ export async function activate(context: vscode.ExtensionContext) { if (!(leftFile instanceof vscode.Uri && rightFile instanceof vscode.Uri)) { return; } - openCompareSelected(leftFile, rightFile, registry, workerMessageHandler!); + openCompareSelected(leftFile, rightFile); }, ); @@ -180,9 +148,6 @@ export async function activate(context: vscode.ExtensionContext) { context.subscriptions.push( HexEditorProvider.register(context, telemetryReporter, dataInspectorProvider, registry), ); - context.subscriptions.push({ - dispose: () => worker.terminate(), - }); } export function deactivate(): void { diff --git a/src/hexDocument.ts b/src/hexDocument.ts index 7ad2db29..5fb73419 100644 --- a/src/hexDocument.ts +++ b/src/hexDocument.ts @@ -12,6 +12,7 @@ import { HexDocumentEditReference, HexDocumentModel, } from "../shared/hexDocumentModel"; +import { parseQuery } from "../shared/util/uri"; import { Backup } from "./backup"; import { Disposable } from "./dispose"; import { accessFile } from "./fileSystemAdaptor"; @@ -41,9 +42,9 @@ export class HexDocument extends Disposable implements vscode.CustomDocument { : undefined, }); - const queries = HexDocument.parseQuery(uri.query); - const baseAddress: number = queries["baseAddress"] - ? HexDocument.parseHexOrDecInt(queries["baseAddress"]) + const queries = parseQuery(uri.query); + const baseAddress: number = queries.baseAddress + ? HexDocument.parseHexOrDecInt(queries.baseAddress) : 0; const fileSize = await accessor.getSize(); @@ -59,7 +60,10 @@ export class HexDocument extends Disposable implements vscode.CustomDocument { const isLargeFile = !backupId && !accessor.supportsIncremetalAccess && (fileSize ?? 0) > maxFileSize; - const diffModel = diffModelBuilder ? await diffModelBuilder.setModel(model).build() : undefined; + const diffModel = + queries.side && diffModelBuilder + ? await diffModelBuilder.setModel(queries.side, model).build() + : undefined; return { document: new HexDocument(model, isLargeFile, baseAddress, diffModel), accessor }; } @@ -360,24 +364,6 @@ export class HexDocument extends Disposable implements vscode.CustomDocument { }; } - /** - * Utility function to convert a Uri query string into a map - */ - private static parseQuery(queryString: string): { [key: string]: string } { - const queries: { [key: string]: string } = {}; - if (queryString) { - const pairs = (queryString[0] === "?" ? queryString.substr(1) : queryString).split("&"); - for (const q of pairs) { - const pair = q.split("="); - const name = pair.shift(); - if (name) { - queries[name] = pair.join("="); - } - } - } - return queries; - } - /** * Utility function to parse a number. Only hex and decimal supported */ diff --git a/src/hexEditorProvider.ts b/src/hexEditorProvider.ts index 503434fa..46b6c2f2 100644 --- a/src/hexEditorProvider.ts +++ b/src/hexEditorProvider.ts @@ -69,14 +69,16 @@ export class HexEditorProvider implements vscode.CustomEditorProvider { + const diff = this._registry.getDiff(uri); + const { document, accessor } = await HexDocument.create( uri, openContext, this._telemetryReporter, - this._registry.getDiff(uri) + diff.builder, ); const disposables: vscode.Disposable[] = []; - + disposables.push(diff); disposables.push( document.onDidRevert(async () => { const replaceFileSize = (await document.size()) ?? null; diff --git a/src/hexEditorRegistry.ts b/src/hexEditorRegistry.ts index b4e52c5c..19c48d2b 100644 --- a/src/hexEditorRegistry.ts +++ b/src/hexEditorRegistry.ts @@ -2,8 +2,10 @@ // Licensed under the MIT license. import * as vscode from "vscode"; -import { HexDiffModelBuilder } from "../shared/hexDiffModel"; +import { DiffExtensionHostMessageHandler } from "../shared/diffWorkerProtocol"; +import { HexDiffModel, HexDiffModelBuilder } from "../shared/hexDiffModel"; import { ExtensionHostMessageHandler } from "../shared/protocol"; +import { parseQuery } from "../shared/util/uri"; import { Disposable } from "./dispose"; import { HexDocument } from "./hexDocument"; @@ -11,7 +13,10 @@ const EMPTY: never[] = []; export class HexEditorRegistry extends Disposable { private readonly docs = new Map>(); - private readonly diffsBuilder = new Map(); + private readonly diffsBuilder = new Map< + string, + { refCount: number; value: HexDiffModelBuilder } + >(); private onChangeEmitter = new vscode.EventEmitter(); private _activeDocument?: HexDocument; @@ -34,7 +39,7 @@ export class HexEditorRegistry extends Disposable { return (this._activeDocument && this.docs.get(this._activeDocument)) || EMPTY; } - constructor() { + constructor(private readonly initDiffWorker: () => DiffExtensionHostMessageHandler) { super(); this._register(vscode.window.tabGroups.onDidChangeTabs(this.onChangedTabs, this)); this._register(vscode.window.tabGroups.onDidChangeTabGroups(this.onChangedTabs, this)); @@ -70,18 +75,38 @@ export class HexEditorRegistry extends Disposable { }; } - /** adds a diff model */ - public addDiff(diffModelBuilder: HexDiffModelBuilder) { - this.diffsBuilder.set(diffModelBuilder.modifiedUri.toString(), diffModelBuilder); - this.diffsBuilder.set(diffModelBuilder.originalUri.toString(), diffModelBuilder); - diffModelBuilder.onBuild = () => { - this.diffsBuilder.delete(diffModelBuilder.modifiedUri.toString()); - this.diffsBuilder.delete(diffModelBuilder.originalUri.toString()); - }; - } /** returns a diff model using the file uri */ - public getDiff(uri: vscode.Uri): HexDiffModelBuilder | undefined { - return this.diffsBuilder.get(uri.toString()); + public getDiff(uri: vscode.Uri): { + builder: HexDiffModelBuilder | undefined; + dispose: () => void; + } { + const { token } = parseQuery(uri.query); + if (token === undefined) { + return { builder: undefined, dispose: () => {} }; + } + // Lazily initializes the diff worker, if it isn't + // iniitalized already + const messageHandler = this.initDiffWorker(); + + // Creates a new diff model + if (!this.diffsBuilder.has(token)) { + this.diffsBuilder.set(token, { + refCount: 0, + value: new HexDiffModel.Builder(messageHandler), + }); + } + const builder = this.diffsBuilder.get(token)!; + builder.refCount++; + + return { + builder: builder.value, + dispose: () => { + builder.refCount--; + if (builder.refCount === 0) { + this.diffsBuilder.delete(token); + } + }, + }; } private onChangedTabs() { diff --git a/src/initWorker.ts b/src/initWorker.ts new file mode 100644 index 00000000..ba95876e --- /dev/null +++ b/src/initWorker.ts @@ -0,0 +1,62 @@ +import * as vscode from "vscode"; +import { + DiffExtensionHostMessageHandler, + FromDiffWorkerMessage, + ToDiffWorkerMessage, +} from "../shared/diffWorkerProtocol"; +import { MessageHandler } from "../shared/protocol"; + +/** Prepares diff worker to be lazily initialized and instantiated once*/ +export function prepareLazyInitDiffWorker( + extensionUri: vscode.Uri, + addDispose: (dispose: vscode.Disposable) => void, +) { + let messageHandler: DiffExtensionHostMessageHandler; + return () => { + if (!messageHandler) { + const { msgHandler, dispose } = initDiffWorker(extensionUri); + messageHandler = msgHandler; + addDispose({ dispose: dispose }); + } + return messageHandler; + }; +} + +/** Initializes the diff worker */ +function initDiffWorker(extensionUri: vscode.Uri): { + msgHandler: DiffExtensionHostMessageHandler; + dispose: () => void; +} { + let worker: Worker; + const workerFilePath = vscode.Uri.joinPath(extensionUri, "dist", "diffWorker.js").toString(); + + try { + worker = new Worker(workerFilePath); + } catch { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { Worker } = require("worker_threads") as typeof import("worker_threads"); + const nodeWorker = new Worker(new URL(workerFilePath)); + // Web and node js have different worker interfaces, so we share a function + // to initialize both workers the same way. + const ref = nodeWorker.addListener; + (nodeWorker as any).addEventListener = ref; + worker = nodeWorker as any; + } + + const workerMessageHandler = new MessageHandler( + // Always return undefined as the diff worker + // does not request anything from extension host + async () => undefined, + // worker.postMessage's transfer parameter type looks to be wrong because + // it should be set as optional. + (message, transfer) => worker.postMessage(message, transfer!), + ); + + worker.addEventListener("message", e => + // e.data is used in web worker and e is used in node js worker + e.data + ? workerMessageHandler.handleMessage(e.data) + : workerMessageHandler.handleMessage(e as any), + ); + return { msgHandler: workerMessageHandler, dispose: () => worker.terminate() }; +}