From c74522eb11cf7e4c83333736664fbe32f9d9e2fb Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Wed, 6 Sep 2023 17:44:10 +0200 Subject: [PATCH] wip --- src/__tests__/compat.proto | 14 ++ src/__tests__/compatibility.test.ts | 40 +++++ src/__tests__/index.test.ts | 243 +++++++++++++++++++++++----- src/binary.ts | 147 ++++++++++++++--- src/index.ts | 10 +- src/{bigint.ts => number.ts} | 38 ++++- src/types.ts | 14 +- 7 files changed, 440 insertions(+), 66 deletions(-) create mode 100644 src/__tests__/compat.proto create mode 100644 src/__tests__/compatibility.test.ts rename src/{bigint.ts => number.ts} (86%) diff --git a/src/__tests__/compat.proto b/src/__tests__/compat.proto new file mode 100644 index 00000000..07e87c54 --- /dev/null +++ b/src/__tests__/compat.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +enum Enum { + X = 1; + Y = 2; + Z = 3; +} + +message Message { + Enum enum = 1; + uint32 uint32 = 2; + sint32 sint32 = 3; + int32 int32 = 4; +} \ No newline at end of file diff --git a/src/__tests__/compatibility.test.ts b/src/__tests__/compatibility.test.ts new file mode 100644 index 00000000..b987c2bf --- /dev/null +++ b/src/__tests__/compatibility.test.ts @@ -0,0 +1,40 @@ +import protobuf from "protobufjs"; +import { field, serialize, variant } from "../index.js"; +const protoRoot = protobuf.loadSync("src/__tests__/compat.proto"); +const ProtoMessage = protoRoot.lookupType("Message"); + +enum Enum { + X = 0, + Y = 1, + Z = 2, +} +class Message { + @field({ type: "vi32" }) + enum: number; + + @field({ type: "vu32" }) + uint32: number; + + @field({ type: "vsi32" }) + sint32: number; + + @field({ type: "vi32" }) + int32: number; + + constructor(message: Message) { + Object.assign(this, message); + } +} +describe("protobuf", () => { + it("protobuf compat", () => { + const obj = { + enum: 1, + uint32: 567, + sint32: -1234, + int32: -5677, + }; + const proto = ProtoMessage.encode(obj).finish(); + const borsh = serialize(new Message(obj)); + expect(proto).toEqual(borsh); + }); +}); diff --git a/src/__tests__/index.test.ts b/src/__tests__/index.test.ts index 609e3ead..450ee7bb 100644 --- a/src/__tests__/index.test.ts +++ b/src/__tests__/index.test.ts @@ -401,60 +401,131 @@ describe("arrays", () => { expect(deserialized.a).toEqual(arr); }); - test("override size type", () => { - class TestStruct { - @field({ type: vec("u16", "u8") }) - public a: number[]; + describe("size type", () => { + test("u8", () => { + class TestStruct { + @field({ type: vec("u16", "u8") }) + public a: number[]; + + constructor(properties?: { a: number[] }) { + if (properties) { + this.a = properties.a; + } + } + } - constructor(properties?: { a: number[] }) { - if (properties) { + validate(TestStruct); + const buf = serialize(new TestStruct({ a: [1, 2, 3] })); + expect(new Uint8Array(buf)).toEqual( + new Uint8Array([3, 1, 0, 2, 0, 3, 0]) + ); + const deserialized = deserialize(buf, TestStruct); + expect(deserialized.a).toEqual([1, 2, 3]); + }); + }); + + describe("vu32", () => { + test("u8 value", () => { + class TestStruct { + @field({ type: vec("u8", "vu32") }) + public a: Uint8Array; + + constructor(properties: { a: Uint8Array }) { this.a = properties.a; } } + + validate(TestStruct); + const buf = serialize(new TestStruct({ a: new Uint8Array([1, 2, 3]) })); + expect(new Uint8Array(buf)).toEqual(new Uint8Array([3, 1, 2, 3])); + const deserialized = deserialize(buf, TestStruct); + expect(new Uint8Array(deserialized.a)).toEqual( + new Uint8Array([1, 2, 3]) + ); + }); + + test("u16 value", () => { + class TestStruct { + @field({ type: vec("u16", "vu32") }) + public a: number[]; + + constructor(properties: { a: number[] }) { + this.a = properties.a; + } + } + + validate(TestStruct); + const buf = serialize(new TestStruct({ a: [1, 2, 3] })); + expect(new Uint8Array(buf)).toEqual( + new Uint8Array([3, 1, 0, 2, 0, 3, 0]) + ); + const deserialized = deserialize(buf, TestStruct); + expect(deserialized.a).toEqual([1, 2, 3]); + }); + }); + + test("handles offsets correctly", () => { + class TestStruct { + @field({ type: vec("u16", "u32") }) + public a: number[]; + + @field({ type: vec("u32", "vu32") }) + public b: number[]; + + @field({ type: vec("u32", "u16") }) + public c: number[]; + + constructor(properties: { a: number[]; b: number[]; c: number[] }) { + this.a = properties.a; + this.b = properties.b; + this.c = properties.c; + } } validate(TestStruct); - const buf = serialize(new TestStruct({ a: [1, 2, 3] })); + const struct = new TestStruct({ a: [1], b: [2], c: [3] }); + const buf = serialize(struct); expect(new Uint8Array(buf)).toEqual( - new Uint8Array([3, 1, 0, 2, 0, 3, 0]) + new Uint8Array([1, 0, 0, 0, 1, 0, 1, 2, 0, 0, 0, 1, 0, 3, 0, 0, 0]) ); + const deserialized = deserialize(buf, TestStruct); - expect(deserialized.a).toEqual([1, 2, 3]); + expect(JSON.stringify(deserialized)).toEqual(JSON.stringify(struct)); }); + }); - test("will not allocate unless there is data to deserialize", () => { - class Inner { - @field({ type: "u8" }) - number: number; - } + test("will not allocate unless there is data to deserialize", () => { + class Inner { + @field({ type: "u8" }) + number: number; + } - class TestStruct { - @field({ type: vec(Inner) }) - public a: Inner[]; - } + class TestStruct { + @field({ type: vec(Inner) }) + public a: Inner[]; + } - expect(() => - deserialize(new Uint8Array([255, 255, 255, 255]), TestStruct) - ).toThrowError(); - }); + expect(() => + deserialize(new Uint8Array([255, 255, 255, 255]), TestStruct) + ).toThrowError(); + }); - test("can deserialize large arrays", () => { - class TestStruct { - @field({ type: vec("string") }) - public a: string[]; - } - const size = 1024 * 1024 + 100; - const struct = new TestStruct(); - struct.a = new Array(size).fill("a"); - const deserialized = deserialize(serialize(struct), TestStruct); - expect(deserialized.a).toHaveLength(size); - for (const a of struct.a) { - // we do this instead of expect(...).toEqual() because this is faster - if (a !== "a") { - throw new Error("Unexpected"); - } + test("can deserialize large arrays", () => { + class TestStruct { + @field({ type: vec("string") }) + public a: string[]; + } + const size = 1024 * 1024 + 100; + const struct = new TestStruct(); + struct.a = new Array(size).fill("a"); + const deserialized = deserialize(serialize(struct), TestStruct); + expect(deserialized.a).toHaveLength(size); + for (const a of struct.a) { + // we do this instead of expect(...).toEqual() because this is faster + if (a !== "a") { + throw new Error("Unexpected"); } - }); + } }); }); @@ -712,6 +783,102 @@ describe("number", () => { const deserialized = deserialize(buf, Struct); expect(deserialized.a).toEqual(n); }); + + describe("varint", () => { + describe("vu32", () => { + class Struct { + @field({ type: "vu32" }) + public a: number; + + constructor(a: number) { + this.a = a; + } + } + it("123", () => { + const ser = serialize(new Struct(123)); + expect(new Uint8Array(ser)).toEqual(new Uint8Array([123])); + expect(deserialize(ser, Struct).a).toEqual(123); + }); + + it("min", () => { + const ser = serialize(new Struct(0)); + expect(new Uint8Array(ser)).toEqual(new Uint8Array([0])); + expect(deserialize(ser, Struct).a).toEqual(0); + }); + + it("max", () => { + const ser = serialize(new Struct(4294967295)); + expect(new Uint8Array(ser)).toEqual( + new Uint8Array([255, 255, 255, 255, 15]) + ); + expect(deserialize(ser, Struct).a).toEqual(4294967295); + }); + }); + + describe("vi32", () => { + class Struct { + @field({ type: "vi32" }) + public a: number; + + constructor(a: number) { + this.a = a; + } + } + it("min", () => { + const ser = serialize(new Struct(-2147483648)); + expect(new Uint8Array(ser)).toEqual( + new Uint8Array([128, 128, 128, 128, 248, 255, 255, 255, 255, 1]) + ); + expect(deserialize(ser, Struct).a).toEqual(-2147483648); + }); + + it("0", () => { + const ser = serialize(new Struct(0)); + expect(new Uint8Array(ser)).toEqual(new Uint8Array([0])); + expect(deserialize(ser, Struct).a).toEqual(0); + }); + + it("max", () => { + const ser = serialize(new Struct(2147483647)); + expect(new Uint8Array(ser)).toEqual( + new Uint8Array([255, 255, 255, 255, 7]) + ); + expect(deserialize(ser, Struct).a).toEqual(2147483647); + }); + }); + + describe("vsi32", () => { + class Struct { + @field({ type: "vsi32" }) + public a: number; + + constructor(a: number) { + this.a = a; + } + } + it("min", () => { + const ser = serialize(new Struct(-2147483648)); + expect(new Uint8Array(ser)).toEqual( + new Uint8Array([255, 255, 255, 255, 15]) + ); + expect(deserialize(ser, Struct).a).toEqual(-2147483648); + }); + + it("0", () => { + const ser = serialize(new Struct(0)); + expect(new Uint8Array(ser)).toEqual(new Uint8Array([0])); + expect(deserialize(ser, Struct).a).toEqual(0); + }); + + it("max", () => { + const ser = serialize(new Struct(2147483647)); + expect(new Uint8Array(ser)).toEqual( + new Uint8Array([254, 255, 255, 255, 15]) + ); + expect(deserialize(ser, Struct).a).toEqual(2147483647); + }); + }); + }); describe("f32", () => { class Struct { @field({ type: "f32" }) diff --git a/src/binary.ts b/src/binary.ts index e5e7aae0..46ca95ca 100644 --- a/src/binary.ts +++ b/src/binary.ts @@ -1,8 +1,9 @@ -import { toBigIntLE, writeBufferLEBigInt, writeUInt32LE, readUInt32LE, readUInt16LE, writeUInt16LE, readBigUInt64LE, readUIntLE, checkInt, writeBigUint64Le } from './bigint.js'; +import { toBigIntLE, writeBufferLEBigInt, writeUInt32LE, readUInt32LE, readUInt16LE, writeUInt16LE, readBigUInt64LE, readUIntLE, checkInt, writeBigUint64Le } from './number.js'; import { BorshError } from "./error.js"; import utf8 from '@protobufjs/utf8'; -import { PrimitiveType, SmallIntegerType } from './types.js'; +import { PrimitiveType, SmallIntegerType, SmallUnsignedIntegerType } from './types.js'; import { readFloatLE, writeFloatLE, readDoubleLE, writeDoubleLE } from '@protobufjs/float' +import { writeVarint64 } from './number.js'; const allocUnsafeFn = (): (len: number) => Uint8Array => { if ((globalThis as any).Buffer) { @@ -35,14 +36,14 @@ const stringLengthFn: () => ((str: string) => number) = () => { type ChainedWrite = (() => any) & { next?: ChainedWrite } export class BinaryWriter { - totalSize: number; + totalSize: number = 0; + counter: number = 0; private _writes: ChainedWrite; private _writesTail: ChainedWrite; private _buf: Uint8Array; public constructor() { - this.totalSize = 0; this._writes = () => this._buf = allocUnsafe(this.totalSize); this._writesTail = this._writes; } @@ -84,9 +85,23 @@ export class BinaryWriter { public static u32(value: number, writer: BinaryWriter) { let offset = writer.totalSize; - writer._writes = writer._writes.next = () => writeUInt32LE(value, writer._buf, offset) + let prev = writer._writes; + writer.counter += 1; + if (writer.counter > 100) + writer._writesTail = () => { + prev() + writeUInt32LE(value, writer._buf, offset) + } + else { + writer._writes = writer._writes.next = () => writeUInt32LE(value, writer._buf, offset) + //writer.counter = 0; + } + // writer.totalSize += 4; + /* writer._writes = writer._writes.next = () => writeUInt32LE(value, writer._buf, offset) + */ + } @@ -135,6 +150,46 @@ export class BinaryWriter { } + public static vu32(value: number, writer: BinaryWriter) { + let offset = writer.totalSize; + let len = (value = value >>> 0) + < 128 ? 1 + : value < 16384 ? 2 + : value < 2097152 ? 3 + : value < 268435456 ? 4 + : 5; + writer._writes = writer._writes.next = () => { + while (value > 127) { + writer._buf[offset++] = value & 127 | 128; + value >>>= 7; + } + writer._buf[offset] = value; + } + writer.totalSize += len; + } + + vu32(value: number) { + return BinaryWriter.vu32(value, this) + } + + static vi32(value: number, writer: BinaryWriter) { + + if (value < 0) { + let offset = writer.totalSize; + writer._writes = writer._writes.next = () => writeVarint64(value, writer._buf, offset) + writer.totalSize += 10; // 10 bytes per spec + } + else { + return BinaryWriter.vu32(value, writer); + } + } + static vsi32(value: number, writer: BinaryWriter) { + return BinaryWriter.vu32((value << 1 ^ value >> 31) >>> 0, writer); + } + vsi32(value: number) { + return BinaryWriter.vsi32(value, this); + } + public f32(value: number) { return BinaryWriter.f32(value, this) } @@ -175,15 +230,14 @@ export class BinaryWriter { writer.totalSize += 4 + len; } - public static stringCustom(str: string, writer: BinaryWriter, lengthWriter: (len: number | bigint, buf: Uint8Array, offset: number) => void = writeUInt32LE, lengthSize = 4) { + public static stringCustom(str: string, writer: BinaryWriter, lengthWriter: (len: number | bigint, writer: BinaryWriter) => void = BinaryWriter.u32) { const len = utf8.length(str); + lengthWriter(len, writer) let offset = writer.totalSize; writer._writes = writer._writes.next = () => { - lengthWriter(len, writer._buf, offset) - writeStringBufferFn(len)(str, writer._buf, offset + lengthSize); + writeStringBufferFn(len)(str, writer._buf, offset); } - - writer.totalSize += lengthSize + len; + writer.totalSize += len; } public set(array: Uint8Array) { @@ -208,14 +262,13 @@ export class BinaryWriter { writer.totalSize += array.length + 4; } - public static uint8ArrayCustom(array: Uint8Array, writer: BinaryWriter, lengthWriter: (len: number | bigint, buf: Uint8Array, offset: number) => void = writeUInt32LE, lengthSize = 4) { + public static uint8ArrayCustom(array: Uint8Array, writer: BinaryWriter, lengthWriter: (len: number | bigint, writer: BinaryWriter) => void) { + lengthWriter(array.length, writer); let offset = writer.totalSize; writer._writes = writer._writes.next = () => { - lengthWriter(array.length, writer._buf, offset) - writer._buf.set(array, offset + lengthSize); + writer._buf.set(array, offset); } - - writer.totalSize += array.length + lengthSize; + writer.totalSize += array.length; } public static uint8ArrayFixed(array: Uint8Array, writer: BinaryWriter) { @@ -228,15 +281,18 @@ export class BinaryWriter { } - public static smallNumberEncoding(encoding: SmallIntegerType): [((value: number, buf: Uint8Array, offset: number) => void), number] { + public static smallNumberEncoding(encoding: SmallUnsignedIntegerType): ((value: number, writer: BinaryWriter) => void) { if (encoding === 'u8') { - return [(value: number, buf: Uint8Array, offset: number) => buf[offset] = value as number, 1] + return BinaryWriter.u8 } else if (encoding === 'u16') { - return [writeUInt16LE, 2] + return BinaryWriter.u16 } else if (encoding === 'u32') { - return [writeUInt32LE, 4] + return BinaryWriter.u32 + } + else if (encoding === 'vu32') { + return BinaryWriter.vu32 } else { throw new Error("Unsupported encoding: " + encoding) @@ -266,6 +322,15 @@ export class BinaryWriter { else if (encoding === 'u512') { return BinaryWriter.u512 } + else if (encoding === 'vu32') { + return BinaryWriter.vu32 + } + else if (encoding === 'vi32') { + return BinaryWriter.vi32 + } + else if (encoding === 'vsi32') { + return BinaryWriter.vsi32 + } else if (encoding === 'bool') { return BinaryWriter.bool } @@ -390,6 +455,40 @@ export class BinaryReader { return toBigIntLE(buf) } + public static vu32(reader: BinaryReader) { + let value = (reader._buf[reader._offset] & 127) >>> 0; if (reader._buf[reader._offset++] < 128) return value; + value = (value | (reader._buf[reader._offset] & 127) << 7) >>> 0; if (reader._buf[reader._offset++] < 128) return value; + value = (value | (reader._buf[reader._offset] & 127) << 14) >>> 0; if (reader._buf[reader._offset++] < 128) return value; + value = (value | (reader._buf[reader._offset] & 127) << 21) >>> 0; if (reader._buf[reader._offset++] < 128) return value; + value = (value | (reader._buf[reader._offset] & 15) << 28) >>> 0; if (reader._buf[reader._offset++] < 128) return value; + + if ((reader._offset += 5) > reader._buf.length) { + throw new Error('Out of bounds'); + } + return value; + } + + vu32() { + return BinaryReader.vu32(this); + } + + static vi32(reader: BinaryReader) { + return reader.vu32() | 0 + } + + vi32() { + return BinaryReader.vi32(this); + } + + static vsi32(reader: BinaryReader) { + var value = reader.vu32(); + return value >>> 1 ^ -(value & 1) | 0; + } + + vsi32() { + return BinaryReader.vsi32(this); + } + f32(): number { return BinaryReader.f32(this) @@ -508,6 +607,16 @@ export class BinaryReader { else if (encoding === 'u512') { return BinaryReader.u512 } + else if (encoding === 'vu32') { + return BinaryReader.vu32 + } + else if (encoding === 'vi32') { + return BinaryReader.vi32 + } + else if (encoding === 'vsi32') { + return BinaryReader.vsi32 + } + else if (encoding === 'string') { return fromBuffer ? BinaryReader.bufferString : BinaryReader.string } diff --git a/src/index.ts b/src/index.ts index 99a69278..c40a7c38 100644 --- a/src/index.ts +++ b/src/index.ts @@ -134,8 +134,8 @@ function serializeField( if (fieldType.sizeEncoding === 'u32') return BinaryWriter.uint8Array else { - const [sizeHandle, width] = BinaryWriter.smallNumberEncoding(fieldType.sizeEncoding) - return (obj, writer) => BinaryWriter.uint8ArrayCustom(obj, writer, sizeHandle, width) + const sizeHandle = BinaryWriter.smallNumberEncoding(fieldType.sizeEncoding) + return (obj, writer) => BinaryWriter.uint8ArrayCustom(obj, writer, sizeHandle) } } } @@ -162,8 +162,8 @@ function serializeField( } else if (fieldType instanceof StringType) { - const [sizeHandle, width] = BinaryWriter.smallNumberEncoding(fieldType.sizeEncoding) - return (obj, writer) => BinaryWriter.stringCustom(obj, writer, sizeHandle, width) + const sizeHandle = BinaryWriter.smallNumberEncoding(fieldType.sizeEncoding) + return (obj, writer) => BinaryWriter.stringCustom(obj, writer, sizeHandle) } @@ -830,7 +830,7 @@ const validateIterator = (clazzes: AbstractType | AbstractType[], allo } // Validate field - validateIterator(field.type, allowUndefined, visited); + validateIterator(field.type as Constructor, allowUndefined, visited); // TODO types } }); }) diff --git a/src/bigint.ts b/src/number.ts similarity index 86% rename from src/bigint.ts rename to src/number.ts index 41a971ef..f09230f1 100644 --- a/src/bigint.ts +++ b/src/number.ts @@ -75,6 +75,40 @@ export const writeBigUint64Le = (bigIntOrNumber: bigint | number, buf: Uint8Arra } +export function hl(value: number) { + if (value === 0) + return [0, 0]; + var sign = value < 0; + if (sign) + value = -value; + var lo = value >>> 0, + hi = (value - lo) / 4294967296 >>> 0; + if (sign) { + hi = ~hi >>> 0; + lo = ~lo >>> 0; + if (++lo > 4294967295) { + lo = 0; + if (++hi > 4294967295) + hi = 0; + } + } + return [hi, lo] +} +export function writeVarint64(val: number, buf: Uint8Array, pos: number) { + let [hi, lo] = hl(val); + while (hi) { + buf[pos++] = lo & 127 | 128; + lo = (lo >>> 7 | hi << 25) >>> 0; + hi >>>= 7; + } + while (lo > 127) { + buf[pos++] = lo & 127 | 128; + lo = lo >>> 7; + } + buf[pos++] = lo; +} + + export const readBigUInt64LE = (buf: Uint8Array, offset: number) => { const first = buf[offset]; const last = buf[offset + 7]; @@ -140,4 +174,6 @@ export const checkInt = (value: number | bigint, min: number | bigint, max: numb } throw new Error("Out of range value: " + range + ", " + value); } -} \ No newline at end of file +} + + diff --git a/src/types.ts b/src/types.ts index caefaf3f..d7c07513 100644 --- a/src/types.ts +++ b/src/types.ts @@ -39,8 +39,16 @@ export type SmallIntegerType = "u8" | "u16" | "u32" +export type SmallUnsignedIntegerType = "u8" | "u16" | "u32" | 'vu32' + + +export type SmallVarintType = "vu32" | "vi32" | "vsi32" + +/* export type VarintType = SmallVarintType | 'vu64' | 'vi64' */ + export type IntegerType = SmallIntegerType + | SmallVarintType | "u64" | "u128" | "u256" @@ -100,14 +108,14 @@ export const option = (type: FieldType): OptionKind => { }; export class VecKind extends WrappedType { - sizeEncoding: SmallIntegerType - constructor(elementType: FieldType, sizeEncoding: SmallIntegerType) { + sizeEncoding: SmallUnsignedIntegerType + constructor(elementType: FieldType, sizeEncoding: SmallUnsignedIntegerType) { super(elementType) this.sizeEncoding = sizeEncoding; } } -export const vec = (type: FieldType, sizeEncoding: SmallIntegerType = 'u32'): VecKind => { +export const vec = (type: FieldType, sizeEncoding: SmallUnsignedIntegerType = 'u32'): VecKind => { return new VecKind(type, sizeEncoding); };