From a14a46191678a1a2cd397404dba72443cc7fa5b6 Mon Sep 17 00:00:00 2001 From: Babak Date: Fri, 14 Jun 2024 14:54:06 -0600 Subject: [PATCH] condensed path impl checkpoint (wip) --- skipledger-base/pathpack_format.md | 16 +- .../main/java/io/crums/sldg/BaggedRow.java | 25 +- .../main/java/io/crums/sldg/CondensedRow.java | 66 ++++++ .../java/io/crums/sldg/LevelsPointer.java | 172 +++++++++++++- .../src/main/java/io/crums/sldg/Path.java | 46 +++- .../src/main/java/io/crums/sldg/PathBag.java | 71 +++++- .../src/main/java/io/crums/sldg/PathPack.java | 216 ++++++++++++------ .../java/io/crums/sldg/PathPackBuilder.java | 8 +- .../src/main/java/io/crums/sldg/Row.java | 83 ++++--- .../src/main/java/io/crums/sldg/RowBag.java | 15 +- .../main/java/io/crums/sldg/SerialRow.java | 22 +- .../main/java/io/crums/sldg/SkipLedger.java | 21 +- .../io/crums/sldg/json/PathPackParser.java | 2 +- .../test/java/io/crums/sldg/PathPackTest.java | 23 ++ 14 files changed, 613 insertions(+), 173 deletions(-) create mode 100644 skipledger-base/src/main/java/io/crums/sldg/CondensedRow.java diff --git a/skipledger-base/pathpack_format.md b/skipledger-base/pathpack_format.md index 0f45fc4..e8f5294 100644 --- a/skipledger-base/pathpack_format.md +++ b/skipledger-base/pathpack_format.md @@ -32,15 +32,15 @@ The components of the structure are first defined, and the final object `PATH_PA TYPE := BYTE // 0 means full; 1 means condensed - RS_COUNT := INT // stitch row number count - STITCH_RNS := LONG ^RS_COUNT // stitch row numbers in strictly ascending order + SR_COUNT := INT // stitch row no. count + STITCH_RNS := LONG ^SR_COUNT // stitch row no.s in strictly ascending order - I_COUNT := INT // number of rows with full info (have input hash) - // inferred from SkipLedger#stitch(..) - // The full row no.s themselves (I_COUNT -many of them) - // are also known at this point. + I_COUNT := INT // no. of rows in the path (have input hashes) + // The path's row no.s are inferred from + // SkipLedger#stitch(STITCH_RNS) + // I_COUNT is the size of that list - R_COUNT := INT // number of rows with only ref-hashes inferred from + R_COUNT := INT // no. of rows with only ref-hashes inferred from // depending on type, the size of: // // TYPE [0] (full) @@ -71,6 +71,6 @@ The components of the structure are first defined, and the final object `PATH_PA // In this way, fs the byte-size of the funnel // block is determined from the STITCH_RNS. - PATH_PACK := TYPE RS_COUNT STITCH_RNS R_TBL I_TBL [FUNNELS] + PATH_PACK := SR_COUNT STITCH_RNS TYPE R_TBL I_TBL [FUNNELS] \ No newline at end of file diff --git a/skipledger-base/src/main/java/io/crums/sldg/BaggedRow.java b/skipledger-base/src/main/java/io/crums/sldg/BaggedRow.java index 03ef315..3c4d074 100644 --- a/skipledger-base/src/main/java/io/crums/sldg/BaggedRow.java +++ b/skipledger-base/src/main/java/io/crums/sldg/BaggedRow.java @@ -4,7 +4,6 @@ package io.crums.sldg; import java.nio.ByteBuffer; -import java.util.Objects; /** * Row backed by data in a {@linkplain RowBag}. @@ -14,7 +13,9 @@ */ public class BaggedRow extends Row { - private final long rowNumber; + private final long rowNo; + private final LevelsPointer levelsPtr; + private final RowBag bag; /** @@ -22,27 +23,25 @@ public class BaggedRow extends Row { * In order to create a subclass that does validate, invoke {@linkplain #hash()} * in the constructor. */ - public BaggedRow(long rowNumber, RowBag bag) { - this.rowNumber = rowNumber; - this.bag = Objects.requireNonNull(bag, "null bag"); - SkipLedger.checkRealRowNumber(rowNumber); + public BaggedRow(long rowNo, RowBag bag) { + this.rowNo = rowNo; + this.levelsPtr = bag.levelsPointer(rowNo); + this.bag = bag; } @Override public LevelsPointer levelsPointer() { - return bag.levelsPointer(rowNumber); + return levelsPtr; } - // @Override - // public final long no() { - // return rowNumber; - // } - @Override public ByteBuffer inputHash() { - return bag.inputHash(rowNumber); + return bag.inputHash(rowNo); } + + + } diff --git a/skipledger-base/src/main/java/io/crums/sldg/CondensedRow.java b/skipledger-base/src/main/java/io/crums/sldg/CondensedRow.java new file mode 100644 index 0000000..f8445ef --- /dev/null +++ b/skipledger-base/src/main/java/io/crums/sldg/CondensedRow.java @@ -0,0 +1,66 @@ +/* + * Copyright 2024 Babak Farhang + */ +package io.crums.sldg; + +import static io.crums.sldg.SkipLedger.levelLinked; + +import java.nio.ByteBuffer; + +/** + * + */ +public class CondensedRow extends Row { + + + public static Row compressToLevelRowNo(Row row, long levelRn) { + + int level = levelLinked(levelRn, row.no()); + if (level < 0) + throw new IllegalArgumentException( + "levelRn " + levelRn + " not covered by row " + row); + + if (row.levelsPointer().coversLevel(level)) + return row.isCompressed() ? + row : new CondensedRow(row, level, true); + + throw new IllegalArgumentException( + "levelRn " + levelRn + " not covered by row " + row.levelsPointer()); + } + + + + + + private final Row row; + + private final int refLevel; + + + private CondensedRow(Row row, int level, boolean trustMe) { + this.row = row; + this.refLevel = level; + } + + public CondensedRow(Row row, int level) { + this(row, level, true); + + LevelsPointer levelsPtr = row.levelsPointer(); + if (!levelsPtr.coversLevel(level)) + throw new IllegalArgumentException( + "level " + level + " not covered: " + levelsPtr); + } + + + @Override + public LevelsPointer levelsPointer() { + return row.levelsPointer().compressToLevel(refLevel); + } + + @Override + public ByteBuffer inputHash() { + return row.inputHash(); + } + + +} diff --git a/skipledger-base/src/main/java/io/crums/sldg/LevelsPointer.java b/skipledger-base/src/main/java/io/crums/sldg/LevelsPointer.java index 60ee5ac..222828a 100644 --- a/skipledger-base/src/main/java/io/crums/sldg/LevelsPointer.java +++ b/skipledger-base/src/main/java/io/crums/sldg/LevelsPointer.java @@ -5,9 +5,10 @@ import static java.util.Collections.binarySearch; - +import static io.crums.sldg.SkipLedger.alwaysAllLevels; import static io.crums.sldg.SkipLedger.dupAndCheck; import static io.crums.sldg.SkipLedger.skipCount; + import static io.crums.sldg.SldgConstants.DIGEST; import java.nio.ByteBuffer; @@ -21,7 +22,7 @@ * from the row's input-hash and the merklized hash of the previous * row-hashes (the so-called row's "levels"). This class offers 2 ways for * calculating a row's merklized levels' {@linkplain #hash() hash} (which - * doesn't concern the input-hash): + * never involves the input-hash): *
    *
  1. Using all the levels' row-hashes.
    2. *
  2. Using a merkle-proof consisting of a single level row-hash, @@ -34,7 +35,7 @@ * (where {@code n} here is the difference of the highest and lowest row no. in * the path). Using the second option this class offers (its so called * {@linkplain #isCondensed() condensed} version), a skip path typically - * contains {@code log(log(n)) x log(n)} many hashes. + * contains {@code log(log(n)) x log(n)} many 32-byte hashes. *

    */ public class LevelsPointer { @@ -64,7 +65,7 @@ public LevelsPointer( this.rn = rn; this.level = level; this.levelHash = dupAndCheck(levelHash); - this.hashes = funnel; + this.hashes = dupAndCheck(funnel); // check args final int levels = skipCount(rn); @@ -84,6 +85,9 @@ public LevelsPointer( + + + /** * Creates a full-info ("un-"{@linkplain #isCondensed() condensed}) instance. * Note the {@code prevHashes} parameter is indexed by level, so the @@ -92,13 +96,14 @@ public LevelsPointer( * @param rn row no. * @param prevHashes level row hashes of size * {@link SkipLedger#skipCount(long) - * SkipLedger.skipCount(rn)} + * SkipLedger.skipCount(rn)}; contents not copied: + * do not modify */ public LevelsPointer(long rn, List prevHashes) { this.rn = rn; this.level = -1; this.levelHash = null; - this.hashes = prevHashes; + this.hashes = dupAndCheck(prevHashes); final int levels = skipCount(rn); if (prevHashes.size() != levels) @@ -111,13 +116,47 @@ public LevelsPointer(long rn, List prevHashes) { throw new IllegalArgumentException( "hash for the sentinel row[0] not zeroed -- required by the protocol"); } - - public final boolean isCondensed() { - return level >= 0; + + + /** + * Package-access (trusted caller), full-info constructor. + * (Offers a way to lazy-load the hashes using a functor-list.) + * + * @param rn row no. + * @param prevHashes neither copied not verified; + * a perhaps functor list + * @param trustMe un-used constructor disambiguator + */ + LevelsPointer(long rn, List prevHashes, boolean trustMe) { + this.rn = rn; + this.level = -1; + this.levelHash = null; + this.hashes = prevHashes; + } + + + + /** + * Package-access (trusted caller), condensed-instance constructor. + * (Offers a way to lazy-load the hashes using a functor-list.) + * + * @param rn row no. + * @param prevHashes neither copied not verified; + * a perhaps functor list + * @param trustMe un-used constructor disambiguator + */ + LevelsPointer( + long rn, int level, ByteBuffer levelHash, List funnel, + boolean trustMe) { + this.rn = rn; + this.level = level; + this.levelHash = levelHash; + this.hashes = funnel; } + /** * Returns the row no. this instance generates the merklized pointer * {@linkplain #hash() hash} for. @@ -127,6 +166,92 @@ public final boolean isCondensed() { public final long rowNo() { return rn; } + + + /** + * Determines whether the instance uses a condensed hash proof. + * When condensed, the instance references the hash of a single + * previous level. (Technically, these being Merkle proofs, there + * are usually 2 level row hashes referenced in each proof; for now + * we haven't supplied a method to convert one condensed version + * into the other). + * + * @see #isCompressed() + */ + public final boolean isCondensed() { + return level >= 0; + } + + + /** + * Determines whether the instance is compressed. This codifies + * when an uncondensed instance may be condensed. Specifically, + * if no more than two (non-sentinel) level row-hashes are used + * to calculate the level merkle root hash, there is no point + * in "condensing" the representation. + * + * @return {@code SkipLedger.alwaysAllLevels(rowNo()) || isCondensed()} + */ + public final boolean isCompressed() { + return alwaysAllLevels(rn) || isCondensed(); + } + + + + /** + * Returns a compressed version of this instance referencing + * the given level row no. + * + * @param levelRn one of {@link #coverage()} + * + * @see #compressToLevel(int) + */ + public final LevelsPointer compressToLevelRowNo(long levelRn) { + var coverage = coverage(); + int index = indexOf(coverage, levelRn); + if (index < 0) + throw new IllegalArgumentException(levelRn + " not covered; " + this); + + int level = skipCount(rn) - 1 - index; + return compressToLevel(level); + } + + + /** + * Returns a compressed version of this instance for the specified level. + * This method fails if the instance is already condensed + * at a different level. + * + *

    Implementation Note

    + *

    + * Condensed instances at adjacent {@code even:odd} levels, eg (0,1) or (6,7), + * share exactly the same hash data. We could derive one from the other, + * but for now, it's not a priority. + *

    + * + * @param level level index {@code 0 <=level < SkipLedger.skipCount(rowNo())} + * @return this instance, if the instance is already compressed; + * a condensed version of this instance, o.w. + */ + public final LevelsPointer compressToLevel(int level) { + + if (isCompressed()) { + if (isCondensed() && this.level != level) + throw new IllegalArgumentException( + "level " + level + " not covered; condensed at level() " + + this.level + "; " + this); + return this; + } + + // assert isCondensable(rn); too obvious + + var fun = SkipLedger.levelsMerkleFunnel(level, hashes()); + var levelHash = hashes.get(level).asReadOnlyBuffer(); + + return new LevelsPointer(rn, level, levelHash, fun, true); + } + + @@ -136,8 +261,7 @@ public final long rowNo() { * (or claims to have) their hash. The returned list does not include * {@link #rowNo()}. * - * @return a singleton no. or strictly ascending row no.s less than - * {@link #rowNo()} + * @return not empty, strictly ascending row no.s, < {@link #rowNo()} */ public final List coverage() { final int skipCount = skipCount(rn); @@ -151,14 +275,31 @@ public final List coverage() { /** - * Determines whether the hash of the given row no. is one of the + * Determines whether the hash of the row at the given row no. is one of the * instance's level [hash] pointers. + * + * @param rn the row no. tested for coverage */ public final boolean coversRow(long rn) { return indexOfCoverage(rn) >= 0; } + /** + * Determines whether the hash of row at the given no. is one of the + * instance's level [hash] pointers. + */ + public final boolean coversLevel(int level) { + if (this.level == level) + return true; + + if (level < 0 || level >= skipCount(rn)) + throw new IllegalArgumentException("level: " + level + "; " + this); + + return !isCondensed(); + } + + private int indexOfCoverage(long rn) { return indexOf(coverage(), rn); } @@ -285,6 +426,13 @@ private List hashes() { + @Override + public String toString() { + return "[" + rn + "] --> " + coverage(); + } + + + } diff --git a/skipledger-base/src/main/java/io/crums/sldg/Path.java b/skipledger-base/src/main/java/io/crums/sldg/Path.java index daa844e..eff8726 100644 --- a/skipledger-base/src/main/java/io/crums/sldg/Path.java +++ b/skipledger-base/src/main/java/io/crums/sldg/Path.java @@ -1,18 +1,15 @@ /* - * Copyright 2020-2021 Babak Farhang + * Copyright 2020-2024 Babak Farhang */ package io.crums.sldg; -import static io.crums.sldg.SkipLedger.hiPtrHash; import static io.crums.sldg.SldgConstants.DIGEST; import static java.util.Collections.binarySearch; import java.nio.ByteBuffer; import java.util.HashMap; -// import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.SortedSet; import java.util.TreeSet; @@ -70,9 +67,7 @@ public Path(List path) throws IllegalArgumentException, HashConflictExcepti else if (path.size() > MAX_ROWS) throw new IllegalArgumentException("too many rows: " + path.size()); - Row[] rows = new Row[path.size()]; - rows = path.toArray(rows); - this.rows = Lists.asReadOnlyList(rows); + this.rows = List.copyOf(path); verify(); } @@ -90,7 +85,7 @@ else if (path.size() > MAX_ROWS) * Copy constructor. */ protected Path(Path copy) { - this.rows = Objects.requireNonNull(copy, "null copy").rows; + this.rows = copy.rows; } @@ -175,6 +170,41 @@ public final boolean isSkipPath() { } + public final boolean isCondensed() { + int index = rows.size(); + while (index-- > 0 && !rows.get(index).isCondensed()); + return index != -1; + } + + + + public final boolean isCompressed() { + int index = rows.size(); + while (index-- > 0 && !rows.get(index).isCompressed()); + return index != -1; + } + + + + + public final Path compress() { + if (isCompressed()) + return this; + + Row[] crows = new Row[rows.size()]; + crows[0] = CondensedRow.compressToLevelRowNo(first(), first().no() - 1); + for (int index = 1; index < crows.length; ++index) + crows[index] = + CondensedRow.compressToLevelRowNo( + rows.get(index), + rows.get(index - 1).no()); + + // FIXME: uncomment after adequately tested.. + // return new Path(Lists.asReadOnlyList(crows), null); + return new Path(Lists.asReadOnlyList(crows)); + } + + /** diff --git a/skipledger-base/src/main/java/io/crums/sldg/PathBag.java b/skipledger-base/src/main/java/io/crums/sldg/PathBag.java index 517384f..db97232 100644 --- a/skipledger-base/src/main/java/io/crums/sldg/PathBag.java +++ b/skipledger-base/src/main/java/io/crums/sldg/PathBag.java @@ -3,7 +3,13 @@ */ package io.crums.sldg; +import static io.crums.sldg.SkipLedger.rowHash; + import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; + +import io.crums.util.Lists; /** * Implementation interface of {@linkplain RowBag#rowHash(long)}. This models @@ -30,20 +36,22 @@ * don't need to be validated against each other: they collectively represent some * ledger. *

    - * In retrospect, this observation is a bit obvious: aside from its trailed (witnessed) rows a skip + * In retrospect, this observation is a bit obvious: a skip * ledger's rows can always be regenerated from scratch. A {@linkplain RowBag} implemented this * way, is much like regenerating a subset of a skip ledger from scratch. *

    - *

    Refactoring Note

    - *

    - * This code was lifted from the {@code RecurseHashRowPack}. - *

    */ public interface PathBag extends RowBag { Path path(); + /** + * {@inheritDoc } By default a {@code PathBag} first looks up a + * row's hash using {@link #refOnlyHash(long)}, and if not found + * invokes {@link RowBag#getRow(long) getRow(rowNo)}.{@link BaggedRow#hash() hash()}. + * + */ @Override default ByteBuffer rowHash(long rowNumber) { @@ -68,13 +76,62 @@ default ByteBuffer rowHash(long rowNumber) { " - cascaded internal error msg: " + internal.getMessage(), internal); } } - + + + + @Override + default LevelsPointer levelsPointer(long rowNo) { + + final var pathRns = getFullRowNumbers(); + int index = Collections.binarySearch(pathRns, rowNo); + if (index < 0) + throw new IllegalArgumentException( + "rowNo " + rowNo + " not contained in bag; " + this); + + if (isCondensed() && SkipLedger.isCondensable(rowNo)) { + int level; + long refedRowNo; + if (index == 0) { + level = 0; + refedRowNo = rowNo - 1; + } else { + refedRowNo = pathRns.get(index - 1); + long diff = rowNo - refedRowNo; + level = Long.numberOfTrailingZeros(diff); + if (diff <= 0 || diff != Long.highestOneBit(diff) || + level >= SkipLedger.skipCount(rowNo)) + throw new IllegalArgumentException( + "assertion failure: row [" + rowNo + "]; not linked: " + pathRns + + "; implemenation class: " + getClass()); + } + + var funnel = getFunnel(rowNo, level).orElseThrow( + () -> new IllegalArgumentException( + "expected funnel [" + rowNo + ":" + level + "] not found")); + + return + new LevelsPointer(rowNo, level, rowHash(refedRowNo), funnel); + } + + + final int levels = SkipLedger.skipCount(rowNo); + List levelHashes = + Lists.functorList(levels, li -> rowHash(rowNo - (1L << li))); + return new LevelsPointer(rowNo, levelHashes); + + } /** * Returns the ref-only row-hash from the minimal bag, if found; * {@code null}, otherwise. */ - ByteBuffer refOnlyHash(long rowNumber); + ByteBuffer refOnlyHash(long rowNo); + + + /** + * Tests whether this instance contains condensed rows. + */ + boolean isCondensed(); } diff --git a/skipledger-base/src/main/java/io/crums/sldg/PathPack.java b/skipledger-base/src/main/java/io/crums/sldg/PathPack.java index d045e13..6d113cf 100644 --- a/skipledger-base/src/main/java/io/crums/sldg/PathPack.java +++ b/skipledger-base/src/main/java/io/crums/sldg/PathPack.java @@ -9,11 +9,10 @@ import java.nio.BufferOverflowException; import java.nio.BufferUnderflowException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.SortedSet; -import java.util.TreeSet; import io.crums.io.Serial; import io.crums.io.buffer.BufferUtils; @@ -33,6 +32,7 @@ public class PathPack implements PathBag, Serial { /** int size */ private final static int STITCH_COUNT_SIZE = 4; + private final static int TYPE_SIZE = 1; /** * Loads and returns a new instance by reading the given buffer. @@ -119,10 +119,15 @@ public static PathPack forPath(Path path) { // so the returned pack object doesn't internally // reference the given path argument - List inputRns = Lists.readOnlyCopy(path.rowNumbers()); - - var refOnlyRns = SkipLedger.refOnlyCoverage(inputRns).tailSet(1L); - + List inputRns = List.copyOf(path.rowNumbers()); + + final boolean condensed = path.isCondensed(); + + var refOnlyRns = ( condensed ? + SkipLedger.refOnlyCondensedCoverage(inputRns) : + SkipLedger.refOnlyCoverage(inputRns) ) + .tailSet(1L); + var refHashes = ByteBuffer.allocate( refOnlyRns.size() * SldgConstants.HASH_WIDTH); @@ -141,8 +146,49 @@ public static PathPack forPath(Path path) { assert !inputHashes.hasRemaining(); inputHashes.flip(); + + if (!condensed) + return new PathPack(inputRns, refHashes, inputHashes); + + + - return new PathPack(inputRns, refHashes, inputHashes); + var funnelList = new ArrayList>(); + + + long lastRn = path.loRowNumber() - 1L; + int hashCount = 0; + + try { + for (Row row : path.rows().subList(1, path.length())) { + var levelsPtr = row.levelsPointer().compressToLevelRowNo(lastRn); + lastRn = levelsPtr.rowNo(); + if (!levelsPtr.isCondensed()) + continue; + var funnel = levelsPtr.funnel(); + hashCount += funnel.size(); + funnelList.add(funnel); + } + } catch (IllegalArgumentException iax) { + if (lastRn == path.loRowNumber() - 1L) + throw new IllegalArgumentException( + "not enuf info to pack first row [" + lastRn + + "]: " + iax.getMessage()); + + throw iax; + } + + assert hashCount > 0; + + var funnels = ByteBuffer.allocate(hashCount * HASH_WIDTH); + for (var funnel : funnelList) + for (var hash : funnel) + funnels.put(hash); + + assert !funnels.hasRemaining(); + funnels.flip(); + + return new PathPack(inputRns, refHashes, inputHashes, funnels); } @@ -157,25 +203,17 @@ public static PathPack forPath(Path path) { private final ByteBuffer hashes; private final ByteBuffer inputs; - private final boolean condensed; // redundant, but good for clarity private final ByteBuffer funnels; + private final List funnelRns; private final List funnelOffsets; - -// /** -// * -// */ -// private PathPack() { -// hashRns = inputRns = List.of(); -// hashes = inputs = BufferUtils.NULL_BUFFER; -// } /** * Creates a new "un-condensed" instance. Caller agrees not to change the contents - * of the inputs. + * of the input arguments. * * @param inputRns ascending row no.s with input hashes (full rows) * @param hashes ref-only hashes (row no.s inferred from {@code inputRns} @@ -184,33 +222,85 @@ public static PathPack forPath(Path path) { PathPack(List inputRns, ByteBuffer hashes, ByteBuffer inputs) { this.inputRns = Objects.requireNonNull(inputRns, "null inputRns"); - SortedSet refs = SkipLedger.refOnlyCoverage(inputRns).tailSet(1L); - Objects.requireNonNull(hashes, "null hashes"); - Objects.requireNonNull(inputs, "null inputs"); - this.hashRns = Lists.readOnlyCopy(refs); + this.hashRns = List.copyOf( + SkipLedger.refOnlyCoverage(inputRns).tailSet(1L)); // sans 0L + this.hashes = BufferUtils.readOnlySlice(hashes); this.inputs = BufferUtils.readOnlySlice(inputs); + + this.funnels = BufferUtils.NULL_BUFFER; + this.funnelRns = List.of(); + this.funnelOffsets = List.of(); - this.condensed = false; - this.funnels = null; - this.funnelOffsets = null; - // if the following throw, there's a bug.. + checkCommonArgs(hashes, inputs); + } + + + private void checkCommonArgs(ByteBuffer hashes, ByteBuffer inputs) { + if (inputRns.isEmpty()) throw new IllegalArgumentException("inputRns is empty"); - if (hashRns.size() * HASH_WIDTH != this.hashes.remaining()) - throw new IllegalArgumentException( - "hash row numbers / buffer size mismatch:\n" + - hashRns + "\n " + this.hashes); + checkRnsBuffer( + inputRns, inputs, "input row no.s / buffer size mismatch"); + checkRnsBuffer( + hashRns, hashes, "hash row no.s / buffer size mismatch"); + } - if (inputRns.size() * HASH_WIDTH != this.inputs.remaining()) - throw new IllegalArgumentException( - "input row numbers / buffer size mismatch:\n" + - inputRns + "\n " + this.inputs); + + private void checkRnsBuffer( + List rns, ByteBuffer buffer, String preamble) { + if (rns.size() * HASH_WIDTH != buffer.remaining()) + throw new IllegalArgumentException( + preamble + ":\n " + rns + "\n " + buffer); + } + + + + PathPack( + List inputRns, ByteBuffer hashes, ByteBuffer inputs, + ByteBuffer funnels) { + this.inputRns = inputRns; + this.hashRns = List.copyOf( + SkipLedger.refOnlyCondensedCoverage(inputRns).tailSet(1L)); // sans 0L + + this.hashes = BufferUtils.readOnlySlice(hashes); + this.inputs = BufferUtils.readOnlySlice(inputs); + this.funnels = BufferUtils.readOnlySlice(funnels); + + checkCommonArgs(hashes, inputs); + + var condensedRns = new ArrayList(inputRns.size()); + var funOffs = new ArrayList(inputRns.size()); + + long prevRn = inputs.get(0) - 1L; + assert prevRn >= 0; + int offset = 0; + + for (Long rn : inputRns) { + assert prevRn < rn; + if (SkipLedger.isCondensable(rn)) { + condensedRns.add(rn); + funOffs.add(offset); + offset += SkipLedger.funnelLength(prevRn, rn); + } + prevRn = rn; + } + + if (condensedRns.isEmpty()) + throw new IllegalArgumentException("no condensable rows: " + inputRns); + + if (offset != this.funnels.remaining()) + throw new IllegalArgumentException( + "expected " + offset + " funnel bytes; actual was " + + this.funnels.remaining() + "; funnels " + funnels + ": " + inputRns); + + this.funnelRns = Lists.asReadOnlyList(condensedRns); + this.funnelOffsets = Lists.asReadOnlyList(funOffs); } @@ -227,12 +317,17 @@ public PathPack(PathPack copy) { this.hashes = copy.hashes; this.inputs = copy.inputs; - this.condensed = copy.condensed; this.funnels = copy.funnels; + this.funnelRns = copy.funnelRns; this.funnelOffsets = copy.funnelOffsets; } + @Override + public final boolean isCondensed() { + return !funnelRns.isEmpty(); + } + public final ByteBuffer inputsBlock() { return inputs.asReadOnlyBuffer(); @@ -324,54 +419,25 @@ protected final ByteBuffer emit(ByteBuffer store, int index) { @Override public int serialSize() { - final int inputsCount = inputRns.size(); - final int dataBytes = inputs.capacity() + hashes.capacity(); - - if (inputsCount <= 2) - return STITCH_COUNT_SIZE + inputsCount*8 + dataBytes; + return + STITCH_COUNT_SIZE + + preStitchRowNos().size() * 8 + + hashes.remaining() + + inputs.remaining() + + funnels.remaining(); - final int stitchRns; - { - var candidate = List.of(lo(), hi()); - - List skipRns = SkipLedger.stitch(candidate); - - if (skipRns.equals(inputRns)) - stitchRns = 2; - - else { - var stitchSet = new TreeSet(candidate); - for (int index = inputsCount - 1; index-- > 1; ) { - Long rn = inputRns.get(index); - if (Collections.binarySearch(skipRns, rn) >= 0) - continue; - stitchSet.add(rn); - skipRns = SkipLedger.stitchSet(stitchSet); - } - stitchRns = stitchSet.size(); - } - } - - return STITCH_COUNT_SIZE + stitchRns*8 + dataBytes; } @Override public ByteBuffer writeTo(ByteBuffer out) throws BufferOverflowException { - final int inputsCount = inputRns.size(); - - // handle the corner cases first.. - if (inputsCount <= 2) { - out.putInt(inputsCount); - return inputsCount == 0 ? out : - writeLongs(inputRns, out) - .put(hashes.slice()) - .put(inputs.slice()); - } - // calculate the stitch row no.s.. - List stitchRns = compressedRowNos(); + // write the type + // out.put((byte) (isCondensed() ? 1 : 0)); + + // calculate the stitch row no.s.. + List stitchRns = preStitchRowNos(); // haven't written anything yet.. write it all out out.putInt(stitchRns.size()); @@ -385,7 +451,7 @@ public ByteBuffer writeTo(ByteBuffer out) throws BufferOverflowException { /** * Returns the full row numbers in pre-stitched form. */ - public List compressedRowNos() { + public List preStitchRowNos() { return SkipLedger.stitchCompress(inputRns); } diff --git a/skipledger-base/src/main/java/io/crums/sldg/PathPackBuilder.java b/skipledger-base/src/main/java/io/crums/sldg/PathPackBuilder.java index 1e9b03d..bc2d85f 100644 --- a/skipledger-base/src/main/java/io/crums/sldg/PathPackBuilder.java +++ b/skipledger-base/src/main/java/io/crums/sldg/PathPackBuilder.java @@ -227,6 +227,10 @@ public int addPath(Path path) { } + @Override + public final boolean isCondensed() { + return false; // FIXME + } public int addPack(PathPack pack) { if (pack.isEmpty()) @@ -269,7 +273,7 @@ private int addSansLinkCheck(SerialRow safeRow) { if (memoHash.equals(rowHash)) return 0; // then there's a bug - throw new AssertionError( + throw new IllegalArgumentException( "memo-ed hash at [" + rn + "] conflicts with argument: " + safeRow); } } @@ -298,7 +302,7 @@ private int addSansLinkCheck(SerialRow safeRow) { if (level < levels - 1) this.corrupted = true; - throw new AssertionError("[" + rn + ":" + level + "]"); + throw new IllegalArgumentException("[" + rn + ":" + level + "]"); } } diff --git a/skipledger-base/src/main/java/io/crums/sldg/Row.java b/skipledger-base/src/main/java/io/crums/sldg/Row.java index 376b275..3378db7 100644 --- a/skipledger-base/src/main/java/io/crums/sldg/Row.java +++ b/skipledger-base/src/main/java/io/crums/sldg/Row.java @@ -6,7 +6,6 @@ import java.nio.ByteBuffer; import java.util.List; -import java.util.SortedSet; import io.crums.util.Lists; @@ -20,6 +19,19 @@ * defined in this module are not sealed, so it's conceivable a subclass may * violate the constraint. *

    + *

    Overridable / Abstract Methods

    + *

    + * All but 3 methods in this class are marked final. 2 of those 3 + * {@link #inputHash()} and {@link #levelsPointer()} are abstract, and + * together they drive the return values for the other methods. The only + * non-final method is {@link #hash()} which generates the RHS of the + * hash-relation mentioned above. The reason why it's not made final is + * that an implementation might have memo-ized the result of the hashing. + *

    + * + * @see #inputHash() + * @see #levelsPointer() + * @see #hash() * * @see RowHash#prevLevels() * @see RowHash#prevNo(int) @@ -27,7 +39,12 @@ public abstract class Row extends RowHash { - + /** + * {@inheritDoc} + * + * @return {@code levelsPointer().rowNo()} + */ + @Override public final long no() { return levelsPointer().rowNo(); } @@ -35,7 +52,7 @@ public final long no() { /** * Returns the user-inputed hash. This is the hash of the abstract object - * (whatever it is, we don't know). + * the entered the ledger (whatever it is, we don't know). * * @return non-null, 32-bytes remaining */ @@ -43,18 +60,21 @@ public final long no() { - + /** + * Returns the levels pointer. + */ public abstract LevelsPointer levelsPointer(); /** * {@inheritDoc} - * - * + * The default implementation derives (calculates) the hash value using + * the {@link #inputHash()} and the {@link #levelsPointer()}. * * @return {@code SkipLedger.rowHash(inputHash(), levelsPointer().hash())} */ + @Override public ByteBuffer hash() { return SkipLedger.rowHash(inputHash(), levelsPointer().hash()); } @@ -70,20 +90,42 @@ final List levelHashes() { - // public boolean hasAllLevels() { - // return SkipLedger.alwaysAllLevels(no()) || hasAllPtrs(); - // } - - + /** + * Tests whether the instance's hash is computed using + * condensed level row hashes. + * + * @return {@code levelsPointer().isCondensed()} + */ public final boolean isCondensed() { return - // !SkipLedger.alwaysAllLevels(no()) && levelsPointer().isCondensed(); } + /** + * Determines whether all level row hashes are present. + * If an in an instance is not condensed, then + * all level row hashes are known by the instance. + * + * @return {@code !isCondensed()} + */ + public final boolean hasAllLevels() { + return !isCondensed(); + } - + + + /** + * Tests whether the instance's hash is computed using + * the minimum number of previous row hashes. Note, an instance + * can both have {@linkplain #hasAllLevels() all levels} and + * still be compressed. + * + * @return {@code levelsPointer().isCompressed()} + */ + public final boolean isCompressed() { + return levelsPointer().isCompressed(); + } /** @@ -92,21 +134,6 @@ public final boolean isCondensed() { */ public final ByteBuffer hash(long rowNumber) { return rowNumber == no() ? hash() : levelsPointer().rowHash(rowNumber); - // final long rn = no(); - // final long diff = rn - rowNumber; - // if (diff == 0) - // return hash(); - // var levelsPtr = levelsPointer(); - // if (levelsPtr.coversRow(rowNumber)) - // return levelsPtr. - - - // int referencedRows = prevLevels(); - // if (diff < 0 || Long.highestOneBit(diff) != diff || diff > (1L << (referencedRows - 1))) - // throw new IllegalArgumentException( - // "rowNumber " + rowNumber + " is not covered by this row " + this); - // int ptrLevel = 63 - Long.numberOfLeadingZeros(diff); - // return prevHash(ptrLevel); } diff --git a/skipledger-base/src/main/java/io/crums/sldg/RowBag.java b/skipledger-base/src/main/java/io/crums/sldg/RowBag.java index 08ed194..769a5ad 100644 --- a/skipledger-base/src/main/java/io/crums/sldg/RowBag.java +++ b/skipledger-base/src/main/java/io/crums/sldg/RowBag.java @@ -39,7 +39,11 @@ public interface RowBag { - + /** + * Returns the level pointer. + * @param rowNo + * @return + */ default LevelsPointer levelsPointer(long rowNo) { final int levels = SkipLedger.skipCount(rowNo); List levelHashes = @@ -49,10 +53,13 @@ default LevelsPointer levelsPointer(long rowNo) { /** + * Returns the hash-funnel for the given row no. and level index, + * if it exists. * - * @param rn row no. funnel outputs row's level-merkel hash - * @param level - * @return + * @param rn row no. the funnel outputs level-merkel hash for + * @param level level index + * + * @return by default empty */ default Optional> getFunnel(long rn, int level) { return Optional.empty(); diff --git a/skipledger-base/src/main/java/io/crums/sldg/SerialRow.java b/skipledger-base/src/main/java/io/crums/sldg/SerialRow.java index 422869d..8505d32 100644 --- a/skipledger-base/src/main/java/io/crums/sldg/SerialRow.java +++ b/skipledger-base/src/main/java/io/crums/sldg/SerialRow.java @@ -4,8 +4,8 @@ package io.crums.sldg; +import static io.crums.sldg.SkipLedger.skipCount; import java.nio.ByteBuffer; -import java.util.Objects; import io.crums.util.Lists; @@ -47,14 +47,15 @@ public static ByteBuffer toDataBuffer(Row row) { private final long rowNumber; private final ByteBuffer data; + private final LevelsPointer levelsPtr; /** * Constructs a new instance. This does a defensive copy (since we want a runtime - * reference to be guarantee immutability). + * reference to guarantee immutability). * - * @param rowNumber the data number - * @param data the data's backing data (a sequence of cells representing hashes) + * @param rowNumber the row number + * @param data the data's backing data (a sequence of cells representing hashes) */ public SerialRow(long rowNumber, ByteBuffer data) { this(rowNumber, ByteBuffer.allocate(data.remaining()).put(data).flip(), false); @@ -66,6 +67,7 @@ public SerialRow(long rowNumber, ByteBuffer data) { public SerialRow(SerialRow copy) { this.rowNumber = copy.rowNumber; this.data = copy.data; + this.levelsPtr = copy.levelsPtr; } /** @@ -85,13 +87,15 @@ public SerialRow(Row copy) { throw new IllegalArgumentException( "expected " + expectedBytes + " bytes for rowNumber " + rowNumber + "; actual given is " + data); + + this.levelsPtr = new LevelsPointer( + rowNumber, + Lists.functorList(skipCount(rowNumber), this::prevHashImpl)); } public final LevelsPointer levelsPointer() { - return new LevelsPointer( - rowNumber, - Lists.functorList(SkipLedger.skipCount(rowNumber), this::prevHashImpl)); + return levelsPtr; } @@ -101,10 +105,6 @@ public final ByteBuffer data() { } - // @Override - // public final long no() { - // return rowNumber; - // } @Override public final ByteBuffer inputHash() { diff --git a/skipledger-base/src/main/java/io/crums/sldg/SkipLedger.java b/skipledger-base/src/main/java/io/crums/sldg/SkipLedger.java index c2ef150..828fde7 100644 --- a/skipledger-base/src/main/java/io/crums/sldg/SkipLedger.java +++ b/skipledger-base/src/main/java/io/crums/sldg/SkipLedger.java @@ -371,6 +371,8 @@ public static int countFunnels(List rowNos) { /** * Returns the row no.s that are condensable. In a condensed path, * these are the row no.s that link to previous rows via funnels. + * + * @return immutable list */ public static List filterFunneled(List rowNos) { return rowNos.stream().filter(SkipLedger::isCondensable).toList(); @@ -452,13 +454,22 @@ public static long hiPtrNo(long rn) { - public static ByteBuffer dupAndCheck(ByteBuffer buffer) { - var out = buffer.slice(); + public static ByteBuffer dupAndCheck(ByteBuffer hash) { + var out = hash.slice(); if (out.remaining() != SldgConstants.HASH_WIDTH) throw new IllegalArgumentException( - "expected " + SldgConstants.HASH_WIDTH + " remaining bytes: " + buffer); + "expected " + SldgConstants.HASH_WIDTH + " remaining bytes: " + hash); return out; } + + + public static List dupAndCheck(List hashes) { + ByteBuffer[] array = new ByteBuffer[hashes.size()]; + for (int index = 0; index < array.length; ++index) + array[index] = dupAndCheck(hashes.get(index)); + + return Lists.asReadOnlyList(array); + } /** @@ -812,7 +823,7 @@ public static List stitch(List rowNumbers) { * list. It can then be uncompressed via the {@linkplain #stitch(List)} * method. * - * @param pathRns path row no.s + * @param pathRns path row no.s (positive and stictly ascending order) * @return a subset of the {@code pathRns} * */ @@ -822,12 +833,14 @@ public static List stitchCompress(List pathRns) { return pathRns; var candidate = List.of(pathRns.get(0), pathRns.get(pSize - 1)); + List skipRns = SkipLedger.stitch(candidate); if (skipRns.equals(pathRns)) return candidate; var stitchList = new ArrayList(candidate); // invariants: stitchList.get(0) == lo(); stitchList.last() == hi() + for (int index = 1; index < pSize - 1; ++index) { Long rn = pathRns.get(index); if (Collections.binarySearch(skipRns, rn) >= 0) diff --git a/skipledger-base/src/main/java/io/crums/sldg/json/PathPackParser.java b/skipledger-base/src/main/java/io/crums/sldg/json/PathPackParser.java index c1f8f54..8559a5b 100644 --- a/skipledger-base/src/main/java/io/crums/sldg/json/PathPackParser.java +++ b/skipledger-base/src/main/java/io/crums/sldg/json/PathPackParser.java @@ -49,7 +49,7 @@ public PathPackParser(HashEncoding hashCodec, String targetNosTag, String hashes @Override public JSONObject injectEntity(PathPack pack, JSONObject jObj) { - var targetRns = pack.compressedRowNos(); + var targetRns = pack.preStitchRowNos(); var jTargetNos = new JSONArray(targetRns.size()); jTargetNos.addAll(targetRns); jObj.put(targetNosTag, jTargetNos); diff --git a/skipledger-base/src/test/java/io/crums/sldg/PathPackTest.java b/skipledger-base/src/test/java/io/crums/sldg/PathPackTest.java index 6b08e81..7d91ebb 100644 --- a/skipledger-base/src/test/java/io/crums/sldg/PathPackTest.java +++ b/skipledger-base/src/test/java/io/crums/sldg/PathPackTest.java @@ -4,8 +4,13 @@ package io.crums.sldg; +import static io.crums.sldg.PathTest.newRandomLedger; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.util.List; +import org.junit.jupiter.api.Test; + /** * */ @@ -26,4 +31,22 @@ protected PathPack newBag(SkipLedger ledger, List rowNumbers) { } + + + @Test + public void testSerialSmallest() { + final int size = 1; + SkipLedger ledger = newRandomLedger(size); + Path state = ledger.statePath(); + PathPack pack = PathPack.forPath(state); + var bytes = pack.serialize(); + PathPack rt = PathPack.load(bytes); + assertEquals(state, rt.path()); + } + + + + + + }