v12 SRF (per-chunk PDC support, no double-compression); fix bad bug causing chunks go poof.

SLIME_FORMAT

@@ -1,7 +1,7 @@
 -------------------------------------
 “Slime” file format
 2 bytes - magic = 0xB10B
-1 byte (ubyte) - version, current = 0x0B
+1 byte (ubyte) - version, current = 0x0C
 4 bytes (int) - world version (see version list below)
 4 bytes (int) - compressed chunks size
 4 bytes (int) - uncompressed chunks size
@@ -10,7 +10,7 @@
 
 4 bytes (int) - compressed “extra” size
 4 bytes (int) - uncompressed “extra” size
-[depends] - compound tag compressed using zstd
+[depends] - extra compound tag compressed using zstd (used for PDC, and/or custom data)
 -------------------------------------
 
 Custom chunk format
@@ -31,19 +31,17 @@ Custom chunk format
 4 bytes (int) - heightmaps size
   <array of heightmap nbt compounds>
     same format as mc, uncompressed
-4 bytes (int) - compressed tile entities size
-4 bytes (int) - uncompressed tile entities size
+4 bytes (int) - tile entities size
   <array of tile entity nbt compounds>
     Same format as mc
     inside an nbt list named “tiles”, in a global compound, no gzip anywhere
-    compressed using zstd
-4 bytes (int) compressed entities size
-4 bytes (int) uncompressed entities size
+    uncompressed
+4 bytes (int) entities size
   <array of entity nbt compounds>
     Same format as mc EXCEPT optional “CustomId”
     inside an nbt list named “entities”, in a global compound
     Compressed using zstd
-
+[depends] - compound tag uncompressed (used for PDC, and/or custom data)
 -------------------------------------
 
 World version list:
@@ -68,4 +66,5 @@ Version history:
  - v8: Variable biomes size
  - v9: Fix issue with biomes size, causing old worlds to be corrupted
  - v10: Use minecraft version id, remove legacy version artifacts
- - v11: Move entities and tile entities into the chunk structure
+ - v11: Move entities and tile entities into the chunk structure
+ - v12: Add support for chunk-based PDC

api/src/main/java/com/infernalsuite/aswm/api/utils/SlimeFormat.java

@@ -9,5 +9,5 @@ public class SlimeFormat {
     public static final byte[] SLIME_HEADER = new byte[] { -79, 11 };
 
     /** Latest version of the SRF that SWM supports **/
-    public static final byte SLIME_VERSION = 11;
+    public static final byte SLIME_VERSION = 12;
 }

api/src/main/java/com/infernalsuite/aswm/api/world/SlimeChunk.java

@@ -53,4 +53,17 @@ public interface SlimeChunk {
      */
     List<CompoundTag> getEntities();
 
+    /**
+     * Returns the extra data of the chunk.
+     * Inside this {@link CompoundTag}
+     * can be stored any information to then be retrieved later, as it's
+     * saved alongside the chunk data.
+     * <br>
+     * <b>Beware, a compound tag under the key "ChunkBukkitValues" will be stored here.
+     * It is used for storing chunk-based Bukkit PDC. Do not overwrite it.</b>
+     *
+     * @return A {@link CompoundTag} containing the extra data of the chunk,
+     */
+    CompoundTag getExtraData();
+
 }

core/src/main/java/com/infernalsuite/aswm/serialization/anvil/AnvilWorldReader.java

@@ -381,9 +381,13 @@ private static SlimeChunk readChunk(CompoundTag compound, int worldVersion) {
             sectionArray[index - minSectionY] = new SlimeChunkSectionSkeleton(/*paletteTag, blockStatesArray,*/ blockStatesTag, biomeTag, blockLightArray, skyLightArray);
         }
 
+        var extraTag = new CompoundTag("", new CompoundMap());
+
+        extraTag.getValue().put(compound.getValue().get("ChunkBukkitValues")); // An attempt to migrate PDC from the anvil to the slime format.
+
         for (SlimeChunkSection section : sectionArray) {
             if (section != null) { // Chunk isn't empty
-                return new SlimeChunkSkeleton(chunkX, chunkZ, sectionArray, heightMapsCompound, tileEntities, entities);
+                return new SlimeChunkSkeleton(chunkX, chunkZ, sectionArray, heightMapsCompound, tileEntities, entities, extraTag);
             }
         }
 

core/src/main/java/com/infernalsuite/aswm/serialization/slime/SlimeSerializer.java

@@ -127,21 +127,28 @@ static byte[] serializeChunks(SlimeWorld world, Collection<SlimeChunk> chunks) t
             ListTag<CompoundTag> tileEntitiesNbtList = new ListTag<>("tileEntities", TagType.TAG_COMPOUND, chunk.getTileEntities());
             CompoundTag tileEntitiesCompound = new CompoundTag("", new CompoundMap(Collections.singletonList(tileEntitiesNbtList)));
             byte[] tileEntitiesData = serializeCompoundTag(tileEntitiesCompound);
-            byte[] compressedTileEntitiesData = Zstd.compress(tileEntitiesData);
 
-            outStream.writeInt(compressedTileEntitiesData.length);
             outStream.writeInt(tileEntitiesData.length);
-            outStream.write(compressedTileEntitiesData);
+            outStream.write(tileEntitiesData);
 
             // Entities
             ListTag<CompoundTag> entitiesNbtList = new ListTag<>("entities", TagType.TAG_COMPOUND, chunk.getEntities());
             CompoundTag entitiesCompound = new CompoundTag("", new CompoundMap(Collections.singletonList(entitiesNbtList)));
             byte[] entitiesData = serializeCompoundTag(entitiesCompound);
-            byte[] compressedEntitiesData = Zstd.compress(entitiesData);
 
-            outStream.writeInt(compressedEntitiesData.length);
             outStream.writeInt(entitiesData.length);
-            outStream.write(compressedEntitiesData);
+            outStream.write(entitiesData);
+
+            // Extra Tag
+            {
+                if (chunk.getExtraData() == null) {
+                    System.err.println("Chunk at " + chunk.getX() + ", " + chunk.getZ() + " from world " + world.getName() + " has no extra data! When deserialized, this chunk will have an empty extra data tag!");
+                }
+                byte[] extra = serializeCompoundTag(chunk.getExtraData());
+
+                outStream.writeInt(extra.length);
+                outStream.write(extra);
+            }
         }
 
         return outByteStream.toByteArray();

core/src/main/java/com/infernalsuite/aswm/serialization/slime/reader/SlimeWorldReaderRegistry.java

@@ -7,6 +7,7 @@
 import com.infernalsuite.aswm.api.world.SlimeWorld;
 import com.infernalsuite.aswm.api.world.properties.SlimePropertyMap;
 import com.infernalsuite.aswm.serialization.slime.reader.impl.v11.v11WorldFormat;
+import com.infernalsuite.aswm.serialization.slime.reader.impl.v12.v12WorldFormat;
 import com.infernalsuite.aswm.serialization.slime.reader.impl.v19.v1_9WorldFormat;
 import com.infernalsuite.aswm.serialization.slime.reader.impl.v10.v10WorldFormat;
 
@@ -25,6 +26,7 @@ public class SlimeWorldReaderRegistry {
         register(v1_9WorldFormat.FORMAT, 1, 2, 3, 4, 5, 6, 7, 8, 9);
         register(v10WorldFormat.FORMAT, 10);
         register(v11WorldFormat.FORMAT, 11);
+        register(v12WorldFormat.FORMAT, 12);
     }
 
     private static void register(VersionedByteSlimeWorldReader<SlimeWorld> format, int... bytes) {

core/src/main/java/com/infernalsuite/aswm/serialization/slime/reader/impl/v10/v10SlimeWorldDeSerializer.java

@@ -167,7 +167,7 @@ private static Map<ChunkPos, SlimeChunk> readChunks(SlimePropertyMap slimeProper
                 }
 
                 chunkMap.put(new ChunkPos(x, z),
-                        new SlimeChunkSkeleton(x, z, chunkSectionArray, heightMaps, new ArrayList<>(), new ArrayList<>())
+                        new SlimeChunkSkeleton(x, z, chunkSectionArray, heightMaps, new ArrayList<>(), new ArrayList<>(), new CompoundTag("", new CompoundMap()))
                 );
             }
         }

core/src/main/java/com/infernalsuite/aswm/serialization/slime/reader/impl/v11/v11SlimeWorldDeSerializer.java

@@ -139,7 +139,7 @@ private static Map<ChunkPos, SlimeChunk> readChunks(SlimePropertyMap slimeProper
             List<CompoundTag> serializedEntities = ((ListTag<CompoundTag>) entitiesCompound.getValue().get("entities")).getValue();
 
             chunkMap.put(new ChunkPos(x, z),
-                    new SlimeChunkSkeleton(x, z, chunkSections, heightMaps, serializedTileEntities, serializedEntities));
+                    new SlimeChunkSkeleton(x, z, chunkSections, heightMaps, serializedTileEntities, serializedEntities, new CompoundTag("", new CompoundMap())));
         }
         return chunkMap;
     }

core/src/main/java/com/infernalsuite/aswm/serialization/slime/reader/impl/v12/v12SlimeWorldDeSerializer.java

@@ -0,0 +1,167 @@
+package com.infernalsuite.aswm.serialization.slime.reader.impl.v12;
+
+import com.flowpowered.nbt.CompoundMap;
+import com.flowpowered.nbt.CompoundTag;
+import com.flowpowered.nbt.ListTag;
+import com.flowpowered.nbt.stream.NBTInputStream;
+import com.github.luben.zstd.Zstd;
+import com.infernalsuite.aswm.ChunkPos;
+import com.infernalsuite.aswm.api.exceptions.CorruptedWorldException;
+import com.infernalsuite.aswm.api.exceptions.NewerFormatException;
+import com.infernalsuite.aswm.api.loaders.SlimeLoader;
+import com.infernalsuite.aswm.api.utils.NibbleArray;
+import com.infernalsuite.aswm.api.world.SlimeChunk;
+import com.infernalsuite.aswm.api.world.SlimeChunkSection;
+import com.infernalsuite.aswm.api.world.SlimeWorld;
+import com.infernalsuite.aswm.api.world.properties.SlimeProperties;
+import com.infernalsuite.aswm.api.world.properties.SlimePropertyMap;
+import com.infernalsuite.aswm.serialization.slime.reader.VersionedByteSlimeWorldReader;
+import com.infernalsuite.aswm.skeleton.SkeletonSlimeWorld;
+import com.infernalsuite.aswm.skeleton.SlimeChunkSectionSkeleton;
+import com.infernalsuite.aswm.skeleton.SlimeChunkSkeleton;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.nio.ByteOrder;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+public class v12SlimeWorldDeSerializer implements VersionedByteSlimeWorldReader<SlimeWorld> {
+
+    public static final int ARRAY_SIZE = 16 * 16 * 16 / (8 / 4);
+
+    @Override
+    public SlimeWorld deserializeWorld(byte version, @Nullable SlimeLoader loader, String worldName, DataInputStream dataStream, SlimePropertyMap propertyMap, boolean readOnly) throws IOException, CorruptedWorldException, NewerFormatException {
+        int worldVersion = dataStream.readInt();
+
+        byte[] chunkBytes = readCompressed(dataStream);
+        Map<ChunkPos, SlimeChunk> chunks = readChunks(propertyMap, chunkBytes);
+
+        byte[] extraTagBytes = readCompressed(dataStream);
+        CompoundTag extraTag = readCompound(extraTagBytes);
+
+        SlimePropertyMap worldPropertyMap = propertyMap;
+        Optional<CompoundMap> propertiesMap = extraTag
+                .getAsCompoundTag("properties")
+                .map(CompoundTag::getValue);
+
+        if (propertiesMap.isPresent()) {
+            worldPropertyMap = new SlimePropertyMap(propertiesMap.get());
+            worldPropertyMap.merge(propertyMap);
+        }
+
+        return new SkeletonSlimeWorld(worldName, loader, readOnly, chunks, extraTag, worldPropertyMap, worldVersion);
+    }
+
+    private static Map<ChunkPos, SlimeChunk> readChunks(SlimePropertyMap slimePropertyMap, byte[] chunkBytes) throws IOException {
+        Map<ChunkPos, SlimeChunk> chunkMap = new HashMap<>();
+        DataInputStream chunkData = new DataInputStream(new ByteArrayInputStream(chunkBytes));
+
+        int chunks = chunkData.readInt();
+        for (int i = 0; i < chunks; i++) {
+            // ChunkPos
+            int x = chunkData.readInt();
+            int z = chunkData.readInt();
+
+            // Sections
+            int sectionAmount = slimePropertyMap.getValue(SlimeProperties.CHUNK_SECTION_MAX) - slimePropertyMap.getValue(SlimeProperties.CHUNK_SECTION_MIN) + 1;
+            SlimeChunkSection[] chunkSections = new SlimeChunkSection[sectionAmount];
+
+            int sectionCount = chunkData.readInt();
+            for (int sectionId = 0; sectionId < sectionCount; sectionId++) {
+
+                // Block Light Nibble Array
+                NibbleArray blockLightArray;
+                if (chunkData.readBoolean()) {
+                    byte[] blockLightByteArray = new byte[ARRAY_SIZE];
+                    chunkData.read(blockLightByteArray);
+                    blockLightArray = new NibbleArray(blockLightByteArray);
+                } else {
+                    blockLightArray = null;
+                }
+
+                // Sky Light Nibble Array
+                NibbleArray skyLightArray;
+                if (chunkData.readBoolean()) {
+                    byte[] skyLightByteArray = new byte[ARRAY_SIZE];
+                    chunkData.read(skyLightByteArray);
+                    skyLightArray = new NibbleArray(skyLightByteArray);
+                } else {
+                    skyLightArray = null;
+                }
+
+                // Block Data
+                byte[] blockStateData = new byte[chunkData.readInt()];
+                chunkData.read(blockStateData);
+                CompoundTag blockStateTag = readCompound(blockStateData);
+
+                // Biome Data
+                byte[] biomeData = new byte[chunkData.readInt()];
+                chunkData.read(biomeData);
+                CompoundTag biomeTag = readCompound(biomeData);
+
+                chunkSections[sectionId] = new SlimeChunkSectionSkeleton(blockStateTag, biomeTag, blockLightArray, skyLightArray);
+            }
+
+            // HeightMaps
+            byte[] heightMapData = new byte[chunkData.readInt()];
+            chunkData.read(heightMapData);
+            CompoundTag heightMaps = readCompound(heightMapData);
+
+            // Tile Entities
+
+            var tileEntities = read(chunkData);
+
+            CompoundTag tileEntitiesCompound = readCompound(tileEntities);
+            @SuppressWarnings("unchecked")
+            List<CompoundTag> serializedTileEntities = ((ListTag<CompoundTag>) tileEntitiesCompound.getValue().get("tileEntities")).getValue();
+
+            // Entities
+
+            var entities = read(chunkData);
+
+            CompoundTag entitiesCompound = readCompound(entities);
+            @SuppressWarnings("unchecked")
+            List<CompoundTag> serializedEntities = ((ListTag<CompoundTag>) entitiesCompound.getValue().get("entities")).getValue();
+
+            // Extra Tag
+            var rawExtra = read(chunkData);
+            var extra = readCompound(rawExtra);
+            // If the extra tag is empty, the serializer will save it as null.
+            // So if we deserialize a null extra tag, we will assume it was empty.
+            if (extra == null) extra = new CompoundTag("", new CompoundMap());
+
+            chunkMap.put(new ChunkPos(x, z),
+                    new SlimeChunkSkeleton(x, z, chunkSections, heightMaps, serializedTileEntities, serializedEntities, extra));
+        }
+        return chunkMap;
+    }
+
+    private static byte[] readCompressed(DataInputStream stream) throws IOException {
+        int compressedLength = stream.readInt();
+        int decompressedLength = stream.readInt();
+        byte[] compressedData = new byte[compressedLength];
+        byte[] decompressedData = new byte[decompressedLength];
+        stream.read(compressedData);
+        Zstd.decompress(decompressedData, compressedData);
+        return decompressedData;
+    }
+
+    private static byte[] read(DataInputStream stream) throws IOException {
+        int length = stream.readInt();
+        byte[] data = new byte[length];
+        stream.read(data);
+        return data;
+    }
+
+    private static CompoundTag readCompound(byte[] tagBytes) throws IOException {
+        if (tagBytes.length == 0) return null;
+
+        NBTInputStream nbtStream = new NBTInputStream(new ByteArrayInputStream(tagBytes), NBTInputStream.NO_COMPRESSION, ByteOrder.BIG_ENDIAN);
+        return (CompoundTag) nbtStream.readTag();
+    }
+}

core/src/main/java/com/infernalsuite/aswm/serialization/slime/reader/impl/v12/v12WorldFormat.java

@@ -0,0 +1,10 @@
+package com.infernalsuite.aswm.serialization.slime.reader.impl.v12;
+
+import com.infernalsuite.aswm.api.world.SlimeWorld;
+import com.infernalsuite.aswm.serialization.slime.reader.impl.SimpleWorldFormat;
+
+public interface v12WorldFormat {
+
+    SimpleWorldFormat<SlimeWorld> FORMAT = new SimpleWorldFormat<>(data -> data, new v12SlimeWorldDeSerializer());
+
+}

core/src/main/java/com/infernalsuite/aswm/serialization/slime/reader/impl/v19/v1_9SlimeConverter.java

@@ -1,5 +1,7 @@
 package com.infernalsuite.aswm.serialization.slime.reader.impl.v19;
 
+import com.flowpowered.nbt.CompoundMap;
+import com.flowpowered.nbt.CompoundTag;
 import com.infernalsuite.aswm.ChunkPos;
 import com.infernalsuite.aswm.serialization.SlimeWorldReader;
 import com.infernalsuite.aswm.skeleton.SkeletonSlimeWorld;
@@ -61,7 +63,8 @@ public SlimeWorld readFromData(v1_9SlimeWorld data) {
                     sections,
                     slimeChunk.heightMap,
                     slimeChunk.tileEntities,
-                    slimeChunk.entities
+                    slimeChunk.entities,
+                    new CompoundTag("", new CompoundMap())
             ));
         }
 

core/src/main/java/com/infernalsuite/aswm/skeleton/SkeletonCloning.java

@@ -71,7 +71,8 @@ private static Map<ChunkPos, SlimeChunk> cloneChunkStorage(Collection<SlimeChunk
                             copied,
                             chunk.getHeightMaps().clone(),
                             deepClone(chunk.getTileEntities()),
-                            deepClone(chunk.getEntities())
+                            deepClone(chunk.getEntities()),
+                            chunk.getExtraData().clone()
                     ));
         }
 

core/src/main/java/com/infernalsuite/aswm/skeleton/SlimeChunkSkeleton.java

@@ -9,7 +9,9 @@
 public record SlimeChunkSkeleton(int x, int z, SlimeChunkSection[] sections,
                                  CompoundTag heightMap,
                                  List<CompoundTag> blockEntities,
-                                 List<CompoundTag> entities) implements SlimeChunk {
+                                 List<CompoundTag> entities,
+                                 CompoundTag extra) implements SlimeChunk {
+
     @Override
     public int getX() {
         return this.x;
@@ -39,4 +41,9 @@ public List<CompoundTag> getTileEntities() {
     public List<CompoundTag> getEntities() {
         return this.entities;
     }
+
+    @Override
+    public CompoundTag getExtraData() {
+        return this.extra;
+    }
 }