Fix: schematic encoder with help from chatgpt

src/main/java/io/ib67/sfcraft/util/litematic/LitematicConverter.java

@@ -5,6 +5,7 @@ import io.netty.buffer.ByteBuf;
 import io.netty.buffer.Unpooled;
 import lombok.SneakyThrows;
 import net.minecraft.nbt.*;
+import net.minecraft.network.PacketByteBuf;
 import org.apache.commons.compress.utils.Lists;
 
 import java.io.InputStream;
@@ -63,7 +64,7 @@ public class LitematicConverter implements AutoCloseable {
         worldEditTag.put("BlockEntities", convertToWETileEntities(tileEntities));
         worldEditTag.putInt("Version", 2);
         worldEditTag.putIntArray("Offset", new int[3]);
-        worldEditTag.putByteArray("BlockData", convertToWEBlocks(Math.abs(size.x * size.y * size.z), region));
+        worldEditTag.putByteArray("BlockData", convertToWEBlocks(size, region));
         var schematicsRoot = new NbtCompound();
         schematicsRoot.put("Schematic", worldEditTag);
         return schematicsRoot;
@@ -94,7 +95,7 @@ public class LitematicConverter implements AutoCloseable {
             tE.remove("y");
             tE.remove("z");
             tE.remove("id");
-            weTE.put("Data",tE);
+            weTE.put("Data", tE);
             weTEs.add(weTE);
         }
         return weTEs;
@@ -129,52 +130,32 @@ public class LitematicConverter implements AutoCloseable {
         return nbt;
     }
 
-    protected byte[] convertToWEBlocks(int totalBlocks, NbtCompound region) {
+    protected byte[] convertToWEBlocks(SizeTuple size, NbtCompound region) {
-        var buf = Unpooled.buffer();
+        var blockCount = Math.abs(size.x * size.y * size.z);
         var blockStates = region.getLongArray("BlockStates");
-
         int bitsPerBlock = region.getList("BlockStatePalette", NbtElement.COMPOUND_TYPE).size();
         bitsPerBlock = Math.max(2, Integer.SIZE - Integer.numberOfLeadingZeros(bitsPerBlock - 1));
+        int maxEntryValue = (1 << bitsPerBlock) - 1;
+        var buffer = new PacketByteBuf(Unpooled.buffer());
+        for (int index = 0; index < blockCount; index++) {
+            int startBit = index * bitsPerBlock;
+            int startLongIndex = startBit / 64;
+            int startBitOffset = startBit % 64;
+            int endBit = startBit + bitsPerBlock - 1;
+            int endLongIndex = endBit / 64;
 
-        int bitCounts = 0, blockIterated = 0;
+            int value;
-        long bitMask, bits = 0;
+            if (startLongIndex == endLongIndex) {
-        for (long blockState : blockStates) {
+                value = (int) ((blockStates[startLongIndex] >>> startBitOffset) & maxEntryValue);
-            int remainingBits = bitCounts + 64;
+            } else {
-            if (bitCounts != 0) {
+                int bitsInFirstPart = 64 - startBitOffset; // 第一个 long 提取的位数
-                bitMask = (1 << (bitsPerBlock - bitCounts)) - 1;
+                long firstPart = blockStates[startLongIndex] >>> startBitOffset;
-                long newBits = (blockState & bitMask) << bitCounts;
+                long secondPart = blockStates[endLongIndex] & ((1L << (bitsPerBlock - bitsInFirstPart)) - 1);
-                bits = bits | newBits;
+                value = (int) ((firstPart | (secondPart << bitsInFirstPart)) & maxEntryValue);
-                blockState = blockState >>> (bitsPerBlock - bitCounts);
-                remainingBits -= bitsPerBlock;
-                writeBlocks(buf, (short) bits);
-                blockIterated++;
             }
-
+            buffer.writeVarInt(value);
-            bitMask = (1 << bitsPerBlock) - 1;
-            while (remainingBits >= bitsPerBlock) {
-                bits = blockState & bitMask;
-                blockState = blockState >>> bitsPerBlock;
-                remainingBits -= bitsPerBlock;
-                if (blockIterated >= totalBlocks) break;
-                writeBlocks(buf, (short) bits);
-                blockIterated++;
-            }
-            bits = blockState;
-            bitCounts = remainingBits;
-        }
-        return buf.array();
-    }
-
-    protected static int writeBlocks(ByteBuf buf, short block) {
-        int b = block >>> 7;
-        if (b == 0) {
-            buf.writeByte(block);
-            return 1;
-        } else {
-            buf.writeByte(block | 128);
-            buf.writeByte(block);
-            return 2;
         }
+        return buffer.array();
     }
 
     @Override
@@ -182,7 +163,7 @@ public class LitematicConverter implements AutoCloseable {
         input.close();
     }
 
-    record SizeTuple(
+    protected record SizeTuple(
             int x, int y, int z
     ) {
     }

src/main/java/io/ib67/sfcraft/util/litematic/LitematicConverterV3.java

@@ -1,16 +1,23 @@
 package io.ib67.sfcraft.util.litematic;
 
+import io.netty.buffer.Unpooled;
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
+import lombok.extern.log4j.Log4j2;
 import net.minecraft.nbt.NbtCompound;
 import net.minecraft.nbt.NbtElement;
 import net.minecraft.nbt.NbtSizeTracker;
+import net.minecraft.network.PacketByteBuf;
+import net.minecraft.util.math.Vec3i;
 
 import java.io.InputStream;
 
+@Log4j2
 public class LitematicConverterV3 extends LitematicConverter {
     public LitematicConverterV3(InputStream input, NbtSizeTracker sizeTracker) {
         super(input, sizeTracker);
     }
-    
+
     @Override
     protected NbtCompound convertRegionToSchematic(int dataVersion, NbtCompound region) {
         var schematicsTag = new NbtCompound();
@@ -34,15 +41,55 @@ public class LitematicConverterV3 extends LitematicConverter {
 
         var blocksNbt = new NbtCompound();
         blocksNbt.put("Palette", wePalette);
-        blocksNbt.putByteArray("Data", convertToWEBlocks(Math.abs(size.x() * size.y() * size.z()), region));
+        var weBlockData = convertToWEBlocks(size, region);
+        blocksNbt.putByteArray("Data", weBlockData);
         blocksNbt.put("BlockEntities", convertToWETileEntities(tileEntities));
-
+        validateData(wePalette, weBlockData, Math.abs(size.x()), Math.abs(size.z()));
         schematicsTag.put("Blocks", blocksNbt);
         var schematicsRoot = new NbtCompound();
         schematicsRoot.put("Schematic", schematicsTag);
         return schematicsRoot;
     }
 
+    private void validateData(
+            NbtCompound wePalette,
+            byte[] weBlockData,
+            int width,
+            int length
+    ) {
+        // find invalid palette ids
+        var buf = new PacketByteBuf(Unpooled.wrappedBuffer(weBlockData));
+        var paletteIds = new Int2ObjectOpenHashMap<>();
+        var touchedIds = new IntOpenHashSet();
+        for (String key : wePalette.getKeys()) {
+            paletteIds.put(wePalette.getInt(key), key);
+        }
+        int counter = 0;
+        try {
+            while (buf.isReadable()) {
+                counter++;
+                var id = buf.readVarInt();
+                if (!paletteIds.containsKey(id)) {
+                    log.error("Cannot find id {} at {}th block in palette, pos: {}", id, counter, decodePositionFromDataIndex(width, length, counter));
+                } else {
+                    touchedIds.add(id);
+                }
+            }
+        } catch (Exception e) {
+            log.error("Error validing schematic", e);
+        }
+        paletteIds.keySet().intStream().filter(it->!touchedIds.contains(it))
+                .forEach(it-> log.error("Unused palette id: {}, blockState: {}", it, paletteIds.get(it)));
+    }
+    private static Vec3i decodePositionFromDataIndex(int width, int length, int index) {
+        // index = (y * width * length) + (z * width) + x
+        int y = index / (width * length);
+        int remainder = index - (y * width * length);
+        int z = remainder / width;
+        int x = remainder - z * width;
+        return new Vec3i(x, y, z);
+    }
+
     @Override
     protected NbtElement convertToWeMeta(SizeTuple size, NbtCompound region) {
         var pos = (NbtCompound) region.get("Position");