feat: implement backup system with compression and bundling

- Add zstd-jni dependency for compression support
- Define Backup record with name, key, incremental, and size fields
- Implement BackupManager with backup listing, deletion, and creation methods
- Create BackupWorker to handle backup operations with server state coordination
- Refactor WorldDir to simplify dimension handling and file listing
- Introduce bundling system with SerialBundlerReader^[[1;5P/Writer for efficient storage
- Add BackupStrategy interface with LocalBackupStrategy implementation
- Support both compressed and uncompressed backup formats
- Add config option for parallel backup operations
- Remove old backup implementation from JustBackupMod
- Update backup file format with metadata footer and magic number validation

build.gradle

@@ -41,6 +41,7 @@ dependencies {
 	// Fabric API. This is technically optional, but you probably want it anyway.
 	modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}"
 	implementation("io.minio:minio:8.5.17")
+    implementation 'com.github.luben:zstd-jni:1.5.7-4'
 
 }
 

src/main/java/io/ib67/sfcraft/Backup.java

@@ -0,0 +1,9 @@
+package io.ib67.sfcraft;
+
+public record Backup(
+        String name,
+        String backupKey,
+        boolean incremental,
+        long sizeTotal
+) {
+}

src/main/java/io/ib67/sfcraft/BackupManager.java

@@ -0,0 +1,14 @@
+package io.ib67.sfcraft;
+
+import java.util.List;
+
+public class BackupManager {
+    public List<Backup> getAllBackup() {
+    }
+
+    public void deleteBackup(Backup backup) {
+    }
+
+    public Backup performBackup() {
+    }
+}

src/main/java/io/ib67/sfcraft/BackupWorker.java

@@ -0,0 +1,45 @@
+package io.ib67.sfcraft;
+
+import io.ib67.sfcraft.mixin.LevelStorageAccessor;
+import io.ib67.sfcraft.strategy.BackupStrategy;
+import lombok.SneakyThrows;
+import lombok.extern.log4j.Log4j2;
+import net.minecraft.server.MinecraftServer;
+
+import java.nio.file.Path;
+
+import static io.ib67.sfcraft.JustBackupMod.PERFORMING_BACKUP;
+
+@Log4j2
+public record BackupWorker(
+        MinecraftServer server,
+        BackupManager manager
+) implements Runnable {
+
+    @Override
+    @SneakyThrows
+    public void run() {
+        if (server == null) {
+            log.warn("MinecraftServer is not initialized yet, not doing backup");
+            return;
+        }
+        int i = 0;
+        while (!PERFORMING_BACKUP.compareAndSet(false, true)) {
+            if (i++ % 10 == 0) log.info("Server is saving world while performing backup! waiting...");
+            Thread.sleep(2000);
+        }
+        // performing backup is now true
+        try {
+            if (i == 0) {
+                // not saved yet.
+                server.saveAll(true, true, false);
+            }
+            var session = ((LevelStorageAccessor) server).getSession();
+            var path = session.getDirectory().getRootPath();
+            manager.performBackup();
+            strategy.createBackup(new WorldDir(Path.of(path)));
+        } finally {
+            PERFORMING_BACKUP.set(false);
+        }
+    }
+}

src/main/java/io/ib67/sfcraft/JustBackupMod.java

@@ -7,6 +7,7 @@ import io.ib67.sfcraft.config.StorageOption;
 import io.ib67.sfcraft.config.serializer.StorageOptionSerializer;
 import io.ib67.sfcraft.mixin.LevelStorageAccessor;
 import io.ib67.sfcraft.mixin.MixinMinecraftServer;
+import io.ib67.sfcraft.strategy.BackupStrategy;
 import lombok.SneakyThrows;
 import net.fabricmc.api.ModInitializer;
 
@@ -32,7 +33,7 @@ public class JustBackupMod implements ModInitializer {
     public static final String MOD_ID = "justbackup";
     public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID);
     public static final AtomicBoolean PERFORMING_BACKUP = new AtomicBoolean(false);
-    private static final Gson GSON = new GsonBuilder().setPrettyPrinting()
+    public static final Gson GSON = new GsonBuilder().setPrettyPrinting()
             .registerTypeAdapter(StorageOption.class, new StorageOptionSerializer(
                     Map.of("s3", StorageOption.S3.class,
                             "local", StorageOption.Local.class)
@@ -40,6 +41,7 @@ public class JustBackupMod implements ModInitializer {
             .create();
     protected ScheduledExecutorService scheduledBackupExecutor;
     protected JustBackupConfig config;
+    protected BackupStrategy strategy;
     protected volatile MinecraftServer server;
 
     @Override
@@ -54,71 +56,25 @@ public class JustBackupMod implements ModInitializer {
             saveConfig(configPath);
         }
         config = GSON.fromJson(Files.readString(configPath), JustBackupConfig.class);
+        this.strategy = createStrategy();
         LOGGER.info("Configuration has been loaded. Using storage option: {}", config.option().type());
         scheduledBackupExecutor = Executors.newSingleThreadScheduledExecutor();
         ServerLifecycleEvents.SERVER_STARTED.register(server -> {
             LOGGER.info("Launching background threads...");
             this.server = server;
-            scheduledBackupExecutor.scheduleAtFixedRate(this::performBackup,
+            scheduledBackupExecutor.scheduleAtFixedRate(new BackupWorker(server, strategy),
                     config.backupIntervalMinutes(), config.backupIntervalMinutes(), TimeUnit.MINUTES);
         });
     }
 
-    @SneakyThrows
-    private void performBackup() {
-        if (server == null) {
-            LOGGER.warn("MinecraftServer is not initialized yet, not doing backup");
-            return;
-        }
-        int i = 0;
-        while (!PERFORMING_BACKUP.compareAndSet(false, true)) {
-            if (i++ % 10 == 0) LOGGER.info("Server is saving world while performing backup! waiting...");
-            Thread.sleep(2000);
-        }
-        // performing backup is now true
-        try {
-            if (i == 0) {
-                // not saved yet.
-                server.saveAll(true, true, false);
+    private BackupStrategy createStrategy() {
+        switch (config.option()) {
+            case StorageOption.Local local -> {
+
             }
-            var session = ((LevelStorageAccessor) server).getSession();
-            var path = session.getDirectory().getRootPath();
-            // !compress
-            if (!config.compress()) {
-                performBackupUncompressed(new WorldDir(Path.of(path)));
-            }
-        } finally {
-            PERFORMING_BACKUP.set(false);
         }
     }
 
-    @SneakyThrows
-    private void performBackupUncompressed(WorldDir worldDir) {
-        var everything = worldDir.everything();
-        var opt = (StorageOption.Local) config.option();
-        var dst = Path.of(opt.saveDir()).resolve("Backup_" + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now()));
-        Files.createDirectories(dst);
-        try(var f = Files.walk(worldDir.region())){
-            bundleUncompressed(dst.resolve("regions.gz"), f.toList());
-        }
-        try(var f = Files.walk(worldDir.poi())) {
-            bundleUncompressed(dst.resolve("poi.gz"), f.toList());
-        }
-        var other = dst.resolve("other.zip", )
-    }
-
-    @SneakyThrows
-    private static void bundleUncompressed(Path dst, Collection<Path> toBundle){
-        try(var fileOut = new RandomAccessFile(dst.toFile(), "rw");){
-            var fileOutDst = fileOut.getChannel();
-            for (Path path : toBundle) {
-                var file = path.toFile();
-                try(var in = new FileInputStream(path.toFile())){
-                    in.getChannel().transferTo(0, file.length(), fileOutDst);
-                }
-            }
-        }
-    }
 
     @SneakyThrows
     private void saveConfig(Path configPath) {

src/main/java/io/ib67/sfcraft/WorldDir.java

@@ -26,24 +26,12 @@ public record WorldDir(
         return root.resolve("entities");
     }
 
-    public Path playerdata() {
-        return root.resolve("playerdata");
-    }
-
     @SneakyThrows
-    public Map<String, WorldDir> otherDimensions() {
-        var begin = root.resolve("dimensions").toAbsolutePath();
-        try (var w = Files.walk(begin, 3)) {
-            var _r = w.filter(Files::isDirectory)
-                    .filter(it -> Files.exists(it.resolve("region")) && Files.exists(it.resolve("poi")))
-                    .collect(Collectors.groupingBy(it -> it.relativize(root).toString(),
-                            Collectors.mapping(WorldDir::new, Collectors.toList())));
-            var r = new HashMap<String, WorldDir>();
-            _r.forEach((k, v) -> {
-                assert v.size() <= 1;
-                r.put(k, v.getFirst());
-            });
-            return r;
+    public List<WorldDir> otherDimensions() {
+        try (var w = Files.walk(root)) {
+            return w.filter(it -> Files.exists(it.resolve("region")) && Files.exists(it.resolve("poi")))
+                    .map(WorldDir::new)
+                    .toList();
         }
     }
 
@@ -52,9 +40,10 @@ public record WorldDir(
      */
     @SneakyThrows
     public List<Path> otherFiles() {
-        var exclude = List.of(playerdata(), entities(), region(), poi(), root.resolve("dimensions"));
+        var otherDimensions = otherDimensions();
         try (var f = Files.walk(root)) {
-            return f.filter(it -> exclude.stream().noneMatch(it::startsWith)).toList();
+            return f.filter(it -> otherDimensions.stream().noneMatch(a -> it.startsWith(a.root)))
+                    .toList();
         }
     }
 

src/main/java/io/ib67/sfcraft/bundler/BundlerEntry.java

@@ -0,0 +1,10 @@
+package io.ib67.sfcraft.bundler;
+
+public record BundlerEntry(
+        String fileName,
+        boolean unGzipped,
+        long fileLength,
+        long offset
+) {
+    public static final int MAGIC = 0xBACEBACE;
+}

src/main/java/io/ib67/sfcraft/bundler/EntryInputStream.java

@@ -0,0 +1,33 @@
+package io.ib67.sfcraft.bundler;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+public class EntryInputStream extends FilterInputStream {
+    /**
+     * Creates a {@code FilterInputStream}
+     * by assigning the  argument {@code in}
+     * to the field {@code this.in} so as
+     * to remember it for later use.
+     *
+     * @param in the underlying input stream, or {@code null} if
+     *           this instance is to be created without an underlying stream.
+     */
+    protected EntryInputStream(InputStream in) {
+        super(in);
+    }
+
+    public BundlerEntry readEntry() throws IOException {
+        var len = ByteBuffer.wrap(readNBytes(4)).getInt();
+        var gunzipped = len < 0;
+        len = Math.abs(len);
+        var name = new String(readNBytes(len)).intern();
+        var buf = ByteBuffer.wrap(readNBytes(8));
+        var fileLen = buf.getLong();
+        var fileOffset = buf.getLong();
+        return new BundlerEntry(name, gunzipped, fileLen, fileOffset);
+
+    }
+}

src/main/java/io/ib67/sfcraft/bundler/EntryOutputStream.java

@@ -0,0 +1,37 @@
+package io.ib67.sfcraft.bundler;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+
+public class EntryOutputStream extends FilterOutputStream {
+    /**
+     * Creates an output stream filter built on top of the specified
+     * underlying output stream.
+     *
+     * @param out the underlying output stream to be assigned to
+     *            the field {@code this.out} for later use, or
+     *            {@code null} if this instance is to be
+     *            created without an underlying stream.
+     */
+    public EntryOutputStream(OutputStream out) {
+        super(out);
+    }
+
+    public void writeMagic() throws IOException {
+        out.write(BundlerEntry.MAGIC);
+    }
+
+    public void write(BundlerEntry entry) throws IOException {
+        var fileName = entry.fileName().getBytes(StandardCharsets.UTF_8);
+        var buf = ByteBuffer.allocate(4).putInt(entry.unGzipped() ? fileName.length * -1 : fileName.length);
+        out.write(buf.array());
+        out.write(fileName);
+        buf = ByteBuffer.allocate(8 + 8);
+        buf.putLong(entry.fileLength());
+        buf.putLong(entry.offset());
+        out.write(buf.array());
+    }
+}

src/main/java/io/ib67/sfcraft/bundler/SerialBundlerReader.java

@@ -0,0 +1,92 @@
+package io.ib67.sfcraft.bundler;
+
+import com.github.luben.zstd.Zstd;
+import com.github.luben.zstd.ZstdInputStreamNoFinalizer;
+import lombok.SneakyThrows;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Objects;
+
+public class SerialBundlerReader implements Closeable {
+    protected final InputStream in;
+    protected final FileChannel inChannel; // for direct copy
+
+    protected final EntryInputStream metadataIn;
+    protected final byte[] buffer = new byte[4096];
+    protected boolean entryRead;
+
+    public SerialBundlerReader(InputStream in, InputStream metadataIn) {
+        this.in = Objects.requireNonNull(in);
+        this.metadataIn = new EntryInputStream(Objects.requireNonNull(metadataIn));
+        inChannel = null;
+    }
+
+    public SerialBundlerReader(Path path) throws IOException {
+        var raf = new RandomAccessFile(path.toFile(), "r");
+        var totalLen = Files.size(path);
+        raf.seek(totalLen - 4);
+        var metadataLength = raf.readInt();
+        var metadataBegin = totalLen - 4 - metadataLength;
+        var metadataBytes = new byte[metadataLength];
+        raf.seek(metadataBegin);
+        raf.readFully(metadataBytes);
+
+        this.metadataIn = new EntryInputStream(new ZstdInputStreamNoFinalizer(new ByteArrayInputStream(metadataBytes)));
+        if (ByteBuffer.wrap(metadataIn.readNBytes(4)).getInt() != BundlerEntry.MAGIC) {
+            throw new IllegalArgumentException("Invalid Bundler entry");
+        }
+        // check compressed
+        raf.seek(0);
+        var fis = Channels.newInputStream(raf.getChannel());
+        if (raf.readInt() == Zstd.magicNumber()) {
+            in = new ZstdInputStreamNoFinalizer(fis);
+            inChannel = null;
+        } else {
+            in = fis;
+            inChannel = raf.getChannel();
+        }
+    }
+
+    @SneakyThrows
+    public boolean hasEntry() {
+        return metadataIn.available() > 0;
+    }
+
+    public BundlerEntry readEntry() throws IOException {
+        var entry = metadataIn.readEntry();
+        entryRead = true;
+        return entry;
+    }
+
+    public void writeTo(FileChannel channel, BundlerEntry entry) throws IOException {
+        if (inChannel != null) {
+            inChannel.transferTo(entry.offset(), entry.fileLength(), channel);
+        } else {
+            writeTo(Channels.newOutputStream(channel), entry);
+        }
+    }
+
+    public void writeTo(OutputStream out, BundlerEntry entry) throws IOException {
+        Objects.requireNonNull(entry);
+        var remaining = entry.fileLength();
+        var buffer = this.buffer;
+        var totalRead = 0;
+        do {
+            var read = in.read(buffer, 0, (int) Math.min(remaining - totalRead, buffer.length));
+            out.write(buffer, 0, read);
+            totalRead += read;
+        } while (totalRead < remaining);
+    }
+
+
+    @Override
+    public void close() throws IOException {
+        in.close();
+        metadataIn.close();
+    }
+}

src/main/java/io/ib67/sfcraft/bundler/SerialBundlerWriter.java

@@ -0,0 +1,80 @@
+package io.ib67.sfcraft.bundler;
+
+import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.List;
+import java.util.Objects;
+import java.util.zip.GZIPInputStream;
+
+@AllArgsConstructor
+public class SerialBundlerWriter implements Closeable {
+    protected final FileChannel channel;
+    protected final OutputStream out;
+    protected long offset;
+    @Getter
+    protected List<BundlerEntry> entries;
+
+    public SerialBundlerWriter(OutputStream out) {
+        Objects.requireNonNull(out);
+        this.out = out;
+        channel = null;
+    }
+
+    public SerialBundlerWriter(FileChannel channel, long offset) {
+        Objects.requireNonNull(channel);
+        this.offset = offset;
+        this.channel = channel;
+        out = null;
+    }
+
+    public void writeGunzip(Path path, String fileName) throws IOException {
+        try (var gzip = new GZIPInputStream(Files.newInputStream(path, StandardOpenOption.READ))) {
+            var len = channel != null ? gzip.transferTo(Channels.newOutputStream(channel)) :
+                    gzip.transferTo(out);
+            entries.add(new BundlerEntry(fileName, true, len, offset));
+            offset += len;
+        }
+    }
+
+    public void writePlain(Path path, String fileName) throws IOException {
+        if (channel != null) {
+            writeZC(path, fileName);
+            return;
+        }
+        try (var in = Files.newInputStream(path)) {
+            in.transferTo(out);
+        }
+    }
+
+    private void writeZC(Path path, String fileName) throws IOException {
+        try (var in = FileChannel.open(path, StandardOpenOption.READ)) {
+            var begin = offset;
+            var currentPosition = offset;
+            long transferred = 0;
+            while ((transferred = in.transferTo(currentPosition, in.size() - transferred, channel)) != 0) {
+                currentPosition += transferred;
+            }
+            offset = currentPosition;
+            entries.add(new BundlerEntry(
+                    fileName, false, offset - begin, begin
+            ));
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (out != null) out.close();
+        if (channel != null) channel.close();
+    }
+}

src/main/java/io/ib67/sfcraft/config/JustBackupConfig.java

@@ -7,6 +7,7 @@ import java.util.List;
 public record JustBackupConfig(
         long backupIntervalMinutes,
         int keepBackups,
+        boolean allowParallel,
         boolean compress,
         StorageOption option
 ) {

src/main/java/io/ib67/sfcraft/strategy/BackupStrategy.java

@@ -0,0 +1,17 @@
+package io.ib67.sfcraft.strategy;
+
+import io.ib67.sfcraft.Backup;
+import io.ib67.sfcraft.WorldDir;
+
+import java.nio.file.Path;
+
+public interface BackupStrategy {
+    /**
+     * create a backup
+     * @param dir world to be bundled
+     * @return a key to this backup
+     */
+    Backup createBackup(WorldDir dir, boolean compress);
+
+    void recoverBackup(Backup backup, Path restorePath);
+}

src/main/java/io/ib67/sfcraft/strategy/LocalBackupStrategy.java

@@ -0,0 +1,104 @@
+package io.ib67.sfcraft.strategy;
+
+import com.github.luben.zstd.Zstd;
+import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
+import io.ib67.sfcraft.Backup;
+import io.ib67.sfcraft.WorldDir;
+import io.ib67.sfcraft.bundler.BundlerEntry;
+import io.ib67.sfcraft.bundler.EntryOutputStream;
+import io.ib67.sfcraft.bundler.SerialBundlerReader;
+import io.ib67.sfcraft.bundler.SerialBundlerWriter;
+import io.ib67.sfcraft.config.StorageOption;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.log4j.Log4j2;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+
+@Log4j2
+@RequiredArgsConstructor
+public class LocalBackupStrategy implements BackupStrategy {
+    protected final StorageOption.Local option;
+    protected final Path backupParentRoot = Path.of(option.saveDir());
+    protected final boolean allowGunzip;
+
+    @Override
+    @SneakyThrows
+    public Backup createBackup(WorldDir dir, boolean compress) {
+        var sortedEntries = dir.everything().stream().sorted((a, b) -> {
+            boolean aMatch = a.endsWith(".mca") || a.endsWith(".dat");
+            boolean bMatch = b.endsWith(".mca") || b.endsWith(".dat");
+
+            if (aMatch && !bMatch) return -1;
+            if (!aMatch && bMatch) return 1;
+            return a.compareTo(b);
+        }).map(dir.root()::relativize).toList();
+        var backupName = "Backup_" + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now());
+        var backupFile = backupParentRoot.resolve(backupName + ".jbp");
+        SerialBundlerWriter writer;
+        if (compress) {
+            var out = new ZstdOutputStreamNoFinalizer(Files.newOutputStream(backupFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE));
+            writer = new SerialBundlerWriter(out);
+        } else {
+            var chan = FileChannel.open(backupFile);
+            writer = new SerialBundlerWriter(chan, 0);
+        }
+        writeEntries(sortedEntries, writer, allowGunzip && compress);
+        writer.close();
+
+        try (var backup = FileChannel.open(backupFile);
+             var bos = new ByteArrayOutputStream();
+             var metadata = new EntryOutputStream(new ZstdOutputStreamNoFinalizer(bos))) {
+            metadata.writeMagic();
+            for (BundlerEntry entry : writer.getEntries()) {
+                metadata.write(entry);
+            }
+            var result = bos.toByteArray();
+            var buffer = ByteBuffer.allocate(result.length + 4);
+            buffer.put(result);
+            buffer.putLong(result.length);
+            backup.write(buffer);
+        }
+        return new Backup(backupName, backupFile.getFileName().toString(), false, Files.size(backupFile));
+    }
+
+    private void writeEntries(
+            List<Path> sortedEntries,
+            SerialBundlerWriter writer,
+            boolean allowGunzip) throws IOException {
+        for (Path sortedEntry : sortedEntries) {
+            var fileName = sortedEntry.getFileName();
+            var shouldGunzip = allowGunzip && (fileName.endsWith("mca") || fileName.endsWith("dat"));
+            if (shouldGunzip) {
+                writer.writeGunzip(sortedEntry, sortedEntry.toString());
+            } else {
+                writer.writePlain(sortedEntry, sortedEntry.toString());
+            }
+        }
+    }
+
+
+    @Override
+    @SneakyThrows
+    public void recoverBackup(Backup backup, Path restorePath) {
+        // check if it is compressed
+        var backupFile = backupParentRoot.resolve(backup.backupKey());
+        if (Files.notExists(backupFile)) throw new IllegalArgumentException("Backup " + backupFile + " does not exist");
+        try (var bundlerReader = new SerialBundlerReader(backupFile)) {
+            while (bundlerReader.hasEntry()) {
+                var entry = bundlerReader.readEntry();
+                var target = restorePath.resolve(entry.fileName());
+                bundlerReader.writeTo(FileChannel.open(target), entry);
+            }
+        }
+    }
+}