Refactor backup system and migrate from submodule to shadow dependencies

- Remove s3-lite submodule and replace with AWS SDK and shadow plugin
- Implement comprehensive backup commands with list, delete, create, restore, and suspend functionality
- Add bundle format with compression support using Zstd and gunzip optimization
- Implement local and S3 backup strategies with configurable options
- Add backup tracking system with JSON persistence
- Improve world directory handling and file exclusion logic
- Add standalone bundler tool for bundle creation/extraction
- Update Gradle configuration with shadow jar support and dependency management
- Enhance configuration system with additional options for compression and temporary directories

build.gradle

@@ -2,7 +2,7 @@ plugins {
     id 'fabric-loom' version "${loom_version}"
     id 'maven-publish'
     id("io.freefair.lombok") version "9.0.0-rc2"
-
+    id("com.gradleup.shadow") version '9.1.0'
 }
 
 version = project.mod_version
@@ -18,13 +18,17 @@ repositories {
     // Loom adds the essential maven repositories to download Minecraft and libraries from automatically.
     // See https://docs.gradle.org/current/userguide/declaring_repositories.html
     // for more information about repositories.
+    maven {
+        name = "Kalculos"
+        url = "https://repo.sfclub.cc/snapshots"
+    }
 }
 
 loom {
     splitEnvironmentSourceSets()
 
     mods {
-		"modid" {
+        "justbackup" {
             sourceSet sourceSets.main
             sourceSet sourceSets.client
         }
@@ -40,9 +44,17 @@ dependencies {
 
     // Fabric API. This is technically optional, but you probably want it anyway.
     modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}"
-	implementation("io.minio:minio:8.5.17")
-    implementation 'com.github.luben:zstd-jni:1.5.7-4'
+    // https://mvnrepository.com/artifact/software.amazon.awssdk/s3
+    shadow 'software.amazon.awssdk:s3:2.34.0'
+    shadow("io.ib67.kiwi:lang:1.2.1-SNAPSHOT")
+    shadow 'com.github.luben:zstd-jni:1.5.7-4'
+}
 
+configurations.implementation.extendsFrom(configurations.shadow)
+shadowJar {
+    configurations = [project.configurations.shadow]
+    exclude("META-INF")
+    //relocate 'com.google', 'sfcraft.relocation.google'
 }
 
 processResources {
@@ -69,12 +81,18 @@ java {
 
 jar {
     inputs.property "archivesName", project.base.archivesName
-
+    manifest {
+        attributes(Map.of("Main-Class", "io.ib67.sfcraft.bundler.Main"))
+    }
     from("LICENSE") {
         rename { "${it}_${inputs.properties.archivesName}" }
     }
 }
 
+remapJar {
+    input = tasks.shadowJar.archiveFile
+}
+
 // configure the maven publication
 publishing {
     publications {

gradle.properties

@@ -1,5 +1,5 @@
 # Done to increase the memory available to gradle.
-org.gradle.jvmargs=-Xmx1G
+org.gradle.jvmargs=-Xmx4G
 org.gradle.parallel=true
 
 # Fabric Properties
@@ -11,8 +11,8 @@ loom_version=1.11-SNAPSHOT
 
 # Mod Properties
 mod_version=1.0.0
-maven_group=com.example
-archives_base_name=modid
+maven_group=io.ib67.sfcraft
+archives_base_name=justbackup
 
 # Dependencies
 fabric_version=0.129.0+1.21.8

src/main/java/io/ib67/sfcraft/Backup.java

@@ -3,6 +3,8 @@ package io.ib67.sfcraft;
 public record Backup(
         String name,
         String backupKey,
+        String type,
+        String from,
         boolean incremental,
         long sizeTotal
 ) {

src/main/java/io/ib67/sfcraft/BackupCommands.java

@@ -0,0 +1,106 @@
+package io.ib67.sfcraft;
+
+import com.mojang.brigadier.Command;
+import com.mojang.brigadier.context.CommandContext;
+import net.minecraft.server.command.ServerCommandSource;
+import net.minecraft.text.ClickEvent;
+import net.minecraft.text.Text;
+
+import java.awt.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public class BackupCommands {
+    protected final JustBackupMod mod;
+    protected Runnable restoreIssued;
+
+    public BackupCommands(JustBackupMod mod) {
+        this.mod = mod;
+    }
+
+    public int cmdList(CommandContext<ServerCommandSource> ctx) {
+        var s = ctx.getSource();
+        s.sendFeedback(() -> Text.of("Tracked backups:"), false);
+        for (Backup value : mod.tracker.getTrackedBackups().values()) {
+            var restoreClick = Text.literal("[RESTORE]").withColor(Color.GREEN.getRGB())
+                    .styled(it -> it.withClickEvent(new ClickEvent.SuggestCommand("/backup restore " + value.backupKey())));
+            var deleteClick = Text.literal("[DELETE]").withColor(Color.RED.getRGB())
+                    .styled(it -> it.withClickEvent(new ClickEvent.SuggestCommand("/backup delete " + value.backupKey())));
+            var message = Text.literal(" - " + value.name() + " (" + value.backupKey() + ") " +
+                    (value.incremental() ? "incremental" : "full") + "  ").append(restoreClick).append("  ").append(deleteClick);
+            s.sendMessage(message);
+            s.sendMessage(Text.of("    Size: " + Math.floorDiv(value.sizeTotal(), 1024 * 1024) + "MiB"));
+        }
+        return Command.SINGLE_SUCCESS;
+    }
+
+    public int cmdDelete(CommandContext<ServerCommandSource> ctx) {
+        var _backup = ctx.getArgument("backupName", String.class);
+        var backup = mod.tracker.getTrackedBackups().get(_backup);
+        mod.tracker.deleteBackup(backup);
+        ctx.getSource().sendMessage(Text.of("Backup " + _backup + " is deleted successfully."));
+        return Command.SINGLE_SUCCESS;
+    }
+
+    public int cmdIssueBackup(CommandContext<ServerCommandSource> ctx) {
+        var s = ctx.getSource();
+        if (JustBackupMod.PERFORMING_BACKUP.get()) {
+            s.sendMessage(Text.of("You can't issue a backup because another backup is in progress."));
+            return Command.SINGLE_SUCCESS;
+        }
+        mod.issueBackup()
+                .whenComplete((backup, error) -> {
+                    if (error != null) {
+                        s.sendMessage(Text.of("Cannot backup, error: " + error.getMessage()));
+                    } else {
+                        s.sendMessage(Text.of("Backup created successfully! name: " + backup.name()));
+                    }
+                });
+        return Command.SINGLE_SUCCESS;
+    }
+
+    public int cmdSuspend(CommandContext<ServerCommandSource> ctx) {
+        mod.suspend = !mod.suspend;
+        ctx.getSource().sendMessage(Text.of("Backup disabled: " + mod.suspend));
+        return Command.SINGLE_SUCCESS;
+    }
+
+    public int cmdRestore(CommandContext<ServerCommandSource> ctx) {
+        var s = ctx.getSource();
+        var pm = s.getServer().getPlayerManager();
+        var _backup = ctx.getArgument("backupName", String.class);
+        var backup = mod.tracker.getTrackedBackups().get(_backup);
+        if (backup == null) {
+            s.sendMessage(Text.of("Invalid backup " + _backup));
+            return 0;
+        }
+        if (restoreIssued != null) {
+            s.sendMessage(Text.of("Another restoration is in progress. try /backup cancel to cancel it"));
+            return 0;
+        }
+        s.sendMessage(Text.literal("Please restart your server to take changes"));
+        restoreIssued = () -> {
+            System.out.println("Recovering backup... ");
+            try{
+                var toOverride = Path.of(backup.from());
+                var old = Path.of(backup.from()+"_old");
+                Files.deleteIfExists(old);
+                Files.move(toOverride, old);
+                System.out.println("Moved your old save to "+old);
+                mod.tracker.recoverBackup(backup, toOverride);
+                System.out.println("Recover successfully!");
+            } catch (Exception e) {
+                throw new RuntimeException(e);
+            }
+        };
+        return Command.SINGLE_SUCCESS;
+    }
+
+    public int cmdHelp(CommandContext<ServerCommandSource> serverCommandSourceCommandContext) {
+        return 0;
+    }
+
+    public int cmdCancelRestore(CommandContext<ServerCommandSource> serverCommandSourceCommandContext) {
+        return 0;
+    }
+}

src/main/java/io/ib67/sfcraft/BackupManager.java

@@ -1,14 +0,0 @@
-package io.ib67.sfcraft;
-
-import java.util.List;
-
-public class BackupManager {
-    public List<Backup> getAllBackup() {
-    }
-
-    public void deleteBackup(Backup backup) {
-    }
-
-    public Backup performBackup() {
-    }
-}

src/main/java/io/ib67/sfcraft/BackupWorker.java

@@ -1,27 +1,34 @@
 package io.ib67.sfcraft;
 
+import io.ib67.sfcraft.bundler.Bundle;
+import io.ib67.sfcraft.config.JustBackupConfig;
 import io.ib67.sfcraft.mixin.LevelStorageAccessor;
 import io.ib67.sfcraft.strategy.BackupStrategy;
 import lombok.SneakyThrows;
 import lombok.extern.log4j.Log4j2;
 import net.minecraft.server.MinecraftServer;
 
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.concurrent.Callable;
+import java.util.function.Supplier;
 
 import static io.ib67.sfcraft.JustBackupMod.PERFORMING_BACKUP;
 
 @Log4j2
 public record BackupWorker(
         MinecraftServer server,
-        BackupManager manager
-) implements Runnable {
+        BackupStrategy strategy,
+        JustBackupConfig config
+) implements Supplier<Backup> {
 
     @Override
     @SneakyThrows
-    public void run() {
+    public Backup get() {
         if (server == null) {
-            log.warn("MinecraftServer is not initialized yet, not doing backup");
-            return;
+            throw new IllegalStateException("MinecraftServer is not initialized yet, not doing backup");
         }
         int i = 0;
         while (!PERFORMING_BACKUP.compareAndSet(false, true)) {
@@ -36,8 +43,20 @@ public record BackupWorker(
             }
             var session = ((LevelStorageAccessor) server).getSession();
             var path = session.getDirectory().getRootPath();
-            manager.performBackup();
-            strategy.createBackup(new WorldDir(Path.of(path)));
+            var backupName = "Backup_" + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now());
+            var backupParentRoot = Path.of(config.temporaryBackupDir());
+            if (Files.notExists(backupParentRoot)) Files.createDirectories(backupParentRoot);
+            var backupFile = backupParentRoot.resolve(backupName + ".jpack");
+            log.info("Creating bundle {}", backupName);
+            //todo diff
+            Path worldDir = Path.of(path);
+            new Bundle(new WorldDir(worldDir))
+                    .compress(config.compress())
+                    .allowGunzip(config.allowGunzip())
+                    .buildBundle(backupFile);
+            var backup = strategy.createBackup(worldDir, backupFile);
+            Files.deleteIfExists(backupFile);
+            return backup;
         } finally {
             PERFORMING_BACKUP.set(false);
         }

src/main/java/io/ib67/sfcraft/JustBackupMod.java

@@ -2,36 +2,58 @@ package io.ib67.sfcraft;
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
+import com.mojang.brigadier.CommandDispatcher;
+import com.mojang.brigadier.arguments.StringArgumentType;
 import io.ib67.sfcraft.config.JustBackupConfig;
 import io.ib67.sfcraft.config.StorageOption;
 import io.ib67.sfcraft.config.serializer.StorageOptionSerializer;
 import io.ib67.sfcraft.mixin.LevelStorageAccessor;
 import io.ib67.sfcraft.mixin.MixinMinecraftServer;
 import io.ib67.sfcraft.strategy.BackupStrategy;
+import io.ib67.sfcraft.strategy.BackupTracker;
+import io.ib67.sfcraft.strategy.LocalBackupStrategy;
+import io.ib67.sfcraft.strategy.S3BackupStrategy;
 import lombok.SneakyThrows;
 import net.fabricmc.api.ModInitializer;
 
+import net.fabricmc.fabric.api.command.v2.CommandRegistrationCallback;
 import net.fabricmc.fabric.api.event.lifecycle.v1.ServerLifecycleEvents;
+import net.minecraft.command.CommandRegistryAccess;
 import net.minecraft.server.MinecraftServer;
+import net.minecraft.server.command.CommandManager;
+import net.minecraft.server.command.ServerCommandSource;
 import net.minecraft.server.world.ServerWorld;
+import net.minecraft.text.Text;
+import org.jetbrains.annotations.UnknownNullability;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.FileInputStream;
+import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
+import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static net.minecraft.server.command.CommandManager.argument;
+import static net.minecraft.server.command.CommandManager.literal;
+
 public class JustBackupMod implements ModInitializer {
     public static final String MOD_ID = "justbackup";
     public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID);
+    /**
+     * This state indicates:
+     * 1. A minecraft auto-save is in-progress.
+     * 2. The backup is in-progress
+     */
     public static final AtomicBoolean PERFORMING_BACKUP = new AtomicBoolean(false);
     public static final Gson GSON = new GsonBuilder().setPrettyPrinting()
             .registerTypeAdapter(StorageOption.class, new StorageOptionSerializer(
@@ -41,38 +63,95 @@ public class JustBackupMod implements ModInitializer {
             .create();
     protected ScheduledExecutorService scheduledBackupExecutor;
     protected JustBackupConfig config;
-    protected BackupStrategy strategy;
+    protected BackupTracker tracker;
+    protected Path backupIndexPath;
     protected volatile MinecraftServer server;
+    protected volatile boolean suspend;
 
     @Override
     @SneakyThrows
     public void onInitialize() {
         LOGGER.info("Loading backup configuration");
+        backupIndexPath = Path.of("config").resolve("just_backup_index.json");
         var configPath = Path.of("config").resolve("just_backup.json");
-        if (Files.notExists(configPath.getParent())) {
+        if (Files.notExists(configPath.getParent()))
             Files.createDirectories(configPath.getParent());
-        }
-        if (Files.notExists(configPath)) {
+        if (Files.notExists(configPath))
             saveConfig(configPath);
-        }
+        if (Files.notExists(backupIndexPath))
+            Files.createFile(backupIndexPath);
+
         config = GSON.fromJson(Files.readString(configPath), JustBackupConfig.class);
-        this.strategy = createStrategy();
+        var strategy = createStrategy();
+        this.tracker = new BackupTracker(strategy, GSON.fromJson(Files.readString(backupIndexPath), new TypeToken<>() {
+        }));
         LOGGER.info("Configuration has been loaded. Using storage option: {}", config.option().type());
         scheduledBackupExecutor = Executors.newSingleThreadScheduledExecutor();
+        registerHandlers();
+    }
+
+    private void registerHandlers() {
         ServerLifecycleEvents.SERVER_STARTED.register(server -> {
             LOGGER.info("Launching background threads...");
             this.server = server;
-            scheduledBackupExecutor.scheduleAtFixedRate(new BackupWorker(server, strategy),
+            scheduledBackupExecutor.scheduleAtFixedRate(this::issueBackup,
                     config.backupIntervalMinutes(), config.backupIntervalMinutes(), TimeUnit.MINUTES);
         });
+        ServerLifecycleEvents.SERVER_STOPPED.register(server -> {
+            try {
+                Files.deleteIfExists(Path.of(config.temporaryBackupDir()));
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        });
+        CommandRegistrationCallback.EVENT.register(this::registerCommand);
+    }
+
+    public CompletableFuture<Backup> issueBackup() {
+        if (suspend) {
+            return CompletableFuture.failedFuture(new IllegalStateException("Backup is temporarily disabled"));
+        }
+        server.getPlayerManager().broadcast(Text.literal("The server will begin the backup shortly, you may experience some lag."), false);
+        var worker = new BackupWorker(server, tracker, config);
+        return CompletableFuture.supplyAsync(worker, scheduledBackupExecutor).whenComplete(this::onBackupComplete);
+    }
+
+    @SneakyThrows
+    private void onBackupComplete(@UnknownNullability Backup backup, @UnknownNullability Throwable throwable) {
+        if (throwable != null) {
+            server.getPlayerManager().broadcast(Text.literal("Backup failed. For administrators, please check your server console."), false);
+            throwable.printStackTrace();
+        } else {
+            Files.writeString(backupIndexPath, GSON.toJson(tracker.getTrackedBackups()));
+            server.getPlayerManager().broadcast(Text.literal("Backup success. " + backup), false);
+        }
+    }
+
+    private void registerCommand(
+            CommandDispatcher<ServerCommandSource> dispatcher,
+            CommandRegistryAccess registry, CommandManager.RegistrationEnvironment env) {
+        var commandHandler = new BackupCommands(this);
+        dispatcher.register(literal("backup")
+                .requires(it -> it.hasPermissionLevel(3))
+                .then(literal("help").executes(commandHandler::cmdHelp))
+                .then(literal("list").executes(commandHandler::cmdList))
+                .then(literal("cancelrestore").executes(commandHandler::cmdCancelRestore))
+                .then(literal("delete")
+                        .then(argument("backupName", StringArgumentType.greedyString())
+                                .executes(commandHandler::cmdDelete)))
+                .then(literal("restore")
+                        .then(argument("backupName", StringArgumentType.greedyString())
+                                .executes(commandHandler::cmdRestore)))
+                .then(literal("create").executes(commandHandler::cmdIssueBackup))
+                .then(literal("suspend").executes(commandHandler::cmdSuspend))
+        );
     }
 
     private BackupStrategy createStrategy() {
-        switch (config.option()) {
-            case StorageOption.Local local -> {
-
-            }
-        }
+        return switch (config.option()) {
+            case StorageOption.Local local -> new LocalBackupStrategy(local);
+            case StorageOption.S3 s3 -> new S3BackupStrategy(s3);
+        };
     }
 
 
@@ -83,8 +162,14 @@ public class JustBackupMod implements ModInitializer {
                 60,
                 5,
                 true,
-                new StorageOption.Local("backups")
+                true,
+                "./.backup_cache",
+                new StorageOption.Local("backups", 0)
         );
+        var path = Path.of(cfg.temporaryBackupDir());
+        if (Files.notExists(path)) {
+            Files.createDirectories(path);
+        }
         Files.writeString(configPath, GSON.toJson(cfg));
     }
 }

src/main/java/io/ib67/sfcraft/WorldDir.java

@@ -42,7 +42,9 @@ public record WorldDir(
     public List<Path> otherFiles() {
         var otherDimensions = otherDimensions();
         try (var f = Files.walk(root)) {
-            return f.filter(it -> otherDimensions.stream().noneMatch(a -> it.startsWith(a.root)))
+            return f.filter(it -> Files.isRegularFile(it)
+                            && otherDimensions.stream().noneMatch(a -> it.startsWith(a.root)))
+                    .filter(it -> !it.getFileName().toString().contains("session.lock"))
                     .toList();
         }
     }
@@ -50,7 +52,9 @@ public record WorldDir(
     @SneakyThrows
     public List<Path> everything() {
         try (var f = Files.walk(root)) {
-            return f.toList();
+            return f.filter(Files::isRegularFile)
+                    .filter(it -> !it.getFileName().toString().contains("session.lock"))
+                    .toList();
         }
     }
 }

src/main/java/io/ib67/sfcraft/bundler/Bundle.java

@@ -0,0 +1,133 @@
+package io.ib67.sfcraft.bundler;
+
+import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
+import io.ib67.sfcraft.WorldDir;
+import io.netty.buffer.Unpooled;
+import lombok.Data;
+import lombok.SneakyThrows;
+import lombok.experimental.Accessors;
+import org.apache.commons.lang3.Validate;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.List;
+
+@Accessors(fluent = true)
+@Data
+public class Bundle {
+    public static final int MAGIC = 0xBACEBACE;
+    public static final int COMPRESSED_MAGIC = 0xBACEAAAA;
+    protected final WorldDir dir;
+    protected boolean compress = true;
+    protected boolean allowGunzip = true;
+    protected boolean allowParallel = false;
+
+
+    public Bundle(WorldDir dir) {
+        this.dir = dir;
+    }
+
+    @SneakyThrows
+    public static void unbundleFile(Path bundle, Path out) {
+        try (var bundlerReader = new SerialBundlerReader(bundle)) {
+            while (bundlerReader.hasEntry()) {
+                var entry = bundlerReader.readEntry();
+                var target = out.resolve(entry.fileName());
+                Files.createDirectories(target.getParent());
+                bundlerReader.writeTo(FileChannel.open(target, StandardOpenOption.WRITE,
+                        StandardOpenOption.CREATE_NEW), entry);
+            }
+        }
+    }
+
+    @SneakyThrows
+    public void buildBundle(Path backupFile) {
+        checkCondition();
+        var sortedEntries = dir.everything().stream().sorted((a, b) -> {
+            boolean aMatch = a.endsWith(".mca") || a.endsWith(".dat");
+            boolean bMatch = b.endsWith(".mca") || b.endsWith(".dat");
+
+            if (aMatch && !bMatch) return -1;
+            if (!aMatch && bMatch) return 1;
+            return a.compareTo(b);
+        }).map(dir.root()::relativize).toList();
+        if (Files.notExists(backupFile.getParent()))
+            Files.createDirectories(backupFile.getParent());
+        SerialBundlerWriter writer;
+        if (compress) {
+            var fout = Files.newOutputStream(backupFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
+            writeMagic(fout, null, true);
+            var out = new ZstdOutputStreamNoFinalizer(fout);
+            writer = new SerialBundlerWriter(out);
+        } else {
+            var chan = FileChannel.open(backupFile);
+            writer = new SerialBundlerWriter(chan, 0);
+            writeMagic(null, chan, false);
+        }
+        // todo parallelism
+        writeEntries(sortedEntries, writer, allowGunzip && compress);
+        writer.close();
+
+        try (var backup = FileChannel.open(backupFile, StandardOpenOption.WRITE, StandardOpenOption.APPEND);
+             var bos = new ByteArrayOutputStream();
+             var metadata = new EntryOutputStream(new ZstdOutputStreamNoFinalizer(bos))) {
+            var entries = writer.getEntries();
+            for (BundlerEntry entry : entries) {
+                metadata.write(entry);
+            }
+            metadata.flush();
+            var result = bos.toByteArray();
+            var buffer = ByteBuffer.allocate(result.length +4 + 4);
+            buffer.put(result);
+            buffer.putInt(entries.size());
+            buffer.putInt(result.length);
+            writeFully(backup, buffer);
+        }
+    }
+
+
+    private static void writeMagic(OutputStream out, FileChannel channel, boolean compress) throws IOException {
+        var magic = compress ? Bundle.COMPRESSED_MAGIC : Bundle.MAGIC;
+        if (channel != null) {
+            channel.write(ByteBuffer.allocate(4).putInt(magic).flip());
+        } else {
+            out.write(ByteBuffer.allocate(4).putInt(magic).array());
+        }
+    }
+
+    private void writeFully(FileChannel chan, ByteBuffer buf) throws IOException {
+        buf.flip();
+        while (buf.hasRemaining()) {
+            chan.write(buf);
+        }
+    }
+
+    private void checkCondition() {
+        if (allowParallel) {
+            Validate.isTrue(!compress && !allowGunzip);
+        }
+    }
+
+    private void writeEntries(
+            List<Path> sortedEntries,
+            SerialBundlerWriter writer,
+            boolean allowGunzip) throws IOException {
+        for (Path sortedEntry : sortedEntries) {
+            var fileName = sortedEntry.getFileName();
+            var shouldGunzip = allowGunzip && (fileName.endsWith("mca") || fileName.endsWith("dat"));
+            if (shouldGunzip) {
+                writer.writeGunzip(dir.root().resolve(sortedEntry), sortedEntry.toString());
+            } else {
+                writer.writePlain(dir.root().resolve(sortedEntry), sortedEntry.toString());
+            }
+        }
+    }
+
+}

src/main/java/io/ib67/sfcraft/bundler/BundlerEntry.java

@@ -6,5 +6,4 @@ public record BundlerEntry(
         long fileLength,
         long offset
 ) {
-    public static final int MAGIC = 0xBACEBACE;
 }

src/main/java/io/ib67/sfcraft/bundler/EntryInputStream.java

@@ -24,7 +24,7 @@ public class EntryInputStream extends FilterInputStream {
         var gunzipped = len < 0;
         len = Math.abs(len);
         var name = new String(readNBytes(len)).intern();
-        var buf = ByteBuffer.wrap(readNBytes(8));
+        var buf = ByteBuffer.wrap(readNBytes(8 + 8));
         var fileLen = buf.getLong();
         var fileOffset = buf.getLong();
         return new BundlerEntry(name, gunzipped, fileLen, fileOffset);

src/main/java/io/ib67/sfcraft/bundler/EntryOutputStream.java

@@ -20,10 +20,6 @@ public class EntryOutputStream extends FilterOutputStream {
         super(out);
     }
 
-    public void writeMagic() throws IOException {
-        out.write(BundlerEntry.MAGIC);
-    }
-
     public void write(BundlerEntry entry) throws IOException {
         var fileName = entry.fileName().getBytes(StandardCharsets.UTF_8);
         var buf = ByteBuffer.allocate(4).putInt(entry.unGzipped() ? fileName.length * -1 : fileName.length);

src/main/java/io/ib67/sfcraft/bundler/Main.java

@@ -0,0 +1,56 @@
+package io.ib67.sfcraft.bundler;
+
+import io.ib67.kiwi.ArgOpts;
+import io.ib67.sfcraft.WorldDir;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public class Main {
+    public static void main(String[] args) {
+        var opts = ArgOpts.builder()
+                .args(args).description("Tools for .jpack format bundle").programName("jpack").build();
+        var in = opts.string("in", "Input. Can be directory or jpack bundle", null);
+        var out = opts.string("out", "Output. Can be jpack bundle or directory", null);
+        var help = opts.bool("help", false);
+        var allowGunzip = opts.bool("gunzip",
+                "Should we recompress NBT files for maximum compression rate (depends on compress)", true);
+        var compress = opts.bool("compress", "Should we compress?", true);
+        var parallel = opts.bool("parallel", "Use parallelized bundler. This conflicts with compress options", false);
+        if (help) {
+            opts.printHelp(System.out);
+            return;
+        }
+        if (allowGunzip && !compress) {
+            System.out.println("allowGunzip depends on compress feature.");
+            return;
+        }
+        if (in == null || out == null) {
+            System.out.println("Both in and out should be provided");
+            return;
+        }
+        var inPath = Path.of(in);
+        var outPath = Path.of(out);
+        if (!Files.exists(inPath)) {
+            System.out.println("The input isn't exists.");
+            return;
+        }
+        if (Files.exists(outPath)) {
+            System.out.println("The output is already exists. Please move it to other places");
+            return;
+        }
+        if (in.endsWith(".jpack")) {
+            Bundle.unbundleFile(inPath, outPath);
+        } else if (out.endsWith(".jpack")) {
+            if (!Files.isDirectory(inPath))
+                throw new IllegalArgumentException("Your input should be a world directory.");
+            new Bundle(new WorldDir(inPath))
+                    .allowGunzip(allowGunzip)
+                    .compress(compress)
+                    .allowParallel(parallel)
+                    .buildBundle(outPath);
+        } else {
+            System.out.println("Invalid arguments. You should provide a pair of jpack and directory to act as in/output.");
+        }
+    }
+}

src/main/java/io/ib67/sfcraft/bundler/SerialBundlerReader.java

@@ -19,31 +19,37 @@ public class SerialBundlerReader implements Closeable {
     protected final EntryInputStream metadataIn;
     protected final byte[] buffer = new byte[4096];
     protected boolean entryRead;
+    protected int entries;
 
-    public SerialBundlerReader(InputStream in, InputStream metadataIn) {
-        this.in = Objects.requireNonNull(in);
-        this.metadataIn = new EntryInputStream(Objects.requireNonNull(metadataIn));
-        inChannel = null;
-    }
+//    public SerialBundlerReader(InputStream in, InputStream metadataIn) {
+//        this.in = Objects.requireNonNull(in);
+//        this.metadataIn = new EntryInputStream(Objects.requireNonNull(metadataIn));
+//        inChannel = null;
+//    }
 
     public SerialBundlerReader(Path path) throws IOException {
         var raf = new RandomAccessFile(path.toFile(), "r");
         var totalLen = Files.size(path);
+        var magic = raf.readInt();
+        var compressed = magic == Bundle.COMPRESSED_MAGIC;
+        if (!compressed && magic != Bundle.MAGIC) throw new IOException("Invalid bundle file");
+
+        // read length
         raf.seek(totalLen - 4);
         var metadataLength = raf.readInt();
-        var metadataBegin = totalLen - 4 - metadataLength;
+        raf.seek(totalLen - 8);
+        entries = raf.readInt();
+        // read metadata
+        var metadataBegin = totalLen - 8 - metadataLength;
         var metadataBytes = new byte[metadataLength];
         raf.seek(metadataBegin);
         raf.readFully(metadataBytes);
 
         this.metadataIn = new EntryInputStream(new ZstdInputStreamNoFinalizer(new ByteArrayInputStream(metadataBytes)));
-        if (ByteBuffer.wrap(metadataIn.readNBytes(4)).getInt() != BundlerEntry.MAGIC) {
-            throw new IllegalArgumentException("Invalid Bundler entry");
-        }
         // check compressed
-        raf.seek(0);
+        raf.seek(4);
         var fis = Channels.newInputStream(raf.getChannel());
-        if (raf.readInt() == Zstd.magicNumber()) {
+        if (compressed) {
             in = new ZstdInputStreamNoFinalizer(fis);
             inChannel = null;
         } else {
@@ -54,12 +60,14 @@ public class SerialBundlerReader implements Closeable {
 
     @SneakyThrows
     public boolean hasEntry() {
-        return metadataIn.available() > 0;
+        return entries > 0;
     }
 
     public BundlerEntry readEntry() throws IOException {
+        if (!hasEntry()) throw new IOException("All entries are read");
         var entry = metadataIn.readEntry();
         entryRead = true;
+        entries--;
         return entry;
     }
 

src/main/java/io/ib67/sfcraft/bundler/SerialBundlerWriter.java

@@ -13,6 +13,7 @@ import java.nio.channels.FileChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 import java.util.zip.GZIPInputStream;
@@ -28,12 +29,14 @@ public class SerialBundlerWriter implements Closeable {
     public SerialBundlerWriter(OutputStream out) {
         Objects.requireNonNull(out);
         this.out = out;
+        this.entries = new ArrayList<>();
         channel = null;
     }
 
     public SerialBundlerWriter(FileChannel channel, long offset) {
         Objects.requireNonNull(channel);
         this.offset = offset;
+        this.entries = new ArrayList<>();
         this.channel = channel;
         out = null;
     }
@@ -50,11 +53,13 @@ public class SerialBundlerWriter implements Closeable {
     public void writePlain(Path path, String fileName) throws IOException {
         if (channel != null) {
             writeZC(path, fileName);
-            return;
-        }
+        } else {
+            var current = offset;
             try (var in = Files.newInputStream(path)) {
                 in.transferTo(out);
             }
+            entries.add(new BundlerEntry(fileName, false, Files.size(path), current));
+        }
     }
 
     private void writeZC(Path path, String fileName) throws IOException {

src/main/java/io/ib67/sfcraft/config/JustBackupConfig.java

@@ -7,8 +7,9 @@ import java.util.List;
 public record JustBackupConfig(
         long backupIntervalMinutes,
         int keepBackups,
-        boolean allowParallel,
+        boolean allowGunzip,
         boolean compress,
+        String temporaryBackupDir,
         StorageOption option
 ) {
     public JustBackupConfig {

src/main/java/io/ib67/sfcraft/config/StorageOption.java

@@ -1,5 +1,7 @@
 package io.ib67.sfcraft.config;
 
+import org.apache.commons.lang3.Validate;
+
 public sealed interface StorageOption {
     String type();
 
@@ -8,20 +10,32 @@ public sealed interface StorageOption {
             String accessKey,
             String secretKey,
             String endpoint,
-            String bucket
+            String bucket,
+            String prefix,
+            boolean enforcePathStyle,
+            int retryAmount
     ) implements StorageOption {
         @Override
         public String type() {
             return "s3";
         }
+
+        public S3 {
+            Validate.isTrue(retryAmount >= 0);
+        }
     }
 
     record Local(
-            String saveDir
+            String saveDir,
+            long diskSizeQuotaBytes
     ) implements StorageOption {
         @Override
         public String type() {
             return "local";
         }
+
+        public Local {
+            Validate.isTrue(diskSizeQuotaBytes >= 0);
+        }
     }
 }

src/main/java/io/ib67/sfcraft/strategy/BackupStrategy.java

@@ -2,16 +2,20 @@ package io.ib67.sfcraft.strategy;
 
 import io.ib67.sfcraft.Backup;
 import io.ib67.sfcraft.WorldDir;
+import io.ib67.sfcraft.bundler.Bundle;
 
 import java.nio.file.Path;
 
 public interface BackupStrategy {
     /**
      * create a backup
-     * @param dir world to be bundled
      * @return a key to this backup
      */
-    Backup createBackup(WorldDir dir, boolean compress);
+    Backup createBackup(Path source, Path bundle);
 
     void recoverBackup(Backup backup, Path restorePath);
+
+    void deleteBackup(Backup backup);
+
+    boolean isAvailable();
 }

src/main/java/io/ib67/sfcraft/strategy/BackupTracker.java

@@ -0,0 +1,44 @@
+package io.ib67.sfcraft.strategy;
+
+import io.ib67.sfcraft.Backup;
+import lombok.Getter;
+import lombok.RequiredArgsConstructor;
+
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+
+public class BackupTracker implements BackupStrategy {
+    protected final BackupStrategy underlying;
+    @Getter
+    protected final Map<String, Backup> trackedBackups;
+
+    public BackupTracker(BackupStrategy underlying, Map<String, Backup> trackedBackups) {
+        this.underlying = Objects.requireNonNull(underlying);
+        this.trackedBackups = trackedBackups == null ? new HashMap<>() : trackedBackups;
+    }
+
+    @Override
+    public Backup createBackup(Path from, Path pathToBundle) {
+        var result = underlying.createBackup(from, pathToBundle);
+        trackedBackups.put(result.backupKey(), result);
+        return result;
+    }
+
+    @Override
+    public void recoverBackup(Backup backup, Path restorePath) {
+        underlying.recoverBackup(backup, restorePath);
+    }
+
+    @Override
+    public void deleteBackup(Backup backup) {
+        underlying.deleteBackup(backup);
+        trackedBackups.remove(backup.backupKey());
+    }
+
+    @Override
+    public boolean isAvailable() {
+        return underlying.isAvailable();
+    }
+}

src/main/java/io/ib67/sfcraft/strategy/LocalBackupStrategy.java

@@ -4,14 +4,12 @@ import com.github.luben.zstd.Zstd;
 import com.github.luben.zstd.ZstdOutputStreamNoFinalizer;
 import io.ib67.sfcraft.Backup;
 import io.ib67.sfcraft.WorldDir;
-import io.ib67.sfcraft.bundler.BundlerEntry;
-import io.ib67.sfcraft.bundler.EntryOutputStream;
-import io.ib67.sfcraft.bundler.SerialBundlerReader;
-import io.ib67.sfcraft.bundler.SerialBundlerWriter;
+import io.ib67.sfcraft.bundler.*;
 import io.ib67.sfcraft.config.StorageOption;
 import lombok.RequiredArgsConstructor;
 import lombok.SneakyThrows;
 import lombok.extern.log4j.Log4j2;
+import org.apache.commons.lang3.Validate;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -25,67 +23,27 @@ import java.time.format.DateTimeFormatter;
 import java.util.*;
 
 @Log4j2
-@RequiredArgsConstructor
 public class LocalBackupStrategy implements BackupStrategy {
     protected final StorageOption.Local option;
-    protected final Path backupParentRoot = Path.of(option.saveDir());
-    protected final boolean allowGunzip;
+    protected final Path backupParentRoot;
+
+    @SneakyThrows
+    public LocalBackupStrategy(StorageOption.Local option) {
+        this.option = option;
+        backupParentRoot = Path.of(option.saveDir());
+        Files.createDirectories(backupParentRoot);
+    }
 
     @Override
     @SneakyThrows
-    public Backup createBackup(WorldDir dir, boolean compress) {
-        var sortedEntries = dir.everything().stream().sorted((a, b) -> {
-            boolean aMatch = a.endsWith(".mca") || a.endsWith(".dat");
-            boolean bMatch = b.endsWith(".mca") || b.endsWith(".dat");
-
-            if (aMatch && !bMatch) return -1;
-            if (!aMatch && bMatch) return 1;
-            return a.compareTo(b);
-        }).map(dir.root()::relativize).toList();
-        var backupName = "Backup_" + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now());
-        var backupFile = backupParentRoot.resolve(backupName + ".jbp");
-        SerialBundlerWriter writer;
-        if (compress) {
-            var out = new ZstdOutputStreamNoFinalizer(Files.newOutputStream(backupFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE));
-            writer = new SerialBundlerWriter(out);
-        } else {
-            var chan = FileChannel.open(backupFile);
-            writer = new SerialBundlerWriter(chan, 0);
+    public Backup createBackup(Path from, Path pathToBundle) {
+        var result = new Backup(
+                pathToBundle.getFileName().toString(),
+                pathToBundle.getFileName().toString(),
+                "local", from.toString(), false, Files.size(pathToBundle));
+        Files.move(pathToBundle, backupParentRoot.resolve(pathToBundle.getFileName()));
+        return result;
     }
-        writeEntries(sortedEntries, writer, allowGunzip && compress);
-        writer.close();
-
-        try (var backup = FileChannel.open(backupFile);
-             var bos = new ByteArrayOutputStream();
-             var metadata = new EntryOutputStream(new ZstdOutputStreamNoFinalizer(bos))) {
-            metadata.writeMagic();
-            for (BundlerEntry entry : writer.getEntries()) {
-                metadata.write(entry);
-            }
-            var result = bos.toByteArray();
-            var buffer = ByteBuffer.allocate(result.length + 4);
-            buffer.put(result);
-            buffer.putLong(result.length);
-            backup.write(buffer);
-        }
-        return new Backup(backupName, backupFile.getFileName().toString(), false, Files.size(backupFile));
-    }
-
-    private void writeEntries(
-            List<Path> sortedEntries,
-            SerialBundlerWriter writer,
-            boolean allowGunzip) throws IOException {
-        for (Path sortedEntry : sortedEntries) {
-            var fileName = sortedEntry.getFileName();
-            var shouldGunzip = allowGunzip && (fileName.endsWith("mca") || fileName.endsWith("dat"));
-            if (shouldGunzip) {
-                writer.writeGunzip(sortedEntry, sortedEntry.toString());
-            } else {
-                writer.writePlain(sortedEntry, sortedEntry.toString());
-            }
-        }
-    }
-
 
     @Override
     @SneakyThrows
@@ -93,12 +51,33 @@ public class LocalBackupStrategy implements BackupStrategy {
         // check if it is compressed
         var backupFile = backupParentRoot.resolve(backup.backupKey());
         if (Files.notExists(backupFile)) throw new IllegalArgumentException("Backup " + backupFile + " does not exist");
-        try (var bundlerReader = new SerialBundlerReader(backupFile)) {
-            while (bundlerReader.hasEntry()) {
-                var entry = bundlerReader.readEntry();
-                var target = restorePath.resolve(entry.fileName());
-                bundlerReader.writeTo(FileChannel.open(target), entry);
+        Bundle.unbundleFile(backupFile, restorePath);
+    }
+
+    @SneakyThrows
+    @Override
+    public void deleteBackup(Backup backup) {
+        Validate.isTrue("local".equals(backup.backupKey()));
+        var bundlePath = backupParentRoot.resolve(backup.backupKey());
+        Files.deleteIfExists(bundlePath);
+    }
+
+    @Override
+    @SneakyThrows
+    public boolean isAvailable() {
+        if (option.diskSizeQuotaBytes() == 0) return true;
+        try (var f = Files.walk(backupParentRoot)) {
+            if (f.mapToLong(it->{
+                try {
+                    return Files.size(it);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+            }).sum() >= option.diskSizeQuotaBytes()) {
+                log.warn("Have no space to set another backup! ");
+                return false;
             }
         }
+        return true;
     }
 }

src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java

@@ -0,0 +1,104 @@
+package io.ib67.sfcraft.strategy;
+
+import io.ib67.sfcraft.Backup;
+import io.ib67.sfcraft.WorldDir;
+import io.ib67.sfcraft.bundler.Bundle;
+import io.ib67.sfcraft.config.StorageOption;
+import io.minio.BucketExistsArgs;
+import io.minio.MinioClient;
+import io.minio.RemoveObjectArgs;
+import io.minio.UploadObjectArgs;
+import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
+import lombok.extern.log4j.Log4j2;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+@Log4j2
+public class S3BackupStrategy implements BackupStrategy {
+    protected final StorageOption.S3 option;
+    protected final MinioClient minio;
+    protected final AtomicBoolean uploading = new AtomicBoolean(false);
+
+    @SneakyThrows
+    public S3BackupStrategy(StorageOption.S3 option) {
+        this.option = option;
+        this.minio = MinioClient.builder()
+                .endpoint(URI.create(option.endpoint()).toURL())
+                .credentials(option.accessKey(), option.secretKey())
+                .region(option.region())
+                .build();
+        if (option.enforcePathStyle()) {
+            minio.disableVirtualStyleEndpoint();
+        } else {
+            minio.enableVirtualStyleEndpoint();
+        }
+        log.info("Testing S3 credentials");
+        var exists = minio.bucketExists(BucketExistsArgs.builder().bucket(option.bucket()).build());
+        if (!exists) throw new IllegalStateException("Bucket '" + option.bucket() + "' does not exist");
+    }
+
+    @Override
+    @SneakyThrows
+    public Backup createBackup(Path from, Path pathToBundle) {
+        if (!uploading.compareAndSet(false, true)) {
+            throw new IllegalStateException("Last upload is not done yet! keep waiting...");
+        }
+        int i = 0;
+        do {
+            try {
+                var object = option.prefix() + "/" + pathToBundle.getFileName().toString();
+                minio.uploadObject(UploadObjectArgs.builder()
+                        .contentType("application/octet-stream")
+                        .object(object)
+                        .filename(pathToBundle.toString())
+                        .build());
+                //todo also calculate sha?
+                return new Backup(pathToBundle.getFileName().toString(),
+                        object, "s3",
+                        from.toString(),
+                        false,
+                        Files.size(pathToBundle));
+            } catch (IOException e) {
+                if (i >= option.retryAmount()) {
+                    throw new IllegalStateException("Cannot upload " + pathToBundle + " to S3");
+                }
+                var nextTry = Math.pow(2, i++);
+                log.error("Error uploading S3. trying in {}s later. ({}/{})", nextTry, i, option.retryAmount(), e);
+                Thread.sleep((long) (nextTry * 1000));
+            } catch (Exception e) {
+                throw new IllegalStateException("Unrecoverable exception occurred when uploading " + pathToBundle, e);
+            } finally {
+                uploading.set(false);
+            }
+        } while (true);
+    }
+
+    @Override
+    public void recoverBackup(Backup backup, Path restorePath) {
+        //todo http range
+    }
+
+    @Override
+    @SneakyThrows
+    public void deleteBackup(Backup backup) {
+        minio.removeObject(RemoveObjectArgs.builder()
+                        .bucket(backup.backupKey()).build());
+    }
+
+    @Override
+    public boolean isAvailable() {
+        if (uploading.get()) {
+            log.warn("Last S3 Backup is still uploading!");
+            return false;
+        }
+        return true;
+    }
+}