From fb42be81550926b81ed691ed2ae4167b94633c01 Mon Sep 17 00:00:00 2001 From: iceBear67 Date: Mon, 22 Sep 2025 17:20:48 +0800 Subject: [PATCH] Refactor backup system and migrate from submodule to shadow dependencies - Remove s3-lite submodule and replace with AWS SDK and shadow plugin - Implement comprehensive backup commands with list, delete, create, restore, and suspend functionality - Add bundle format with compression support using Zstd and gunzip optimization - Implement local and S3 backup strategies with configurable options - Add backup tracking system with JSON persistence - Improve world directory handling and file exclusion logic - Add standalone bundler tool for bundle creation/extraction - Update Gradle configuration with shadow jar support and dependency management - Enhance configuration system with additional options for compression and temporary directories --- build.gradle | 124 +++++++++------- gradle.properties | 6 +- src/main/java/io/ib67/sfcraft/Backup.java | 2 + .../java/io/ib67/sfcraft/BackupCommands.java | 106 ++++++++++++++ .../java/io/ib67/sfcraft/BackupManager.java | 14 -- .../java/io/ib67/sfcraft/BackupWorker.java | 33 ++++- .../java/io/ib67/sfcraft/JustBackupMod.java | 111 +++++++++++++-- src/main/java/io/ib67/sfcraft/WorldDir.java | 8 +- .../java/io/ib67/sfcraft/bundler/Bundle.java | 133 ++++++++++++++++++ .../io/ib67/sfcraft/bundler/BundlerEntry.java | 1 - .../sfcraft/bundler/EntryInputStream.java | 2 +- .../sfcraft/bundler/EntryOutputStream.java | 4 - .../java/io/ib67/sfcraft/bundler/Main.java | 56 ++++++++ .../sfcraft/bundler/SerialBundlerReader.java | 32 +++-- .../sfcraft/bundler/SerialBundlerWriter.java | 13 +- .../ib67/sfcraft/config/JustBackupConfig.java | 3 +- .../io/ib67/sfcraft/config/StorageOption.java | 18 ++- .../ib67/sfcraft/strategy/BackupStrategy.java | 8 +- .../ib67/sfcraft/strategy/BackupTracker.java | 44 ++++++ .../sfcraft/strategy/LocalBackupStrategy.java | 107 ++++++-------- .../sfcraft/strategy/S3BackupStrategy.java | 104 ++++++++++++++ 21 files changed, 746 insertions(+), 183 deletions(-) create mode 100644 src/main/java/io/ib67/sfcraft/BackupCommands.java delete mode 100644 src/main/java/io/ib67/sfcraft/BackupManager.java create mode 100644 src/main/java/io/ib67/sfcraft/bundler/Bundle.java create mode 100644 src/main/java/io/ib67/sfcraft/bundler/Main.java create mode 100644 src/main/java/io/ib67/sfcraft/strategy/BackupTracker.java create mode 100644 src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java diff --git a/build.gradle b/build.gradle index 60828dd..6ffcfe8 100644 --- a/build.gradle +++ b/build.gradle @@ -1,94 +1,112 @@ plugins { - id 'fabric-loom' version "${loom_version}" - id 'maven-publish' - id("io.freefair.lombok") version "9.0.0-rc2" - + id 'fabric-loom' version "${loom_version}" + id 'maven-publish' + id("io.freefair.lombok") version "9.0.0-rc2" + id("com.gradleup.shadow") version '9.1.0' } version = project.mod_version group = project.maven_group base { - archivesName = project.archives_base_name + archivesName = project.archives_base_name } repositories { - // Add repositories to retrieve artifacts from in here. - // You should only use this when depending on other mods because - // Loom adds the essential maven repositories to download Minecraft and libraries from automatically. - // See https://docs.gradle.org/current/userguide/declaring_repositories.html - // for more information about repositories. + // Add repositories to retrieve artifacts from in here. + // You should only use this when depending on other mods because + // Loom adds the essential maven repositories to download Minecraft and libraries from automatically. + // See https://docs.gradle.org/current/userguide/declaring_repositories.html + // for more information about repositories. + maven { + name = "Kalculos" + url = "https://repo.sfclub.cc/snapshots" + } } loom { - splitEnvironmentSourceSets() + splitEnvironmentSourceSets() - mods { - "modid" { - sourceSet sourceSets.main - sourceSet sourceSets.client - } - } + mods { + "justbackup" { + sourceSet sourceSets.main + sourceSet sourceSets.client + } + } } dependencies { - // To change the versions see the gradle.properties file - minecraft "com.mojang:minecraft:${project.minecraft_version}" - mappings "net.fabricmc:yarn:${project.yarn_mappings}:v2" - modImplementation "net.fabricmc:fabric-loader:${project.loader_version}" + // To change the versions see the gradle.properties file + minecraft "com.mojang:minecraft:${project.minecraft_version}" + mappings "net.fabricmc:yarn:${project.yarn_mappings}:v2" + modImplementation "net.fabricmc:fabric-loader:${project.loader_version}" - // Fabric API. This is technically optional, but you probably want it anyway. - modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}" - implementation("io.minio:minio:8.5.17") - implementation 'com.github.luben:zstd-jni:1.5.7-4' + // Fabric API. This is technically optional, but you probably want it anyway. + modImplementation "net.fabricmc.fabric-api:fabric-api:${project.fabric_version}" + // https://mvnrepository.com/artifact/software.amazon.awssdk/s3 + shadow 'software.amazon.awssdk:s3:2.34.0' + shadow("io.ib67.kiwi:lang:1.2.1-SNAPSHOT") + shadow 'com.github.luben:zstd-jni:1.5.7-4' +} +configurations.implementation.extendsFrom(configurations.shadow) +shadowJar { + configurations = [project.configurations.shadow] + exclude("META-INF") + //relocate 'com.google', 'sfcraft.relocation.google' } processResources { - inputs.property "version", project.version + inputs.property "version", project.version - filesMatching("fabric.mod.json") { - expand "version": inputs.properties.version - } + filesMatching("fabric.mod.json") { + expand "version": inputs.properties.version + } } tasks.withType(JavaCompile).configureEach { - it.options.release = 21 + it.options.release = 21 } java { - // Loom will automatically attach sourcesJar to a RemapSourcesJar task and to the "build" task - // if it is present. - // If you remove this line, sources will not be generated. - withSourcesJar() + // Loom will automatically attach sourcesJar to a RemapSourcesJar task and to the "build" task + // if it is present. + // If you remove this line, sources will not be generated. + withSourcesJar() - sourceCompatibility = JavaVersion.VERSION_21 - targetCompatibility = JavaVersion.VERSION_21 + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 } jar { - inputs.property "archivesName", project.base.archivesName + inputs.property "archivesName", project.base.archivesName + manifest { + attributes(Map.of("Main-Class", "io.ib67.sfcraft.bundler.Main")) + } + from("LICENSE") { + rename { "${it}_${inputs.properties.archivesName}" } + } +} - from("LICENSE") { - rename { "${it}_${inputs.properties.archivesName}"} - } +remapJar { + input = tasks.shadowJar.archiveFile } // configure the maven publication publishing { - publications { - create("mavenJava", MavenPublication) { - artifactId = project.archives_base_name - from components.java - } - } + publications { + create("mavenJava", MavenPublication) { + artifactId = project.archives_base_name + from components.java + } + } - // See https://docs.gradle.org/current/userguide/publishing_maven.html for information on how to set up publishing. - repositories { - // Add repositories to publish to here. - // Notice: This block does NOT have the same function as the block in the top level. - // The repositories here will be used for publishing your artifact, not for - // retrieving dependencies. - } + // See https://docs.gradle.org/current/userguide/publishing_maven.html for information on how to set up publishing. + repositories { + // Add repositories to publish to here. + // Notice: This block does NOT have the same function as the block in the top level. + // The repositories here will be used for publishing your artifact, not for + // retrieving dependencies. + } } \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index bf96b0c..13c286d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,5 @@ # Done to increase the memory available to gradle. -org.gradle.jvmargs=-Xmx1G +org.gradle.jvmargs=-Xmx4G org.gradle.parallel=true # Fabric Properties @@ -11,8 +11,8 @@ loom_version=1.11-SNAPSHOT # Mod Properties mod_version=1.0.0 -maven_group=com.example -archives_base_name=modid +maven_group=io.ib67.sfcraft +archives_base_name=justbackup # Dependencies fabric_version=0.129.0+1.21.8 \ No newline at end of file diff --git a/src/main/java/io/ib67/sfcraft/Backup.java b/src/main/java/io/ib67/sfcraft/Backup.java index 9177022..ce4704a 100644 --- a/src/main/java/io/ib67/sfcraft/Backup.java +++ b/src/main/java/io/ib67/sfcraft/Backup.java @@ -3,6 +3,8 @@ package io.ib67.sfcraft; public record Backup( String name, String backupKey, + String type, + String from, boolean incremental, long sizeTotal ) { diff --git a/src/main/java/io/ib67/sfcraft/BackupCommands.java b/src/main/java/io/ib67/sfcraft/BackupCommands.java new file mode 100644 index 0000000..b151d64 --- /dev/null +++ b/src/main/java/io/ib67/sfcraft/BackupCommands.java @@ -0,0 +1,106 @@ +package io.ib67.sfcraft; + +import com.mojang.brigadier.Command; +import com.mojang.brigadier.context.CommandContext; +import net.minecraft.server.command.ServerCommandSource; +import net.minecraft.text.ClickEvent; +import net.minecraft.text.Text; + +import java.awt.*; +import java.nio.file.Files; +import java.nio.file.Path; + +public class BackupCommands { + protected final JustBackupMod mod; + protected Runnable restoreIssued; + + public BackupCommands(JustBackupMod mod) { + this.mod = mod; + } + + public int cmdList(CommandContext ctx) { + var s = ctx.getSource(); + s.sendFeedback(() -> Text.of("Tracked backups:"), false); + for (Backup value : mod.tracker.getTrackedBackups().values()) { + var restoreClick = Text.literal("[RESTORE]").withColor(Color.GREEN.getRGB()) + .styled(it -> it.withClickEvent(new ClickEvent.SuggestCommand("/backup restore " + value.backupKey()))); + var deleteClick = Text.literal("[DELETE]").withColor(Color.RED.getRGB()) + .styled(it -> it.withClickEvent(new ClickEvent.SuggestCommand("/backup delete " + value.backupKey()))); + var message = Text.literal(" - " + value.name() + " (" + value.backupKey() + ") " + + (value.incremental() ? "incremental" : "full") + " ").append(restoreClick).append(" ").append(deleteClick); + s.sendMessage(message); + s.sendMessage(Text.of(" Size: " + Math.floorDiv(value.sizeTotal(), 1024 * 1024) + "MiB")); + } + return Command.SINGLE_SUCCESS; + } + + public int cmdDelete(CommandContext ctx) { + var _backup = ctx.getArgument("backupName", String.class); + var backup = mod.tracker.getTrackedBackups().get(_backup); + mod.tracker.deleteBackup(backup); + ctx.getSource().sendMessage(Text.of("Backup " + _backup + " is deleted successfully.")); + return Command.SINGLE_SUCCESS; + } + + public int cmdIssueBackup(CommandContext ctx) { + var s = ctx.getSource(); + if (JustBackupMod.PERFORMING_BACKUP.get()) { + s.sendMessage(Text.of("You can't issue a backup because another backup is in progress.")); + return Command.SINGLE_SUCCESS; + } + mod.issueBackup() + .whenComplete((backup, error) -> { + if (error != null) { + s.sendMessage(Text.of("Cannot backup, error: " + error.getMessage())); + } else { + s.sendMessage(Text.of("Backup created successfully! name: " + backup.name())); + } + }); + return Command.SINGLE_SUCCESS; + } + + public int cmdSuspend(CommandContext ctx) { + mod.suspend = !mod.suspend; + ctx.getSource().sendMessage(Text.of("Backup disabled: " + mod.suspend)); + return Command.SINGLE_SUCCESS; + } + + public int cmdRestore(CommandContext ctx) { + var s = ctx.getSource(); + var pm = s.getServer().getPlayerManager(); + var _backup = ctx.getArgument("backupName", String.class); + var backup = mod.tracker.getTrackedBackups().get(_backup); + if (backup == null) { + s.sendMessage(Text.of("Invalid backup " + _backup)); + return 0; + } + if (restoreIssued != null) { + s.sendMessage(Text.of("Another restoration is in progress. try /backup cancel to cancel it")); + return 0; + } + s.sendMessage(Text.literal("Please restart your server to take changes")); + restoreIssued = () -> { + System.out.println("Recovering backup... "); + try{ + var toOverride = Path.of(backup.from()); + var old = Path.of(backup.from()+"_old"); + Files.deleteIfExists(old); + Files.move(toOverride, old); + System.out.println("Moved your old save to "+old); + mod.tracker.recoverBackup(backup, toOverride); + System.out.println("Recover successfully!"); + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + return Command.SINGLE_SUCCESS; + } + + public int cmdHelp(CommandContext serverCommandSourceCommandContext) { + return 0; + } + + public int cmdCancelRestore(CommandContext serverCommandSourceCommandContext) { + return 0; + } +} diff --git a/src/main/java/io/ib67/sfcraft/BackupManager.java b/src/main/java/io/ib67/sfcraft/BackupManager.java deleted file mode 100644 index 14ca26a..0000000 --- a/src/main/java/io/ib67/sfcraft/BackupManager.java +++ /dev/null @@ -1,14 +0,0 @@ -package io.ib67.sfcraft; - -import java.util.List; - -public class BackupManager { - public List getAllBackup() { - } - - public void deleteBackup(Backup backup) { - } - - public Backup performBackup() { - } -} \ No newline at end of file diff --git a/src/main/java/io/ib67/sfcraft/BackupWorker.java b/src/main/java/io/ib67/sfcraft/BackupWorker.java index a929207..5b34983 100644 --- a/src/main/java/io/ib67/sfcraft/BackupWorker.java +++ b/src/main/java/io/ib67/sfcraft/BackupWorker.java @@ -1,27 +1,34 @@ package io.ib67.sfcraft; +import io.ib67.sfcraft.bundler.Bundle; +import io.ib67.sfcraft.config.JustBackupConfig; import io.ib67.sfcraft.mixin.LevelStorageAccessor; import io.ib67.sfcraft.strategy.BackupStrategy; import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; import net.minecraft.server.MinecraftServer; +import java.nio.file.Files; import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.concurrent.Callable; +import java.util.function.Supplier; import static io.ib67.sfcraft.JustBackupMod.PERFORMING_BACKUP; @Log4j2 public record BackupWorker( MinecraftServer server, - BackupManager manager -) implements Runnable { + BackupStrategy strategy, + JustBackupConfig config +) implements Supplier { @Override @SneakyThrows - public void run() { + public Backup get() { if (server == null) { - log.warn("MinecraftServer is not initialized yet, not doing backup"); - return; + throw new IllegalStateException("MinecraftServer is not initialized yet, not doing backup"); } int i = 0; while (!PERFORMING_BACKUP.compareAndSet(false, true)) { @@ -36,8 +43,20 @@ public record BackupWorker( } var session = ((LevelStorageAccessor) server).getSession(); var path = session.getDirectory().getRootPath(); - manager.performBackup(); - strategy.createBackup(new WorldDir(Path.of(path))); + var backupName = "Backup_" + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now()); + var backupParentRoot = Path.of(config.temporaryBackupDir()); + if (Files.notExists(backupParentRoot)) Files.createDirectories(backupParentRoot); + var backupFile = backupParentRoot.resolve(backupName + ".jpack"); + log.info("Creating bundle {}", backupName); + //todo diff + Path worldDir = Path.of(path); + new Bundle(new WorldDir(worldDir)) + .compress(config.compress()) + .allowGunzip(config.allowGunzip()) + .buildBundle(backupFile); + var backup = strategy.createBackup(worldDir, backupFile); + Files.deleteIfExists(backupFile); + return backup; } finally { PERFORMING_BACKUP.set(false); } diff --git a/src/main/java/io/ib67/sfcraft/JustBackupMod.java b/src/main/java/io/ib67/sfcraft/JustBackupMod.java index 1170c00..7b5a180 100644 --- a/src/main/java/io/ib67/sfcraft/JustBackupMod.java +++ b/src/main/java/io/ib67/sfcraft/JustBackupMod.java @@ -2,36 +2,58 @@ package io.ib67.sfcraft; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import com.mojang.brigadier.CommandDispatcher; +import com.mojang.brigadier.arguments.StringArgumentType; import io.ib67.sfcraft.config.JustBackupConfig; import io.ib67.sfcraft.config.StorageOption; import io.ib67.sfcraft.config.serializer.StorageOptionSerializer; import io.ib67.sfcraft.mixin.LevelStorageAccessor; import io.ib67.sfcraft.mixin.MixinMinecraftServer; import io.ib67.sfcraft.strategy.BackupStrategy; +import io.ib67.sfcraft.strategy.BackupTracker; +import io.ib67.sfcraft.strategy.LocalBackupStrategy; +import io.ib67.sfcraft.strategy.S3BackupStrategy; import lombok.SneakyThrows; import net.fabricmc.api.ModInitializer; +import net.fabricmc.fabric.api.command.v2.CommandRegistrationCallback; import net.fabricmc.fabric.api.event.lifecycle.v1.ServerLifecycleEvents; +import net.minecraft.command.CommandRegistryAccess; import net.minecraft.server.MinecraftServer; +import net.minecraft.server.command.CommandManager; +import net.minecraft.server.command.ServerCommandSource; import net.minecraft.server.world.ServerWorld; +import net.minecraft.text.Text; +import org.jetbrains.annotations.UnknownNullability; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.FileInputStream; +import java.io.IOException; import java.io.RandomAccessFile; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.*; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import static net.minecraft.server.command.CommandManager.argument; +import static net.minecraft.server.command.CommandManager.literal; + public class JustBackupMod implements ModInitializer { public static final String MOD_ID = "justbackup"; public static final Logger LOGGER = LoggerFactory.getLogger(MOD_ID); + /** + * This state indicates: + * 1. A minecraft auto-save is in-progress. + * 2. The backup is in-progress + */ public static final AtomicBoolean PERFORMING_BACKUP = new AtomicBoolean(false); public static final Gson GSON = new GsonBuilder().setPrettyPrinting() .registerTypeAdapter(StorageOption.class, new StorageOptionSerializer( @@ -41,38 +63,95 @@ public class JustBackupMod implements ModInitializer { .create(); protected ScheduledExecutorService scheduledBackupExecutor; protected JustBackupConfig config; - protected BackupStrategy strategy; + protected BackupTracker tracker; + protected Path backupIndexPath; protected volatile MinecraftServer server; + protected volatile boolean suspend; @Override @SneakyThrows public void onInitialize() { LOGGER.info("Loading backup configuration"); + backupIndexPath = Path.of("config").resolve("just_backup_index.json"); var configPath = Path.of("config").resolve("just_backup.json"); - if (Files.notExists(configPath.getParent())) { + if (Files.notExists(configPath.getParent())) Files.createDirectories(configPath.getParent()); - } - if (Files.notExists(configPath)) { + if (Files.notExists(configPath)) saveConfig(configPath); - } + if (Files.notExists(backupIndexPath)) + Files.createFile(backupIndexPath); + config = GSON.fromJson(Files.readString(configPath), JustBackupConfig.class); - this.strategy = createStrategy(); + var strategy = createStrategy(); + this.tracker = new BackupTracker(strategy, GSON.fromJson(Files.readString(backupIndexPath), new TypeToken<>() { + })); LOGGER.info("Configuration has been loaded. Using storage option: {}", config.option().type()); scheduledBackupExecutor = Executors.newSingleThreadScheduledExecutor(); + registerHandlers(); + } + + private void registerHandlers() { ServerLifecycleEvents.SERVER_STARTED.register(server -> { LOGGER.info("Launching background threads..."); this.server = server; - scheduledBackupExecutor.scheduleAtFixedRate(new BackupWorker(server, strategy), + scheduledBackupExecutor.scheduleAtFixedRate(this::issueBackup, config.backupIntervalMinutes(), config.backupIntervalMinutes(), TimeUnit.MINUTES); }); + ServerLifecycleEvents.SERVER_STOPPED.register(server -> { + try { + Files.deleteIfExists(Path.of(config.temporaryBackupDir())); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + CommandRegistrationCallback.EVENT.register(this::registerCommand); + } + + public CompletableFuture issueBackup() { + if (suspend) { + return CompletableFuture.failedFuture(new IllegalStateException("Backup is temporarily disabled")); + } + server.getPlayerManager().broadcast(Text.literal("The server will begin the backup shortly, you may experience some lag."), false); + var worker = new BackupWorker(server, tracker, config); + return CompletableFuture.supplyAsync(worker, scheduledBackupExecutor).whenComplete(this::onBackupComplete); + } + + @SneakyThrows + private void onBackupComplete(@UnknownNullability Backup backup, @UnknownNullability Throwable throwable) { + if (throwable != null) { + server.getPlayerManager().broadcast(Text.literal("Backup failed. For administrators, please check your server console."), false); + throwable.printStackTrace(); + } else { + Files.writeString(backupIndexPath, GSON.toJson(tracker.getTrackedBackups())); + server.getPlayerManager().broadcast(Text.literal("Backup success. " + backup), false); + } + } + + private void registerCommand( + CommandDispatcher dispatcher, + CommandRegistryAccess registry, CommandManager.RegistrationEnvironment env) { + var commandHandler = new BackupCommands(this); + dispatcher.register(literal("backup") + .requires(it -> it.hasPermissionLevel(3)) + .then(literal("help").executes(commandHandler::cmdHelp)) + .then(literal("list").executes(commandHandler::cmdList)) + .then(literal("cancelrestore").executes(commandHandler::cmdCancelRestore)) + .then(literal("delete") + .then(argument("backupName", StringArgumentType.greedyString()) + .executes(commandHandler::cmdDelete))) + .then(literal("restore") + .then(argument("backupName", StringArgumentType.greedyString()) + .executes(commandHandler::cmdRestore))) + .then(literal("create").executes(commandHandler::cmdIssueBackup)) + .then(literal("suspend").executes(commandHandler::cmdSuspend)) + ); } private BackupStrategy createStrategy() { - switch (config.option()) { - case StorageOption.Local local -> { - - } - } + return switch (config.option()) { + case StorageOption.Local local -> new LocalBackupStrategy(local); + case StorageOption.S3 s3 -> new S3BackupStrategy(s3); + }; } @@ -83,8 +162,14 @@ public class JustBackupMod implements ModInitializer { 60, 5, true, - new StorageOption.Local("backups") + true, + "./.backup_cache", + new StorageOption.Local("backups", 0) ); + var path = Path.of(cfg.temporaryBackupDir()); + if (Files.notExists(path)) { + Files.createDirectories(path); + } Files.writeString(configPath, GSON.toJson(cfg)); } } \ No newline at end of file diff --git a/src/main/java/io/ib67/sfcraft/WorldDir.java b/src/main/java/io/ib67/sfcraft/WorldDir.java index 789c685..dc6d917 100644 --- a/src/main/java/io/ib67/sfcraft/WorldDir.java +++ b/src/main/java/io/ib67/sfcraft/WorldDir.java @@ -42,7 +42,9 @@ public record WorldDir( public List otherFiles() { var otherDimensions = otherDimensions(); try (var f = Files.walk(root)) { - return f.filter(it -> otherDimensions.stream().noneMatch(a -> it.startsWith(a.root))) + return f.filter(it -> Files.isRegularFile(it) + && otherDimensions.stream().noneMatch(a -> it.startsWith(a.root))) + .filter(it -> !it.getFileName().toString().contains("session.lock")) .toList(); } } @@ -50,7 +52,9 @@ public record WorldDir( @SneakyThrows public List everything() { try (var f = Files.walk(root)) { - return f.toList(); + return f.filter(Files::isRegularFile) + .filter(it -> !it.getFileName().toString().contains("session.lock")) + .toList(); } } } diff --git a/src/main/java/io/ib67/sfcraft/bundler/Bundle.java b/src/main/java/io/ib67/sfcraft/bundler/Bundle.java new file mode 100644 index 0000000..6574352 --- /dev/null +++ b/src/main/java/io/ib67/sfcraft/bundler/Bundle.java @@ -0,0 +1,133 @@ +package io.ib67.sfcraft.bundler; + +import com.github.luben.zstd.ZstdOutputStreamNoFinalizer; +import io.ib67.sfcraft.WorldDir; +import io.netty.buffer.Unpooled; +import lombok.Data; +import lombok.SneakyThrows; +import lombok.experimental.Accessors; +import org.apache.commons.lang3.Validate; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; +import java.util.List; + +@Accessors(fluent = true) +@Data +public class Bundle { + public static final int MAGIC = 0xBACEBACE; + public static final int COMPRESSED_MAGIC = 0xBACEAAAA; + protected final WorldDir dir; + protected boolean compress = true; + protected boolean allowGunzip = true; + protected boolean allowParallel = false; + + + public Bundle(WorldDir dir) { + this.dir = dir; + } + + @SneakyThrows + public static void unbundleFile(Path bundle, Path out) { + try (var bundlerReader = new SerialBundlerReader(bundle)) { + while (bundlerReader.hasEntry()) { + var entry = bundlerReader.readEntry(); + var target = out.resolve(entry.fileName()); + Files.createDirectories(target.getParent()); + bundlerReader.writeTo(FileChannel.open(target, StandardOpenOption.WRITE, + StandardOpenOption.CREATE_NEW), entry); + } + } + } + + @SneakyThrows + public void buildBundle(Path backupFile) { + checkCondition(); + var sortedEntries = dir.everything().stream().sorted((a, b) -> { + boolean aMatch = a.endsWith(".mca") || a.endsWith(".dat"); + boolean bMatch = b.endsWith(".mca") || b.endsWith(".dat"); + + if (aMatch && !bMatch) return -1; + if (!aMatch && bMatch) return 1; + return a.compareTo(b); + }).map(dir.root()::relativize).toList(); + if (Files.notExists(backupFile.getParent())) + Files.createDirectories(backupFile.getParent()); + SerialBundlerWriter writer; + if (compress) { + var fout = Files.newOutputStream(backupFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + writeMagic(fout, null, true); + var out = new ZstdOutputStreamNoFinalizer(fout); + writer = new SerialBundlerWriter(out); + } else { + var chan = FileChannel.open(backupFile); + writer = new SerialBundlerWriter(chan, 0); + writeMagic(null, chan, false); + } + // todo parallelism + writeEntries(sortedEntries, writer, allowGunzip && compress); + writer.close(); + + try (var backup = FileChannel.open(backupFile, StandardOpenOption.WRITE, StandardOpenOption.APPEND); + var bos = new ByteArrayOutputStream(); + var metadata = new EntryOutputStream(new ZstdOutputStreamNoFinalizer(bos))) { + var entries = writer.getEntries(); + for (BundlerEntry entry : entries) { + metadata.write(entry); + } + metadata.flush(); + var result = bos.toByteArray(); + var buffer = ByteBuffer.allocate(result.length +4 + 4); + buffer.put(result); + buffer.putInt(entries.size()); + buffer.putInt(result.length); + writeFully(backup, buffer); + } + } + + + private static void writeMagic(OutputStream out, FileChannel channel, boolean compress) throws IOException { + var magic = compress ? Bundle.COMPRESSED_MAGIC : Bundle.MAGIC; + if (channel != null) { + channel.write(ByteBuffer.allocate(4).putInt(magic).flip()); + } else { + out.write(ByteBuffer.allocate(4).putInt(magic).array()); + } + } + + private void writeFully(FileChannel chan, ByteBuffer buf) throws IOException { + buf.flip(); + while (buf.hasRemaining()) { + chan.write(buf); + } + } + + private void checkCondition() { + if (allowParallel) { + Validate.isTrue(!compress && !allowGunzip); + } + } + + private void writeEntries( + List sortedEntries, + SerialBundlerWriter writer, + boolean allowGunzip) throws IOException { + for (Path sortedEntry : sortedEntries) { + var fileName = sortedEntry.getFileName(); + var shouldGunzip = allowGunzip && (fileName.endsWith("mca") || fileName.endsWith("dat")); + if (shouldGunzip) { + writer.writeGunzip(dir.root().resolve(sortedEntry), sortedEntry.toString()); + } else { + writer.writePlain(dir.root().resolve(sortedEntry), sortedEntry.toString()); + } + } + } + +} diff --git a/src/main/java/io/ib67/sfcraft/bundler/BundlerEntry.java b/src/main/java/io/ib67/sfcraft/bundler/BundlerEntry.java index ec1e5a9..d6a8b1b 100644 --- a/src/main/java/io/ib67/sfcraft/bundler/BundlerEntry.java +++ b/src/main/java/io/ib67/sfcraft/bundler/BundlerEntry.java @@ -6,5 +6,4 @@ public record BundlerEntry( long fileLength, long offset ) { - public static final int MAGIC = 0xBACEBACE; } diff --git a/src/main/java/io/ib67/sfcraft/bundler/EntryInputStream.java b/src/main/java/io/ib67/sfcraft/bundler/EntryInputStream.java index e9ce306..8c5f9a4 100644 --- a/src/main/java/io/ib67/sfcraft/bundler/EntryInputStream.java +++ b/src/main/java/io/ib67/sfcraft/bundler/EntryInputStream.java @@ -24,7 +24,7 @@ public class EntryInputStream extends FilterInputStream { var gunzipped = len < 0; len = Math.abs(len); var name = new String(readNBytes(len)).intern(); - var buf = ByteBuffer.wrap(readNBytes(8)); + var buf = ByteBuffer.wrap(readNBytes(8 + 8)); var fileLen = buf.getLong(); var fileOffset = buf.getLong(); return new BundlerEntry(name, gunzipped, fileLen, fileOffset); diff --git a/src/main/java/io/ib67/sfcraft/bundler/EntryOutputStream.java b/src/main/java/io/ib67/sfcraft/bundler/EntryOutputStream.java index 3672204..def6289 100644 --- a/src/main/java/io/ib67/sfcraft/bundler/EntryOutputStream.java +++ b/src/main/java/io/ib67/sfcraft/bundler/EntryOutputStream.java @@ -20,10 +20,6 @@ public class EntryOutputStream extends FilterOutputStream { super(out); } - public void writeMagic() throws IOException { - out.write(BundlerEntry.MAGIC); - } - public void write(BundlerEntry entry) throws IOException { var fileName = entry.fileName().getBytes(StandardCharsets.UTF_8); var buf = ByteBuffer.allocate(4).putInt(entry.unGzipped() ? fileName.length * -1 : fileName.length); diff --git a/src/main/java/io/ib67/sfcraft/bundler/Main.java b/src/main/java/io/ib67/sfcraft/bundler/Main.java new file mode 100644 index 0000000..0b2ff51 --- /dev/null +++ b/src/main/java/io/ib67/sfcraft/bundler/Main.java @@ -0,0 +1,56 @@ +package io.ib67.sfcraft.bundler; + +import io.ib67.kiwi.ArgOpts; +import io.ib67.sfcraft.WorldDir; + +import java.nio.file.Files; +import java.nio.file.Path; + +public class Main { + public static void main(String[] args) { + var opts = ArgOpts.builder() + .args(args).description("Tools for .jpack format bundle").programName("jpack").build(); + var in = opts.string("in", "Input. Can be directory or jpack bundle", null); + var out = opts.string("out", "Output. Can be jpack bundle or directory", null); + var help = opts.bool("help", false); + var allowGunzip = opts.bool("gunzip", + "Should we recompress NBT files for maximum compression rate (depends on compress)", true); + var compress = opts.bool("compress", "Should we compress?", true); + var parallel = opts.bool("parallel", "Use parallelized bundler. This conflicts with compress options", false); + if (help) { + opts.printHelp(System.out); + return; + } + if (allowGunzip && !compress) { + System.out.println("allowGunzip depends on compress feature."); + return; + } + if (in == null || out == null) { + System.out.println("Both in and out should be provided"); + return; + } + var inPath = Path.of(in); + var outPath = Path.of(out); + if (!Files.exists(inPath)) { + System.out.println("The input isn't exists."); + return; + } + if (Files.exists(outPath)) { + System.out.println("The output is already exists. Please move it to other places"); + return; + } + if (in.endsWith(".jpack")) { + Bundle.unbundleFile(inPath, outPath); + } else if (out.endsWith(".jpack")) { + if (!Files.isDirectory(inPath)) + throw new IllegalArgumentException("Your input should be a world directory."); + new Bundle(new WorldDir(inPath)) + .allowGunzip(allowGunzip) + .compress(compress) + .allowParallel(parallel) + .buildBundle(outPath); + } else { + System.out.println("Invalid arguments. You should provide a pair of jpack and directory to act as in/output."); + } + } +} diff --git a/src/main/java/io/ib67/sfcraft/bundler/SerialBundlerReader.java b/src/main/java/io/ib67/sfcraft/bundler/SerialBundlerReader.java index 334f507..cb25764 100644 --- a/src/main/java/io/ib67/sfcraft/bundler/SerialBundlerReader.java +++ b/src/main/java/io/ib67/sfcraft/bundler/SerialBundlerReader.java @@ -19,31 +19,37 @@ public class SerialBundlerReader implements Closeable { protected final EntryInputStream metadataIn; protected final byte[] buffer = new byte[4096]; protected boolean entryRead; + protected int entries; - public SerialBundlerReader(InputStream in, InputStream metadataIn) { - this.in = Objects.requireNonNull(in); - this.metadataIn = new EntryInputStream(Objects.requireNonNull(metadataIn)); - inChannel = null; - } +// public SerialBundlerReader(InputStream in, InputStream metadataIn) { +// this.in = Objects.requireNonNull(in); +// this.metadataIn = new EntryInputStream(Objects.requireNonNull(metadataIn)); +// inChannel = null; +// } public SerialBundlerReader(Path path) throws IOException { var raf = new RandomAccessFile(path.toFile(), "r"); var totalLen = Files.size(path); + var magic = raf.readInt(); + var compressed = magic == Bundle.COMPRESSED_MAGIC; + if (!compressed && magic != Bundle.MAGIC) throw new IOException("Invalid bundle file"); + + // read length raf.seek(totalLen - 4); var metadataLength = raf.readInt(); - var metadataBegin = totalLen - 4 - metadataLength; + raf.seek(totalLen - 8); + entries = raf.readInt(); + // read metadata + var metadataBegin = totalLen - 8 - metadataLength; var metadataBytes = new byte[metadataLength]; raf.seek(metadataBegin); raf.readFully(metadataBytes); this.metadataIn = new EntryInputStream(new ZstdInputStreamNoFinalizer(new ByteArrayInputStream(metadataBytes))); - if (ByteBuffer.wrap(metadataIn.readNBytes(4)).getInt() != BundlerEntry.MAGIC) { - throw new IllegalArgumentException("Invalid Bundler entry"); - } // check compressed - raf.seek(0); + raf.seek(4); var fis = Channels.newInputStream(raf.getChannel()); - if (raf.readInt() == Zstd.magicNumber()) { + if (compressed) { in = new ZstdInputStreamNoFinalizer(fis); inChannel = null; } else { @@ -54,12 +60,14 @@ public class SerialBundlerReader implements Closeable { @SneakyThrows public boolean hasEntry() { - return metadataIn.available() > 0; + return entries > 0; } public BundlerEntry readEntry() throws IOException { + if (!hasEntry()) throw new IOException("All entries are read"); var entry = metadataIn.readEntry(); entryRead = true; + entries--; return entry; } diff --git a/src/main/java/io/ib67/sfcraft/bundler/SerialBundlerWriter.java b/src/main/java/io/ib67/sfcraft/bundler/SerialBundlerWriter.java index 7ca2306..6dae16f 100644 --- a/src/main/java/io/ib67/sfcraft/bundler/SerialBundlerWriter.java +++ b/src/main/java/io/ib67/sfcraft/bundler/SerialBundlerWriter.java @@ -13,6 +13,7 @@ import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.zip.GZIPInputStream; @@ -28,12 +29,14 @@ public class SerialBundlerWriter implements Closeable { public SerialBundlerWriter(OutputStream out) { Objects.requireNonNull(out); this.out = out; + this.entries = new ArrayList<>(); channel = null; } public SerialBundlerWriter(FileChannel channel, long offset) { Objects.requireNonNull(channel); this.offset = offset; + this.entries = new ArrayList<>(); this.channel = channel; out = null; } @@ -50,10 +53,12 @@ public class SerialBundlerWriter implements Closeable { public void writePlain(Path path, String fileName) throws IOException { if (channel != null) { writeZC(path, fileName); - return; - } - try (var in = Files.newInputStream(path)) { - in.transferTo(out); + } else { + var current = offset; + try (var in = Files.newInputStream(path)) { + in.transferTo(out); + } + entries.add(new BundlerEntry(fileName, false, Files.size(path), current)); } } diff --git a/src/main/java/io/ib67/sfcraft/config/JustBackupConfig.java b/src/main/java/io/ib67/sfcraft/config/JustBackupConfig.java index 0d1b46d..2a78954 100644 --- a/src/main/java/io/ib67/sfcraft/config/JustBackupConfig.java +++ b/src/main/java/io/ib67/sfcraft/config/JustBackupConfig.java @@ -7,8 +7,9 @@ import java.util.List; public record JustBackupConfig( long backupIntervalMinutes, int keepBackups, - boolean allowParallel, + boolean allowGunzip, boolean compress, + String temporaryBackupDir, StorageOption option ) { public JustBackupConfig { diff --git a/src/main/java/io/ib67/sfcraft/config/StorageOption.java b/src/main/java/io/ib67/sfcraft/config/StorageOption.java index bdfc27a..20726d3 100644 --- a/src/main/java/io/ib67/sfcraft/config/StorageOption.java +++ b/src/main/java/io/ib67/sfcraft/config/StorageOption.java @@ -1,5 +1,7 @@ package io.ib67.sfcraft.config; +import org.apache.commons.lang3.Validate; + public sealed interface StorageOption { String type(); @@ -8,20 +10,32 @@ public sealed interface StorageOption { String accessKey, String secretKey, String endpoint, - String bucket + String bucket, + String prefix, + boolean enforcePathStyle, + int retryAmount ) implements StorageOption { @Override public String type() { return "s3"; } + + public S3 { + Validate.isTrue(retryAmount >= 0); + } } record Local( - String saveDir + String saveDir, + long diskSizeQuotaBytes ) implements StorageOption { @Override public String type() { return "local"; } + + public Local { + Validate.isTrue(diskSizeQuotaBytes >= 0); + } } } diff --git a/src/main/java/io/ib67/sfcraft/strategy/BackupStrategy.java b/src/main/java/io/ib67/sfcraft/strategy/BackupStrategy.java index 89be876..4745c8f 100644 --- a/src/main/java/io/ib67/sfcraft/strategy/BackupStrategy.java +++ b/src/main/java/io/ib67/sfcraft/strategy/BackupStrategy.java @@ -2,16 +2,20 @@ package io.ib67.sfcraft.strategy; import io.ib67.sfcraft.Backup; import io.ib67.sfcraft.WorldDir; +import io.ib67.sfcraft.bundler.Bundle; import java.nio.file.Path; public interface BackupStrategy { /** * create a backup - * @param dir world to be bundled * @return a key to this backup */ - Backup createBackup(WorldDir dir, boolean compress); + Backup createBackup(Path source, Path bundle); void recoverBackup(Backup backup, Path restorePath); + + void deleteBackup(Backup backup); + + boolean isAvailable(); } diff --git a/src/main/java/io/ib67/sfcraft/strategy/BackupTracker.java b/src/main/java/io/ib67/sfcraft/strategy/BackupTracker.java new file mode 100644 index 0000000..8ab1885 --- /dev/null +++ b/src/main/java/io/ib67/sfcraft/strategy/BackupTracker.java @@ -0,0 +1,44 @@ +package io.ib67.sfcraft.strategy; + +import io.ib67.sfcraft.Backup; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public class BackupTracker implements BackupStrategy { + protected final BackupStrategy underlying; + @Getter + protected final Map trackedBackups; + + public BackupTracker(BackupStrategy underlying, Map trackedBackups) { + this.underlying = Objects.requireNonNull(underlying); + this.trackedBackups = trackedBackups == null ? new HashMap<>() : trackedBackups; + } + + @Override + public Backup createBackup(Path from, Path pathToBundle) { + var result = underlying.createBackup(from, pathToBundle); + trackedBackups.put(result.backupKey(), result); + return result; + } + + @Override + public void recoverBackup(Backup backup, Path restorePath) { + underlying.recoverBackup(backup, restorePath); + } + + @Override + public void deleteBackup(Backup backup) { + underlying.deleteBackup(backup); + trackedBackups.remove(backup.backupKey()); + } + + @Override + public boolean isAvailable() { + return underlying.isAvailable(); + } +} diff --git a/src/main/java/io/ib67/sfcraft/strategy/LocalBackupStrategy.java b/src/main/java/io/ib67/sfcraft/strategy/LocalBackupStrategy.java index cdc1d9c..8a39fc7 100644 --- a/src/main/java/io/ib67/sfcraft/strategy/LocalBackupStrategy.java +++ b/src/main/java/io/ib67/sfcraft/strategy/LocalBackupStrategy.java @@ -4,14 +4,12 @@ import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdOutputStreamNoFinalizer; import io.ib67.sfcraft.Backup; import io.ib67.sfcraft.WorldDir; -import io.ib67.sfcraft.bundler.BundlerEntry; -import io.ib67.sfcraft.bundler.EntryOutputStream; -import io.ib67.sfcraft.bundler.SerialBundlerReader; -import io.ib67.sfcraft.bundler.SerialBundlerWriter; +import io.ib67.sfcraft.bundler.*; import io.ib67.sfcraft.config.StorageOption; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang3.Validate; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -25,80 +23,61 @@ import java.time.format.DateTimeFormatter; import java.util.*; @Log4j2 -@RequiredArgsConstructor public class LocalBackupStrategy implements BackupStrategy { protected final StorageOption.Local option; - protected final Path backupParentRoot = Path.of(option.saveDir()); - protected final boolean allowGunzip; + protected final Path backupParentRoot; + + @SneakyThrows + public LocalBackupStrategy(StorageOption.Local option) { + this.option = option; + backupParentRoot = Path.of(option.saveDir()); + Files.createDirectories(backupParentRoot); + } @Override @SneakyThrows - public Backup createBackup(WorldDir dir, boolean compress) { - var sortedEntries = dir.everything().stream().sorted((a, b) -> { - boolean aMatch = a.endsWith(".mca") || a.endsWith(".dat"); - boolean bMatch = b.endsWith(".mca") || b.endsWith(".dat"); - - if (aMatch && !bMatch) return -1; - if (!aMatch && bMatch) return 1; - return a.compareTo(b); - }).map(dir.root()::relativize).toList(); - var backupName = "Backup_" + DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(LocalDateTime.now()); - var backupFile = backupParentRoot.resolve(backupName + ".jbp"); - SerialBundlerWriter writer; - if (compress) { - var out = new ZstdOutputStreamNoFinalizer(Files.newOutputStream(backupFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE)); - writer = new SerialBundlerWriter(out); - } else { - var chan = FileChannel.open(backupFile); - writer = new SerialBundlerWriter(chan, 0); - } - writeEntries(sortedEntries, writer, allowGunzip && compress); - writer.close(); - - try (var backup = FileChannel.open(backupFile); - var bos = new ByteArrayOutputStream(); - var metadata = new EntryOutputStream(new ZstdOutputStreamNoFinalizer(bos))) { - metadata.writeMagic(); - for (BundlerEntry entry : writer.getEntries()) { - metadata.write(entry); - } - var result = bos.toByteArray(); - var buffer = ByteBuffer.allocate(result.length + 4); - buffer.put(result); - buffer.putLong(result.length); - backup.write(buffer); - } - return new Backup(backupName, backupFile.getFileName().toString(), false, Files.size(backupFile)); + public Backup createBackup(Path from, Path pathToBundle) { + var result = new Backup( + pathToBundle.getFileName().toString(), + pathToBundle.getFileName().toString(), + "local", from.toString(), false, Files.size(pathToBundle)); + Files.move(pathToBundle, backupParentRoot.resolve(pathToBundle.getFileName())); + return result; } - private void writeEntries( - List sortedEntries, - SerialBundlerWriter writer, - boolean allowGunzip) throws IOException { - for (Path sortedEntry : sortedEntries) { - var fileName = sortedEntry.getFileName(); - var shouldGunzip = allowGunzip && (fileName.endsWith("mca") || fileName.endsWith("dat")); - if (shouldGunzip) { - writer.writeGunzip(sortedEntry, sortedEntry.toString()); - } else { - writer.writePlain(sortedEntry, sortedEntry.toString()); - } - } - } - - @Override @SneakyThrows public void recoverBackup(Backup backup, Path restorePath) { // check if it is compressed var backupFile = backupParentRoot.resolve(backup.backupKey()); if (Files.notExists(backupFile)) throw new IllegalArgumentException("Backup " + backupFile + " does not exist"); - try (var bundlerReader = new SerialBundlerReader(backupFile)) { - while (bundlerReader.hasEntry()) { - var entry = bundlerReader.readEntry(); - var target = restorePath.resolve(entry.fileName()); - bundlerReader.writeTo(FileChannel.open(target), entry); + Bundle.unbundleFile(backupFile, restorePath); + } + + @SneakyThrows + @Override + public void deleteBackup(Backup backup) { + Validate.isTrue("local".equals(backup.backupKey())); + var bundlePath = backupParentRoot.resolve(backup.backupKey()); + Files.deleteIfExists(bundlePath); + } + + @Override + @SneakyThrows + public boolean isAvailable() { + if (option.diskSizeQuotaBytes() == 0) return true; + try (var f = Files.walk(backupParentRoot)) { + if (f.mapToLong(it->{ + try { + return Files.size(it); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).sum() >= option.diskSizeQuotaBytes()) { + log.warn("Have no space to set another backup! "); + return false; } } + return true; } } diff --git a/src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java b/src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java new file mode 100644 index 0000000..e67f6fb --- /dev/null +++ b/src/main/java/io/ib67/sfcraft/strategy/S3BackupStrategy.java @@ -0,0 +1,104 @@ +package io.ib67.sfcraft.strategy; + +import io.ib67.sfcraft.Backup; +import io.ib67.sfcraft.WorldDir; +import io.ib67.sfcraft.bundler.Bundle; +import io.ib67.sfcraft.config.StorageOption; +import io.minio.BucketExistsArgs; +import io.minio.MinioClient; +import io.minio.RemoveObjectArgs; +import io.minio.UploadObjectArgs; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +@Log4j2 +public class S3BackupStrategy implements BackupStrategy { + protected final StorageOption.S3 option; + protected final MinioClient minio; + protected final AtomicBoolean uploading = new AtomicBoolean(false); + + @SneakyThrows + public S3BackupStrategy(StorageOption.S3 option) { + this.option = option; + this.minio = MinioClient.builder() + .endpoint(URI.create(option.endpoint()).toURL()) + .credentials(option.accessKey(), option.secretKey()) + .region(option.region()) + .build(); + if (option.enforcePathStyle()) { + minio.disableVirtualStyleEndpoint(); + } else { + minio.enableVirtualStyleEndpoint(); + } + log.info("Testing S3 credentials"); + var exists = minio.bucketExists(BucketExistsArgs.builder().bucket(option.bucket()).build()); + if (!exists) throw new IllegalStateException("Bucket '" + option.bucket() + "' does not exist"); + } + + @Override + @SneakyThrows + public Backup createBackup(Path from, Path pathToBundle) { + if (!uploading.compareAndSet(false, true)) { + throw new IllegalStateException("Last upload is not done yet! keep waiting..."); + } + int i = 0; + do { + try { + var object = option.prefix() + "/" + pathToBundle.getFileName().toString(); + minio.uploadObject(UploadObjectArgs.builder() + .contentType("application/octet-stream") + .object(object) + .filename(pathToBundle.toString()) + .build()); + //todo also calculate sha? + return new Backup(pathToBundle.getFileName().toString(), + object, "s3", + from.toString(), + false, + Files.size(pathToBundle)); + } catch (IOException e) { + if (i >= option.retryAmount()) { + throw new IllegalStateException("Cannot upload " + pathToBundle + " to S3"); + } + var nextTry = Math.pow(2, i++); + log.error("Error uploading S3. trying in {}s later. ({}/{})", nextTry, i, option.retryAmount(), e); + Thread.sleep((long) (nextTry * 1000)); + } catch (Exception e) { + throw new IllegalStateException("Unrecoverable exception occurred when uploading " + pathToBundle, e); + } finally { + uploading.set(false); + } + } while (true); + } + + @Override + public void recoverBackup(Backup backup, Path restorePath) { + //todo http range + } + + @Override + @SneakyThrows + public void deleteBackup(Backup backup) { + minio.removeObject(RemoveObjectArgs.builder() + .bucket(backup.backupKey()).build()); + } + + @Override + public boolean isAvailable() { + if (uploading.get()) { + log.warn("Last S3 Backup is still uploading!"); + return false; + } + return true; + } +}